Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

33 files changed, 17092 insertions, 0 deletions

diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
new file mode 100644
index 000000000..30f90e383
--- /dev/null
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "rustc_mir_build"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc_arena = { path = "../rustc_arena" }
+tracing = "0.1"
+rustc_middle = { path = "../rustc_middle" }
+rustc_apfloat = { path = "../rustc_apfloat" }
+rustc_attr = { path = "../rustc_attr" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_index = { path = "../rustc_index" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_infer = { path = "../rustc_infer" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+rustc_trait_selection = { path = "../rustc_trait_selection" }
+rustc_ast = { path = "../rustc_ast" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
new file mode 100644
index 000000000..687560012
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -0,0 +1,240 @@
+use crate::build::matches::ArmHasGuard;
+use crate::build::ForGuard::OutsideGuard;
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use rustc_middle::thir::*;
+use rustc_middle::{mir::*, ty};
+use rustc_span::Span;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    pub(crate) fn ast_block(
+        &mut self,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+        ast_block: &Block,
+        source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let Block {
+            region_scope,
+            opt_destruction_scope,
+            span,
+            ref stmts,
+            expr,
+            targeted_by_break,
+            safety_mode,
+        } = *ast_block;
+        let expr = expr.map(|expr| &self.thir[expr]);
+        self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
+            this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
+                if targeted_by_break {
+                    this.in_breakable_scope(None, destination, span, |this| {
+                        Some(this.ast_block_stmts(
+                            destination,
+                            block,
+                            span,
+                            &stmts,
+                            expr,
+                            safety_mode,
+                        ))
+                    })
+                } else {
+                    this.ast_block_stmts(destination, block, span, &stmts, expr, safety_mode)
+                }
+            })
+        })
+    }
+
+    fn ast_block_stmts(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        span: Span,
+        stmts: &[StmtId],
+        expr: Option<&Expr<'tcx>>,
+        safety_mode: BlockSafety,
+    ) -> BlockAnd<()> {
+        let this = self;
+
+        // This convoluted structure is to avoid using recursion as we walk down a list
+        // of statements. Basically, the structure we get back is something like:
+        //
+        //    let x = <init> in {
+        //       expr1;
+        //       let y = <init> in {
+        //           expr2;
+        //           expr3;
+        //           ...
+        //       }
+        //    }
+        //
+        // The let bindings are valid till the end of block so all we have to do is to pop all
+        // the let-scopes at the end.
+        //
+        // First we build all the statements in the block.
+        let mut let_scope_stack = Vec::with_capacity(8);
+        let outer_source_scope = this.source_scope;
+        let outer_in_scope_unsafe = this.in_scope_unsafe;
+        this.update_source_scope_for_safety_mode(span, safety_mode);
+
+        let source_info = this.source_info(span);
+        for stmt in stmts {
+            let Stmt { ref kind, opt_destruction_scope } = this.thir[*stmt];
+            match kind {
+                StmtKind::Expr { scope, expr } => {
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
+                    unpack!(
+                        block = this.in_opt_scope(
+                            opt_destruction_scope.map(|de| (de, source_info)),
+                            |this| {
+                                let si = (*scope, source_info);
+                                this.in_scope(si, LintLevel::Inherited, |this| {
+                                    this.stmt_expr(block, &this.thir[*expr], Some(*scope))
+                                })
+                            }
+                        )
+                    );
+                }
+                StmtKind::Let {
+                    remainder_scope,
+                    init_scope,
+                    ref pattern,
+                    initializer,
+                    lint_level,
+                    else_block,
+                } => {
+                    let ignores_expr_result = matches!(*pattern.kind, PatKind::Wild);
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result });
+
+                    // Enter the remainder scope, i.e., the bindings' destruction scope.
+                    this.push_scope((*remainder_scope, source_info));
+                    let_scope_stack.push(remainder_scope);
+
+                    // Declare the bindings, which may create a source scope.
+                    let remainder_span = remainder_scope.span(this.tcx, this.region_scope_tree);
+
+                    let visibility_scope =
+                        Some(this.new_source_scope(remainder_span, LintLevel::Inherited, None));
+
+                    // Evaluate the initializer, if present.
+                    if let Some(init) = initializer {
+                        let init = &this.thir[*init];
+                        let initializer_span = init.span;
+
+                        unpack!(
+                            block = this.in_opt_scope(
+                                opt_destruction_scope.map(|de| (de, source_info)),
+                                |this| {
+                                    let scope = (*init_scope, source_info);
+                                    this.in_scope(scope, *lint_level, |this| {
+                                        if let Some(else_block) = else_block {
+                                            this.ast_let_else(
+                                                block,
+                                                init,
+                                                initializer_span,
+                                                else_block,
+                                                visibility_scope,
+                                                *remainder_scope,
+                                                remainder_span,
+                                                pattern,
+                                            )
+                                        } else {
+                                            this.declare_bindings(
+                                                visibility_scope,
+                                                remainder_span,
+                                                pattern,
+                                                ArmHasGuard(false),
+                                                Some((None, initializer_span)),
+                                            );
+                                            this.expr_into_pattern(block, pattern.clone(), init) // irrefutable pattern
+                                        }
+                                    })
+                                },
+                            )
+                        )
+                    } else {
+                        let scope = (*init_scope, source_info);
+                        unpack!(this.in_scope(scope, *lint_level, |this| {
+                            this.declare_bindings(
+                                visibility_scope,
+                                remainder_span,
+                                pattern,
+                                ArmHasGuard(false),
+                                None,
+                            );
+                            block.unit()
+                        }));
+
+                        debug!("ast_block_stmts: pattern={:?}", pattern);
+                        this.visit_primary_bindings(
+                            pattern,
+                            UserTypeProjections::none(),
+                            &mut |this, _, _, _, node, span, _, _| {
+                                this.storage_live_binding(block, node, span, OutsideGuard, true);
+                                this.schedule_drop_for_binding(node, span, OutsideGuard);
+                            },
+                        )
+                    }
+
+                    // Enter the visibility scope, after evaluating the initializer.
+                    if let Some(source_scope) = visibility_scope {
+                        this.source_scope = source_scope;
+                    }
+                }
+            }
+
+            let popped = this.block_context.pop();
+            assert!(popped.map_or(false, |bf| bf.is_statement()));
+        }
+
+        // Then, the block may have an optional trailing expression which is a “return” value
+        // of the block, which is stored into `destination`.
+        let tcx = this.tcx;
+        let destination_ty = destination.ty(&this.local_decls, tcx).ty;
+        if let Some(expr) = expr {
+            let tail_result_is_ignored =
+                destination_ty.is_unit() || this.block_context.currently_ignores_tail_results();
+            this.block_context
+                .push(BlockFrame::TailExpr { tail_result_is_ignored, span: expr.span });
+
+            unpack!(block = this.expr_into_dest(destination, block, expr));
+            let popped = this.block_context.pop();
+
+            assert!(popped.map_or(false, |bf| bf.is_tail_expr()));
+        } else {
+            // If a block has no trailing expression, then it is given an implicit return type.
+            // This return type is usually `()`, unless the block is diverging, in which case the
+            // return type is `!`. For the unit type, we need to actually return the unit, but in
+            // the case of `!`, no return value is required, as the block will never return.
+            // Opaque types of empty bodies also need this unit assignment, in order to infer that their
+            // type is actually unit. Otherwise there will be no defining use found in the MIR.
+            if destination_ty.is_unit() || matches!(destination_ty.kind(), ty::Opaque(..)) {
+                // We only want to assign an implicit `()` as the return value of the block if the
+                // block does not diverge. (Otherwise, we may try to assign a unit to a `!`-type.)
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+            }
+        }
+        // Finally, we pop all the let scopes before exiting out from the scope of block
+        // itself.
+        for scope in let_scope_stack.into_iter().rev() {
+            unpack!(block = this.pop_scope((*scope, source_info), block));
+        }
+        // Restore the original source scope.
+        this.source_scope = outer_source_scope;
+        this.in_scope_unsafe = outer_in_scope_unsafe;
+        block.unit()
+    }
+
+    /// If we are entering an unsafe block, create a new source scope
+    fn update_source_scope_for_safety_mode(&mut self, span: Span, safety_mode: BlockSafety) {
+        debug!("update_source_scope_for({:?}, {:?})", span, safety_mode);
+        let new_unsafety = match safety_mode {
+            BlockSafety::Safe => return,
+            BlockSafety::BuiltinUnsafe => Safety::BuiltinUnsafe,
+            BlockSafety::ExplicitUnsafe(hir_id) => {
+                self.in_scope_unsafe = Safety::ExplicitUnsafe(hir_id);
+                Safety::ExplicitUnsafe(hir_id)
+            }
+        };
+
+        self.source_scope = self.new_source_scope(span, LintLevel::Inherited, Some(new_unsafety));
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
new file mode 100644
index 000000000..d7b4b1f73
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -0,0 +1,113 @@
+//! Routines for manipulating the control-flow graph.
+
+use crate::build::CFG;
+use rustc_middle::mir::*;
+use rustc_middle::ty::TyCtxt;
+
+impl<'tcx> CFG<'tcx> {
+    pub(crate) fn block_data(&self, blk: BasicBlock) -> &BasicBlockData<'tcx> {
+        &self.basic_blocks[blk]
+    }
+
+    pub(crate) fn block_data_mut(&mut self, blk: BasicBlock) -> &mut BasicBlockData<'tcx> {
+        &mut self.basic_blocks[blk]
+    }
+
+    // llvm.org/PR32488 makes this function use an excess of stack space. Mark
+    // it as #[inline(never)] to keep rustc's stack use in check.
+    #[inline(never)]
+    pub(crate) fn start_new_block(&mut self) -> BasicBlock {
+        self.basic_blocks.push(BasicBlockData::new(None))
+    }
+
+    pub(crate) fn start_new_cleanup_block(&mut self) -> BasicBlock {
+        let bb = self.start_new_block();
+        self.block_data_mut(bb).is_cleanup = true;
+        bb
+    }
+
+    pub(crate) fn push(&mut self, block: BasicBlock, statement: Statement<'tcx>) {
+        debug!("push({:?}, {:?})", block, statement);
+        self.block_data_mut(block).statements.push(statement);
+    }
+
+    pub(crate) fn push_assign(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        rvalue: Rvalue<'tcx>,
+    ) {
+        self.push(
+            block,
+            Statement { source_info, kind: StatementKind::Assign(Box::new((place, rvalue))) },
+        );
+    }
+
+    pub(crate) fn push_assign_constant(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        temp: Place<'tcx>,
+        constant: Constant<'tcx>,
+    ) {
+        self.push_assign(
+            block,
+            source_info,
+            temp,
+            Rvalue::Use(Operand::Constant(Box::new(constant))),
+        );
+    }
+
+    pub(crate) fn push_assign_unit(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) {
+        self.push_assign(
+            block,
+            source_info,
+            place,
+            Rvalue::Use(Operand::Constant(Box::new(Constant {
+                span: source_info.span,
+                user_ty: None,
+                literal: ConstantKind::zero_sized(tcx.types.unit),
+            }))),
+        );
+    }
+
+    pub(crate) fn push_fake_read(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        cause: FakeReadCause,
+        place: Place<'tcx>,
+    ) {
+        let kind = StatementKind::FakeRead(Box::new((cause, place)));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
+    pub(crate) fn terminate(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        kind: TerminatorKind<'tcx>,
+    ) {
+        debug!("terminating block {:?} <- {:?}", block, kind);
+        debug_assert!(
+            self.block_data(block).terminator.is_none(),
+            "terminate: block {:?}={:?} already has a terminator set",
+            block,
+            self.block_data(block)
+        );
+        self.block_data_mut(block).terminator = Some(Terminator { source_info, kind });
+    }
+
+    /// In the `origin` block, push a `goto -> target` terminator.
+    pub(crate) fn goto(&mut self, origin: BasicBlock, source_info: SourceInfo, target: BasicBlock) {
+        self.terminate(origin, source_info, TerminatorKind::Goto { target })
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
new file mode 100644
index 000000000..648d10b9e
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -0,0 +1,152 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::{parse_float_into_constval, Builder};
+use rustc_ast as ast;
+use rustc_hir::def_id::DefId;
+use rustc_middle::mir::interpret::{
+    Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
+};
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt};
+use rustc_target::abi::Size;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a compile-time constant. Assumes that
+    /// `expr` is a valid compile-time constant!
+    pub(crate) fn as_constant(&mut self, expr: &Expr<'tcx>) -> Constant<'tcx> {
+        let create_uneval_from_def_id =
+            |tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, substs: SubstsRef<'tcx>| {
+                let uneval = ty::Unevaluated::new(ty::WithOptConstParam::unknown(def_id), substs);
+                tcx.mk_const(ty::ConstS { kind: ty::ConstKind::Unevaluated(uneval), ty })
+            };
+
+        let this = self;
+        let tcx = this.tcx;
+        let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
+        match *kind {
+            ExprKind::Scope { region_scope: _, lint_level: _, value } => {
+                this.as_constant(&this.thir[value])
+            }
+            ExprKind::Literal { lit, neg } => {
+                let literal =
+                    match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg }) {
+                        Ok(c) => c,
+                        Err(LitToConstError::Reported) => ConstantKind::Ty(tcx.const_error(ty)),
+                        Err(LitToConstError::TypeError) => {
+                            bug!("encountered type error in `lit_to_mir_constant")
+                        }
+                    };
+
+                Constant { span, user_ty: None, literal }
+            }
+            ExprKind::NonHirLiteral { lit, user_ty } => {
+                let user_ty = user_ty.map(|user_ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span,
+                        user_ty,
+                        inferred_ty: ty,
+                    })
+                });
+                let literal = ConstantKind::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
+
+                Constant { span, user_ty: user_ty, literal }
+            }
+            ExprKind::ZstLiteral { user_ty } => {
+                let user_ty = user_ty.map(|user_ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span,
+                        user_ty,
+                        inferred_ty: ty,
+                    })
+                });
+                let literal = ConstantKind::Val(ConstValue::ZeroSized, ty);
+
+                Constant { span, user_ty: user_ty, literal }
+            }
+            ExprKind::NamedConst { def_id, substs, user_ty } => {
+                let user_ty = user_ty.map(|user_ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span,
+                        user_ty,
+                        inferred_ty: ty,
+                    })
+                });
+                let literal = ConstantKind::Ty(create_uneval_from_def_id(tcx, def_id, ty, substs));
+
+                Constant { user_ty, span, literal }
+            }
+            ExprKind::ConstParam { param, def_id: _ } => {
+                let const_param =
+                    tcx.mk_const(ty::ConstS { kind: ty::ConstKind::Param(param), ty: expr.ty });
+                let literal = ConstantKind::Ty(const_param);
+
+                Constant { user_ty: None, span, literal }
+            }
+            ExprKind::ConstBlock { did: def_id, substs } => {
+                let literal = ConstantKind::Ty(create_uneval_from_def_id(tcx, def_id, ty, substs));
+
+                Constant { user_ty: None, span, literal }
+            }
+            ExprKind::StaticRef { alloc_id, ty, .. } => {
+                let const_val = ConstValue::Scalar(Scalar::from_pointer(alloc_id.into(), &tcx));
+                let literal = ConstantKind::Val(const_val, ty);
+
+                Constant { span, user_ty: None, literal }
+            }
+            _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
+        }
+    }
+}
+
+#[instrument(skip(tcx, lit_input))]
+pub(crate) fn lit_to_mir_constant<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    lit_input: LitToConstInput<'tcx>,
+) -> Result<ConstantKind<'tcx>, LitToConstError> {
+    let LitToConstInput { lit, ty, neg } = lit_input;
+    let trunc = |n| {
+        let param_ty = ty::ParamEnv::reveal_all().and(ty);
+        let width = tcx.layout_of(param_ty).map_err(|_| LitToConstError::Reported)?.size;
+        trace!("trunc {} with size {} and shift {}", n, width.bits(), 128 - width.bits());
+        let result = width.truncate(n);
+        trace!("trunc result: {}", result);
+        Ok(ConstValue::Scalar(Scalar::from_uint(result, width)))
+    };
+
+    let value = match (lit, &ty.kind()) {
+        (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
+            let s = s.as_str();
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
+            let allocation = tcx.intern_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, start: 0, end: s.len() }
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _))
+            if matches!(inner_ty.kind(), ty::Slice(_)) =>
+        {
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = tcx.intern_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, start: 0, end: data.len() }
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
+            let id = tcx.allocate_bytes(data);
+            ConstValue::Scalar(Scalar::from_pointer(id.into(), &tcx))
+        }
+        (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
+            ConstValue::Scalar(Scalar::from_uint(*n, Size::from_bytes(1)))
+        }
+        (ast::LitKind::Int(n, _), ty::Uint(_)) | (ast::LitKind::Int(n, _), ty::Int(_)) => {
+            trunc(if neg { (*n as i128).overflowing_neg().0 as u128 } else { *n })?
+        }
+        (ast::LitKind::Float(n, _), ty::Float(fty)) => {
+            parse_float_into_constval(*n, *fty, neg).ok_or(LitToConstError::Reported)?
+        }
+        (ast::LitKind::Bool(b), ty::Bool) => ConstValue::Scalar(Scalar::from_bool(*b)),
+        (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
+        (ast::LitKind::Err(_), _) => return Err(LitToConstError::Reported),
+        _ => return Err(LitToConstError::TypeError),
+    };
+
+    Ok(ConstantKind::Val(value, ty))
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
new file mode 100644
index 000000000..e707c373f
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
@@ -0,0 +1,184 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::Category;
+use crate::build::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Returns an operand suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    pub(crate) fn as_local_operand(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let local_scope = self.local_scope();
+        self.as_operand(block, Some(local_scope), expr, None, NeedsTemporary::Maybe)
+    }
+
+    /// Returns an operand suitable for use until the end of the current scope expression and
+    /// suitable also to be passed as function arguments.
+    ///
+    /// The operand returned from this function will *not be valid* after an ExprKind::Scope is
+    /// passed, so please do *not* return it from functions to avoid bad miscompiles.  Returns an
+    /// operand suitable for use as a call argument. This is almost always equivalent to
+    /// `as_operand`, except for the particular case of passing values of (potentially) unsized
+    /// types "by value" (see details below).
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    ///
+    /// # Parameters of unsized types
+    ///
+    /// We tweak the handling of parameters of unsized type slightly to avoid the need to create a
+    /// local variable of unsized type. For example, consider this program:
+    ///
+    /// ```
+    /// #![feature(unsized_locals, unsized_fn_params)]
+    /// # use core::fmt::Debug;
+    /// fn foo(p: dyn Debug) { dbg!(p); }
+    ///
+    /// fn bar(box_p: Box<dyn Debug>) { foo(*box_p); }
+    /// ```
+    ///
+    /// Ordinarily, for sized types, we would compile the call `foo(*p)` like so:
+    ///
+    /// ```ignore (illustrative)
+    /// let tmp0 = *box_p; // tmp0 would be the operand returned by this function call
+    /// foo(tmp0)
+    /// ```
+    ///
+    /// But because the parameter to `foo` is of the unsized type `dyn Debug`, and because it is
+    /// being moved the deref of a box, we compile it slightly differently. The temporary `tmp0`
+    /// that we create *stores the entire box*, and the parameter to the call itself will be
+    /// `*tmp0`:
+    ///
+    /// ```ignore (illustrative)
+    /// let tmp0 = box_p; call foo(*tmp0)
+    /// ```
+    ///
+    /// This way, the temporary `tmp0` that we create has type `Box<dyn Debug>`, which is sized.
+    /// The value passed to the call (`*tmp0`) still has the `dyn Debug` type -- but the way that
+    /// calls are compiled means that this parameter will be passed "by reference", meaning that we
+    /// will actually provide a pointer to the interior of the box, and not move the `dyn Debug`
+    /// value to the stack.
+    ///
+    /// See #68034 for more details.
+    pub(crate) fn as_local_call_operand(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let local_scope = self.local_scope();
+        self.as_call_operand(block, Some(local_scope), expr)
+    }
+
+    /// Compile `expr` into a value that can be used as an operand.
+    /// If `expr` is a place like `x`, this will introduce a
+    /// temporary `tmp = x`, so that we capture the value of `x` at
+    /// this time.
+    ///
+    /// If we end up needing to create a temporary, then we will use
+    /// `local_info` as its `LocalInfo`, unless `as_temporary`
+    /// has already assigned it a non-`None` `LocalInfo`.
+    /// Normally, you should use `None` for `local_info`
+    ///
+    /// The operand is known to be live until the end of `scope`.
+    ///
+    /// Like `as_local_call_operand`, except that the argument will
+    /// not be valid once `scope` ends.
+    #[instrument(level = "debug", skip(self, scope))]
+    pub(crate) fn as_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: &Expr<'tcx>,
+        local_info: Option<Box<LocalInfo<'tcx>>>,
+        needs_temporary: NeedsTemporary,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this.in_scope(region_scope, lint_level, |this| {
+                this.as_operand(block, scope, &this.thir[value], local_info, needs_temporary)
+            });
+        }
+
+        let category = Category::of(&expr.kind).unwrap();
+        debug!(?category, ?expr.kind);
+        match category {
+            Category::Constant if let NeedsTemporary::No = needs_temporary || !expr.ty.needs_drop(this.tcx, this.param_env) => {
+                let constant = this.as_constant(expr);
+                block.and(Operand::Constant(Box::new(constant)))
+            }
+            Category::Constant | Category::Place | Category::Rvalue(..) => {
+                let operand = unpack!(block = this.as_temp(block, scope, expr, Mutability::Mut));
+                if this.local_decls[operand].local_info.is_none() {
+                    this.local_decls[operand].local_info = local_info;
+                }
+                block.and(Operand::Move(Place::from(operand)))
+            }
+        }
+    }
+
+    pub(crate) fn as_call_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        debug!("as_call_operand(block={:?}, expr={:?})", block, expr);
+        let this = self;
+
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this.in_scope(region_scope, lint_level, |this| {
+                this.as_call_operand(block, scope, &this.thir[value])
+            });
+        }
+
+        let tcx = this.tcx;
+
+        if tcx.features().unsized_fn_params {
+            let ty = expr.ty;
+            let span = expr.span;
+            let param_env = this.param_env;
+
+            if !ty.is_sized(tcx.at(span), param_env) {
+                // !sized means !copy, so this is an unsized move
+                assert!(!ty.is_copy_modulo_regions(tcx.at(span), param_env));
+
+                // As described above, detect the case where we are passing a value of unsized
+                // type, and that value is coming from the deref of a box.
+                if let ExprKind::Deref { arg } = expr.kind {
+                    // Generate let tmp0 = arg0
+                    let operand = unpack!(
+                        block = this.as_temp(block, scope, &this.thir[arg], Mutability::Mut)
+                    );
+
+                    // Return the operand *tmp0 to be used as the call argument
+                    let place = Place {
+                        local: operand,
+                        projection: tcx.intern_place_elems(&[PlaceElem::Deref]),
+                    };
+
+                    return block.and(Operand::Move(place));
+                }
+            }
+        }
+
+        this.as_operand(block, scope, expr, None, NeedsTemporary::Maybe)
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
new file mode 100644
index 000000000..0c06aad4e
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -0,0 +1,820 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::Category;
+use crate::build::ForGuard::{OutsideGuard, RefWithinGuard};
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::hir::place::Projection as HirProjection;
+use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
+use rustc_middle::middle::region;
+use rustc_middle::mir::AssertKind::BoundsCheck;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::AdtDef;
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance};
+use rustc_span::Span;
+use rustc_target::abi::VariantIdx;
+
+use rustc_index::vec::Idx;
+
+use std::iter;
+
+/// The "outermost" place that holds this value.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub(crate) enum PlaceBase {
+    /// Denotes the start of a `Place`.
+    Local(Local),
+
+    /// When building place for an expression within a closure, the place might start off a
+    /// captured path. When `capture_disjoint_fields` is enabled, we might not know the capture
+    /// index (within the desugared closure) of the captured path until most of the projections
+    /// are applied. We use `PlaceBase::Upvar` to keep track of the root variable off of which the
+    /// captured path starts, the closure the capture belongs to and the trait the closure
+    /// implements.
+    ///
+    /// Once we have figured out the capture index, we can convert the place builder to start from
+    /// `PlaceBase::Local`.
+    ///
+    /// Consider the following example
+    /// ```rust
+    /// let t = (((10, 10), 10), 10);
+    ///
+    /// let c = || {
+    ///     println!("{}", t.0.0.0);
+    /// };
+    /// ```
+    /// Here the THIR expression for `t.0.0.0` will be something like
+    ///
+    /// ```ignore (illustrative)
+    /// * Field(0)
+    ///     * Field(0)
+    ///         * Field(0)
+    ///             * UpvarRef(t)
+    /// ```
+    ///
+    /// When `capture_disjoint_fields` is enabled, `t.0.0.0` is captured and we won't be able to
+    /// figure out that it is captured until all the `Field` projections are applied.
+    Upvar {
+        /// HirId of the upvar
+        var_hir_id: LocalVarId,
+        /// DefId of the closure
+        closure_def_id: LocalDefId,
+        /// The trait closure implements, `Fn`, `FnMut`, `FnOnce`
+        closure_kind: ty::ClosureKind,
+    },
+}
+
+/// `PlaceBuilder` is used to create places during MIR construction. It allows you to "build up" a
+/// place by pushing more and more projections onto the end, and then convert the final set into a
+/// place using the `into_place` method.
+///
+/// This is used internally when building a place for an expression like `a.b.c`. The fields `b`
+/// and `c` can be progressively pushed onto the place builder that is created when converting `a`.
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct PlaceBuilder<'tcx> {
+    base: PlaceBase,
+    projection: Vec<PlaceElem<'tcx>>,
+}
+
+/// Given a list of MIR projections, convert them to list of HIR ProjectionKind.
+/// The projections are truncated to represent a path that might be captured by a
+/// closure/generator. This implies the vector returned from this function doesn't contain
+/// ProjectionElems `Downcast`, `ConstantIndex`, `Index`, or `Subslice` because those will never be
+/// part of a path that is captured by a closure. We stop applying projections once we see the first
+/// projection that isn't captured by a closure.
+fn convert_to_hir_projections_and_truncate_for_capture<'tcx>(
+    mir_projections: &[PlaceElem<'tcx>],
+) -> Vec<HirProjectionKind> {
+    let mut hir_projections = Vec::new();
+    let mut variant = None;
+
+    for mir_projection in mir_projections {
+        let hir_projection = match mir_projection {
+            ProjectionElem::Deref => HirProjectionKind::Deref,
+            ProjectionElem::Field(field, _) => {
+                let variant = variant.unwrap_or(VariantIdx::new(0));
+                HirProjectionKind::Field(field.index() as u32, variant)
+            }
+            ProjectionElem::Downcast(.., idx) => {
+                // We don't expect to see multi-variant enums here, as earlier
+                // phases will have truncated them already. However, there can
+                // still be downcasts, thanks to single-variant enums.
+                // We keep track of VariantIdx so we can use this information
+                // if the next ProjectionElem is a Field.
+                variant = Some(*idx);
+                continue;
+            }
+            ProjectionElem::Index(..)
+            | ProjectionElem::ConstantIndex { .. }
+            | ProjectionElem::Subslice { .. } => {
+                // We don't capture array-access projections.
+                // We can stop here as arrays are captured completely.
+                break;
+            }
+        };
+        variant = None;
+        hir_projections.push(hir_projection);
+    }
+
+    hir_projections
+}
+
+/// Return true if the `proj_possible_ancestor` represents an ancestor path
+/// to `proj_capture` or `proj_possible_ancestor` is same as `proj_capture`,
+/// assuming they both start off of the same root variable.
+///
+/// **Note:** It's the caller's responsibility to ensure that both lists of projections
+///           start off of the same root variable.
+///
+/// Eg: 1. `foo.x` which is represented using `projections=[Field(x)]` is an ancestor of
+///        `foo.x.y` which is represented using `projections=[Field(x), Field(y)]`.
+///        Note both `foo.x` and `foo.x.y` start off of the same root variable `foo`.
+///     2. Since we only look at the projections here function will return `bar.x` as an a valid
+///        ancestor of `foo.x.y`. It's the caller's responsibility to ensure that both projections
+///        list are being applied to the same root variable.
+fn is_ancestor_or_same_capture(
+    proj_possible_ancestor: &[HirProjectionKind],
+    proj_capture: &[HirProjectionKind],
+) -> bool {
+    // We want to make sure `is_ancestor_or_same_capture("x.0.0", "x.0")` to return false.
+    // Therefore we can't just check if all projections are same in the zipped iterator below.
+    if proj_possible_ancestor.len() > proj_capture.len() {
+        return false;
+    }
+
+    iter::zip(proj_possible_ancestor, proj_capture).all(|(a, b)| a == b)
+}
+
+/// Computes the index of a capture within the desugared closure provided the closure's
+/// `closure_min_captures` and the capture's index of the capture in the
+/// `ty::MinCaptureList` of the root variable `var_hir_id`.
+fn compute_capture_idx<'tcx>(
+    closure_min_captures: &ty::RootVariableMinCaptureList<'tcx>,
+    var_hir_id: LocalVarId,
+    root_var_idx: usize,
+) -> usize {
+    let mut res = 0;
+    for (var_id, capture_list) in closure_min_captures {
+        if *var_id == var_hir_id.0 {
+            res += root_var_idx;
+            break;
+        } else {
+            res += capture_list.len();
+        }
+    }
+
+    res
+}
+
+/// Given a closure, returns the index of a capture within the desugared closure struct and the
+/// `ty::CapturedPlace` which is the ancestor of the Place represented using the `var_hir_id`
+/// and `projection`.
+///
+/// Note there will be at most one ancestor for any given Place.
+///
+/// Returns None, when the ancestor is not found.
+fn find_capture_matching_projections<'a, 'tcx>(
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    var_hir_id: LocalVarId,
+    closure_def_id: LocalDefId,
+    projections: &[PlaceElem<'tcx>],
+) -> Option<(usize, &'a ty::CapturedPlace<'tcx>)> {
+    let closure_min_captures = typeck_results.closure_min_captures.get(&closure_def_id)?;
+    let root_variable_min_captures = closure_min_captures.get(&var_hir_id.0)?;
+
+    let hir_projections = convert_to_hir_projections_and_truncate_for_capture(projections);
+
+    // If an ancestor is found, `idx` is the index within the list of captured places
+    // for root variable `var_hir_id` and `capture` is the `ty::CapturedPlace` itself.
+    let (idx, capture) = root_variable_min_captures.iter().enumerate().find(|(_, capture)| {
+        let possible_ancestor_proj_kinds: Vec<_> =
+            capture.place.projections.iter().map(|proj| proj.kind).collect();
+        is_ancestor_or_same_capture(&possible_ancestor_proj_kinds, &hir_projections)
+    })?;
+
+    // Convert index to be from the perspective of the entire closure_min_captures map
+    // instead of just the root variable capture list
+    Some((compute_capture_idx(closure_min_captures, var_hir_id, idx), capture))
+}
+
+/// Takes a PlaceBuilder and resolves the upvar (if any) within it, so that the
+/// `PlaceBuilder` now starts from `PlaceBase::Local`.
+///
+/// Returns a Result with the error being the PlaceBuilder (`from_builder`) that was not found.
+fn to_upvars_resolved_place_builder<'a, 'tcx>(
+    from_builder: PlaceBuilder<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
+    match from_builder.base {
+        PlaceBase::Local(_) => Ok(from_builder),
+        PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind } => {
+            let mut upvar_resolved_place_builder = PlaceBuilder::from(ty::CAPTURE_STRUCT_LOCAL);
+            match closure_kind {
+                ty::ClosureKind::Fn | ty::ClosureKind::FnMut => {
+                    upvar_resolved_place_builder = upvar_resolved_place_builder.deref();
+                }
+                ty::ClosureKind::FnOnce => {}
+            }
+
+            let Some((capture_index, capture)) =
+                find_capture_matching_projections(
+                    typeck_results,
+                    var_hir_id,
+                    closure_def_id,
+                    &from_builder.projection,
+                ) else {
+                let closure_span = tcx.def_span(closure_def_id);
+                if !enable_precise_capture(tcx, closure_span) {
+                    bug!(
+                        "No associated capture found for {:?}[{:#?}] even though \
+                            capture_disjoint_fields isn't enabled",
+                        var_hir_id,
+                        from_builder.projection
+                    )
+                } else {
+                    debug!(
+                        "No associated capture found for {:?}[{:#?}]",
+                        var_hir_id, from_builder.projection,
+                    );
+                }
+                return Err(from_builder);
+            };
+
+            // We won't be building MIR if the closure wasn't local
+            let closure_hir_id = tcx.hir().local_def_id_to_hir_id(closure_def_id);
+            let closure_ty = typeck_results.node_type(closure_hir_id);
+
+            let substs = match closure_ty.kind() {
+                ty::Closure(_, substs) => ty::UpvarSubsts::Closure(substs),
+                ty::Generator(_, substs, _) => ty::UpvarSubsts::Generator(substs),
+                _ => bug!("Lowering capture for non-closure type {:?}", closure_ty),
+            };
+
+            // Access the capture by accessing the field within the Closure struct.
+            //
+            // We must have inferred the capture types since we are building MIR, therefore
+            // it's safe to call `tuple_element_ty` and we can unwrap here because
+            // we know that the capture exists and is the `capture_index`-th capture.
+            let var_ty = substs.tupled_upvars_ty().tuple_fields()[capture_index];
+
+            upvar_resolved_place_builder =
+                upvar_resolved_place_builder.field(Field::new(capture_index), var_ty);
+
+            // If the variable is captured via ByRef(Immutable/Mutable) Borrow,
+            // we need to deref it
+            upvar_resolved_place_builder = match capture.info.capture_kind {
+                ty::UpvarCapture::ByRef(_) => upvar_resolved_place_builder.deref(),
+                ty::UpvarCapture::ByValue => upvar_resolved_place_builder,
+            };
+
+            // We used some of the projections to build the capture itself,
+            // now we apply the remaining to the upvar resolved place.
+            let remaining_projections = strip_prefix(
+                capture.place.base_ty,
+                from_builder.projection,
+                &capture.place.projections,
+            );
+            upvar_resolved_place_builder.projection.extend(remaining_projections);
+
+            Ok(upvar_resolved_place_builder)
+        }
+    }
+}
+
+/// Returns projections remaining after stripping an initial prefix of HIR
+/// projections.
+///
+/// Supports only HIR projection kinds that represent a path that might be
+/// captured by a closure or a generator, i.e., an `Index` or a `Subslice`
+/// projection kinds are unsupported.
+fn strip_prefix<'tcx>(
+    mut base_ty: Ty<'tcx>,
+    projections: Vec<PlaceElem<'tcx>>,
+    prefix_projections: &[HirProjection<'tcx>],
+) -> impl Iterator<Item = PlaceElem<'tcx>> {
+    let mut iter = projections.into_iter();
+    for projection in prefix_projections {
+        match projection.kind {
+            HirProjectionKind::Deref => {
+                assert!(matches!(iter.next(), Some(ProjectionElem::Deref)));
+            }
+            HirProjectionKind::Field(..) => {
+                if base_ty.is_enum() {
+                    assert!(matches!(iter.next(), Some(ProjectionElem::Downcast(..))));
+                }
+                assert!(matches!(iter.next(), Some(ProjectionElem::Field(..))));
+            }
+            HirProjectionKind::Index | HirProjectionKind::Subslice => {
+                bug!("unexpected projection kind: {:?}", projection);
+            }
+        }
+        base_ty = projection.ty;
+    }
+    iter
+}
+
+impl<'tcx> PlaceBuilder<'tcx> {
+    pub(crate) fn into_place<'a>(
+        self,
+        tcx: TyCtxt<'tcx>,
+        typeck_results: &'a ty::TypeckResults<'tcx>,
+    ) -> Place<'tcx> {
+        if let PlaceBase::Local(local) = self.base {
+            Place { local, projection: tcx.intern_place_elems(&self.projection) }
+        } else {
+            self.expect_upvars_resolved(tcx, typeck_results).into_place(tcx, typeck_results)
+        }
+    }
+
+    fn expect_upvars_resolved<'a>(
+        self,
+        tcx: TyCtxt<'tcx>,
+        typeck_results: &'a ty::TypeckResults<'tcx>,
+    ) -> PlaceBuilder<'tcx> {
+        to_upvars_resolved_place_builder(self, tcx, typeck_results).unwrap()
+    }
+
+    /// Attempts to resolve the `PlaceBuilder`.
+    /// On success, it will return the resolved `PlaceBuilder`.
+    /// On failure, it will return itself.
+    ///
+    /// Upvars resolve may fail for a `PlaceBuilder` when attempting to
+    /// resolve a disjoint field whose root variable is not captured
+    /// (destructured assignments) or when attempting to resolve a root
+    /// variable (discriminant matching with only wildcard arm) that is
+    /// not captured. This can happen because the final mir that will be
+    /// generated doesn't require a read for this place. Failures will only
+    /// happen inside closures.
+    pub(crate) fn try_upvars_resolved<'a>(
+        self,
+        tcx: TyCtxt<'tcx>,
+        typeck_results: &'a ty::TypeckResults<'tcx>,
+    ) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
+        to_upvars_resolved_place_builder(self, tcx, typeck_results)
+    }
+
+    pub(crate) fn base(&self) -> PlaceBase {
+        self.base
+    }
+
+    pub(crate) fn field(self, f: Field, ty: Ty<'tcx>) -> Self {
+        self.project(PlaceElem::Field(f, ty))
+    }
+
+    pub(crate) fn deref(self) -> Self {
+        self.project(PlaceElem::Deref)
+    }
+
+    pub(crate) fn downcast(self, adt_def: AdtDef<'tcx>, variant_index: VariantIdx) -> Self {
+        self.project(PlaceElem::Downcast(Some(adt_def.variant(variant_index).name), variant_index))
+    }
+
+    fn index(self, index: Local) -> Self {
+        self.project(PlaceElem::Index(index))
+    }
+
+    pub(crate) fn project(mut self, elem: PlaceElem<'tcx>) -> Self {
+        self.projection.push(elem);
+        self
+    }
+}
+
+impl<'tcx> From<Local> for PlaceBuilder<'tcx> {
+    fn from(local: Local) -> Self {
+        Self { base: PlaceBase::Local(local), projection: Vec::new() }
+    }
+}
+
+impl<'tcx> From<PlaceBase> for PlaceBuilder<'tcx> {
+    fn from(base: PlaceBase) -> Self {
+        Self { base, projection: Vec::new() }
+    }
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a place that we can move from etc.
+    ///
+    /// WARNING: Any user code might:
+    /// * Invalidate any slice bounds checks performed.
+    /// * Change the address that this `Place` refers to.
+    /// * Modify the memory that this place refers to.
+    /// * Invalidate the memory that this place refers to, this will be caught
+    ///   by borrow checking.
+    ///
+    /// Extra care is needed if any user code is allowed to run between calling
+    /// this method and using it, as is the case for `match` and index
+    /// expressions.
+    pub(crate) fn as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Place<'tcx>> {
+        let place_builder = unpack!(block = self.as_place_builder(block, expr));
+        block.and(place_builder.into_place(self.tcx, self.typeck_results))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    pub(crate) fn as_place_builder(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        self.expr_as_place(block, expr, Mutability::Mut, None)
+    }
+
+    /// Compile `expr`, yielding a place that we can move from etc.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    pub(crate) fn as_read_only_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Place<'tcx>> {
+        let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr));
+        block.and(place_builder.into_place(self.tcx, self.typeck_results))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    fn as_read_only_place_builder(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        self.expr_as_place(block, expr, Mutability::Not, None)
+    }
+
+    fn expr_as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        debug!("expr_as_place(block={:?}, expr={:?}, mutability={:?})", block, expr, mutability);
+
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    this.expr_as_place(block, &this.thir[value], mutability, fake_borrow_temps)
+                })
+            }
+            ExprKind::Field { lhs, variant_index, name } => {
+                let lhs = &this.thir[lhs];
+                let mut place_builder =
+                    unpack!(block = this.expr_as_place(block, lhs, mutability, fake_borrow_temps,));
+                if let ty::Adt(adt_def, _) = lhs.ty.kind() {
+                    if adt_def.is_enum() {
+                        place_builder = place_builder.downcast(*adt_def, variant_index);
+                    }
+                }
+                block.and(place_builder.field(name, expr.ty))
+            }
+            ExprKind::Deref { arg } => {
+                let place_builder = unpack!(
+                    block =
+                        this.expr_as_place(block, &this.thir[arg], mutability, fake_borrow_temps,)
+                );
+                block.and(place_builder.deref())
+            }
+            ExprKind::Index { lhs, index } => this.lower_index_expression(
+                block,
+                &this.thir[lhs],
+                &this.thir[index],
+                mutability,
+                fake_borrow_temps,
+                expr.temp_lifetime,
+                expr_span,
+                source_info,
+            ),
+            ExprKind::UpvarRef { closure_def_id, var_hir_id } => {
+                this.lower_captured_upvar(block, closure_def_id.expect_local(), var_hir_id)
+            }
+
+            ExprKind::VarRef { id } => {
+                let place_builder = if this.is_bound_var_in_guard(id) {
+                    let index = this.var_local_id(id, RefWithinGuard);
+                    PlaceBuilder::from(index).deref()
+                } else {
+                    let index = this.var_local_id(id, OutsideGuard);
+                    PlaceBuilder::from(index)
+                };
+                block.and(place_builder)
+            }
+
+            ExprKind::PlaceTypeAscription { source, user_ty } => {
+                let place_builder = unpack!(
+                    block = this.expr_as_place(
+                        block,
+                        &this.thir[source],
+                        mutability,
+                        fake_borrow_temps,
+                    )
+                );
+                if let Some(user_ty) = user_ty {
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: source_info.span,
+                            user_ty,
+                            inferred_ty: expr.ty,
+                        });
+
+                    let place = place_builder.clone().into_place(this.tcx, this.typeck_results);
+                    this.cfg.push(
+                        block,
+                        Statement {
+                            source_info,
+                            kind: StatementKind::AscribeUserType(
+                                Box::new((
+                                    place,
+                                    UserTypeProjection { base: annotation_index, projs: vec![] },
+                                )),
+                                Variance::Invariant,
+                            ),
+                        },
+                    );
+                }
+                block.and(place_builder)
+            }
+            ExprKind::ValueTypeAscription { source, user_ty } => {
+                let source = &this.thir[source];
+                let temp =
+                    unpack!(block = this.as_temp(block, source.temp_lifetime, source, mutability));
+                if let Some(user_ty) = user_ty {
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: source_info.span,
+                            user_ty,
+                            inferred_ty: expr.ty,
+                        });
+                    this.cfg.push(
+                        block,
+                        Statement {
+                            source_info,
+                            kind: StatementKind::AscribeUserType(
+                                Box::new((
+                                    Place::from(temp),
+                                    UserTypeProjection { base: annotation_index, projs: vec![] },
+                                )),
+                                Variance::Invariant,
+                            ),
+                        },
+                    );
+                }
+                block.and(PlaceBuilder::from(temp))
+            }
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::Call { .. } => {
+                // these are not places, so we need to make a temporary.
+                debug_assert!(!matches!(Category::of(&expr.kind), Some(Category::Place)));
+                let temp =
+                    unpack!(block = this.as_temp(block, expr.temp_lifetime, expr, mutability));
+                block.and(PlaceBuilder::from(temp))
+            }
+        }
+    }
+
+    /// Lower a captured upvar. Note we might not know the actual capture index,
+    /// so we create a place starting from `PlaceBase::Upvar`, which will be resolved
+    /// once all projections that allow us to identify a capture have been applied.
+    fn lower_captured_upvar(
+        &mut self,
+        block: BasicBlock,
+        closure_def_id: LocalDefId,
+        var_hir_id: LocalVarId,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let closure_ty =
+            self.typeck_results.node_type(self.tcx.hir().local_def_id_to_hir_id(closure_def_id));
+
+        let closure_kind = if let ty::Closure(_, closure_substs) = closure_ty.kind() {
+            self.infcx.closure_kind(closure_substs).unwrap()
+        } else {
+            // Generators are considered FnOnce.
+            ty::ClosureKind::FnOnce
+        };
+
+        block.and(PlaceBuilder::from(PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind }))
+    }
+
+    /// Lower an index expression
+    ///
+    /// This has two complications;
+    ///
+    /// * We need to do a bounds check.
+    /// * We need to ensure that the bounds check can't be invalidated using an
+    ///   expression like `x[1][{x = y; 2}]`. We use fake borrows here to ensure
+    ///   that this is the case.
+    fn lower_index_expression(
+        &mut self,
+        mut block: BasicBlock,
+        base: &Expr<'tcx>,
+        index: &Expr<'tcx>,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+        temp_lifetime: Option<region::Scope>,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let base_fake_borrow_temps = &mut Vec::new();
+        let is_outermost_index = fake_borrow_temps.is_none();
+        let fake_borrow_temps = fake_borrow_temps.unwrap_or(base_fake_borrow_temps);
+
+        let mut base_place =
+            unpack!(block = self.expr_as_place(block, base, mutability, Some(fake_borrow_temps),));
+
+        // Making this a *fresh* temporary means we do not have to worry about
+        // the index changing later: Nothing will ever change this temporary.
+        // The "retagging" transformation (for Stacked Borrows) relies on this.
+        let idx = unpack!(block = self.as_temp(block, temp_lifetime, index, Mutability::Not,));
+
+        block = self.bounds_check(block, base_place.clone(), idx, expr_span, source_info);
+
+        if is_outermost_index {
+            self.read_fake_borrows(block, fake_borrow_temps, source_info)
+        } else {
+            base_place = base_place.expect_upvars_resolved(self.tcx, self.typeck_results);
+            self.add_fake_borrows_of_base(
+                &base_place,
+                block,
+                fake_borrow_temps,
+                expr_span,
+                source_info,
+            );
+        }
+
+        block.and(base_place.index(idx))
+    }
+
+    fn bounds_check(
+        &mut self,
+        block: BasicBlock,
+        slice: PlaceBuilder<'tcx>,
+        index: Local,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BasicBlock {
+        let usize_ty = self.tcx.types.usize;
+        let bool_ty = self.tcx.types.bool;
+        // bounds check:
+        let len = self.temp(usize_ty, expr_span);
+        let lt = self.temp(bool_ty, expr_span);
+
+        // len = len(slice)
+        self.cfg.push_assign(
+            block,
+            source_info,
+            len,
+            Rvalue::Len(slice.into_place(self.tcx, self.typeck_results)),
+        );
+        // lt = idx < len
+        self.cfg.push_assign(
+            block,
+            source_info,
+            lt,
+            Rvalue::BinaryOp(
+                BinOp::Lt,
+                Box::new((Operand::Copy(Place::from(index)), Operand::Copy(len))),
+            ),
+        );
+        let msg = BoundsCheck { len: Operand::Move(len), index: Operand::Copy(Place::from(index)) };
+        // assert!(lt, "...")
+        self.assert(block, Operand::Move(lt), true, msg, expr_span)
+    }
+
+    fn add_fake_borrows_of_base(
+        &mut self,
+        base_place: &PlaceBuilder<'tcx>,
+        block: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) {
+        let tcx = self.tcx;
+        let local = match base_place.base {
+            PlaceBase::Local(local) => local,
+            PlaceBase::Upvar { .. } => bug!("Expected PlacseBase::Local found Upvar"),
+        };
+
+        let place_ty = Place::ty_from(local, &base_place.projection, &self.local_decls, tcx);
+        if let ty::Slice(_) = place_ty.ty.kind() {
+            // We need to create fake borrows to ensure that the bounds
+            // check that we just did stays valid. Since we can't assign to
+            // unsized values, we only need to ensure that none of the
+            // pointers in the base place are modified.
+            for (idx, elem) in base_place.projection.iter().enumerate().rev() {
+                match elem {
+                    ProjectionElem::Deref => {
+                        let fake_borrow_deref_ty = Place::ty_from(
+                            local,
+                            &base_place.projection[..idx],
+                            &self.local_decls,
+                            tcx,
+                        )
+                        .ty;
+                        let fake_borrow_ty =
+                            tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
+                        let fake_borrow_temp =
+                            self.local_decls.push(LocalDecl::new(fake_borrow_ty, expr_span));
+                        let projection = tcx.intern_place_elems(&base_place.projection[..idx]);
+                        self.cfg.push_assign(
+                            block,
+                            source_info,
+                            fake_borrow_temp.into(),
+                            Rvalue::Ref(
+                                tcx.lifetimes.re_erased,
+                                BorrowKind::Shallow,
+                                Place { local, projection },
+                            ),
+                        );
+                        fake_borrow_temps.push(fake_borrow_temp);
+                    }
+                    ProjectionElem::Index(_) => {
+                        let index_ty = Place::ty_from(
+                            local,
+                            &base_place.projection[..idx],
+                            &self.local_decls,
+                            tcx,
+                        );
+                        match index_ty.ty.kind() {
+                            // The previous index expression has already
+                            // done any index expressions needed here.
+                            ty::Slice(_) => break,
+                            ty::Array(..) => (),
+                            _ => bug!("unexpected index base"),
+                        }
+                    }
+                    ProjectionElem::Field(..)
+                    | ProjectionElem::Downcast(..)
+                    | ProjectionElem::ConstantIndex { .. }
+                    | ProjectionElem::Subslice { .. } => (),
+                }
+            }
+        }
+    }
+
+    fn read_fake_borrows(
+        &mut self,
+        bb: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        source_info: SourceInfo,
+    ) {
+        // All indexes have been evaluated now, read all of the
+        // fake borrows so that they are live across those index
+        // expressions.
+        for temp in fake_borrow_temps {
+            self.cfg.push_fake_read(bb, source_info, FakeReadCause::ForIndex, Place::from(*temp));
+        }
+    }
+}
+
+/// Precise capture is enabled if the feature gate `capture_disjoint_fields` is enabled or if
+/// user is using Rust Edition 2021 or higher.
+fn enable_precise_capture(tcx: TyCtxt<'_>, closure_span: Span) -> bool {
+    tcx.features().capture_disjoint_fields || closure_span.rust_2021()
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
new file mode 100644
index 000000000..15f2d17c4
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -0,0 +1,694 @@
+//! See docs in `build/expr/mod.rs`.
+
+use rustc_index::vec::Idx;
+use rustc_middle::ty::util::IntTypeExt;
+
+use crate::build::expr::as_place::PlaceBase;
+use crate::build::expr::category::{Category, RvalueFunc};
+use crate::build::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
+use rustc_hir::lang_items::LangItem;
+use rustc_middle::middle::region;
+use rustc_middle::mir::AssertKind;
+use rustc_middle::mir::Place;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::cast::CastTy;
+use rustc_middle::ty::{self, Ty, UpvarSubsts};
+use rustc_span::Span;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Returns an rvalue suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid* after
+    /// an ExprKind::Scope is passed, so please do *not* return it from
+    /// functions to avoid bad miscompiles.
+    pub(crate) fn as_local_rvalue(
+        &mut self,
+        block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let local_scope = self.local_scope();
+        self.as_rvalue(block, Some(local_scope), expr)
+    }
+
+    /// Compile `expr`, yielding an rvalue.
+    pub(crate) fn as_rvalue(
+        &mut self,
+        mut block: BasicBlock,
+        scope: Option<region::Scope>,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        debug!("expr_as_rvalue(block={:?}, scope={:?}, expr={:?})", block, scope, expr);
+
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        match expr.kind {
+            ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)),
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                this.in_scope(region_scope, lint_level, |this| {
+                    this.as_rvalue(block, scope, &this.thir[value])
+                })
+            }
+            ExprKind::Repeat { value, count } => {
+                if Some(0) == count.try_eval_usize(this.tcx, this.param_env) {
+                    this.build_zero_repeat(block, value, scope, source_info)
+                } else {
+                    let value_operand = unpack!(
+                        block = this.as_operand(
+                            block,
+                            scope,
+                            &this.thir[value],
+                            None,
+                            NeedsTemporary::No
+                        )
+                    );
+                    block.and(Rvalue::Repeat(value_operand, count))
+                }
+            }
+            ExprKind::Binary { op, lhs, rhs } => {
+                let lhs = unpack!(
+                    block =
+                        this.as_operand(block, scope, &this.thir[lhs], None, NeedsTemporary::Maybe)
+                );
+                let rhs = unpack!(
+                    block =
+                        this.as_operand(block, scope, &this.thir[rhs], None, NeedsTemporary::No)
+                );
+                this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs)
+            }
+            ExprKind::Unary { op, arg } => {
+                let arg = unpack!(
+                    block =
+                        this.as_operand(block, scope, &this.thir[arg], None, NeedsTemporary::No)
+                );
+                // Check for -MIN on signed integers
+                if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() {
+                    let bool_ty = this.tcx.types.bool;
+
+                    let minval = this.minval_literal(expr_span, expr.ty);
+                    let is_min = this.temp(bool_ty, expr_span);
+
+                    this.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((arg.to_copy(), minval))),
+                    );
+
+                    block = this.assert(
+                        block,
+                        Operand::Move(is_min),
+                        false,
+                        AssertKind::OverflowNeg(arg.to_copy()),
+                        expr_span,
+                    );
+                }
+                block.and(Rvalue::UnaryOp(op, arg))
+            }
+            ExprKind::Box { value } => {
+                let value = &this.thir[value];
+                let tcx = this.tcx;
+
+                // `exchange_malloc` is unsafe but box is safe, so need a new scope.
+                let synth_scope = this.new_source_scope(
+                    expr_span,
+                    LintLevel::Inherited,
+                    Some(Safety::BuiltinUnsafe),
+                );
+                let synth_info = SourceInfo { span: expr_span, scope: synth_scope };
+
+                let size = this.temp(tcx.types.usize, expr_span);
+                this.cfg.push_assign(
+                    block,
+                    synth_info,
+                    size,
+                    Rvalue::NullaryOp(NullOp::SizeOf, value.ty),
+                );
+
+                let align = this.temp(tcx.types.usize, expr_span);
+                this.cfg.push_assign(
+                    block,
+                    synth_info,
+                    align,
+                    Rvalue::NullaryOp(NullOp::AlignOf, value.ty),
+                );
+
+                // malloc some memory of suitable size and align:
+                let exchange_malloc = Operand::function_handle(
+                    tcx,
+                    tcx.require_lang_item(LangItem::ExchangeMalloc, Some(expr_span)),
+                    ty::List::empty(),
+                    expr_span,
+                );
+                let storage = this.temp(tcx.mk_mut_ptr(tcx.types.u8), expr_span);
+                let success = this.cfg.start_new_block();
+                this.cfg.terminate(
+                    block,
+                    synth_info,
+                    TerminatorKind::Call {
+                        func: exchange_malloc,
+                        args: vec![Operand::Move(size), Operand::Move(align)],
+                        destination: storage,
+                        target: Some(success),
+                        cleanup: None,
+                        from_hir_call: false,
+                        fn_span: expr_span,
+                    },
+                );
+                this.diverge_from(block);
+                block = success;
+
+                // The `Box<T>` temporary created here is not a part of the HIR,
+                // and therefore is not considered during generator auto-trait
+                // determination. See the comment about `box` at `yield_in_scope`.
+                let result = this.local_decls.push(LocalDecl::new(expr.ty, expr_span).internal());
+                this.cfg.push(
+                    block,
+                    Statement { source_info, kind: StatementKind::StorageLive(result) },
+                );
+                if let Some(scope) = scope {
+                    // schedule a shallow free of that memory, lest we unwind:
+                    this.schedule_drop_storage_and_value(expr_span, scope, result);
+                }
+
+                // Transmute `*mut u8` to the box (thus far, uninitialized):
+                let box_ = Rvalue::ShallowInitBox(Operand::Move(storage), value.ty);
+                this.cfg.push_assign(block, source_info, Place::from(result), box_);
+
+                // initialize the box contents:
+                unpack!(
+                    block = this.expr_into_dest(
+                        this.tcx.mk_place_deref(Place::from(result)),
+                        block,
+                        value
+                    )
+                );
+                block.and(Rvalue::Use(Operand::Move(Place::from(result))))
+            }
+            ExprKind::Cast { source } => {
+                let source = &this.thir[source];
+
+                // Casting an enum to an integer is equivalent to computing the discriminant and casting the
+                // discriminant. Previously every backend had to repeat the logic for this operation. Now we
+                // create all the steps directly in MIR with operations all backends need to support anyway.
+                let (source, ty) = if let ty::Adt(adt_def, ..) = source.ty.kind() && adt_def.is_enum() {
+                    let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
+                    let place = unpack!(block = this.as_place(block, source));
+                    let discr = this.temp(discr_ty, source.span);
+                    this.cfg.push_assign(
+                        block,
+                        source_info,
+                        discr,
+                        Rvalue::Discriminant(place),
+                    );
+
+                    (Operand::Move(discr), discr_ty)
+                } else {
+                    let ty = source.ty;
+                    let source = unpack!(
+                        block = this.as_operand(block, scope, source, None, NeedsTemporary::No)
+                    );
+                    (source, ty)
+                };
+                let from_ty = CastTy::from_ty(ty);
+                let cast_ty = CastTy::from_ty(expr.ty);
+                let cast_kind = match (from_ty, cast_ty) {
+                    (Some(CastTy::Ptr(_) | CastTy::FnPtr), Some(CastTy::Int(_))) => {
+                        CastKind::PointerExposeAddress
+                    }
+                    (Some(CastTy::Int(_)), Some(CastTy::Ptr(_))) => {
+                        CastKind::PointerFromExposedAddress
+                    }
+                    (_, _) => CastKind::Misc,
+                };
+                block.and(Rvalue::Cast(cast_kind, source, expr.ty))
+            }
+            ExprKind::Pointer { cast, source } => {
+                let source = unpack!(
+                    block =
+                        this.as_operand(block, scope, &this.thir[source], None, NeedsTemporary::No)
+                );
+                block.and(Rvalue::Cast(CastKind::Pointer(cast), source, expr.ty))
+            }
+            ExprKind::Array { ref fields } => {
+                // (*) We would (maybe) be closer to codegen if we
+                // handled this and other aggregate cases via
+                // `into()`, not `as_rvalue` -- in that case, instead
+                // of generating
+                //
+                //     let tmp1 = ...1;
+                //     let tmp2 = ...2;
+                //     dest = Rvalue::Aggregate(Foo, [tmp1, tmp2])
+                //
+                // we could just generate
+                //
+                //     dest.f = ...1;
+                //     dest.g = ...2;
+                //
+                // The problem is that then we would need to:
+                //
+                // (a) have a more complex mechanism for handling
+                //     partial cleanup;
+                // (b) distinguish the case where the type `Foo` has a
+                //     destructor, in which case creating an instance
+                //     as a whole "arms" the destructor, and you can't
+                //     write individual fields; and,
+                // (c) handle the case where the type Foo has no
+                //     fields. We don't want `let x: ();` to compile
+                //     to the same MIR as `let x = ();`.
+
+                // first process the set of fields
+                let el_ty = expr.ty.sequence_element_type(this.tcx);
+                let fields: Vec<_> = fields
+                    .into_iter()
+                    .copied()
+                    .map(|f| {
+                        unpack!(
+                            block = this.as_operand(
+                                block,
+                                scope,
+                                &this.thir[f],
+                                None,
+                                NeedsTemporary::Maybe
+                            )
+                        )
+                    })
+                    .collect();
+
+                block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(el_ty)), fields))
+            }
+            ExprKind::Tuple { ref fields } => {
+                // see (*) above
+                // first process the set of fields
+                let fields: Vec<_> = fields
+                    .into_iter()
+                    .copied()
+                    .map(|f| {
+                        unpack!(
+                            block = this.as_operand(
+                                block,
+                                scope,
+                                &this.thir[f],
+                                None,
+                                NeedsTemporary::Maybe
+                            )
+                        )
+                    })
+                    .collect();
+
+                block.and(Rvalue::Aggregate(Box::new(AggregateKind::Tuple), fields))
+            }
+            ExprKind::Closure { closure_id, substs, ref upvars, movability, ref fake_reads } => {
+                // Convert the closure fake reads, if any, from `ExprRef` to mir `Place`
+                // and push the fake reads.
+                // This must come before creating the operands. This is required in case
+                // there is a fake read and a borrow of the same path, since otherwise the
+                // fake read might interfere with the borrow. Consider an example like this
+                // one:
+                // ```
+                // let mut x = 0;
+                // let c = || {
+                //     &mut x; // mutable borrow of `x`
+                //     match x { _ => () } // fake read of `x`
+                // };
+                // ```
+                //
+                for (thir_place, cause, hir_id) in fake_reads.into_iter() {
+                    let place_builder =
+                        unpack!(block = this.as_place_builder(block, &this.thir[*thir_place]));
+
+                    if let Ok(place_builder_resolved) =
+                        place_builder.try_upvars_resolved(this.tcx, this.typeck_results)
+                    {
+                        let mir_place =
+                            place_builder_resolved.into_place(this.tcx, this.typeck_results);
+                        this.cfg.push_fake_read(
+                            block,
+                            this.source_info(this.tcx.hir().span(*hir_id)),
+                            *cause,
+                            mir_place,
+                        );
+                    }
+                }
+
+                // see (*) above
+                let operands: Vec<_> = upvars
+                    .into_iter()
+                    .copied()
+                    .map(|upvar| {
+                        let upvar = &this.thir[upvar];
+                        match Category::of(&upvar.kind) {
+                            // Use as_place to avoid creating a temporary when
+                            // moving a variable into a closure, so that
+                            // borrowck knows which variables to mark as being
+                            // used as mut. This is OK here because the upvar
+                            // expressions have no side effects and act on
+                            // disjoint places.
+                            // This occurs when capturing by copy/move, while
+                            // by reference captures use as_operand
+                            Some(Category::Place) => {
+                                let place = unpack!(block = this.as_place(block, upvar));
+                                this.consume_by_copy_or_move(place)
+                            }
+                            _ => {
+                                // Turn mutable borrow captures into unique
+                                // borrow captures when capturing an immutable
+                                // variable. This is sound because the mutation
+                                // that caused the capture will cause an error.
+                                match upvar.kind {
+                                    ExprKind::Borrow {
+                                        borrow_kind:
+                                            BorrowKind::Mut { allow_two_phase_borrow: false },
+                                        arg,
+                                    } => unpack!(
+                                        block = this.limit_capture_mutability(
+                                            upvar.span,
+                                            upvar.ty,
+                                            scope,
+                                            block,
+                                            &this.thir[arg],
+                                        )
+                                    ),
+                                    _ => {
+                                        unpack!(
+                                            block = this.as_operand(
+                                                block,
+                                                scope,
+                                                upvar,
+                                                None,
+                                                NeedsTemporary::Maybe
+                                            )
+                                        )
+                                    }
+                                }
+                            }
+                        }
+                    })
+                    .collect();
+
+                let result = match substs {
+                    UpvarSubsts::Generator(substs) => {
+                        // We implicitly set the discriminant to 0. See
+                        // librustc_mir/transform/deaggregator.rs for details.
+                        let movability = movability.unwrap();
+                        Box::new(AggregateKind::Generator(closure_id, substs, movability))
+                    }
+                    UpvarSubsts::Closure(substs) => {
+                        Box::new(AggregateKind::Closure(closure_id, substs))
+                    }
+                };
+                block.and(Rvalue::Aggregate(result, operands))
+            }
+            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+                block = unpack!(this.stmt_expr(block, expr, None));
+                block.and(Rvalue::Use(Operand::Constant(Box::new(Constant {
+                    span: expr_span,
+                    user_ty: None,
+                    literal: ConstantKind::zero_sized(this.tcx.types.unit),
+                }))))
+            }
+
+            ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::StaticRef { .. } => {
+                let constant = this.as_constant(expr);
+                block.and(Rvalue::Use(Operand::Constant(Box::new(constant))))
+            }
+
+            ExprKind::Yield { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::Field { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                // these do not have corresponding `Rvalue` variants,
+                // so make an operand and then return that
+                debug_assert!(!matches!(
+                    Category::of(&expr.kind),
+                    Some(Category::Rvalue(RvalueFunc::AsRvalue) | Category::Constant)
+                ));
+                let operand =
+                    unpack!(block = this.as_operand(block, scope, expr, None, NeedsTemporary::No));
+                block.and(Rvalue::Use(operand))
+            }
+        }
+    }
+
+    pub(crate) fn build_binary_op(
+        &mut self,
+        mut block: BasicBlock,
+        op: BinOp,
+        span: Span,
+        ty: Ty<'tcx>,
+        lhs: Operand<'tcx>,
+        rhs: Operand<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let source_info = self.source_info(span);
+        let bool_ty = self.tcx.types.bool;
+        if self.check_overflow && op.is_checkable() && ty.is_integral() {
+            let result_tup = self.tcx.intern_tup(&[ty, bool_ty]);
+            let result_value = self.temp(result_tup, span);
+
+            self.cfg.push_assign(
+                block,
+                source_info,
+                result_value,
+                Rvalue::CheckedBinaryOp(op, Box::new((lhs.to_copy(), rhs.to_copy()))),
+            );
+            let val_fld = Field::new(0);
+            let of_fld = Field::new(1);
+
+            let tcx = self.tcx;
+            let val = tcx.mk_place_field(result_value, val_fld, ty);
+            let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
+
+            let err = AssertKind::Overflow(op, lhs, rhs);
+
+            block = self.assert(block, Operand::Move(of), false, err, span);
+
+            block.and(Rvalue::Use(Operand::Move(val)))
+        } else {
+            if ty.is_integral() && (op == BinOp::Div || op == BinOp::Rem) {
+                // Checking division and remainder is more complex, since we 1. always check
+                // and 2. there are two possible failure cases, divide-by-zero and overflow.
+
+                let zero_err = if op == BinOp::Div {
+                    AssertKind::DivisionByZero(lhs.to_copy())
+                } else {
+                    AssertKind::RemainderByZero(lhs.to_copy())
+                };
+                let overflow_err = AssertKind::Overflow(op, lhs.to_copy(), rhs.to_copy());
+
+                // Check for / 0
+                let is_zero = self.temp(bool_ty, span);
+                let zero = self.zero_literal(span, ty);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    is_zero,
+                    Rvalue::BinaryOp(BinOp::Eq, Box::new((rhs.to_copy(), zero))),
+                );
+
+                block = self.assert(block, Operand::Move(is_zero), false, zero_err, span);
+
+                // We only need to check for the overflow in one case:
+                // MIN / -1, and only for signed values.
+                if ty.is_signed() {
+                    let neg_1 = self.neg_1_literal(span, ty);
+                    let min = self.minval_literal(span, ty);
+
+                    let is_neg_1 = self.temp(bool_ty, span);
+                    let is_min = self.temp(bool_ty, span);
+                    let of = self.temp(bool_ty, span);
+
+                    // this does (rhs == -1) & (lhs == MIN). It could short-circuit instead
+
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_neg_1,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((rhs.to_copy(), neg_1))),
+                    );
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((lhs.to_copy(), min))),
+                    );
+
+                    let is_neg_1 = Operand::Move(is_neg_1);
+                    let is_min = Operand::Move(is_min);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        of,
+                        Rvalue::BinaryOp(BinOp::BitAnd, Box::new((is_neg_1, is_min))),
+                    );
+
+                    block = self.assert(block, Operand::Move(of), false, overflow_err, span);
+                }
+            }
+
+            block.and(Rvalue::BinaryOp(op, Box::new((lhs, rhs))))
+        }
+    }
+
+    fn build_zero_repeat(
+        &mut self,
+        mut block: BasicBlock,
+        value: ExprId,
+        scope: Option<region::Scope>,
+        outer_source_info: SourceInfo,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let this = self;
+        let value = &this.thir[value];
+        let elem_ty = value.ty;
+        if let Some(Category::Constant) = Category::of(&value.kind) {
+            // Repeating a const does nothing
+        } else {
+            // For a non-const, we may need to generate an appropriate `Drop`
+            let value_operand =
+                unpack!(block = this.as_operand(block, scope, value, None, NeedsTemporary::No));
+            if let Operand::Move(to_drop) = value_operand {
+                let success = this.cfg.start_new_block();
+                this.cfg.terminate(
+                    block,
+                    outer_source_info,
+                    TerminatorKind::Drop { place: to_drop, target: success, unwind: None },
+                );
+                this.diverge_from(block);
+                block = success;
+            }
+            this.record_operands_moved(&[value_operand]);
+        }
+        block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(elem_ty)), Vec::new()))
+    }
+
+    fn limit_capture_mutability(
+        &mut self,
+        upvar_span: Span,
+        upvar_ty: Ty<'tcx>,
+        temp_lifetime: Option<region::Scope>,
+        mut block: BasicBlock,
+        arg: &Expr<'tcx>,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+
+        let source_info = this.source_info(upvar_span);
+        let temp = this.local_decls.push(LocalDecl::new(upvar_ty, upvar_span));
+
+        this.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+        let arg_place_builder = unpack!(block = this.as_place_builder(block, arg));
+
+        let mutability = match arg_place_builder.base() {
+            // We are capturing a path that starts off a local variable in the parent.
+            // The mutability of the current capture is same as the mutability
+            // of the local declaration in the parent.
+            PlaceBase::Local(local) => this.local_decls[local].mutability,
+            // Parent is a closure and we are capturing a path that is captured
+            // by the parent itself. The mutability of the current capture
+            // is same as that of the capture in the parent closure.
+            PlaceBase::Upvar { .. } => {
+                let enclosing_upvars_resolved =
+                    arg_place_builder.clone().into_place(this.tcx, this.typeck_results);
+
+                match enclosing_upvars_resolved.as_ref() {
+                    PlaceRef {
+                        local,
+                        projection: &[ProjectionElem::Field(upvar_index, _), ..],
+                    }
+                    | PlaceRef {
+                        local,
+                        projection:
+                            &[ProjectionElem::Deref, ProjectionElem::Field(upvar_index, _), ..],
+                    } => {
+                        // Not in a closure
+                        debug_assert!(
+                            local == ty::CAPTURE_STRUCT_LOCAL,
+                            "Expected local to be Local(1), found {:?}",
+                            local
+                        );
+                        // Not in a closure
+                        debug_assert!(
+                            this.upvar_mutbls.len() > upvar_index.index(),
+                            "Unexpected capture place, upvar_mutbls={:#?}, upvar_index={:?}",
+                            this.upvar_mutbls,
+                            upvar_index
+                        );
+                        this.upvar_mutbls[upvar_index.index()]
+                    }
+                    _ => bug!("Unexpected capture place"),
+                }
+            }
+        };
+
+        let borrow_kind = match mutability {
+            Mutability::Not => BorrowKind::Unique,
+            Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
+        };
+
+        let arg_place = arg_place_builder.into_place(this.tcx, this.typeck_results);
+
+        this.cfg.push_assign(
+            block,
+            source_info,
+            Place::from(temp),
+            Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place),
+        );
+
+        // See the comment in `expr_as_temp` and on the `rvalue_scopes` field for why
+        // this can be `None`.
+        if let Some(temp_lifetime) = temp_lifetime {
+            this.schedule_drop_storage_and_value(upvar_span, temp_lifetime, temp);
+        }
+
+        block.and(Operand::Move(Place::from(temp)))
+    }
+
+    // Helper to get a `-1` value of the appropriate type
+    fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let param_ty = ty::ParamEnv::empty().and(ty);
+        let size = self.tcx.layout_of(param_ty).unwrap().size;
+        let literal = ConstantKind::from_bits(self.tcx, size.unsigned_int_max(), param_ty);
+
+        self.literal_operand(span, literal)
+    }
+
+    // Helper to get the minimum value of the appropriate type
+    fn minval_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        assert!(ty.is_signed());
+        let param_ty = ty::ParamEnv::empty().and(ty);
+        let bits = self.tcx.layout_of(param_ty).unwrap().size.bits();
+        let n = 1 << (bits - 1);
+        let literal = ConstantKind::from_bits(self.tcx, n, param_ty);
+
+        self.literal_operand(span, literal)
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
new file mode 100644
index 000000000..724b72f87
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -0,0 +1,119 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::scope::DropKind;
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr` into a fresh temporary. This is used when building
+    /// up rvalues so as to freeze the value that will be consumed.
+    pub(crate) fn as_temp(
+        &mut self,
+        block: BasicBlock,
+        temp_lifetime: Option<region::Scope>,
+        expr: &Expr<'tcx>,
+        mutability: Mutability,
+    ) -> BlockAnd<Local> {
+        // this is the only place in mir building that we need to truly need to worry about
+        // infinite recursion. Everything else does recurse, too, but it always gets broken up
+        // at some point by inserting an intermediate temporary
+        ensure_sufficient_stack(|| self.as_temp_inner(block, temp_lifetime, expr, mutability))
+    }
+
+    fn as_temp_inner(
+        &mut self,
+        mut block: BasicBlock,
+        temp_lifetime: Option<region::Scope>,
+        expr: &Expr<'tcx>,
+        mutability: Mutability,
+    ) -> BlockAnd<Local> {
+        debug!(
+            "as_temp(block={:?}, temp_lifetime={:?}, expr={:?}, mutability={:?})",
+            block, temp_lifetime, expr, mutability
+        );
+        let this = self;
+
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            return this.in_scope((region_scope, source_info), lint_level, |this| {
+                this.as_temp(block, temp_lifetime, &this.thir[value], mutability)
+            });
+        }
+
+        let expr_ty = expr.ty;
+        let temp = {
+            let mut local_decl = LocalDecl::new(expr_ty, expr_span);
+            if mutability == Mutability::Not {
+                local_decl = local_decl.immutable();
+            }
+
+            debug!("creating temp {:?} with block_context: {:?}", local_decl, this.block_context);
+            // Find out whether this temp is being created within the
+            // tail expression of a block whose result is ignored.
+            if let Some(tail_info) = this.block_context.currently_in_block_tail() {
+                local_decl = local_decl.block_tail(tail_info);
+            }
+            match expr.kind {
+                ExprKind::StaticRef { def_id, .. } => {
+                    assert!(!this.tcx.is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info =
+                        Some(Box::new(LocalInfo::StaticRef { def_id, is_thread_local: false }));
+                }
+                ExprKind::ThreadLocalRef(def_id) => {
+                    assert!(this.tcx.is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info =
+                        Some(Box::new(LocalInfo::StaticRef { def_id, is_thread_local: true }));
+                }
+                ExprKind::NamedConst { def_id, .. } | ExprKind::ConstParam { def_id, .. } => {
+                    local_decl.local_info = Some(Box::new(LocalInfo::ConstRef { def_id }));
+                }
+                _ => {}
+            }
+            this.local_decls.push(local_decl)
+        };
+        let temp_place = Place::from(temp);
+
+        match expr.kind {
+            // Don't bother with StorageLive and Dead for these temporaries,
+            // they are never assigned.
+            ExprKind::Break { .. } | ExprKind::Continue { .. } | ExprKind::Return { .. } => (),
+            ExprKind::Block { body: Block { expr: None, targeted_by_break: false, .. } }
+                if expr_ty.is_never() => {}
+            _ => {
+                this.cfg
+                    .push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+                // In constants, `temp_lifetime` is `None` for temporaries that
+                // live for the `'static` lifetime. Thus we do not drop these
+                // temporaries and simply leak them.
+                // This is equivalent to what `let x = &foo();` does in
+                // functions. The temporary is lifted to their surrounding
+                // scope. In a function that means the temporary lives until
+                // just before the function returns. In constants that means it
+                // outlives the constant's initialization value computation.
+                // Anything outliving a constant must have the `'static`
+                // lifetime and live forever.
+                // Anything with a shorter lifetime (e.g the `&foo()` in
+                // `bar(&foo())` or anything within a block will keep the
+                // regular drops just like runtime code.
+                if let Some(temp_lifetime) = temp_lifetime {
+                    this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Storage);
+                }
+            }
+        }
+
+        unpack!(block = this.expr_into_dest(temp_place, block, expr));
+
+        if let Some(temp_lifetime) = temp_lifetime {
+            this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Value);
+        }
+
+        block.and(temp)
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/category.rs b/compiler/rustc_mir_build/src/build/expr/category.rs
new file mode 100644
index 000000000..a4386319d
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/category.rs
@@ -0,0 +1,92 @@
+use rustc_middle::thir::*;
+
+#[derive(Debug, PartialEq)]
+pub(crate) enum Category {
+    // An assignable memory location like `x`, `x.f`, `foo()[3]`, that
+    // sort of thing. Something that could appear on the LHS of an `=`
+    // sign.
+    Place,
+
+    // A literal like `23` or `"foo"`. Does not include constant
+    // expressions like `3 + 5`.
+    Constant,
+
+    // Something that generates a new value at runtime, like `x + y`
+    // or `foo()`.
+    Rvalue(RvalueFunc),
+}
+
+// Rvalues fall into different "styles" that will determine which fn
+// is best suited to generate them.
+#[derive(Debug, PartialEq)]
+pub(crate) enum RvalueFunc {
+    // Best generated by `into`. This is generally exprs that
+    // cause branching, like `match`, but also includes calls.
+    Into,
+
+    // Best generated by `as_rvalue`. This is usually the case.
+    AsRvalue,
+}
+
+/// Determines the category for a given expression. Note that scope
+/// and paren expressions have no category.
+impl Category {
+    pub(crate) fn of(ek: &ExprKind<'_>) -> Option<Category> {
+        match *ek {
+            ExprKind::Scope { .. } => None,
+
+            ExprKind::Field { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => Some(Category::Place),
+
+            ExprKind::LogicalOp { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::AddressOf { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::InlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::Into)),
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::ThreadLocalRef(_) => Some(Category::Rvalue(RvalueFunc::AsRvalue)),
+
+            ExprKind::ConstBlock { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::NamedConst { .. } => Some(Category::Constant),
+
+            ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. } =>
+            // FIXME(#27840) these probably want their own
+            // category, like "nonterminating"
+            {
+                Some(Category::Rvalue(RvalueFunc::Into))
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
new file mode 100644
index 000000000..017d43d10
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -0,0 +1,599 @@
+//! See docs in build/expr/mod.rs
+
+use crate::build::expr::category::{Category, RvalueFunc};
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, NeedsTemporary};
+use rustc_ast::InlineAsmOptions;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir as hir;
+use rustc_index::vec::Idx;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::CanonicalUserTypeAnnotation;
+use std::iter;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, storing the result into `destination`, which
+    /// is assumed to be uninitialized.
+    pub(crate) fn expr_into_dest(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<()> {
+        debug!("expr_into_dest(destination={:?}, block={:?}, expr={:?})", destination, block, expr);
+
+        // since we frequently have to reference `self` from within a
+        // closure, where `self` would be shadowed, it's easier to
+        // just use the name `this` uniformly
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        let expr_is_block_or_scope =
+            matches!(expr.kind, ExprKind::Block { .. } | ExprKind::Scope { .. });
+
+        if !expr_is_block_or_scope {
+            this.block_context.push(BlockFrame::SubExpr);
+        }
+
+        let block_and = match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                ensure_sufficient_stack(|| {
+                    this.in_scope(region_scope, lint_level, |this| {
+                        this.expr_into_dest(destination, block, &this.thir[value])
+                    })
+                })
+            }
+            ExprKind::Block { body: ref ast_block } => {
+                this.ast_block(destination, block, ast_block, source_info)
+            }
+            ExprKind::Match { scrutinee, ref arms } => {
+                this.match_expr(destination, expr_span, block, &this.thir[scrutinee], arms)
+            }
+            ExprKind::If { cond, then, else_opt, if_then_scope } => {
+                let then_blk;
+                let then_expr = &this.thir[then];
+                let then_source_info = this.source_info(then_expr.span);
+                let condition_scope = this.local_scope();
+
+                let mut else_blk = unpack!(
+                    then_blk = this.in_scope(
+                        (if_then_scope, then_source_info),
+                        LintLevel::Inherited,
+                        |this| {
+                            let source_info = if this.is_let(cond) {
+                                let variable_scope = this.new_source_scope(
+                                    then_expr.span,
+                                    LintLevel::Inherited,
+                                    None,
+                                );
+                                this.source_scope = variable_scope;
+                                SourceInfo { span: then_expr.span, scope: variable_scope }
+                            } else {
+                                this.source_info(then_expr.span)
+                            };
+                            let (then_block, else_block) =
+                                this.in_if_then_scope(condition_scope, |this| {
+                                    let then_blk = unpack!(this.then_else_break(
+                                        block,
+                                        &this.thir[cond],
+                                        Some(condition_scope),
+                                        condition_scope,
+                                        source_info
+                                    ));
+
+                                    this.expr_into_dest(destination, then_blk, then_expr)
+                                });
+                            then_block.and(else_block)
+                        },
+                    )
+                );
+
+                else_blk = if let Some(else_opt) = else_opt {
+                    unpack!(this.expr_into_dest(destination, else_blk, &this.thir[else_opt]))
+                } else {
+                    // Body of the `if` expression without an `else` clause must return `()`, thus
+                    // we implicitly generate an `else {}` if it is not specified.
+                    let correct_si = this.source_info(expr_span.shrink_to_hi());
+                    this.cfg.push_assign_unit(else_blk, correct_si, destination, this.tcx);
+                    else_blk
+                };
+
+                let join_block = this.cfg.start_new_block();
+                this.cfg.goto(then_blk, source_info, join_block);
+                this.cfg.goto(else_blk, source_info, join_block);
+                join_block.unit()
+            }
+            ExprKind::Let { expr, ref pat } => {
+                let scope = this.local_scope();
+                let (true_block, false_block) = this.in_if_then_scope(scope, |this| {
+                    this.lower_let_expr(block, &this.thir[expr], pat, scope, None, expr_span)
+                });
+
+                this.cfg.push_assign_constant(
+                    true_block,
+                    source_info,
+                    destination,
+                    Constant {
+                        span: expr_span,
+                        user_ty: None,
+                        literal: ConstantKind::from_bool(this.tcx, true),
+                    },
+                );
+
+                this.cfg.push_assign_constant(
+                    false_block,
+                    source_info,
+                    destination,
+                    Constant {
+                        span: expr_span,
+                        user_ty: None,
+                        literal: ConstantKind::from_bool(this.tcx, false),
+                    },
+                );
+
+                let join_block = this.cfg.start_new_block();
+                this.cfg.goto(true_block, source_info, join_block);
+                this.cfg.goto(false_block, source_info, join_block);
+                join_block.unit()
+            }
+            ExprKind::NeverToAny { source } => {
+                let source = &this.thir[source];
+                let is_call =
+                    matches!(source.kind, ExprKind::Call { .. } | ExprKind::InlineAsm { .. });
+
+                // (#66975) Source could be a const of type `!`, so has to
+                // exist in the generated MIR.
+                unpack!(
+                    block = this.as_temp(block, Some(this.local_scope()), source, Mutability::Mut,)
+                );
+
+                // This is an optimization. If the expression was a call then we already have an
+                // unreachable block. Don't bother to terminate it and create a new one.
+                if is_call {
+                    block.unit()
+                } else {
+                    this.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
+                    let end_block = this.cfg.start_new_block();
+                    end_block.unit()
+                }
+            }
+            ExprKind::LogicalOp { op, lhs, rhs } => {
+                // And:
+                //
+                // [block: If(lhs)] -true-> [else_block: dest = (rhs)]
+                //        | (false)
+                //  [shortcurcuit_block: dest = false]
+                //
+                // Or:
+                //
+                // [block: If(lhs)] -false-> [else_block: dest = (rhs)]
+                //        | (true)
+                //  [shortcurcuit_block: dest = true]
+
+                let (shortcircuit_block, mut else_block, join_block) = (
+                    this.cfg.start_new_block(),
+                    this.cfg.start_new_block(),
+                    this.cfg.start_new_block(),
+                );
+
+                let lhs = unpack!(block = this.as_local_operand(block, &this.thir[lhs]));
+                let blocks = match op {
+                    LogicalOp::And => (else_block, shortcircuit_block),
+                    LogicalOp::Or => (shortcircuit_block, else_block),
+                };
+                let term = TerminatorKind::if_(this.tcx, lhs, blocks.0, blocks.1);
+                this.cfg.terminate(block, source_info, term);
+
+                this.cfg.push_assign_constant(
+                    shortcircuit_block,
+                    source_info,
+                    destination,
+                    Constant {
+                        span: expr_span,
+                        user_ty: None,
+                        literal: match op {
+                            LogicalOp::And => ConstantKind::from_bool(this.tcx, false),
+                            LogicalOp::Or => ConstantKind::from_bool(this.tcx, true),
+                        },
+                    },
+                );
+                this.cfg.goto(shortcircuit_block, source_info, join_block);
+
+                let rhs = unpack!(else_block = this.as_local_operand(else_block, &this.thir[rhs]));
+                this.cfg.push_assign(else_block, source_info, destination, Rvalue::Use(rhs));
+                this.cfg.goto(else_block, source_info, join_block);
+
+                join_block.unit()
+            }
+            ExprKind::Loop { body } => {
+                // [block]
+                //    |
+                //   [loop_block] -> [body_block] -/eval. body/-> [body_block_end]
+                //    |        ^                                         |
+                // false link  |                                         |
+                //    |        +-----------------------------------------+
+                //    +-> [diverge_cleanup]
+                // The false link is required to make sure borrowck considers unwinds through the
+                // body, even when the exact code in the body cannot unwind
+
+                let loop_block = this.cfg.start_new_block();
+
+                // Start the loop.
+                this.cfg.goto(block, source_info, loop_block);
+
+                this.in_breakable_scope(Some(loop_block), destination, expr_span, move |this| {
+                    // conduct the test, if necessary
+                    let body_block = this.cfg.start_new_block();
+                    this.cfg.terminate(
+                        loop_block,
+                        source_info,
+                        TerminatorKind::FalseUnwind { real_target: body_block, unwind: None },
+                    );
+                    this.diverge_from(loop_block);
+
+                    // The “return” value of the loop body must always be a unit. We therefore
+                    // introduce a unit temporary as the destination for the loop body.
+                    let tmp = this.get_unit_temp();
+                    // Execute the body, branching back to the test.
+                    let body_block_end =
+                        unpack!(this.expr_into_dest(tmp, body_block, &this.thir[body]));
+                    this.cfg.goto(body_block_end, source_info, loop_block);
+
+                    // Loops are only exited by `break` expressions.
+                    None
+                })
+            }
+            ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => {
+                let fun = unpack!(block = this.as_local_operand(block, &this.thir[fun]));
+                let args: Vec<_> = args
+                    .into_iter()
+                    .copied()
+                    .map(|arg| unpack!(block = this.as_local_call_operand(block, &this.thir[arg])))
+                    .collect();
+
+                let success = this.cfg.start_new_block();
+
+                this.record_operands_moved(&args);
+
+                debug!("expr_into_dest: fn_span={:?}", fn_span);
+
+                this.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::Call {
+                        func: fun,
+                        args,
+                        cleanup: None,
+                        destination,
+                        // The presence or absence of a return edge affects control-flow sensitive
+                        // MIR checks and ultimately whether code is accepted or not. We can only
+                        // omit the return edge if a return type is visibly uninhabited to a module
+                        // that makes the call.
+                        target: if this.tcx.is_ty_uninhabited_from(
+                            this.parent_module,
+                            expr.ty,
+                            this.param_env,
+                        ) {
+                            None
+                        } else {
+                            Some(success)
+                        },
+                        from_hir_call,
+                        fn_span,
+                    },
+                );
+                this.diverge_from(block);
+                success.unit()
+            }
+            ExprKind::Use { source } => this.expr_into_dest(destination, block, &this.thir[source]),
+            ExprKind::Borrow { arg, borrow_kind } => {
+                let arg = &this.thir[arg];
+                // We don't do this in `as_rvalue` because we use `as_place`
+                // for borrow expressions, so we cannot create an `RValue` that
+                // remains valid across user code. `as_rvalue` is usually called
+                // by this method anyway, so this shouldn't cause too many
+                // unnecessary temporaries.
+                let arg_place = match borrow_kind {
+                    BorrowKind::Shared => unpack!(block = this.as_read_only_place(block, arg)),
+                    _ => unpack!(block = this.as_place(block, arg)),
+                };
+                let borrow = Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place);
+                this.cfg.push_assign(block, source_info, destination, borrow);
+                block.unit()
+            }
+            ExprKind::AddressOf { mutability, arg } => {
+                let arg = &this.thir[arg];
+                let place = match mutability {
+                    hir::Mutability::Not => this.as_read_only_place(block, arg),
+                    hir::Mutability::Mut => this.as_place(block, arg),
+                };
+                let address_of = Rvalue::AddressOf(mutability, unpack!(block = place));
+                this.cfg.push_assign(block, source_info, destination, address_of);
+                block.unit()
+            }
+            ExprKind::Adt(box Adt {
+                adt_def,
+                variant_index,
+                substs,
+                user_ty,
+                ref fields,
+                ref base,
+            }) => {
+                // See the notes for `ExprKind::Array` in `as_rvalue` and for
+                // `ExprKind::Borrow` above.
+                let is_union = adt_def.is_union();
+                let active_field_index = if is_union { Some(fields[0].name.index()) } else { None };
+
+                let scope = this.local_scope();
+
+                // first process the set of fields that were provided
+                // (evaluating them in order given by user)
+                let fields_map: FxHashMap<_, _> = fields
+                    .into_iter()
+                    .map(|f| {
+                        let local_info = Box::new(LocalInfo::AggregateTemp);
+                        (
+                            f.name,
+                            unpack!(
+                                block = this.as_operand(
+                                    block,
+                                    Some(scope),
+                                    &this.thir[f.expr],
+                                    Some(local_info),
+                                    NeedsTemporary::Maybe,
+                                )
+                            ),
+                        )
+                    })
+                    .collect();
+
+                let field_names: Vec<_> =
+                    (0..adt_def.variant(variant_index).fields.len()).map(Field::new).collect();
+
+                let fields: Vec<_> = if let Some(FruInfo { base, field_types }) = base {
+                    let place_builder =
+                        unpack!(block = this.as_place_builder(block, &this.thir[*base]));
+
+                    // MIR does not natively support FRU, so for each
+                    // base-supplied field, generate an operand that
+                    // reads it from the base.
+                    iter::zip(field_names, &**field_types)
+                        .map(|(n, ty)| match fields_map.get(&n) {
+                            Some(v) => v.clone(),
+                            None => {
+                                let place_builder = place_builder.clone();
+                                this.consume_by_copy_or_move(
+                                    place_builder
+                                        .field(n, *ty)
+                                        .into_place(this.tcx, this.typeck_results),
+                                )
+                            }
+                        })
+                        .collect()
+                } else {
+                    field_names.iter().filter_map(|n| fields_map.get(n).cloned()).collect()
+                };
+
+                let inferred_ty = expr.ty;
+                let user_ty = user_ty.map(|ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span: source_info.span,
+                        user_ty: ty,
+                        inferred_ty,
+                    })
+                });
+                let adt = Box::new(AggregateKind::Adt(
+                    adt_def.did(),
+                    variant_index,
+                    substs,
+                    user_ty,
+                    active_field_index,
+                ));
+                this.cfg.push_assign(
+                    block,
+                    source_info,
+                    destination,
+                    Rvalue::Aggregate(adt, fields),
+                );
+                block.unit()
+            }
+            ExprKind::InlineAsm { template, ref operands, options, line_spans } => {
+                use rustc_middle::{mir, thir};
+                let operands = operands
+                    .into_iter()
+                    .map(|op| match *op {
+                        thir::InlineAsmOperand::In { reg, expr } => mir::InlineAsmOperand::In {
+                            reg,
+                            value: unpack!(block = this.as_local_operand(block, &this.thir[expr])),
+                        },
+                        thir::InlineAsmOperand::Out { reg, late, expr } => {
+                            mir::InlineAsmOperand::Out {
+                                reg,
+                                late,
+                                place: expr.map(|expr| {
+                                    unpack!(block = this.as_place(block, &this.thir[expr]))
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::InOut { reg, late, expr } => {
+                            let place = unpack!(block = this.as_place(block, &this.thir[expr]));
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                // This works because asm operands must be Copy
+                                in_value: Operand::Copy(place),
+                                out_place: Some(place),
+                            }
+                        }
+                        thir::InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                in_value: unpack!(
+                                    block = this.as_local_operand(block, &this.thir[in_expr])
+                                ),
+                                out_place: out_expr.map(|out_expr| {
+                                    unpack!(block = this.as_place(block, &this.thir[out_expr]))
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::Const { value, span } => {
+                            mir::InlineAsmOperand::Const {
+                                value: Box::new(Constant { span, user_ty: None, literal: value }),
+                            }
+                        }
+                        thir::InlineAsmOperand::SymFn { value, span } => {
+                            mir::InlineAsmOperand::SymFn {
+                                value: Box::new(Constant { span, user_ty: None, literal: value }),
+                            }
+                        }
+                        thir::InlineAsmOperand::SymStatic { def_id } => {
+                            mir::InlineAsmOperand::SymStatic { def_id }
+                        }
+                    })
+                    .collect();
+
+                if !options.contains(InlineAsmOptions::NORETURN) {
+                    this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+                }
+
+                let destination_block = this.cfg.start_new_block();
+                this.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::InlineAsm {
+                        template,
+                        operands,
+                        options,
+                        line_spans,
+                        destination: if options.contains(InlineAsmOptions::NORETURN) {
+                            None
+                        } else {
+                            Some(destination_block)
+                        },
+                        cleanup: None,
+                    },
+                );
+                if options.contains(InlineAsmOptions::MAY_UNWIND) {
+                    this.diverge_from(block);
+                }
+                destination_block.unit()
+            }
+
+            // These cases don't actually need a destination
+            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+                unpack!(block = this.stmt_expr(block, expr, None));
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+                block.unit()
+            }
+
+            ExprKind::Continue { .. } | ExprKind::Break { .. } | ExprKind::Return { .. } => {
+                unpack!(block = this.stmt_expr(block, expr, None));
+                // No assign, as these have type `!`.
+                block.unit()
+            }
+
+            // Avoid creating a temporary
+            ExprKind::VarRef { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+
+                let place = unpack!(block = this.as_place(block, expr));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+            ExprKind::Index { .. } | ExprKind::Deref { .. } | ExprKind::Field { .. } => {
+                debug_assert_eq!(Category::of(&expr.kind), Some(Category::Place));
+
+                // Create a "fake" temporary variable so that we check that the
+                // value is Sized. Usually, this is caught in type checking, but
+                // in the case of box expr there is no such check.
+                if !destination.projection.is_empty() {
+                    this.local_decls.push(LocalDecl::new(expr.ty, expr.span));
+                }
+
+                let place = unpack!(block = this.as_place(block, expr));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+
+            ExprKind::Yield { value } => {
+                let scope = this.local_scope();
+                let value = unpack!(
+                    block = this.as_operand(
+                        block,
+                        Some(scope),
+                        &this.thir[value],
+                        None,
+                        NeedsTemporary::No
+                    )
+                );
+                let resume = this.cfg.start_new_block();
+                this.cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::Yield { value, resume, resume_arg: destination, drop: None },
+                );
+                this.generator_drop_cleanup(block);
+                resume.unit()
+            }
+
+            // these are the cases that are more naturally handled by some other mode
+            ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::StaticRef { .. } => {
+                debug_assert!(match Category::of(&expr.kind).unwrap() {
+                    // should be handled above
+                    Category::Rvalue(RvalueFunc::Into) => false,
+
+                    // must be handled above or else we get an
+                    // infinite loop in the builder; see
+                    // e.g., `ExprKind::VarRef` above
+                    Category::Place => false,
+
+                    _ => true,
+                });
+
+                let rvalue = unpack!(block = this.as_local_rvalue(block, expr));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+        };
+
+        if !expr_is_block_or_scope {
+            let popped = this.block_context.pop();
+            assert!(popped.is_some());
+        }
+
+        block_and
+    }
+
+    fn is_let(&self, expr: ExprId) -> bool {
+        match self.thir[expr].kind {
+            ExprKind::Let { .. } => true,
+            ExprKind::Scope { value, .. } => self.is_let(value),
+            _ => false,
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/mod.rs b/compiler/rustc_mir_build/src/build/expr/mod.rs
new file mode 100644
index 000000000..f5ae060d6
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/mod.rs
@@ -0,0 +1,70 @@
+//! Builds MIR from expressions. As a caller into this module, you
+//! have many options, but the first thing you have to decide is
+//! whether you are evaluating this expression for its *value*, its
+//! *location*, or as a *constant*.
+//!
+//! Typically, you want the value: e.g., if you are doing `expr_a +
+//! expr_b`, you want the values of those expressions. In that case,
+//! you want one of the following functions. Note that if the expr has
+//! a type that is not `Copy`, then using any of these functions will
+//! "move" the value out of its current home (if any).
+//!
+//! - `expr_into_dest` -- writes the value into a specific location, which
+//!   should be uninitialized
+//! - `as_operand` -- evaluates the value and yields an `Operand`,
+//!   suitable for use as an argument to an `Rvalue`
+//! - `as_temp` -- evaluates into a temporary; this is similar to `as_operand`
+//!   except it always returns a fresh place, even for constants
+//! - `as_rvalue` -- yields an `Rvalue`, suitable for use in an assignment;
+//!   as of this writing, never needed outside of the `expr` module itself
+//!
+//! Sometimes though want the expression's *location*. An example
+//! would be during a match statement, or the operand of the `&`
+//! operator. In that case, you want `as_place`. This will create a
+//! temporary if necessary.
+//!
+//! Finally, if it's a constant you seek, then call
+//! `as_constant`. This creates a `Constant<H>`, but naturally it can
+//! only be used on constant expressions and hence is needed only in
+//! very limited contexts.
+//!
+//! ### Implementation notes
+//!
+//! For any given kind of expression, there is generally one way that
+//! can be lowered most naturally. This is specified by the
+//! `Category::of` function in the `category` module. For example, a
+//! struct expression (or other expression that creates a new value)
+//! is typically easiest to write in terms of `as_rvalue` or `into`,
+//! whereas a reference to a field is easiest to write in terms of
+//! `as_place`. (The exception to this is scope and paren
+//! expressions, which have no category.)
+//!
+//! Therefore, the various functions above make use of one another in
+//! a descending fashion. For any given expression, you should pick
+//! the most suitable spot to implement it, and then just let the
+//! other fns cycle around. The handoff works like this:
+//!
+//! - `into(place)` -> fallback is to create a rvalue with `as_rvalue` and assign it to `place`
+//! - `as_rvalue` -> fallback is to create an Operand with `as_operand` and use `Rvalue::use`
+//! - `as_operand` -> either invokes `as_constant` or `as_temp`
+//! - `as_constant` -> (no fallback)
+//! - `as_temp` -> creates a temporary and either calls `as_place` or `into`
+//! - `as_place` -> for rvalues, falls back to `as_temp` and returns that
+//!
+//! As you can see, there is a cycle where `into` can (in theory) fallback to `as_temp`
+//! which can fallback to `into`. So if one of the `ExprKind` variants is not, in fact,
+//! implemented in the category where it is supposed to be, there will be a problem.
+//!
+//! Of those fallbacks, the most interesting one is `into`, because
+//! it discriminates based on the category of the expression. This is
+//! basically the point where the "by value" operations are bridged
+//! over to the "by reference" mode (`as_place`).
+
+pub(crate) mod as_constant;
+mod as_operand;
+pub mod as_place;
+mod as_rvalue;
+mod as_temp;
+pub mod category;
+mod into;
+mod stmt;
diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs
new file mode 100644
index 000000000..a7e1331aa
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs
@@ -0,0 +1,149 @@
+use crate::build::scope::BreakableTarget;
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Builds a block of MIR statements to evaluate the THIR `expr`.
+    /// If the original expression was an AST statement,
+    /// (e.g., `some().code(&here());`) then `opt_stmt_span` is the
+    /// span of that statement (including its semicolon, if any).
+    /// The scope is used if a statement temporary must be dropped.
+    pub(crate) fn stmt_expr(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+        statement_scope: Option<region::Scope>,
+    ) -> BlockAnd<()> {
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr.span);
+        // Handle a number of expressions that don't need a destination at all. This
+        // avoids needing a mountain of temporary `()` variables.
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    this.stmt_expr(block, &this.thir[value], statement_scope)
+                })
+            }
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs = &this.thir[lhs];
+                let rhs = &this.thir[rhs];
+                let lhs_span = lhs.span;
+
+                // Note: we evaluate assignments right-to-left. This
+                // is better for borrowck interaction with overloaded
+                // operators like x[j] = x[i].
+
+                debug!("stmt_expr Assign block_context.push(SubExpr) : {:?}", expr);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // Generate better code for things that don't need to be
+                // dropped.
+                if lhs.ty.needs_drop(this.tcx, this.param_env) {
+                    let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    unpack!(block = this.build_drop_and_replace(block, lhs_span, lhs, rhs));
+                } else {
+                    let rhs = unpack!(block = this.as_local_rvalue(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    this.cfg.push_assign(block, source_info, lhs, rhs);
+                }
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::AssignOp { op, lhs, rhs } => {
+                // FIXME(#28160) there is an interesting semantics
+                // question raised here -- should we "freeze" the
+                // value of the lhs here?  I'm inclined to think not,
+                // since it seems closer to the semantics of the
+                // overloaded version, which takes `&mut self`.  This
+                // only affects weird things like `x += {x += 1; x}`
+                // -- is that equal to `x + (x + 1)` or `2*(x+1)`?
+
+                let lhs = &this.thir[lhs];
+                let rhs = &this.thir[rhs];
+                let lhs_ty = lhs.ty;
+
+                debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // As above, RTL.
+                let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                let lhs = unpack!(block = this.as_place(block, lhs));
+
+                // we don't have to drop prior contents or anything
+                // because AssignOp is only legal for Copy types
+                // (overloaded ops should be desugared into a call).
+                let result = unpack!(
+                    block =
+                        this.build_binary_op(block, op, expr_span, lhs_ty, Operand::Copy(lhs), rhs)
+                );
+                this.cfg.push_assign(block, source_info, lhs, result);
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::Continue { label } => {
+                this.break_scope(block, None, BreakableTarget::Continue(label), source_info)
+            }
+            ExprKind::Break { label, value } => this.break_scope(
+                block,
+                value.map(|value| &this.thir[value]),
+                BreakableTarget::Break(label),
+                source_info,
+            ),
+            ExprKind::Return { value } => this.break_scope(
+                block,
+                value.map(|value| &this.thir[value]),
+                BreakableTarget::Return,
+                source_info,
+            ),
+            _ => {
+                assert!(
+                    statement_scope.is_some(),
+                    "Should not be calling `stmt_expr` on a general expression \
+                     without a statement scope",
+                );
+
+                // Issue #54382: When creating temp for the value of
+                // expression like:
+                //
+                // `{ side_effects(); { let l = stuff(); the_value } }`
+                //
+                // it is usually better to focus on `the_value` rather
+                // than the entirety of block(s) surrounding it.
+                let adjusted_span = (|| {
+                    if let ExprKind::Block { body } = &expr.kind && let Some(tail_ex) = body.expr {
+                        let mut expr = &this.thir[tail_ex];
+                        while let ExprKind::Block {
+                            body: Block { expr: Some(nested_expr), .. },
+                        }
+                        | ExprKind::Scope { value: nested_expr, .. } = expr.kind
+                        {
+                            expr = &this.thir[nested_expr];
+                        }
+                        this.block_context.push(BlockFrame::TailExpr {
+                            tail_result_is_ignored: true,
+                            span: expr.span,
+                        });
+                        return Some(expr.span);
+                    }
+                    None
+                })();
+
+                let temp =
+                    unpack!(block = this.as_temp(block, statement_scope, expr, Mutability::Not));
+
+                if let Some(span) = adjusted_span {
+                    this.local_decls[temp].source_info.span = span;
+                    this.block_context.pop();
+                }
+
+                block.unit()
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
new file mode 100644
index 000000000..58b1564cc
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -0,0 +1,2354 @@
+//! Code related to match expressions. These are sufficiently complex to
+//! warrant their own module and submodules. :) This main module includes the
+//! high-level algorithm, the submodules contain the details.
+//!
+//! This also includes code for pattern bindings in `let` statements and
+//! function parameters.
+
+use crate::build::expr::as_place::PlaceBuilder;
+use crate::build::scope::DropKind;
+use crate::build::ForGuard::{self, OutsideGuard, RefWithinGuard};
+use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::build::{GuardFrame, GuardFrameLocal, LocalsForNode};
+use rustc_data_structures::{
+    fx::{FxHashSet, FxIndexMap, FxIndexSet},
+    stack::ensure_sufficient_stack,
+};
+use rustc_index::bit_set::BitSet;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::thir::{self, *};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty};
+use rustc_span::symbol::Symbol;
+use rustc_span::{BytePos, Pos, Span};
+use rustc_target::abi::VariantIdx;
+use smallvec::{smallvec, SmallVec};
+
+// helper functions, broken out by category:
+mod simplify;
+mod test;
+mod util;
+
+use std::borrow::Borrow;
+use std::convert::TryFrom;
+use std::mem;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    pub(crate) fn then_else_break(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+        temp_scope_override: Option<region::Scope>,
+        break_scope: region::Scope,
+        variable_source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let this = self;
+        let expr_span = expr.span;
+
+        match expr.kind {
+            ExprKind::LogicalOp { op: LogicalOp::And, lhs, rhs } => {
+                let lhs_then_block = unpack!(this.then_else_break(
+                    block,
+                    &this.thir[lhs],
+                    temp_scope_override,
+                    break_scope,
+                    variable_source_info,
+                ));
+
+                let rhs_then_block = unpack!(this.then_else_break(
+                    lhs_then_block,
+                    &this.thir[rhs],
+                    temp_scope_override,
+                    break_scope,
+                    variable_source_info,
+                ));
+
+                rhs_then_block.unit()
+            }
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, this.source_info(expr_span));
+                this.in_scope(region_scope, lint_level, |this| {
+                    this.then_else_break(
+                        block,
+                        &this.thir[value],
+                        temp_scope_override,
+                        break_scope,
+                        variable_source_info,
+                    )
+                })
+            }
+            ExprKind::Let { expr, ref pat } => this.lower_let_expr(
+                block,
+                &this.thir[expr],
+                pat,
+                break_scope,
+                Some(variable_source_info.scope),
+                variable_source_info.span,
+            ),
+            _ => {
+                let temp_scope = temp_scope_override.unwrap_or_else(|| this.local_scope());
+                let mutability = Mutability::Mut;
+                let place =
+                    unpack!(block = this.as_temp(block, Some(temp_scope), expr, mutability));
+                let operand = Operand::Move(Place::from(place));
+
+                let then_block = this.cfg.start_new_block();
+                let else_block = this.cfg.start_new_block();
+                let term = TerminatorKind::if_(this.tcx, operand, then_block, else_block);
+
+                let source_info = this.source_info(expr_span);
+                this.cfg.terminate(block, source_info, term);
+                this.break_for_else(else_block, break_scope, source_info);
+
+                then_block.unit()
+            }
+        }
+    }
+
+    /// Generates MIR for a `match` expression.
+    ///
+    /// The MIR that we generate for a match looks like this.
+    ///
+    /// ```text
+    /// [ 0. Pre-match ]
+    ///        |
+    /// [ 1. Evaluate Scrutinee (expression being matched on) ]
+    /// [ (fake read of scrutinee) ]
+    ///        |
+    /// [ 2. Decision tree -- check discriminants ] <--------+
+    ///        |                                             |
+    ///        | (once a specific arm is chosen)             |
+    ///        |                                             |
+    /// [pre_binding_block]                           [otherwise_block]
+    ///        |                                             |
+    /// [ 3. Create "guard bindings" for arm ]               |
+    /// [ (create fake borrows) ]                            |
+    ///        |                                             |
+    /// [ 4. Execute guard code ]                            |
+    /// [ (read fake borrows) ] --(guard is false)-----------+
+    ///        |
+    ///        | (guard results in true)
+    ///        |
+    /// [ 5. Create real bindings and execute arm ]
+    ///        |
+    /// [ Exit match ]
+    /// ```
+    ///
+    /// All of the different arms have been stacked on top of each other to
+    /// simplify the diagram. For an arm with no guard the blocks marked 3 and
+    /// 4 and the fake borrows are omitted.
+    ///
+    /// We generate MIR in the following steps:
+    ///
+    /// 1. Evaluate the scrutinee and add the fake read of it ([Builder::lower_scrutinee]).
+    /// 2. Create the decision tree ([Builder::lower_match_tree]).
+    /// 3. Determine the fake borrows that are needed from the places that were
+    ///    matched against and create the required temporaries for them
+    ///    ([Builder::calculate_fake_borrows]).
+    /// 4. Create everything else: the guards and the arms ([Builder::lower_match_arms]).
+    ///
+    /// ## False edges
+    ///
+    /// We don't want to have the exact structure of the decision tree be
+    /// visible through borrow checking. False edges ensure that the CFG as
+    /// seen by borrow checking doesn't encode this. False edges are added:
+    ///
+    /// * From each pre-binding block to the next pre-binding block.
+    /// * From each otherwise block to the next pre-binding block.
+    #[tracing::instrument(level = "debug", skip(self, arms))]
+    pub(crate) fn match_expr(
+        &mut self,
+        destination: Place<'tcx>,
+        span: Span,
+        mut block: BasicBlock,
+        scrutinee: &Expr<'tcx>,
+        arms: &[ArmId],
+    ) -> BlockAnd<()> {
+        let scrutinee_span = scrutinee.span;
+        let scrutinee_place =
+            unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span,));
+
+        let mut arm_candidates = self.create_match_candidates(scrutinee_place.clone(), &arms);
+
+        let match_has_guard = arms.iter().copied().any(|arm| self.thir[arm].guard.is_some());
+        let mut candidates =
+            arm_candidates.iter_mut().map(|(_, candidate)| candidate).collect::<Vec<_>>();
+
+        let match_start_span = span.shrink_to_lo().to(scrutinee.span);
+
+        let fake_borrow_temps = self.lower_match_tree(
+            block,
+            scrutinee_span,
+            match_start_span,
+            match_has_guard,
+            &mut candidates,
+        );
+
+        self.lower_match_arms(
+            destination,
+            scrutinee_place,
+            scrutinee_span,
+            arm_candidates,
+            self.source_info(span),
+            fake_borrow_temps,
+        )
+    }
+
+    /// Evaluate the scrutinee and add the fake read of it.
+    fn lower_scrutinee(
+        &mut self,
+        mut block: BasicBlock,
+        scrutinee: &Expr<'tcx>,
+        scrutinee_span: Span,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let scrutinee_place_builder = unpack!(block = self.as_place_builder(block, scrutinee));
+        // Matching on a `scrutinee_place` with an uninhabited type doesn't
+        // generate any memory reads by itself, and so if the place "expression"
+        // contains unsafe operations like raw pointer dereferences or union
+        // field projections, we wouldn't know to require an `unsafe` block
+        // around a `match` equivalent to `std::intrinsics::unreachable()`.
+        // See issue #47412 for this hole being discovered in the wild.
+        //
+        // HACK(eddyb) Work around the above issue by adding a dummy inspection
+        // of `scrutinee_place`, specifically by applying `ReadForMatch`.
+        //
+        // NOTE: ReadForMatch also checks that the scrutinee is initialized.
+        // This is currently needed to not allow matching on an uninitialized,
+        // uninhabited value. If we get never patterns, those will check that
+        // the place is initialized, and so this read would only be used to
+        // check safety.
+        let cause_matched_place = FakeReadCause::ForMatchedPlace(None);
+        let source_info = self.source_info(scrutinee_span);
+
+        if let Ok(scrutinee_builder) =
+            scrutinee_place_builder.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            let scrutinee_place = scrutinee_builder.into_place(self.tcx, self.typeck_results);
+            self.cfg.push_fake_read(block, source_info, cause_matched_place, scrutinee_place);
+        }
+
+        block.and(scrutinee_place_builder)
+    }
+
+    /// Create the initial `Candidate`s for a `match` expression.
+    fn create_match_candidates<'pat>(
+        &mut self,
+        scrutinee: PlaceBuilder<'tcx>,
+        arms: &'pat [ArmId],
+    ) -> Vec<(&'pat Arm<'tcx>, Candidate<'pat, 'tcx>)>
+    where
+        'a: 'pat,
+    {
+        // Assemble a list of candidates: there is one candidate per pattern,
+        // which means there may be more than one candidate *per arm*.
+        arms.iter()
+            .copied()
+            .map(|arm| {
+                let arm = &self.thir[arm];
+                let arm_has_guard = arm.guard.is_some();
+                let arm_candidate = Candidate::new(scrutinee.clone(), &arm.pattern, arm_has_guard);
+                (arm, arm_candidate)
+            })
+            .collect()
+    }
+
+    /// Create the decision tree for the match expression, starting from `block`.
+    ///
+    /// Modifies `candidates` to store the bindings and type ascriptions for
+    /// that candidate.
+    ///
+    /// Returns the places that need fake borrows because we bind or test them.
+    fn lower_match_tree<'pat>(
+        &mut self,
+        block: BasicBlock,
+        scrutinee_span: Span,
+        match_start_span: Span,
+        match_has_guard: bool,
+        candidates: &mut [&mut Candidate<'pat, 'tcx>],
+    ) -> Vec<(Place<'tcx>, Local)> {
+        // The set of places that we are creating fake borrows of. If there are
+        // no match guards then we don't need any fake borrows, so don't track
+        // them.
+        let mut fake_borrows = match_has_guard.then(FxIndexSet::default);
+
+        let mut otherwise = None;
+
+        // This will generate code to test scrutinee_place and
+        // branch to the appropriate arm block
+        self.match_candidates(
+            match_start_span,
+            scrutinee_span,
+            block,
+            &mut otherwise,
+            candidates,
+            &mut fake_borrows,
+        );
+
+        if let Some(otherwise_block) = otherwise {
+            // See the doc comment on `match_candidates` for why we may have an
+            // otherwise block. Match checking will ensure this is actually
+            // unreachable.
+            let source_info = self.source_info(scrutinee_span);
+            self.cfg.terminate(otherwise_block, source_info, TerminatorKind::Unreachable);
+        }
+
+        // Link each leaf candidate to the `pre_binding_block` of the next one.
+        let mut previous_candidate: Option<&mut Candidate<'_, '_>> = None;
+
+        for candidate in candidates {
+            candidate.visit_leaves(|leaf_candidate| {
+                if let Some(ref mut prev) = previous_candidate {
+                    prev.next_candidate_pre_binding_block = leaf_candidate.pre_binding_block;
+                }
+                previous_candidate = Some(leaf_candidate);
+            });
+        }
+
+        if let Some(ref borrows) = fake_borrows {
+            self.calculate_fake_borrows(borrows, scrutinee_span)
+        } else {
+            Vec::new()
+        }
+    }
+
+    /// Lower the bindings, guards and arm bodies of a `match` expression.
+    ///
+    /// The decision tree should have already been created
+    /// (by [Builder::lower_match_tree]).
+    ///
+    /// `outer_source_info` is the SourceInfo for the whole match.
+    fn lower_match_arms(
+        &mut self,
+        destination: Place<'tcx>,
+        scrutinee_place_builder: PlaceBuilder<'tcx>,
+        scrutinee_span: Span,
+        arm_candidates: Vec<(&'_ Arm<'tcx>, Candidate<'_, 'tcx>)>,
+        outer_source_info: SourceInfo,
+        fake_borrow_temps: Vec<(Place<'tcx>, Local)>,
+    ) -> BlockAnd<()> {
+        let arm_end_blocks: Vec<_> = arm_candidates
+            .into_iter()
+            .map(|(arm, candidate)| {
+                debug!("lowering arm {:?}\ncandidate = {:?}", arm, candidate);
+
+                let arm_source_info = self.source_info(arm.span);
+                let arm_scope = (arm.scope, arm_source_info);
+                let match_scope = self.local_scope();
+                self.in_scope(arm_scope, arm.lint_level, |this| {
+                    // `try_upvars_resolved` may fail if it is unable to resolve the given
+                    // `PlaceBuilder` inside a closure. In this case, we don't want to include
+                    // a scrutinee place. `scrutinee_place_builder` will fail to be resolved
+                    // if the only match arm is a wildcard (`_`).
+                    // Example:
+                    // ```
+                    // let foo = (0, 1);
+                    // let c = || {
+                    //    match foo { _ => () };
+                    // };
+                    // ```
+                    let mut opt_scrutinee_place: Option<(Option<&Place<'tcx>>, Span)> = None;
+                    let scrutinee_place: Place<'tcx>;
+                    if let Ok(scrutinee_builder) = scrutinee_place_builder
+                        .clone()
+                        .try_upvars_resolved(this.tcx, this.typeck_results)
+                    {
+                        scrutinee_place =
+                            scrutinee_builder.into_place(this.tcx, this.typeck_results);
+                        opt_scrutinee_place = Some((Some(&scrutinee_place), scrutinee_span));
+                    }
+                    let scope = this.declare_bindings(
+                        None,
+                        arm.span,
+                        &arm.pattern,
+                        ArmHasGuard(arm.guard.is_some()),
+                        opt_scrutinee_place,
+                    );
+
+                    let arm_block = this.bind_pattern(
+                        outer_source_info,
+                        candidate,
+                        arm.guard.as_ref(),
+                        &fake_borrow_temps,
+                        scrutinee_span,
+                        Some(arm.span),
+                        Some(arm.scope),
+                        Some(match_scope),
+                    );
+
+                    if let Some(source_scope) = scope {
+                        this.source_scope = source_scope;
+                    }
+
+                    this.expr_into_dest(destination, arm_block, &&this.thir[arm.body])
+                })
+            })
+            .collect();
+
+        // all the arm blocks will rejoin here
+        let end_block = self.cfg.start_new_block();
+
+        let end_brace = self.source_info(
+            outer_source_info.span.with_lo(outer_source_info.span.hi() - BytePos::from_usize(1)),
+        );
+        for arm_block in arm_end_blocks {
+            let block = &self.cfg.basic_blocks[arm_block.0];
+            let last_location = block.statements.last().map(|s| s.source_info);
+
+            self.cfg.goto(unpack!(arm_block), last_location.unwrap_or(end_brace), end_block);
+        }
+
+        self.source_scope = outer_source_info.scope;
+
+        end_block.unit()
+    }
+
+    /// Binds the variables and ascribes types for a given `match` arm or
+    /// `let` binding.
+    ///
+    /// Also check if the guard matches, if it's provided.
+    /// `arm_scope` should be `Some` if and only if this is called for a
+    /// `match` arm.
+    fn bind_pattern(
+        &mut self,
+        outer_source_info: SourceInfo,
+        candidate: Candidate<'_, 'tcx>,
+        guard: Option<&Guard<'tcx>>,
+        fake_borrow_temps: &[(Place<'tcx>, Local)],
+        scrutinee_span: Span,
+        arm_span: Option<Span>,
+        arm_scope: Option<region::Scope>,
+        match_scope: Option<region::Scope>,
+    ) -> BasicBlock {
+        if candidate.subcandidates.is_empty() {
+            // Avoid generating another `BasicBlock` when we only have one
+            // candidate.
+            self.bind_and_guard_matched_candidate(
+                candidate,
+                &[],
+                guard,
+                fake_borrow_temps,
+                scrutinee_span,
+                arm_span,
+                match_scope,
+                true,
+            )
+        } else {
+            // It's helpful to avoid scheduling drops multiple times to save
+            // drop elaboration from having to clean up the extra drops.
+            //
+            // If we are in a `let` then we only schedule drops for the first
+            // candidate.
+            //
+            // If we're in a `match` arm then we could have a case like so:
+            //
+            // Ok(x) | Err(x) if return => { /* ... */ }
+            //
+            // In this case we don't want a drop of `x` scheduled when we
+            // return: it isn't bound by move until right before enter the arm.
+            // To handle this we instead unschedule it's drop after each time
+            // we lower the guard.
+            let target_block = self.cfg.start_new_block();
+            let mut schedule_drops = true;
+            // We keep a stack of all of the bindings and type ascriptions
+            // from the parent candidates that we visit, that also need to
+            // be bound for each candidate.
+            traverse_candidate(
+                candidate,
+                &mut Vec::new(),
+                &mut |leaf_candidate, parent_bindings| {
+                    if let Some(arm_scope) = arm_scope {
+                        self.clear_top_scope(arm_scope);
+                    }
+                    let binding_end = self.bind_and_guard_matched_candidate(
+                        leaf_candidate,
+                        parent_bindings,
+                        guard,
+                        &fake_borrow_temps,
+                        scrutinee_span,
+                        arm_span,
+                        match_scope,
+                        schedule_drops,
+                    );
+                    if arm_scope.is_none() {
+                        schedule_drops = false;
+                    }
+                    self.cfg.goto(binding_end, outer_source_info, target_block);
+                },
+                |inner_candidate, parent_bindings| {
+                    parent_bindings.push((inner_candidate.bindings, inner_candidate.ascriptions));
+                    inner_candidate.subcandidates.into_iter()
+                },
+                |parent_bindings| {
+                    parent_bindings.pop();
+                },
+            );
+
+            target_block
+        }
+    }
+
+    pub(super) fn expr_into_pattern(
+        &mut self,
+        mut block: BasicBlock,
+        irrefutable_pat: Pat<'tcx>,
+        initializer: &Expr<'tcx>,
+    ) -> BlockAnd<()> {
+        match *irrefutable_pat.kind {
+            // Optimize the case of `let x = ...` to write directly into `x`
+            PatKind::Binding { mode: BindingMode::ByValue, var, subpattern: None, .. } => {
+                let place =
+                    self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
+                unpack!(block = self.expr_into_dest(place, block, initializer));
+
+                // Inject a fake read, see comments on `FakeReadCause::ForLet`.
+                let source_info = self.source_info(irrefutable_pat.span);
+                self.cfg.push_fake_read(block, source_info, FakeReadCause::ForLet(None), place);
+
+                self.schedule_drop_for_binding(var, irrefutable_pat.span, OutsideGuard);
+                block.unit()
+            }
+
+            // Optimize the case of `let x: T = ...` to write directly
+            // into `x` and then require that `T == typeof(x)`.
+            //
+            // Weirdly, this is needed to prevent the
+            // `intrinsic-move-val.rs` test case from crashing. That
+            // test works with uninitialized values in a rather
+            // dubious way, so it may be that the test is kind of
+            // broken.
+            PatKind::AscribeUserType {
+                subpattern:
+                    Pat {
+                        kind:
+                            box PatKind::Binding {
+                                mode: BindingMode::ByValue,
+                                var,
+                                subpattern: None,
+                                ..
+                            },
+                        ..
+                    },
+                ascription: thir::Ascription { annotation, variance: _ },
+            } => {
+                let place =
+                    self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
+                unpack!(block = self.expr_into_dest(place, block, initializer));
+
+                // Inject a fake read, see comments on `FakeReadCause::ForLet`.
+                let pattern_source_info = self.source_info(irrefutable_pat.span);
+                let cause_let = FakeReadCause::ForLet(None);
+                self.cfg.push_fake_read(block, pattern_source_info, cause_let, place);
+
+                let ty_source_info = self.source_info(annotation.span);
+
+                let base = self.canonical_user_type_annotations.push(annotation);
+                self.cfg.push(
+                    block,
+                    Statement {
+                        source_info: ty_source_info,
+                        kind: StatementKind::AscribeUserType(
+                            Box::new((place, UserTypeProjection { base, projs: Vec::new() })),
+                            // We always use invariant as the variance here. This is because the
+                            // variance field from the ascription refers to the variance to use
+                            // when applying the type to the value being matched, but this
+                            // ascription applies rather to the type of the binding. e.g., in this
+                            // example:
+                            //
+                            // ```
+                            // let x: T = <expr>
+                            // ```
+                            //
+                            // We are creating an ascription that defines the type of `x` to be
+                            // exactly `T` (i.e., with invariance). The variance field, in
+                            // contrast, is intended to be used to relate `T` to the type of
+                            // `<expr>`.
+                            ty::Variance::Invariant,
+                        ),
+                    },
+                );
+
+                self.schedule_drop_for_binding(var, irrefutable_pat.span, OutsideGuard);
+                block.unit()
+            }
+
+            _ => {
+                let place_builder = unpack!(block = self.as_place_builder(block, initializer));
+                self.place_into_pattern(block, irrefutable_pat, place_builder, true)
+            }
+        }
+    }
+
+    pub(crate) fn place_into_pattern(
+        &mut self,
+        block: BasicBlock,
+        irrefutable_pat: Pat<'tcx>,
+        initializer: PlaceBuilder<'tcx>,
+        set_match_place: bool,
+    ) -> BlockAnd<()> {
+        let mut candidate = Candidate::new(initializer.clone(), &irrefutable_pat, false);
+        let fake_borrow_temps = self.lower_match_tree(
+            block,
+            irrefutable_pat.span,
+            irrefutable_pat.span,
+            false,
+            &mut [&mut candidate],
+        );
+        // For matches and function arguments, the place that is being matched
+        // can be set when creating the variables. But the place for
+        // let PATTERN = ... might not even exist until we do the assignment.
+        // so we set it here instead.
+        if set_match_place {
+            let mut candidate_ref = &candidate;
+            while let Some(next) = {
+                for binding in &candidate_ref.bindings {
+                    let local = self.var_local_id(binding.var_id, OutsideGuard);
+
+                    let Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
+                        VarBindingForm { opt_match_place: Some((ref mut match_place, _)), .. },
+                    )))) = self.local_decls[local].local_info else {
+                        bug!("Let binding to non-user variable.")
+                    };
+                    // `try_upvars_resolved` may fail if it is unable to resolve the given
+                    // `PlaceBuilder` inside a closure. In this case, we don't want to include
+                    // a scrutinee place. `scrutinee_place_builder` will fail for destructured
+                    // assignments. This is because a closure only captures the precise places
+                    // that it will read and as a result a closure may not capture the entire
+                    // tuple/struct and rather have individual places that will be read in the
+                    // final MIR.
+                    // Example:
+                    // ```
+                    // let foo = (0, 1);
+                    // let c = || {
+                    //    let (v1, v2) = foo;
+                    // };
+                    // ```
+                    if let Ok(match_pair_resolved) =
+                        initializer.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                    {
+                        let place = match_pair_resolved.into_place(self.tcx, self.typeck_results);
+                        *match_place = Some(place);
+                    }
+                }
+                // All of the subcandidates should bind the same locals, so we
+                // only visit the first one.
+                candidate_ref.subcandidates.get(0)
+            } {
+                candidate_ref = next;
+            }
+        }
+
+        self.bind_pattern(
+            self.source_info(irrefutable_pat.span),
+            candidate,
+            None,
+            &fake_borrow_temps,
+            irrefutable_pat.span,
+            None,
+            None,
+            None,
+        )
+        .unit()
+    }
+
+    /// Declares the bindings of the given patterns and returns the visibility
+    /// scope for the bindings in these patterns, if such a scope had to be
+    /// created. NOTE: Declaring the bindings should always be done in their
+    /// drop scope.
+    pub(crate) fn declare_bindings(
+        &mut self,
+        mut visibility_scope: Option<SourceScope>,
+        scope_span: Span,
+        pattern: &Pat<'tcx>,
+        has_guard: ArmHasGuard,
+        opt_match_place: Option<(Option<&Place<'tcx>>, Span)>,
+    ) -> Option<SourceScope> {
+        debug!("declare_bindings: pattern={:?}", pattern);
+        self.visit_primary_bindings(
+            &pattern,
+            UserTypeProjections::none(),
+            &mut |this, mutability, name, mode, var, span, ty, user_ty| {
+                if visibility_scope.is_none() {
+                    visibility_scope =
+                        Some(this.new_source_scope(scope_span, LintLevel::Inherited, None));
+                }
+                let source_info = SourceInfo { span, scope: this.source_scope };
+                let visibility_scope = visibility_scope.unwrap();
+                this.declare_binding(
+                    source_info,
+                    visibility_scope,
+                    mutability,
+                    name,
+                    mode,
+                    var,
+                    ty,
+                    user_ty,
+                    has_guard,
+                    opt_match_place.map(|(x, y)| (x.cloned(), y)),
+                    pattern.span,
+                );
+            },
+        );
+        visibility_scope
+    }
+
+    pub(crate) fn storage_live_binding(
+        &mut self,
+        block: BasicBlock,
+        var: LocalVarId,
+        span: Span,
+        for_guard: ForGuard,
+        schedule_drop: bool,
+    ) -> Place<'tcx> {
+        let local_id = self.var_local_id(var, for_guard);
+        let source_info = self.source_info(span);
+        self.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(local_id) });
+        // Altough there is almost always scope for given variable in corner cases
+        // like #92893 we might get variable with no scope.
+        if let Some(region_scope) = self.region_scope_tree.var_scope(var.0.local_id) && schedule_drop{
+            self.schedule_drop(span, region_scope, local_id, DropKind::Storage);
+        }
+        Place::from(local_id)
+    }
+
+    pub(crate) fn schedule_drop_for_binding(
+        &mut self,
+        var: LocalVarId,
+        span: Span,
+        for_guard: ForGuard,
+    ) {
+        let local_id = self.var_local_id(var, for_guard);
+        if let Some(region_scope) = self.region_scope_tree.var_scope(var.0.local_id) {
+            self.schedule_drop(span, region_scope, local_id, DropKind::Value);
+        }
+    }
+
+    /// Visit all of the primary bindings in a patterns, that is, visit the
+    /// leftmost occurrence of each variable bound in a pattern. A variable
+    /// will occur more than once in an or-pattern.
+    pub(super) fn visit_primary_bindings(
+        &mut self,
+        pattern: &Pat<'tcx>,
+        pattern_user_ty: UserTypeProjections,
+        f: &mut impl FnMut(
+            &mut Self,
+            Mutability,
+            Symbol,
+            BindingMode,
+            LocalVarId,
+            Span,
+            Ty<'tcx>,
+            UserTypeProjections,
+        ),
+    ) {
+        debug!(
+            "visit_primary_bindings: pattern={:?} pattern_user_ty={:?}",
+            pattern, pattern_user_ty
+        );
+        match *pattern.kind {
+            PatKind::Binding {
+                mutability,
+                name,
+                mode,
+                var,
+                ty,
+                ref subpattern,
+                is_primary,
+                ..
+            } => {
+                if is_primary {
+                    f(self, mutability, name, mode, var, pattern.span, ty, pattern_user_ty.clone());
+                }
+                if let Some(subpattern) = subpattern.as_ref() {
+                    self.visit_primary_bindings(subpattern, pattern_user_ty, f);
+                }
+            }
+
+            PatKind::Array { ref prefix, ref slice, ref suffix }
+            | PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                let from = u64::try_from(prefix.len()).unwrap();
+                let to = u64::try_from(suffix.len()).unwrap();
+                for subpattern in prefix {
+                    self.visit_primary_bindings(subpattern, pattern_user_ty.clone().index(), f);
+                }
+                for subpattern in slice {
+                    self.visit_primary_bindings(
+                        subpattern,
+                        pattern_user_ty.clone().subslice(from, to),
+                        f,
+                    );
+                }
+                for subpattern in suffix {
+                    self.visit_primary_bindings(subpattern, pattern_user_ty.clone().index(), f);
+                }
+            }
+
+            PatKind::Constant { .. } | PatKind::Range { .. } | PatKind::Wild => {}
+
+            PatKind::Deref { ref subpattern } => {
+                self.visit_primary_bindings(subpattern, pattern_user_ty.deref(), f);
+            }
+
+            PatKind::AscribeUserType {
+                ref subpattern,
+                ascription: thir::Ascription { ref annotation, variance: _ },
+            } => {
+                // This corresponds to something like
+                //
+                // ```
+                // let A::<'a>(_): A<'static> = ...;
+                // ```
+                //
+                // Note that the variance doesn't apply here, as we are tracking the effect
+                // of `user_ty` on any bindings contained with subpattern.
+
+                let projection = UserTypeProjection {
+                    base: self.canonical_user_type_annotations.push(annotation.clone()),
+                    projs: Vec::new(),
+                };
+                let subpattern_user_ty =
+                    pattern_user_ty.push_projection(&projection, annotation.span);
+                self.visit_primary_bindings(subpattern, subpattern_user_ty, f)
+            }
+
+            PatKind::Leaf { ref subpatterns } => {
+                for subpattern in subpatterns {
+                    let subpattern_user_ty = pattern_user_ty.clone().leaf(subpattern.field);
+                    debug!("visit_primary_bindings: subpattern_user_ty={:?}", subpattern_user_ty);
+                    self.visit_primary_bindings(&subpattern.pattern, subpattern_user_ty, f);
+                }
+            }
+
+            PatKind::Variant { adt_def, substs: _, variant_index, ref subpatterns } => {
+                for subpattern in subpatterns {
+                    let subpattern_user_ty =
+                        pattern_user_ty.clone().variant(adt_def, variant_index, subpattern.field);
+                    self.visit_primary_bindings(&subpattern.pattern, subpattern_user_ty, f);
+                }
+            }
+            PatKind::Or { ref pats } => {
+                // In cases where we recover from errors the primary bindings
+                // may not all be in the leftmost subpattern. For example in
+                // `let (x | y) = ...`, the primary binding of `y` occurs in
+                // the right subpattern
+                for subpattern in pats {
+                    self.visit_primary_bindings(subpattern, pattern_user_ty.clone(), f);
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Candidate<'pat, 'tcx> {
+    /// [`Span`] of the original pattern that gave rise to this candidate.
+    span: Span,
+
+    /// Whether this `Candidate` has a guard.
+    has_guard: bool,
+
+    /// All of these must be satisfied...
+    match_pairs: SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
+
+    /// ...these bindings established...
+    bindings: Vec<Binding<'tcx>>,
+
+    /// ...and these types asserted...
+    ascriptions: Vec<Ascription<'tcx>>,
+
+    /// ...and if this is non-empty, one of these subcandidates also has to match...
+    subcandidates: Vec<Candidate<'pat, 'tcx>>,
+
+    /// ...and the guard must be evaluated; if it's `false` then branch to `otherwise_block`.
+    otherwise_block: Option<BasicBlock>,
+
+    /// The block before the `bindings` have been established.
+    pre_binding_block: Option<BasicBlock>,
+    /// The pre-binding block of the next candidate.
+    next_candidate_pre_binding_block: Option<BasicBlock>,
+}
+
+impl<'tcx, 'pat> Candidate<'pat, 'tcx> {
+    fn new(place: PlaceBuilder<'tcx>, pattern: &'pat Pat<'tcx>, has_guard: bool) -> Self {
+        Candidate {
+            span: pattern.span,
+            has_guard,
+            match_pairs: smallvec![MatchPair { place, pattern }],
+            bindings: Vec::new(),
+            ascriptions: Vec::new(),
+            subcandidates: Vec::new(),
+            otherwise_block: None,
+            pre_binding_block: None,
+            next_candidate_pre_binding_block: None,
+        }
+    }
+
+    /// Visit the leaf candidates (those with no subcandidates) contained in
+    /// this candidate.
+    fn visit_leaves<'a>(&'a mut self, mut visit_leaf: impl FnMut(&'a mut Self)) {
+        traverse_candidate(
+            self,
+            &mut (),
+            &mut move |c, _| visit_leaf(c),
+            move |c, _| c.subcandidates.iter_mut(),
+            |_| {},
+        );
+    }
+}
+
+/// A depth-first traversal of the `Candidate` and all of its recursive
+/// subcandidates.
+fn traverse_candidate<'pat, 'tcx: 'pat, C, T, I>(
+    candidate: C,
+    context: &mut T,
+    visit_leaf: &mut impl FnMut(C, &mut T),
+    get_children: impl Copy + Fn(C, &mut T) -> I,
+    complete_children: impl Copy + Fn(&mut T),
+) where
+    C: Borrow<Candidate<'pat, 'tcx>>,
+    I: Iterator<Item = C>,
+{
+    if candidate.borrow().subcandidates.is_empty() {
+        visit_leaf(candidate, context)
+    } else {
+        for child in get_children(candidate, context) {
+            traverse_candidate(child, context, visit_leaf, get_children, complete_children);
+        }
+        complete_children(context)
+    }
+}
+
+#[derive(Clone, Debug)]
+struct Binding<'tcx> {
+    span: Span,
+    source: Place<'tcx>,
+    var_id: LocalVarId,
+    binding_mode: BindingMode,
+}
+
+/// Indicates that the type of `source` must be a subtype of the
+/// user-given type `user_ty`; this is basically a no-op but can
+/// influence region inference.
+#[derive(Clone, Debug)]
+struct Ascription<'tcx> {
+    source: Place<'tcx>,
+    annotation: CanonicalUserTypeAnnotation<'tcx>,
+    variance: ty::Variance,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MatchPair<'pat, 'tcx> {
+    // this place...
+    place: PlaceBuilder<'tcx>,
+
+    // ... must match this pattern.
+    pattern: &'pat Pat<'tcx>,
+}
+
+/// See [`Test`] for more.
+#[derive(Clone, Debug, PartialEq)]
+enum TestKind<'tcx> {
+    /// Test what enum variant a value is.
+    Switch {
+        /// The enum type being tested.
+        adt_def: ty::AdtDef<'tcx>,
+        /// The set of variants that we should create a branch for. We also
+        /// create an additional "otherwise" case.
+        variants: BitSet<VariantIdx>,
+    },
+
+    /// Test what value an integer, `bool`, or `char` has.
+    SwitchInt {
+        /// The type of the value that we're testing.
+        switch_ty: Ty<'tcx>,
+        /// The (ordered) set of values that we test for.
+        ///
+        /// For integers and `char`s we create a branch to each of the values in
+        /// `options`, as well as an "otherwise" branch for all other values, even
+        /// in the (rare) case that `options` is exhaustive.
+        ///
+        /// For `bool` we always generate two edges, one for `true` and one for
+        /// `false`.
+        options: FxIndexMap<ConstantKind<'tcx>, u128>,
+    },
+
+    /// Test for equality with value, possibly after an unsizing coercion to
+    /// `ty`,
+    Eq {
+        value: ConstantKind<'tcx>,
+        // Integer types are handled by `SwitchInt`, and constants with ADT
+        // types are converted back into patterns, so this can only be `&str`,
+        // `&[T]`, `f32` or `f64`.
+        ty: Ty<'tcx>,
+    },
+
+    /// Test whether the value falls within an inclusive or exclusive range
+    Range(PatRange<'tcx>),
+
+    /// Test that the length of the slice is equal to `len`.
+    Len { len: u64, op: BinOp },
+}
+
+/// A test to perform to determine which [`Candidate`] matches a value.
+///
+/// [`Test`] is just the test to perform; it does not include the value
+/// to be tested.
+#[derive(Debug)]
+pub(crate) struct Test<'tcx> {
+    span: Span,
+    kind: TestKind<'tcx>,
+}
+
+/// `ArmHasGuard` is a wrapper around a boolean flag. It indicates whether
+/// a match arm has a guard expression attached to it.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct ArmHasGuard(pub(crate) bool);
+
+///////////////////////////////////////////////////////////////////////////
+// Main matching algorithm
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// The main match algorithm. It begins with a set of candidates
+    /// `candidates` and has the job of generating code to determine
+    /// which of these candidates, if any, is the correct one. The
+    /// candidates are sorted such that the first item in the list
+    /// has the highest priority. When a candidate is found to match
+    /// the value, we will set and generate a branch to the appropriate
+    /// pre-binding block.
+    ///
+    /// If we find that *NONE* of the candidates apply, we branch to the
+    /// `otherwise_block`, setting it to `Some` if required. In principle, this
+    /// means that the input list was not exhaustive, though at present we
+    /// sometimes are not smart enough to recognize all exhaustive inputs.
+    ///
+    /// It might be surprising that the input can be non-exhaustive.
+    /// Indeed, initially, it is not, because all matches are
+    /// exhaustive in Rust. But during processing we sometimes divide
+    /// up the list of candidates and recurse with a non-exhaustive
+    /// list. This is important to keep the size of the generated code
+    /// under control. See [`Builder::test_candidates`] for more details.
+    ///
+    /// If `fake_borrows` is `Some`, then places which need fake borrows
+    /// will be added to it.
+    ///
+    /// For an example of a case where we set `otherwise_block`, even for an
+    /// exhaustive match, consider:
+    ///
+    /// ```
+    /// # fn foo(x: (bool, bool)) {
+    /// match x {
+    ///     (true, true) => (),
+    ///     (_, false) => (),
+    ///     (false, true) => (),
+    /// }
+    /// # }
+    /// ```
+    ///
+    /// For this match, we check if `x.0` matches `true` (for the first
+    /// arm). If it doesn't match, we check `x.1`. If `x.1` is `true` we check
+    /// if `x.0` matches `false` (for the third arm). In the (impossible at
+    /// runtime) case when `x.0` is now `true`, we branch to
+    /// `otherwise_block`.
+    fn match_candidates<'pat>(
+        &mut self,
+        span: Span,
+        scrutinee_span: Span,
+        start_block: BasicBlock,
+        otherwise_block: &mut Option<BasicBlock>,
+        candidates: &mut [&mut Candidate<'pat, 'tcx>],
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) {
+        debug!(
+            "matched_candidate(span={:?}, candidates={:?}, start_block={:?}, otherwise_block={:?})",
+            span, candidates, start_block, otherwise_block,
+        );
+
+        // Start by simplifying candidates. Once this process is complete, all
+        // the match pairs which remain require some form of test, whether it
+        // be a switch or pattern comparison.
+        let mut split_or_candidate = false;
+        for candidate in &mut *candidates {
+            split_or_candidate |= self.simplify_candidate(candidate);
+        }
+
+        ensure_sufficient_stack(|| {
+            if split_or_candidate {
+                // At least one of the candidates has been split into subcandidates.
+                // We need to change the candidate list to include those.
+                let mut new_candidates = Vec::new();
+
+                for candidate in candidates {
+                    candidate.visit_leaves(|leaf_candidate| new_candidates.push(leaf_candidate));
+                }
+                self.match_simplified_candidates(
+                    span,
+                    scrutinee_span,
+                    start_block,
+                    otherwise_block,
+                    &mut *new_candidates,
+                    fake_borrows,
+                );
+            } else {
+                self.match_simplified_candidates(
+                    span,
+                    scrutinee_span,
+                    start_block,
+                    otherwise_block,
+                    candidates,
+                    fake_borrows,
+                );
+            }
+        });
+    }
+
+    fn match_simplified_candidates(
+        &mut self,
+        span: Span,
+        scrutinee_span: Span,
+        start_block: BasicBlock,
+        otherwise_block: &mut Option<BasicBlock>,
+        candidates: &mut [&mut Candidate<'_, 'tcx>],
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) {
+        // The candidates are sorted by priority. Check to see whether the
+        // higher priority candidates (and hence at the front of the slice)
+        // have satisfied all their match pairs.
+        let fully_matched = candidates.iter().take_while(|c| c.match_pairs.is_empty()).count();
+        debug!("match_candidates: {:?} candidates fully matched", fully_matched);
+        let (matched_candidates, unmatched_candidates) = candidates.split_at_mut(fully_matched);
+
+        let block = if !matched_candidates.is_empty() {
+            let otherwise_block =
+                self.select_matched_candidates(matched_candidates, start_block, fake_borrows);
+
+            if let Some(last_otherwise_block) = otherwise_block {
+                last_otherwise_block
+            } else {
+                // Any remaining candidates are unreachable.
+                if unmatched_candidates.is_empty() {
+                    return;
+                }
+                self.cfg.start_new_block()
+            }
+        } else {
+            start_block
+        };
+
+        // If there are no candidates that still need testing, we're
+        // done. Since all matches are exhaustive, execution should
+        // never reach this point.
+        if unmatched_candidates.is_empty() {
+            let source_info = self.source_info(span);
+            if let Some(otherwise) = *otherwise_block {
+                self.cfg.goto(block, source_info, otherwise);
+            } else {
+                *otherwise_block = Some(block);
+            }
+            return;
+        }
+
+        // Test for the remaining candidates.
+        self.test_candidates_with_or(
+            span,
+            scrutinee_span,
+            unmatched_candidates,
+            block,
+            otherwise_block,
+            fake_borrows,
+        );
+    }
+
+    /// Link up matched candidates.
+    ///
+    /// For example, if we have something like this:
+    ///
+    /// ```ignore (illustrative)
+    /// ...
+    /// Some(x) if cond1 => ...
+    /// Some(x) => ...
+    /// Some(x) if cond2 => ...
+    /// ...
+    /// ```
+    ///
+    /// We generate real edges from:
+    ///
+    /// * `start_block` to the [pre-binding block] of the first pattern,
+    /// * the [otherwise block] of the first pattern to the second pattern,
+    /// * the [otherwise block] of the third pattern to a block with an
+    ///   [`Unreachable` terminator](TerminatorKind::Unreachable).
+    ///
+    /// In addition, we add fake edges from the otherwise blocks to the
+    /// pre-binding block of the next candidate in the original set of
+    /// candidates.
+    ///
+    /// [pre-binding block]: Candidate::pre_binding_block
+    /// [otherwise block]: Candidate::otherwise_block
+    fn select_matched_candidates(
+        &mut self,
+        matched_candidates: &mut [&mut Candidate<'_, 'tcx>],
+        start_block: BasicBlock,
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) -> Option<BasicBlock> {
+        debug_assert!(
+            !matched_candidates.is_empty(),
+            "select_matched_candidates called with no candidates",
+        );
+        debug_assert!(
+            matched_candidates.iter().all(|c| c.subcandidates.is_empty()),
+            "subcandidates should be empty in select_matched_candidates",
+        );
+
+        // Insert a borrows of prefixes of places that are bound and are
+        // behind a dereference projection.
+        //
+        // These borrows are taken to avoid situations like the following:
+        //
+        // match x[10] {
+        //     _ if { x = &[0]; false } => (),
+        //     y => (), // Out of bounds array access!
+        // }
+        //
+        // match *x {
+        //     // y is bound by reference in the guard and then by copy in the
+        //     // arm, so y is 2 in the arm!
+        //     y if { y == 1 && (x = &2) == () } => y,
+        //     _ => 3,
+        // }
+        if let Some(fake_borrows) = fake_borrows {
+            for Binding { source, .. } in
+                matched_candidates.iter().flat_map(|candidate| &candidate.bindings)
+            {
+                if let Some(i) =
+                    source.projection.iter().rposition(|elem| elem == ProjectionElem::Deref)
+                {
+                    let proj_base = &source.projection[..i];
+
+                    fake_borrows.insert(Place {
+                        local: source.local,
+                        projection: self.tcx.intern_place_elems(proj_base),
+                    });
+                }
+            }
+        }
+
+        let fully_matched_with_guard = matched_candidates
+            .iter()
+            .position(|c| !c.has_guard)
+            .unwrap_or(matched_candidates.len() - 1);
+
+        let (reachable_candidates, unreachable_candidates) =
+            matched_candidates.split_at_mut(fully_matched_with_guard + 1);
+
+        let mut next_prebinding = start_block;
+
+        for candidate in reachable_candidates.iter_mut() {
+            assert!(candidate.otherwise_block.is_none());
+            assert!(candidate.pre_binding_block.is_none());
+            candidate.pre_binding_block = Some(next_prebinding);
+            if candidate.has_guard {
+                // Create the otherwise block for this candidate, which is the
+                // pre-binding block for the next candidate.
+                next_prebinding = self.cfg.start_new_block();
+                candidate.otherwise_block = Some(next_prebinding);
+            }
+        }
+
+        debug!(
+            "match_candidates: add pre_binding_blocks for unreachable {:?}",
+            unreachable_candidates,
+        );
+        for candidate in unreachable_candidates {
+            assert!(candidate.pre_binding_block.is_none());
+            candidate.pre_binding_block = Some(self.cfg.start_new_block());
+        }
+
+        reachable_candidates.last_mut().unwrap().otherwise_block
+    }
+
+    /// Tests a candidate where there are only or-patterns left to test, or
+    /// forwards to [Builder::test_candidates].
+    ///
+    /// Given a pattern `(P | Q, R | S)` we (in principle) generate a CFG like
+    /// so:
+    ///
+    /// ```text
+    /// [ start ]
+    ///      |
+    /// [ match P, Q ]
+    ///      |
+    ///      +----------------------------------------+------------------------------------+
+    ///      |                                        |                                    |
+    ///      V                                        V                                    V
+    /// [ P matches ]                           [ Q matches ]                        [ otherwise ]
+    ///      |                                        |                                    |
+    ///      V                                        V                                    |
+    /// [ match R, S ]                          [ match R, S ]                             |
+    ///      |                                        |                                    |
+    ///      +--------------+------------+            +--------------+------------+        |
+    ///      |              |            |            |              |            |        |
+    ///      V              V            V            V              V            V        |
+    /// [ R matches ] [ S matches ] [otherwise ] [ R matches ] [ S matches ] [otherwise ]  |
+    ///      |              |            |            |              |            |        |
+    ///      +--------------+------------|------------+--------------+            |        |
+    ///      |                           |                                        |        |
+    ///      |                           +----------------------------------------+--------+
+    ///      |                           |
+    ///      V                           V
+    /// [ Success ]                 [ Failure ]
+    /// ```
+    ///
+    /// In practice there are some complications:
+    ///
+    /// * If there's a guard, then the otherwise branch of the first match on
+    ///   `R | S` goes to a test for whether `Q` matches, and the control flow
+    ///   doesn't merge into a single success block until after the guard is
+    ///   tested.
+    /// * If neither `P` or `Q` has any bindings or type ascriptions and there
+    ///   isn't a match guard, then we create a smaller CFG like:
+    ///
+    /// ```text
+    ///     ...
+    ///      +---------------+------------+
+    ///      |               |            |
+    /// [ P matches ] [ Q matches ] [ otherwise ]
+    ///      |               |            |
+    ///      +---------------+            |
+    ///      |                           ...
+    /// [ match R, S ]
+    ///      |
+    ///     ...
+    /// ```
+    fn test_candidates_with_or(
+        &mut self,
+        span: Span,
+        scrutinee_span: Span,
+        candidates: &mut [&mut Candidate<'_, 'tcx>],
+        block: BasicBlock,
+        otherwise_block: &mut Option<BasicBlock>,
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) {
+        let (first_candidate, remaining_candidates) = candidates.split_first_mut().unwrap();
+
+        // All of the or-patterns have been sorted to the end, so if the first
+        // pattern is an or-pattern we only have or-patterns.
+        match *first_candidate.match_pairs[0].pattern.kind {
+            PatKind::Or { .. } => (),
+            _ => {
+                self.test_candidates(
+                    span,
+                    scrutinee_span,
+                    candidates,
+                    block,
+                    otherwise_block,
+                    fake_borrows,
+                );
+                return;
+            }
+        }
+
+        let match_pairs = mem::take(&mut first_candidate.match_pairs);
+        first_candidate.pre_binding_block = Some(block);
+
+        let mut otherwise = None;
+        for match_pair in match_pairs {
+            let PatKind::Or { ref pats } = &*match_pair.pattern.kind else {
+                bug!("Or-patterns should have been sorted to the end");
+            };
+            let or_span = match_pair.pattern.span;
+            let place = match_pair.place;
+
+            first_candidate.visit_leaves(|leaf_candidate| {
+                self.test_or_pattern(
+                    leaf_candidate,
+                    &mut otherwise,
+                    pats,
+                    or_span,
+                    place.clone(),
+                    fake_borrows,
+                );
+            });
+        }
+
+        let remainder_start = otherwise.unwrap_or_else(|| self.cfg.start_new_block());
+
+        self.match_candidates(
+            span,
+            scrutinee_span,
+            remainder_start,
+            otherwise_block,
+            remaining_candidates,
+            fake_borrows,
+        )
+    }
+
+    fn test_or_pattern<'pat>(
+        &mut self,
+        candidate: &mut Candidate<'pat, 'tcx>,
+        otherwise: &mut Option<BasicBlock>,
+        pats: &'pat [Pat<'tcx>],
+        or_span: Span,
+        place: PlaceBuilder<'tcx>,
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) {
+        debug!("test_or_pattern:\ncandidate={:#?}\npats={:#?}", candidate, pats);
+        let mut or_candidates: Vec<_> = pats
+            .iter()
+            .map(|pat| Candidate::new(place.clone(), pat, candidate.has_guard))
+            .collect();
+        let mut or_candidate_refs: Vec<_> = or_candidates.iter_mut().collect();
+        let otherwise = if candidate.otherwise_block.is_some() {
+            &mut candidate.otherwise_block
+        } else {
+            otherwise
+        };
+        self.match_candidates(
+            or_span,
+            or_span,
+            candidate.pre_binding_block.unwrap(),
+            otherwise,
+            &mut or_candidate_refs,
+            fake_borrows,
+        );
+        candidate.subcandidates = or_candidates;
+        self.merge_trivial_subcandidates(candidate, self.source_info(or_span));
+    }
+
+    /// Try to merge all of the subcandidates of the given candidate into one.
+    /// This avoids exponentially large CFGs in cases like `(1 | 2, 3 | 4, ...)`.
+    fn merge_trivial_subcandidates(
+        &mut self,
+        candidate: &mut Candidate<'_, 'tcx>,
+        source_info: SourceInfo,
+    ) {
+        if candidate.subcandidates.is_empty() || candidate.has_guard {
+            // FIXME(or_patterns; matthewjasper) Don't give up if we have a guard.
+            return;
+        }
+
+        let mut can_merge = true;
+
+        // Not `Iterator::all` because we don't want to short-circuit.
+        for subcandidate in &mut candidate.subcandidates {
+            self.merge_trivial_subcandidates(subcandidate, source_info);
+
+            // FIXME(or_patterns; matthewjasper) Try to be more aggressive here.
+            can_merge &= subcandidate.subcandidates.is_empty()
+                && subcandidate.bindings.is_empty()
+                && subcandidate.ascriptions.is_empty();
+        }
+
+        if can_merge {
+            let any_matches = self.cfg.start_new_block();
+            for subcandidate in mem::take(&mut candidate.subcandidates) {
+                let or_block = subcandidate.pre_binding_block.unwrap();
+                self.cfg.goto(or_block, source_info, any_matches);
+            }
+            candidate.pre_binding_block = Some(any_matches);
+        }
+    }
+
+    /// This is the most subtle part of the matching algorithm. At
+    /// this point, the input candidates have been fully simplified,
+    /// and so we know that all remaining match-pairs require some
+    /// sort of test. To decide what test to perform, we take the highest
+    /// priority candidate (the first one in the list, as of January 2021)
+    /// and extract the first match-pair from the list. From this we decide
+    /// what kind of test is needed using [`Builder::test`], defined in the
+    /// [`test` module](mod@test).
+    ///
+    /// *Note:* taking the first match pair is somewhat arbitrary, and
+    /// we might do better here by choosing more carefully what to
+    /// test.
+    ///
+    /// For example, consider the following possible match-pairs:
+    ///
+    /// 1. `x @ Some(P)` -- we will do a [`Switch`] to decide what variant `x` has
+    /// 2. `x @ 22` -- we will do a [`SwitchInt`] to decide what value `x` has
+    /// 3. `x @ 3..5` -- we will do a [`Range`] test to decide what range `x` falls in
+    /// 4. etc.
+    ///
+    /// [`Switch`]: TestKind::Switch
+    /// [`SwitchInt`]: TestKind::SwitchInt
+    /// [`Range`]: TestKind::Range
+    ///
+    /// Once we know what sort of test we are going to perform, this
+    /// test may also help us winnow down our candidates. So we walk over
+    /// the candidates (from high to low priority) and check. This
+    /// gives us, for each outcome of the test, a transformed list of
+    /// candidates. For example, if we are testing `x.0`'s variant,
+    /// and we have a candidate `(x.0 @ Some(v), x.1 @ 22)`,
+    /// then we would have a resulting candidate of `((x.0 as Some).0 @ v, x.1 @ 22)`.
+    /// Note that the first match-pair is now simpler (and, in fact, irrefutable).
+    ///
+    /// But there may also be candidates that the test just doesn't
+    /// apply to. The classical example involves wildcards:
+    ///
+    /// ```
+    /// # let (x, y, z) = (true, true, true);
+    /// match (x, y, z) {
+    ///     (true , _    , true ) => true,  // (0)
+    ///     (_    , true , _    ) => true,  // (1)
+    ///     (false, false, _    ) => false, // (2)
+    ///     (true , _    , false) => false, // (3)
+    /// }
+    /// # ;
+    /// ```
+    ///
+    /// In that case, after we test on `x`, there are 2 overlapping candidate
+    /// sets:
+    ///
+    /// - If the outcome is that `x` is true, candidates 0, 1, and 3
+    /// - If the outcome is that `x` is false, candidates 1 and 2
+    ///
+    /// Here, the traditional "decision tree" method would generate 2
+    /// separate code-paths for the 2 separate cases.
+    ///
+    /// In some cases, this duplication can create an exponential amount of
+    /// code. This is most easily seen by noticing that this method terminates
+    /// with precisely the reachable arms being reachable - but that problem
+    /// is trivially NP-complete:
+    ///
+    /// ```ignore (illustrative)
+    /// match (var0, var1, var2, var3, ...) {
+    ///     (true , _   , _    , false, true, ...) => false,
+    ///     (_    , true, true , false, _   , ...) => false,
+    ///     (false, _   , false, false, _   , ...) => false,
+    ///     ...
+    ///     _ => true
+    /// }
+    /// ```
+    ///
+    /// Here the last arm is reachable only if there is an assignment to
+    /// the variables that does not match any of the literals. Therefore,
+    /// compilation would take an exponential amount of time in some cases.
+    ///
+    /// That kind of exponential worst-case might not occur in practice, but
+    /// our simplistic treatment of constants and guards would make it occur
+    /// in very common situations - for example [#29740]:
+    ///
+    /// ```ignore (illustrative)
+    /// match x {
+    ///     "foo" if foo_guard => ...,
+    ///     "bar" if bar_guard => ...,
+    ///     "baz" if baz_guard => ...,
+    ///     ...
+    /// }
+    /// ```
+    ///
+    /// [#29740]: https://github.com/rust-lang/rust/issues/29740
+    ///
+    /// Here we first test the match-pair `x @ "foo"`, which is an [`Eq` test].
+    ///
+    /// [`Eq` test]: TestKind::Eq
+    ///
+    /// It might seem that we would end up with 2 disjoint candidate
+    /// sets, consisting of the first candidate or the other two, but our
+    /// algorithm doesn't reason about `"foo"` being distinct from the other
+    /// constants; it considers the latter arms to potentially match after
+    /// both outcomes, which obviously leads to an exponential number
+    /// of tests.
+    ///
+    /// To avoid these kinds of problems, our algorithm tries to ensure
+    /// the amount of generated tests is linear. When we do a k-way test,
+    /// we return an additional "unmatched" set alongside the obvious `k`
+    /// sets. When we encounter a candidate that would be present in more
+    /// than one of the sets, we put it and all candidates below it into the
+    /// "unmatched" set. This ensures these `k+1` sets are disjoint.
+    ///
+    /// After we perform our test, we branch into the appropriate candidate
+    /// set and recurse with `match_candidates`. These sub-matches are
+    /// obviously non-exhaustive - as we discarded our otherwise set - so
+    /// we set their continuation to do `match_candidates` on the
+    /// "unmatched" set (which is again non-exhaustive).
+    ///
+    /// If you apply this to the above test, you basically wind up
+    /// with an if-else-if chain, testing each candidate in turn,
+    /// which is precisely what we want.
+    ///
+    /// In addition to avoiding exponential-time blowups, this algorithm
+    /// also has the nice property that each guard and arm is only generated
+    /// once.
+    fn test_candidates<'pat, 'b, 'c>(
+        &mut self,
+        span: Span,
+        scrutinee_span: Span,
+        mut candidates: &'b mut [&'c mut Candidate<'pat, 'tcx>],
+        block: BasicBlock,
+        otherwise_block: &mut Option<BasicBlock>,
+        fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
+    ) {
+        // extract the match-pair from the highest priority candidate
+        let match_pair = &candidates.first().unwrap().match_pairs[0];
+        let mut test = self.test(match_pair);
+        let match_place = match_pair.place.clone();
+
+        // most of the time, the test to perform is simply a function
+        // of the main candidate; but for a test like SwitchInt, we
+        // may want to add cases based on the candidates that are
+        // available
+        match test.kind {
+            TestKind::SwitchInt { switch_ty, ref mut options } => {
+                for candidate in candidates.iter() {
+                    if !self.add_cases_to_switch(&match_place, candidate, switch_ty, options) {
+                        break;
+                    }
+                }
+            }
+            TestKind::Switch { adt_def: _, ref mut variants } => {
+                for candidate in candidates.iter() {
+                    if !self.add_variants_to_switch(&match_place, candidate, variants) {
+                        break;
+                    }
+                }
+            }
+            _ => {}
+        }
+
+        // Insert a Shallow borrow of any places that is switched on.
+        if let Some(fb) = fake_borrows && let Ok(match_place_resolved) =
+            match_place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            let resolved_place = match_place_resolved.into_place(self.tcx, self.typeck_results);
+            fb.insert(resolved_place);
+        }
+
+        // perform the test, branching to one of N blocks. For each of
+        // those N possible outcomes, create a (initially empty)
+        // vector of candidates. Those are the candidates that still
+        // apply if the test has that particular outcome.
+        debug!("test_candidates: test={:?} match_pair={:?}", test, match_pair);
+        let mut target_candidates: Vec<Vec<&mut Candidate<'pat, 'tcx>>> = vec![];
+        target_candidates.resize_with(test.targets(), Default::default);
+
+        let total_candidate_count = candidates.len();
+
+        // Sort the candidates into the appropriate vector in
+        // `target_candidates`. Note that at some point we may
+        // encounter a candidate where the test is not relevant; at
+        // that point, we stop sorting.
+        while let Some(candidate) = candidates.first_mut() {
+            let Some(idx) = self.sort_candidate(&match_place.clone(), &test, candidate) else {
+                break;
+            };
+            let (candidate, rest) = candidates.split_first_mut().unwrap();
+            target_candidates[idx].push(candidate);
+            candidates = rest;
+        }
+        // at least the first candidate ought to be tested
+        assert!(total_candidate_count > candidates.len());
+        debug!("test_candidates: tested_candidates: {}", total_candidate_count - candidates.len());
+        debug!("test_candidates: untested_candidates: {}", candidates.len());
+
+        // HACK(matthewjasper) This is a closure so that we can let the test
+        // create its blocks before the rest of the match. This currently
+        // improves the speed of llvm when optimizing long string literal
+        // matches
+        let make_target_blocks = move |this: &mut Self| -> Vec<BasicBlock> {
+            // The block that we should branch to if none of the
+            // `target_candidates` match. This is either the block where we
+            // start matching the untested candidates if there are any,
+            // otherwise it's the `otherwise_block`.
+            let remainder_start = &mut None;
+            let remainder_start =
+                if candidates.is_empty() { &mut *otherwise_block } else { remainder_start };
+
+            // For each outcome of test, process the candidates that still
+            // apply. Collect a list of blocks where control flow will
+            // branch if one of the `target_candidate` sets is not
+            // exhaustive.
+            let target_blocks: Vec<_> = target_candidates
+                .into_iter()
+                .map(|mut candidates| {
+                    if !candidates.is_empty() {
+                        let candidate_start = this.cfg.start_new_block();
+                        this.match_candidates(
+                            span,
+                            scrutinee_span,
+                            candidate_start,
+                            remainder_start,
+                            &mut *candidates,
+                            fake_borrows,
+                        );
+                        candidate_start
+                    } else {
+                        *remainder_start.get_or_insert_with(|| this.cfg.start_new_block())
+                    }
+                })
+                .collect();
+
+            if !candidates.is_empty() {
+                let remainder_start = remainder_start.unwrap_or_else(|| this.cfg.start_new_block());
+                this.match_candidates(
+                    span,
+                    scrutinee_span,
+                    remainder_start,
+                    otherwise_block,
+                    candidates,
+                    fake_borrows,
+                );
+            };
+
+            target_blocks
+        };
+
+        self.perform_test(span, scrutinee_span, block, match_place, &test, make_target_blocks);
+    }
+
+    /// Determine the fake borrows that are needed from a set of places that
+    /// have to be stable across match guards.
+    ///
+    /// Returns a list of places that need a fake borrow and the temporary
+    /// that's used to store the fake borrow.
+    ///
+    /// Match exhaustiveness checking is not able to handle the case where the
+    /// place being matched on is mutated in the guards. We add "fake borrows"
+    /// to the guards that prevent any mutation of the place being matched.
+    /// There are a some subtleties:
+    ///
+    /// 1. Borrowing `*x` doesn't prevent assigning to `x`. If `x` is a shared
+    ///    reference, the borrow isn't even tracked. As such we have to add fake
+    ///    borrows of any prefixes of a place
+    /// 2. We don't want `match x { _ => (), }` to conflict with mutable
+    ///    borrows of `x`, so we only add fake borrows for places which are
+    ///    bound or tested by the match.
+    /// 3. We don't want the fake borrows to conflict with `ref mut` bindings,
+    ///    so we use a special BorrowKind for them.
+    /// 4. The fake borrows may be of places in inactive variants, so it would
+    ///    be UB to generate code for them. They therefore have to be removed
+    ///    by a MIR pass run after borrow checking.
+    fn calculate_fake_borrows<'b>(
+        &mut self,
+        fake_borrows: &'b FxIndexSet<Place<'tcx>>,
+        temp_span: Span,
+    ) -> Vec<(Place<'tcx>, Local)> {
+        let tcx = self.tcx;
+
+        debug!("add_fake_borrows fake_borrows = {:?}", fake_borrows);
+
+        let mut all_fake_borrows = Vec::with_capacity(fake_borrows.len());
+
+        // Insert a Shallow borrow of the prefixes of any fake borrows.
+        for place in fake_borrows {
+            let mut cursor = place.projection.as_ref();
+            while let [proj_base @ .., elem] = cursor {
+                cursor = proj_base;
+
+                if let ProjectionElem::Deref = elem {
+                    // Insert a shallow borrow after a deref. For other
+                    // projections the borrow of prefix_cursor will
+                    // conflict with any mutation of base.
+                    all_fake_borrows.push(PlaceRef { local: place.local, projection: proj_base });
+                }
+            }
+
+            all_fake_borrows.push(place.as_ref());
+        }
+
+        // Deduplicate
+        let mut dedup = FxHashSet::default();
+        all_fake_borrows.retain(|b| dedup.insert(*b));
+
+        debug!("add_fake_borrows all_fake_borrows = {:?}", all_fake_borrows);
+
+        all_fake_borrows
+            .into_iter()
+            .map(|matched_place_ref| {
+                let matched_place = Place {
+                    local: matched_place_ref.local,
+                    projection: tcx.intern_place_elems(matched_place_ref.projection),
+                };
+                let fake_borrow_deref_ty = matched_place.ty(&self.local_decls, tcx).ty;
+                let fake_borrow_ty = tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
+                let fake_borrow_temp =
+                    self.local_decls.push(LocalDecl::new(fake_borrow_ty, temp_span));
+
+                (matched_place, fake_borrow_temp)
+            })
+            .collect()
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Pat binding - used for `let` and function parameters as well.
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    pub(crate) fn lower_let_expr(
+        &mut self,
+        mut block: BasicBlock,
+        expr: &Expr<'tcx>,
+        pat: &Pat<'tcx>,
+        else_target: region::Scope,
+        source_scope: Option<SourceScope>,
+        span: Span,
+    ) -> BlockAnd<()> {
+        let expr_span = expr.span;
+        let expr_place_builder = unpack!(block = self.lower_scrutinee(block, expr, expr_span));
+        let wildcard = Pat::wildcard_from_ty(pat.ty);
+        let mut guard_candidate = Candidate::new(expr_place_builder.clone(), &pat, false);
+        let mut otherwise_candidate = Candidate::new(expr_place_builder.clone(), &wildcard, false);
+        let fake_borrow_temps = self.lower_match_tree(
+            block,
+            pat.span,
+            pat.span,
+            false,
+            &mut [&mut guard_candidate, &mut otherwise_candidate],
+        );
+        let mut opt_expr_place: Option<(Option<&Place<'tcx>>, Span)> = None;
+        let expr_place: Place<'tcx>;
+        if let Ok(expr_builder) =
+            expr_place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            expr_place = expr_builder.into_place(self.tcx, self.typeck_results);
+            opt_expr_place = Some((Some(&expr_place), expr_span));
+        }
+        let otherwise_post_guard_block = otherwise_candidate.pre_binding_block.unwrap();
+        self.break_for_else(otherwise_post_guard_block, else_target, self.source_info(expr_span));
+
+        self.declare_bindings(
+            source_scope,
+            pat.span.to(span),
+            pat,
+            ArmHasGuard(false),
+            opt_expr_place,
+        );
+
+        let post_guard_block = self.bind_pattern(
+            self.source_info(pat.span),
+            guard_candidate,
+            None,
+            &fake_borrow_temps,
+            expr.span,
+            None,
+            None,
+            None,
+        );
+
+        post_guard_block.unit()
+    }
+
+    /// Initializes each of the bindings from the candidate by
+    /// moving/copying/ref'ing the source as appropriate. Tests the guard, if
+    /// any, and then branches to the arm. Returns the block for the case where
+    /// the guard succeeds.
+    ///
+    /// Note: we do not check earlier that if there is a guard,
+    /// there cannot be move bindings. We avoid a use-after-move by only
+    /// moving the binding once the guard has evaluated to true (see below).
+    fn bind_and_guard_matched_candidate<'pat>(
+        &mut self,
+        candidate: Candidate<'pat, 'tcx>,
+        parent_bindings: &[(Vec<Binding<'tcx>>, Vec<Ascription<'tcx>>)],
+        guard: Option<&Guard<'tcx>>,
+        fake_borrows: &[(Place<'tcx>, Local)],
+        scrutinee_span: Span,
+        arm_span: Option<Span>,
+        match_scope: Option<region::Scope>,
+        schedule_drops: bool,
+    ) -> BasicBlock {
+        debug!("bind_and_guard_matched_candidate(candidate={:?})", candidate);
+
+        debug_assert!(candidate.match_pairs.is_empty());
+
+        let candidate_source_info = self.source_info(candidate.span);
+
+        let mut block = candidate.pre_binding_block.unwrap();
+
+        if candidate.next_candidate_pre_binding_block.is_some() {
+            let fresh_block = self.cfg.start_new_block();
+            self.false_edges(
+                block,
+                fresh_block,
+                candidate.next_candidate_pre_binding_block,
+                candidate_source_info,
+            );
+            block = fresh_block;
+        }
+
+        self.ascribe_types(
+            block,
+            parent_bindings
+                .iter()
+                .flat_map(|(_, ascriptions)| ascriptions)
+                .cloned()
+                .chain(candidate.ascriptions),
+        );
+
+        // rust-lang/rust#27282: The `autoref` business deserves some
+        // explanation here.
+        //
+        // The intent of the `autoref` flag is that when it is true,
+        // then any pattern bindings of type T will map to a `&T`
+        // within the context of the guard expression, but will
+        // continue to map to a `T` in the context of the arm body. To
+        // avoid surfacing this distinction in the user source code
+        // (which would be a severe change to the language and require
+        // far more revision to the compiler), when `autoref` is true,
+        // then any occurrence of the identifier in the guard
+        // expression will automatically get a deref op applied to it.
+        //
+        // So an input like:
+        //
+        // ```
+        // let place = Foo::new();
+        // match place { foo if inspect(foo)
+        //     => feed(foo), ...  }
+        // ```
+        //
+        // will be treated as if it were really something like:
+        //
+        // ```
+        // let place = Foo::new();
+        // match place { Foo { .. } if { let tmp1 = &place; inspect(*tmp1) }
+        //     => { let tmp2 = place; feed(tmp2) }, ... }
+        //
+        // And an input like:
+        //
+        // ```
+        // let place = Foo::new();
+        // match place { ref mut foo if inspect(foo)
+        //     => feed(foo), ...  }
+        // ```
+        //
+        // will be treated as if it were really something like:
+        //
+        // ```
+        // let place = Foo::new();
+        // match place { Foo { .. } if { let tmp1 = & &mut place; inspect(*tmp1) }
+        //     => { let tmp2 = &mut place; feed(tmp2) }, ... }
+        // ```
+        //
+        // In short, any pattern binding will always look like *some*
+        // kind of `&T` within the guard at least in terms of how the
+        // MIR-borrowck views it, and this will ensure that guard
+        // expressions cannot mutate their the match inputs via such
+        // bindings. (It also ensures that guard expressions can at
+        // most *copy* values from such bindings; non-Copy things
+        // cannot be moved via pattern bindings in guard expressions.)
+        //
+        // ----
+        //
+        // Implementation notes (under assumption `autoref` is true).
+        //
+        // To encode the distinction above, we must inject the
+        // temporaries `tmp1` and `tmp2`.
+        //
+        // There are two cases of interest: binding by-value, and binding by-ref.
+        //
+        // 1. Binding by-value: Things are simple.
+        //
+        //    * Establishing `tmp1` creates a reference into the
+        //      matched place. This code is emitted by
+        //      bind_matched_candidate_for_guard.
+        //
+        //    * `tmp2` is only initialized "lazily", after we have
+        //      checked the guard. Thus, the code that can trigger
+        //      moves out of the candidate can only fire after the
+        //      guard evaluated to true. This initialization code is
+        //      emitted by bind_matched_candidate_for_arm.
+        //
+        // 2. Binding by-reference: Things are tricky.
+        //
+        //    * Here, the guard expression wants a `&&` or `&&mut`
+        //      into the original input. This means we need to borrow
+        //      the reference that we create for the arm.
+        //    * So we eagerly create the reference for the arm and then take a
+        //      reference to that.
+        if let Some(guard) = guard {
+            let tcx = self.tcx;
+            let bindings = parent_bindings
+                .iter()
+                .flat_map(|(bindings, _)| bindings)
+                .chain(&candidate.bindings);
+
+            self.bind_matched_candidate_for_guard(block, schedule_drops, bindings.clone());
+            let guard_frame = GuardFrame {
+                locals: bindings.map(|b| GuardFrameLocal::new(b.var_id, b.binding_mode)).collect(),
+            };
+            debug!("entering guard building context: {:?}", guard_frame);
+            self.guard_context.push(guard_frame);
+
+            let re_erased = tcx.lifetimes.re_erased;
+            let scrutinee_source_info = self.source_info(scrutinee_span);
+            for &(place, temp) in fake_borrows {
+                let borrow = Rvalue::Ref(re_erased, BorrowKind::Shallow, place);
+                self.cfg.push_assign(block, scrutinee_source_info, Place::from(temp), borrow);
+            }
+
+            let arm_span = arm_span.unwrap();
+            let match_scope = match_scope.unwrap();
+            let mut guard_span = rustc_span::DUMMY_SP;
+
+            let (post_guard_block, otherwise_post_guard_block) =
+                self.in_if_then_scope(match_scope, |this| match *guard {
+                    Guard::If(e) => {
+                        let e = &this.thir[e];
+                        guard_span = e.span;
+                        this.then_else_break(
+                            block,
+                            e,
+                            None,
+                            match_scope,
+                            this.source_info(arm_span),
+                        )
+                    }
+                    Guard::IfLet(ref pat, scrutinee) => {
+                        let s = &this.thir[scrutinee];
+                        guard_span = s.span;
+                        this.lower_let_expr(block, s, pat, match_scope, None, arm_span)
+                    }
+                });
+
+            let source_info = self.source_info(guard_span);
+            let guard_end = self.source_info(tcx.sess.source_map().end_point(guard_span));
+            let guard_frame = self.guard_context.pop().unwrap();
+            debug!("Exiting guard building context with locals: {:?}", guard_frame);
+
+            for &(_, temp) in fake_borrows {
+                let cause = FakeReadCause::ForMatchGuard;
+                self.cfg.push_fake_read(post_guard_block, guard_end, cause, Place::from(temp));
+            }
+
+            let otherwise_block = candidate.otherwise_block.unwrap_or_else(|| {
+                let unreachable = self.cfg.start_new_block();
+                self.cfg.terminate(unreachable, source_info, TerminatorKind::Unreachable);
+                unreachable
+            });
+            self.false_edges(
+                otherwise_post_guard_block,
+                otherwise_block,
+                candidate.next_candidate_pre_binding_block,
+                source_info,
+            );
+
+            // We want to ensure that the matched candidates are bound
+            // after we have confirmed this candidate *and* any
+            // associated guard; Binding them on `block` is too soon,
+            // because that would be before we've checked the result
+            // from the guard.
+            //
+            // But binding them on the arm is *too late*, because
+            // then all of the candidates for a single arm would be
+            // bound in the same place, that would cause a case like:
+            //
+            // ```rust
+            // match (30, 2) {
+            //     (mut x, 1) | (2, mut x) if { true } => { ... }
+            //     ...                                 // ^^^^^^^ (this is `arm_block`)
+            // }
+            // ```
+            //
+            // would yield an `arm_block` something like:
+            //
+            // ```
+            // StorageLive(_4);        // _4 is `x`
+            // _4 = &mut (_1.0: i32);  // this is handling `(mut x, 1)` case
+            // _4 = &mut (_1.1: i32);  // this is handling `(2, mut x)` case
+            // ```
+            //
+            // and that is clearly not correct.
+            let by_value_bindings = parent_bindings
+                .iter()
+                .flat_map(|(bindings, _)| bindings)
+                .chain(&candidate.bindings)
+                .filter(|binding| matches!(binding.binding_mode, BindingMode::ByValue));
+            // Read all of the by reference bindings to ensure that the
+            // place they refer to can't be modified by the guard.
+            for binding in by_value_bindings.clone() {
+                let local_id = self.var_local_id(binding.var_id, RefWithinGuard);
+                let cause = FakeReadCause::ForGuardBinding;
+                self.cfg.push_fake_read(post_guard_block, guard_end, cause, Place::from(local_id));
+            }
+            assert!(schedule_drops, "patterns with guards must schedule drops");
+            self.bind_matched_candidate_for_arm_body(post_guard_block, true, by_value_bindings);
+
+            post_guard_block
+        } else {
+            // (Here, it is not too early to bind the matched
+            // candidate on `block`, because there is no guard result
+            // that we have to inspect before we bind them.)
+            self.bind_matched_candidate_for_arm_body(
+                block,
+                schedule_drops,
+                parent_bindings
+                    .iter()
+                    .flat_map(|(bindings, _)| bindings)
+                    .chain(&candidate.bindings),
+            );
+            block
+        }
+    }
+
+    /// Append `AscribeUserType` statements onto the end of `block`
+    /// for each ascription
+    fn ascribe_types(
+        &mut self,
+        block: BasicBlock,
+        ascriptions: impl IntoIterator<Item = Ascription<'tcx>>,
+    ) {
+        for ascription in ascriptions {
+            let source_info = self.source_info(ascription.annotation.span);
+
+            let base = self.canonical_user_type_annotations.push(ascription.annotation);
+            self.cfg.push(
+                block,
+                Statement {
+                    source_info,
+                    kind: StatementKind::AscribeUserType(
+                        Box::new((
+                            ascription.source,
+                            UserTypeProjection { base, projs: Vec::new() },
+                        )),
+                        ascription.variance,
+                    ),
+                },
+            );
+        }
+    }
+
+    fn bind_matched_candidate_for_guard<'b>(
+        &mut self,
+        block: BasicBlock,
+        schedule_drops: bool,
+        bindings: impl IntoIterator<Item = &'b Binding<'tcx>>,
+    ) where
+        'tcx: 'b,
+    {
+        debug!("bind_matched_candidate_for_guard(block={:?})", block);
+
+        // Assign each of the bindings. Since we are binding for a
+        // guard expression, this will never trigger moves out of the
+        // candidate.
+        let re_erased = self.tcx.lifetimes.re_erased;
+        for binding in bindings {
+            debug!("bind_matched_candidate_for_guard(binding={:?})", binding);
+            let source_info = self.source_info(binding.span);
+
+            // For each pattern ident P of type T, `ref_for_guard` is
+            // a reference R: &T pointing to the location matched by
+            // the pattern, and every occurrence of P within a guard
+            // denotes *R.
+            let ref_for_guard = self.storage_live_binding(
+                block,
+                binding.var_id,
+                binding.span,
+                RefWithinGuard,
+                schedule_drops,
+            );
+            match binding.binding_mode {
+                BindingMode::ByValue => {
+                    let rvalue = Rvalue::Ref(re_erased, BorrowKind::Shared, binding.source);
+                    self.cfg.push_assign(block, source_info, ref_for_guard, rvalue);
+                }
+                BindingMode::ByRef(borrow_kind) => {
+                    let value_for_arm = self.storage_live_binding(
+                        block,
+                        binding.var_id,
+                        binding.span,
+                        OutsideGuard,
+                        schedule_drops,
+                    );
+
+                    let rvalue = Rvalue::Ref(re_erased, borrow_kind, binding.source);
+                    self.cfg.push_assign(block, source_info, value_for_arm, rvalue);
+                    let rvalue = Rvalue::Ref(re_erased, BorrowKind::Shared, value_for_arm);
+                    self.cfg.push_assign(block, source_info, ref_for_guard, rvalue);
+                }
+            }
+        }
+    }
+
+    fn bind_matched_candidate_for_arm_body<'b>(
+        &mut self,
+        block: BasicBlock,
+        schedule_drops: bool,
+        bindings: impl IntoIterator<Item = &'b Binding<'tcx>>,
+    ) where
+        'tcx: 'b,
+    {
+        debug!("bind_matched_candidate_for_arm_body(block={:?})", block);
+
+        let re_erased = self.tcx.lifetimes.re_erased;
+        // Assign each of the bindings. This may trigger moves out of the candidate.
+        for binding in bindings {
+            let source_info = self.source_info(binding.span);
+            let local = self.storage_live_binding(
+                block,
+                binding.var_id,
+                binding.span,
+                OutsideGuard,
+                schedule_drops,
+            );
+            if schedule_drops {
+                self.schedule_drop_for_binding(binding.var_id, binding.span, OutsideGuard);
+            }
+            let rvalue = match binding.binding_mode {
+                BindingMode::ByValue => Rvalue::Use(self.consume_by_copy_or_move(binding.source)),
+                BindingMode::ByRef(borrow_kind) => {
+                    Rvalue::Ref(re_erased, borrow_kind, binding.source)
+                }
+            };
+            self.cfg.push_assign(block, source_info, local, rvalue);
+        }
+    }
+
+    /// Each binding (`ref mut var`/`ref var`/`mut var`/`var`, where the bound
+    /// `var` has type `T` in the arm body) in a pattern maps to 2 locals. The
+    /// first local is a binding for occurrences of `var` in the guard, which
+    /// will have type `&T`. The second local is a binding for occurrences of
+    /// `var` in the arm body, which will have type `T`.
+    fn declare_binding(
+        &mut self,
+        source_info: SourceInfo,
+        visibility_scope: SourceScope,
+        mutability: Mutability,
+        name: Symbol,
+        mode: BindingMode,
+        var_id: LocalVarId,
+        var_ty: Ty<'tcx>,
+        user_ty: UserTypeProjections,
+        has_guard: ArmHasGuard,
+        opt_match_place: Option<(Option<Place<'tcx>>, Span)>,
+        pat_span: Span,
+    ) {
+        debug!(
+            "declare_binding(var_id={:?}, name={:?}, mode={:?}, var_ty={:?}, \
+             visibility_scope={:?}, source_info={:?})",
+            var_id, name, mode, var_ty, visibility_scope, source_info
+        );
+
+        let tcx = self.tcx;
+        let debug_source_info = SourceInfo { span: source_info.span, scope: visibility_scope };
+        let binding_mode = match mode {
+            BindingMode::ByValue => ty::BindingMode::BindByValue(mutability),
+            BindingMode::ByRef(_) => ty::BindingMode::BindByReference(mutability),
+        };
+        debug!("declare_binding: user_ty={:?}", user_ty);
+        let local = LocalDecl::<'tcx> {
+            mutability,
+            ty: var_ty,
+            user_ty: if user_ty.is_empty() { None } else { Some(Box::new(user_ty)) },
+            source_info,
+            internal: false,
+            is_block_tail: None,
+            local_info: Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
+                VarBindingForm {
+                    binding_mode,
+                    // hypothetically, `visit_primary_bindings` could try to unzip
+                    // an outermost hir::Ty as we descend, matching up
+                    // idents in pat; but complex w/ unclear UI payoff.
+                    // Instead, just abandon providing diagnostic info.
+                    opt_ty_info: None,
+                    opt_match_place,
+                    pat_span,
+                },
+            ))))),
+        };
+        let for_arm_body = self.local_decls.push(local);
+        self.var_debug_info.push(VarDebugInfo {
+            name,
+            source_info: debug_source_info,
+            value: VarDebugInfoContents::Place(for_arm_body.into()),
+        });
+        let locals = if has_guard.0 {
+            let ref_for_guard = self.local_decls.push(LocalDecl::<'tcx> {
+                // This variable isn't mutated but has a name, so has to be
+                // immutable to avoid the unused mut lint.
+                mutability: Mutability::Not,
+                ty: tcx.mk_imm_ref(tcx.lifetimes.re_erased, var_ty),
+                user_ty: None,
+                source_info,
+                internal: false,
+                is_block_tail: None,
+                local_info: Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(
+                    BindingForm::RefForGuard,
+                )))),
+            });
+            self.var_debug_info.push(VarDebugInfo {
+                name,
+                source_info: debug_source_info,
+                value: VarDebugInfoContents::Place(ref_for_guard.into()),
+            });
+            LocalsForNode::ForGuard { ref_for_guard, for_arm_body }
+        } else {
+            LocalsForNode::One(for_arm_body)
+        };
+        debug!("declare_binding: vars={:?}", locals);
+        self.var_indices.insert(var_id, locals);
+    }
+
+    pub(crate) fn ast_let_else(
+        &mut self,
+        mut block: BasicBlock,
+        init: &Expr<'tcx>,
+        initializer_span: Span,
+        else_block: &Block,
+        visibility_scope: Option<SourceScope>,
+        remainder_scope: region::Scope,
+        remainder_span: Span,
+        pattern: &Pat<'tcx>,
+    ) -> BlockAnd<()> {
+        let (matching, failure) = self.in_if_then_scope(remainder_scope, |this| {
+            let scrutinee = unpack!(block = this.lower_scrutinee(block, init, initializer_span));
+            let pat = Pat { ty: init.ty, span: else_block.span, kind: Box::new(PatKind::Wild) };
+            let mut wildcard = Candidate::new(scrutinee.clone(), &pat, false);
+            this.declare_bindings(
+                visibility_scope,
+                remainder_span,
+                pattern,
+                ArmHasGuard(false),
+                Some((None, initializer_span)),
+            );
+            let mut candidate = Candidate::new(scrutinee.clone(), pattern, false);
+            let fake_borrow_temps = this.lower_match_tree(
+                block,
+                initializer_span,
+                pattern.span,
+                false,
+                &mut [&mut candidate, &mut wildcard],
+            );
+            // This block is for the matching case
+            let matching = this.bind_pattern(
+                this.source_info(pattern.span),
+                candidate,
+                None,
+                &fake_borrow_temps,
+                initializer_span,
+                None,
+                None,
+                None,
+            );
+            // This block is for the failure case
+            let failure = this.bind_pattern(
+                this.source_info(else_block.span),
+                wildcard,
+                None,
+                &fake_borrow_temps,
+                initializer_span,
+                None,
+                None,
+                None,
+            );
+            this.break_for_else(failure, remainder_scope, this.source_info(initializer_span));
+            matching.unit()
+        });
+
+        // This place is not really used because this destination place
+        // should never be used to take values at the end of the failure
+        // block.
+        let dummy_place = Place { local: RETURN_PLACE, projection: ty::List::empty() };
+        let failure_block;
+        unpack!(
+            failure_block = self.ast_block(
+                dummy_place,
+                failure,
+                else_block,
+                self.source_info(else_block.span),
+            )
+        );
+        self.cfg.terminate(
+            failure_block,
+            self.source_info(else_block.span),
+            TerminatorKind::Unreachable,
+        );
+        matching.unit()
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
new file mode 100644
index 000000000..c62989041
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -0,0 +1,318 @@
+//! Simplifying Candidates
+//!
+//! *Simplifying* a match pair `place @ pattern` means breaking it down
+//! into bindings or other, simpler match pairs. For example:
+//!
+//! - `place @ (P1, P2)` can be simplified to `[place.0 @ P1, place.1 @ P2]`
+//! - `place @ x` can be simplified to `[]` by binding `x` to `place`
+//!
+//! The `simplify_candidate` routine just repeatedly applies these
+//! sort of simplifications until there is nothing left to
+//! simplify. Match pairs cannot be simplified if they require some
+//! sort of test: for example, testing which variant an enum is, or
+//! testing a value against a constant.
+
+use crate::build::expr::as_place::PlaceBuilder;
+use crate::build::matches::{Ascription, Binding, Candidate, MatchPair};
+use crate::build::Builder;
+use rustc_hir::RangeEnd;
+use rustc_middle::thir::{self, *};
+use rustc_middle::ty;
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_target::abi::{Integer, Size};
+
+use std::mem;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Simplify a candidate so that all match pairs require a test.
+    ///
+    /// This method will also split a candidate, in which the only
+    /// match-pair is an or-pattern, into multiple candidates.
+    /// This is so that
+    ///
+    /// match x {
+    ///     0 | 1 => { ... },
+    ///     2 | 3 => { ... },
+    /// }
+    ///
+    /// only generates a single switch. If this happens this method returns
+    /// `true`.
+    pub(super) fn simplify_candidate<'pat>(
+        &mut self,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> bool {
+        // repeatedly simplify match pairs until fixed point is reached
+        debug!(?candidate, "simplify_candidate");
+
+        // existing_bindings and new_bindings exists to keep the semantics in order.
+        // Reversing the binding order for bindings after `@` changes the binding order in places
+        // it shouldn't be changed, for example `let (Some(a), Some(b)) = (x, y)`
+        //
+        // To avoid this, the binding occurs in the following manner:
+        // * the bindings for one iteration of the following loop occurs in order (i.e. left to
+        // right)
+        // * the bindings from the previous iteration of the loop is prepended to the bindings from
+        // the current iteration (in the implementation this is done by mem::swap and extend)
+        // * after all iterations, these new bindings are then appended to the bindings that were
+        // preexisting (i.e. `candidate.binding` when the function was called).
+        //
+        // example:
+        // candidate.bindings = [1, 2, 3]
+        // binding in iter 1: [4, 5]
+        // binding in iter 2: [6, 7]
+        //
+        // final binding: [1, 2, 3, 6, 7, 4, 5]
+        let mut existing_bindings = mem::take(&mut candidate.bindings);
+        let mut new_bindings = Vec::new();
+        loop {
+            let match_pairs = mem::take(&mut candidate.match_pairs);
+
+            if let [MatchPair { pattern: Pat { kind: box PatKind::Or { pats }, .. }, place }] =
+                &*match_pairs
+            {
+                existing_bindings.extend_from_slice(&new_bindings);
+                mem::swap(&mut candidate.bindings, &mut existing_bindings);
+                candidate.subcandidates =
+                    self.create_or_subcandidates(candidate, place.clone(), pats);
+                return true;
+            }
+
+            let mut changed = false;
+            for match_pair in match_pairs {
+                match self.simplify_match_pair(match_pair, candidate) {
+                    Ok(()) => {
+                        changed = true;
+                    }
+                    Err(match_pair) => {
+                        candidate.match_pairs.push(match_pair);
+                    }
+                }
+            }
+            // Avoid issue #69971: the binding order should be right to left if there are more
+            // bindings after `@` to please the borrow checker
+            // Ex
+            // struct NonCopyStruct {
+            //     copy_field: u32,
+            // }
+            //
+            // fn foo1(x: NonCopyStruct) {
+            //     let y @ NonCopyStruct { copy_field: z } = x;
+            //     // the above should turn into
+            //     let z = x.copy_field;
+            //     let y = x;
+            // }
+            candidate.bindings.extend_from_slice(&new_bindings);
+            mem::swap(&mut candidate.bindings, &mut new_bindings);
+            candidate.bindings.clear();
+
+            if !changed {
+                existing_bindings.extend_from_slice(&new_bindings);
+                mem::swap(&mut candidate.bindings, &mut existing_bindings);
+                // Move or-patterns to the end, because they can result in us
+                // creating additional candidates, so we want to test them as
+                // late as possible.
+                candidate
+                    .match_pairs
+                    .sort_by_key(|pair| matches!(*pair.pattern.kind, PatKind::Or { .. }));
+                debug!(simplified = ?candidate, "simplify_candidate");
+                return false; // if we were not able to simplify any, done.
+            }
+        }
+    }
+
+    /// Given `candidate` that has a single or-pattern for its match-pairs,
+    /// creates a fresh candidate for each of its input subpatterns passed via
+    /// `pats`.
+    fn create_or_subcandidates<'pat>(
+        &mut self,
+        candidate: &Candidate<'pat, 'tcx>,
+        place: PlaceBuilder<'tcx>,
+        pats: &'pat [Pat<'tcx>],
+    ) -> Vec<Candidate<'pat, 'tcx>> {
+        pats.iter()
+            .map(|pat| {
+                let mut candidate = Candidate::new(place.clone(), pat, candidate.has_guard);
+                self.simplify_candidate(&mut candidate);
+                candidate
+            })
+            .collect()
+    }
+
+    /// Tries to simplify `match_pair`, returning `Ok(())` if
+    /// successful. If successful, new match pairs and bindings will
+    /// have been pushed into the candidate. If no simplification is
+    /// possible, `Err` is returned and no changes are made to
+    /// candidate.
+    fn simplify_match_pair<'pat>(
+        &mut self,
+        match_pair: MatchPair<'pat, 'tcx>,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> Result<(), MatchPair<'pat, 'tcx>> {
+        let tcx = self.tcx;
+        match *match_pair.pattern.kind {
+            PatKind::AscribeUserType {
+                ref subpattern,
+                ascription: thir::Ascription { ref annotation, variance },
+            } => {
+                // Apply the type ascription to the value at `match_pair.place`, which is the
+                if let Ok(place_resolved) =
+                    match_pair.place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                {
+                    candidate.ascriptions.push(Ascription {
+                        annotation: annotation.clone(),
+                        source: place_resolved.into_place(self.tcx, self.typeck_results),
+                        variance,
+                    });
+                }
+
+                candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
+
+                Ok(())
+            }
+
+            PatKind::Wild => {
+                // nothing left to do
+                Ok(())
+            }
+
+            PatKind::Binding {
+                name: _,
+                mutability: _,
+                mode,
+                var,
+                ty: _,
+                ref subpattern,
+                is_primary: _,
+            } => {
+                if let Ok(place_resolved) =
+                    match_pair.place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                {
+                    candidate.bindings.push(Binding {
+                        span: match_pair.pattern.span,
+                        source: place_resolved.into_place(self.tcx, self.typeck_results),
+                        var_id: var,
+                        binding_mode: mode,
+                    });
+                }
+
+                if let Some(subpattern) = subpattern.as_ref() {
+                    // this is the `x @ P` case; have to keep matching against `P` now
+                    candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
+                }
+
+                Ok(())
+            }
+
+            PatKind::Constant { .. } => {
+                // FIXME normalize patterns when possible
+                Err(match_pair)
+            }
+
+            PatKind::Range(PatRange { lo, hi, end }) => {
+                let (range, bias) = match *lo.ty().kind() {
+                    ty::Char => {
+                        (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
+                    }
+                    ty::Int(ity) => {
+                        let size = Integer::from_int_ty(&tcx, ity).size();
+                        let max = size.truncate(u128::MAX);
+                        let bias = 1u128 << (size.bits() - 1);
+                        (Some((0, max, size)), bias)
+                    }
+                    ty::Uint(uty) => {
+                        let size = Integer::from_uint_ty(&tcx, uty).size();
+                        let max = size.truncate(u128::MAX);
+                        (Some((0, max, size)), 0)
+                    }
+                    _ => (None, 0),
+                };
+                if let Some((min, max, sz)) = range {
+                    // We want to compare ranges numerically, but the order of the bitwise
+                    // representation of signed integers does not match their numeric order. Thus,
+                    // to correct the ordering, we need to shift the range of signed integers to
+                    // correct the comparison. This is achieved by XORing with a bias (see
+                    // pattern/_match.rs for another pertinent example of this pattern).
+                    //
+                    // Also, for performance, it's important to only do the second `try_to_bits` if
+                    // necessary.
+                    let lo = lo.try_to_bits(sz).unwrap() ^ bias;
+                    if lo <= min {
+                        let hi = hi.try_to_bits(sz).unwrap() ^ bias;
+                        if hi > max || hi == max && end == RangeEnd::Included {
+                            // Irrefutable pattern match.
+                            return Ok(());
+                        }
+                    }
+                }
+                Err(match_pair)
+            }
+
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
+                    // irrefutable
+                    self.prefix_slice_suffix(
+                        &mut candidate.match_pairs,
+                        &match_pair.place,
+                        prefix,
+                        slice.as_ref(),
+                        suffix,
+                    );
+                    Ok(())
+                } else {
+                    Err(match_pair)
+                }
+            }
+
+            PatKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
+                let irrefutable = adt_def.variants().iter_enumerated().all(|(i, v)| {
+                    i == variant_index || {
+                        self.tcx.features().exhaustive_patterns
+                            && !v
+                                .uninhabited_from(
+                                    self.tcx,
+                                    substs,
+                                    adt_def.adt_kind(),
+                                    self.param_env,
+                                )
+                                .is_empty()
+                    }
+                }) && (adt_def.did().is_local()
+                    || !adt_def.is_variant_list_non_exhaustive());
+                if irrefutable {
+                    let place_builder = match_pair.place.downcast(adt_def, variant_index);
+                    candidate
+                        .match_pairs
+                        .extend(self.field_match_pairs(place_builder, subpatterns));
+                    Ok(())
+                } else {
+                    Err(match_pair)
+                }
+            }
+
+            PatKind::Array { ref prefix, ref slice, ref suffix } => {
+                self.prefix_slice_suffix(
+                    &mut candidate.match_pairs,
+                    &match_pair.place,
+                    prefix,
+                    slice.as_ref(),
+                    suffix,
+                );
+                Ok(())
+            }
+
+            PatKind::Leaf { ref subpatterns } => {
+                // tuple struct, match subpats (if any)
+                candidate.match_pairs.extend(self.field_match_pairs(match_pair.place, subpatterns));
+                Ok(())
+            }
+
+            PatKind::Deref { ref subpattern } => {
+                let place_builder = match_pair.place.deref();
+                candidate.match_pairs.push(MatchPair::new(place_builder, subpattern));
+                Ok(())
+            }
+
+            PatKind::Or { .. } => Err(match_pair),
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
new file mode 100644
index 000000000..598da80c5
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -0,0 +1,837 @@
+// Testing candidates
+//
+// After candidates have been simplified, the only match pairs that
+// remain are those that require some sort of test. The functions here
+// identify what tests are needed, perform the tests, and then filter
+// the candidates based on the result.
+
+use crate::build::expr::as_place::PlaceBuilder;
+use crate::build::matches::{Candidate, MatchPair, Test, TestKind};
+use crate::build::Builder;
+use crate::thir::pattern::compare_const_vals;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_hir::{LangItem, RangeEnd};
+use rustc_index::bit_set::BitSet;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::subst::{GenericArg, Subst};
+use rustc_middle::ty::util::IntTypeExt;
+use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt};
+use rustc_span::def_id::DefId;
+use rustc_span::symbol::{sym, Symbol};
+use rustc_span::Span;
+use rustc_target::abi::VariantIdx;
+
+use std::cmp::Ordering;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Identifies what test is needed to decide if `match_pair` is applicable.
+    ///
+    /// It is a bug to call this with a not-fully-simplified pattern.
+    pub(super) fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
+        match *match_pair.pattern.kind {
+            PatKind::Variant { adt_def, substs: _, variant_index: _, subpatterns: _ } => Test {
+                span: match_pair.pattern.span,
+                kind: TestKind::Switch {
+                    adt_def,
+                    variants: BitSet::new_empty(adt_def.variants().len()),
+                },
+            },
+
+            PatKind::Constant { .. } if is_switch_ty(match_pair.pattern.ty) => {
+                // For integers, we use a `SwitchInt` match, which allows
+                // us to handle more cases.
+                Test {
+                    span: match_pair.pattern.span,
+                    kind: TestKind::SwitchInt {
+                        switch_ty: match_pair.pattern.ty,
+
+                        // these maps are empty to start; cases are
+                        // added below in add_cases_to_switch
+                        options: Default::default(),
+                    },
+                }
+            }
+
+            PatKind::Constant { value } => Test {
+                span: match_pair.pattern.span,
+                kind: TestKind::Eq { value, ty: match_pair.pattern.ty },
+            },
+
+            PatKind::Range(range) => {
+                assert_eq!(range.lo.ty(), match_pair.pattern.ty);
+                assert_eq!(range.hi.ty(), match_pair.pattern.ty);
+                Test { span: match_pair.pattern.span, kind: TestKind::Range(range) }
+            }
+
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                let len = prefix.len() + suffix.len();
+                let op = if slice.is_some() { BinOp::Ge } else { BinOp::Eq };
+                Test { span: match_pair.pattern.span, kind: TestKind::Len { len: len as u64, op } }
+            }
+
+            PatKind::Or { .. } => bug!("or-patterns should have already been handled"),
+
+            PatKind::AscribeUserType { .. }
+            | PatKind::Array { .. }
+            | PatKind::Wild
+            | PatKind::Binding { .. }
+            | PatKind::Leaf { .. }
+            | PatKind::Deref { .. } => self.error_simplifyable(match_pair),
+        }
+    }
+
+    pub(super) fn add_cases_to_switch<'pat>(
+        &mut self,
+        test_place: &PlaceBuilder<'tcx>,
+        candidate: &Candidate<'pat, 'tcx>,
+        switch_ty: Ty<'tcx>,
+        options: &mut FxIndexMap<ConstantKind<'tcx>, u128>,
+    ) -> bool {
+        let Some(match_pair) = candidate.match_pairs.iter().find(|mp| mp.place == *test_place) else {
+            return false;
+        };
+
+        match *match_pair.pattern.kind {
+            PatKind::Constant { value } => {
+                options
+                    .entry(value)
+                    .or_insert_with(|| value.eval_bits(self.tcx, self.param_env, switch_ty));
+                true
+            }
+            PatKind::Variant { .. } => {
+                panic!("you should have called add_variants_to_switch instead!");
+            }
+            PatKind::Range(range) => {
+                // Check that none of the switch values are in the range.
+                self.values_not_contained_in_range(range, options).unwrap_or(false)
+            }
+            PatKind::Slice { .. }
+            | PatKind::Array { .. }
+            | PatKind::Wild
+            | PatKind::Or { .. }
+            | PatKind::Binding { .. }
+            | PatKind::AscribeUserType { .. }
+            | PatKind::Leaf { .. }
+            | PatKind::Deref { .. } => {
+                // don't know how to add these patterns to a switch
+                false
+            }
+        }
+    }
+
+    pub(super) fn add_variants_to_switch<'pat>(
+        &mut self,
+        test_place: &PlaceBuilder<'tcx>,
+        candidate: &Candidate<'pat, 'tcx>,
+        variants: &mut BitSet<VariantIdx>,
+    ) -> bool {
+        let Some(match_pair) = candidate.match_pairs.iter().find(|mp| mp.place == *test_place) else {
+            return false;
+        };
+
+        match *match_pair.pattern.kind {
+            PatKind::Variant { adt_def: _, variant_index, .. } => {
+                // We have a pattern testing for variant `variant_index`
+                // set the corresponding index to true
+                variants.insert(variant_index);
+                true
+            }
+            _ => {
+                // don't know how to add these patterns to a switch
+                false
+            }
+        }
+    }
+
+    pub(super) fn perform_test(
+        &mut self,
+        match_start_span: Span,
+        scrutinee_span: Span,
+        block: BasicBlock,
+        place_builder: PlaceBuilder<'tcx>,
+        test: &Test<'tcx>,
+        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+    ) {
+        let place: Place<'tcx>;
+        if let Ok(test_place_builder) =
+            place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
+        {
+            place = test_place_builder.into_place(self.tcx, self.typeck_results);
+        } else {
+            return;
+        }
+        debug!(
+            "perform_test({:?}, {:?}: {:?}, {:?})",
+            block,
+            place,
+            place.ty(&self.local_decls, self.tcx),
+            test
+        );
+
+        let source_info = self.source_info(test.span);
+        match test.kind {
+            TestKind::Switch { adt_def, ref variants } => {
+                let target_blocks = make_target_blocks(self);
+                // Variants is a BitVec of indexes into adt_def.variants.
+                let num_enum_variants = adt_def.variants().len();
+                debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
+                let otherwise_block = *target_blocks.last().unwrap();
+                let tcx = self.tcx;
+                let switch_targets = SwitchTargets::new(
+                    adt_def.discriminants(tcx).filter_map(|(idx, discr)| {
+                        if variants.contains(idx) {
+                            debug_assert_ne!(
+                                target_blocks[idx.index()],
+                                otherwise_block,
+                                "no canididates for tested discriminant: {:?}",
+                                discr,
+                            );
+                            Some((discr.val, target_blocks[idx.index()]))
+                        } else {
+                            debug_assert_eq!(
+                                target_blocks[idx.index()],
+                                otherwise_block,
+                                "found canididates for untested discriminant: {:?}",
+                                discr,
+                            );
+                            None
+                        }
+                    }),
+                    otherwise_block,
+                );
+                debug!("num_enum_variants: {}, variants: {:?}", num_enum_variants, variants);
+                let discr_ty = adt_def.repr().discr_type().to_ty(tcx);
+                let discr = self.temp(discr_ty, test.span);
+                self.cfg.push_assign(
+                    block,
+                    self.source_info(scrutinee_span),
+                    discr,
+                    Rvalue::Discriminant(place),
+                );
+                self.cfg.terminate(
+                    block,
+                    self.source_info(match_start_span),
+                    TerminatorKind::SwitchInt {
+                        discr: Operand::Move(discr),
+                        switch_ty: discr_ty,
+                        targets: switch_targets,
+                    },
+                );
+            }
+
+            TestKind::SwitchInt { switch_ty, ref options } => {
+                let target_blocks = make_target_blocks(self);
+                let terminator = if *switch_ty.kind() == ty::Bool {
+                    assert!(!options.is_empty() && options.len() <= 2);
+                    let [first_bb, second_bb] = *target_blocks else {
+                        bug!("`TestKind::SwitchInt` on `bool` should have two targets")
+                    };
+                    let (true_bb, false_bb) = match options[0] {
+                        1 => (first_bb, second_bb),
+                        0 => (second_bb, first_bb),
+                        v => span_bug!(test.span, "expected boolean value but got {:?}", v),
+                    };
+                    TerminatorKind::if_(self.tcx, Operand::Copy(place), true_bb, false_bb)
+                } else {
+                    // The switch may be inexhaustive so we have a catch all block
+                    debug_assert_eq!(options.len() + 1, target_blocks.len());
+                    let otherwise_block = *target_blocks.last().unwrap();
+                    let switch_targets = SwitchTargets::new(
+                        options.values().copied().zip(target_blocks),
+                        otherwise_block,
+                    );
+                    TerminatorKind::SwitchInt {
+                        discr: Operand::Copy(place),
+                        switch_ty,
+                        targets: switch_targets,
+                    }
+                };
+                self.cfg.terminate(block, self.source_info(match_start_span), terminator);
+            }
+
+            TestKind::Eq { value, ty } => {
+                if !ty.is_scalar() {
+                    // Use `PartialEq::eq` instead of `BinOp::Eq`
+                    // (the binop can only handle primitives)
+                    self.non_scalar_compare(
+                        block,
+                        make_target_blocks,
+                        source_info,
+                        value,
+                        place,
+                        ty,
+                    );
+                } else if let [success, fail] = *make_target_blocks(self) {
+                    assert_eq!(value.ty(), ty);
+                    let expect = self.literal_operand(test.span, value);
+                    let val = Operand::Copy(place);
+                    self.compare(block, success, fail, source_info, BinOp::Eq, expect, val);
+                } else {
+                    bug!("`TestKind::Eq` should have two target blocks");
+                }
+            }
+
+            TestKind::Range(PatRange { lo, hi, ref end }) => {
+                let lower_bound_success = self.cfg.start_new_block();
+                let target_blocks = make_target_blocks(self);
+
+                // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
+                let lo = self.literal_operand(test.span, lo);
+                let hi = self.literal_operand(test.span, hi);
+                let val = Operand::Copy(place);
+
+                let [success, fail] = *target_blocks else {
+                    bug!("`TestKind::Range` should have two target blocks");
+                };
+                self.compare(
+                    block,
+                    lower_bound_success,
+                    fail,
+                    source_info,
+                    BinOp::Le,
+                    lo,
+                    val.clone(),
+                );
+                let op = match *end {
+                    RangeEnd::Included => BinOp::Le,
+                    RangeEnd::Excluded => BinOp::Lt,
+                };
+                self.compare(lower_bound_success, success, fail, source_info, op, val, hi);
+            }
+
+            TestKind::Len { len, op } => {
+                let target_blocks = make_target_blocks(self);
+
+                let usize_ty = self.tcx.types.usize;
+                let actual = self.temp(usize_ty, test.span);
+
+                // actual = len(place)
+                self.cfg.push_assign(block, source_info, actual, Rvalue::Len(place));
+
+                // expected = <N>
+                let expected = self.push_usize(block, source_info, len);
+
+                let [true_bb, false_bb] = *target_blocks else {
+                    bug!("`TestKind::Len` should have two target blocks");
+                };
+                // result = actual == expected OR result = actual < expected
+                // branch based on result
+                self.compare(
+                    block,
+                    true_bb,
+                    false_bb,
+                    source_info,
+                    op,
+                    Operand::Move(actual),
+                    Operand::Move(expected),
+                );
+            }
+        }
+    }
+
+    /// Compare using the provided built-in comparison operator
+    fn compare(
+        &mut self,
+        block: BasicBlock,
+        success_block: BasicBlock,
+        fail_block: BasicBlock,
+        source_info: SourceInfo,
+        op: BinOp,
+        left: Operand<'tcx>,
+        right: Operand<'tcx>,
+    ) {
+        let bool_ty = self.tcx.types.bool;
+        let result = self.temp(bool_ty, source_info.span);
+
+        // result = op(left, right)
+        self.cfg.push_assign(
+            block,
+            source_info,
+            result,
+            Rvalue::BinaryOp(op, Box::new((left, right))),
+        );
+
+        // branch based on result
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::if_(self.tcx, Operand::Move(result), success_block, fail_block),
+        );
+    }
+
+    /// Compare two `&T` values using `<T as std::compare::PartialEq>::eq`
+    fn non_scalar_compare(
+        &mut self,
+        block: BasicBlock,
+        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+        source_info: SourceInfo,
+        value: ConstantKind<'tcx>,
+        place: Place<'tcx>,
+        mut ty: Ty<'tcx>,
+    ) {
+        let mut expect = self.literal_operand(source_info.span, value);
+        let mut val = Operand::Copy(place);
+
+        // If we're using `b"..."` as a pattern, we need to insert an
+        // unsizing coercion, as the byte string has the type `&[u8; N]`.
+        //
+        // We want to do this even when the scrutinee is a reference to an
+        // array, so we can call `<[u8]>::eq` rather than having to find an
+        // `<[u8; N]>::eq`.
+        let unsize = |ty: Ty<'tcx>| match ty.kind() {
+            ty::Ref(region, rty, _) => match rty.kind() {
+                ty::Array(inner_ty, n) => Some((region, inner_ty, n)),
+                _ => None,
+            },
+            _ => None,
+        };
+        let opt_ref_ty = unsize(ty);
+        let opt_ref_test_ty = unsize(value.ty());
+        match (opt_ref_ty, opt_ref_test_ty) {
+            // nothing to do, neither is an array
+            (None, None) => {}
+            (Some((region, elem_ty, _)), _) | (None, Some((region, elem_ty, _))) => {
+                let tcx = self.tcx;
+                // make both a slice
+                ty = tcx.mk_imm_ref(*region, tcx.mk_slice(*elem_ty));
+                if opt_ref_ty.is_some() {
+                    let temp = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        temp,
+                        Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), val, ty),
+                    );
+                    val = Operand::Move(temp);
+                }
+                if opt_ref_test_ty.is_some() {
+                    let slice = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        slice,
+                        Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), expect, ty),
+                    );
+                    expect = Operand::Move(slice);
+                }
+            }
+        }
+
+        let ty::Ref(_, deref_ty, _) = *ty.kind() else {
+            bug!("non_scalar_compare called on non-reference type: {}", ty);
+        };
+
+        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, None);
+        let method = trait_method(self.tcx, eq_def_id, sym::eq, deref_ty, &[deref_ty.into()]);
+
+        let bool_ty = self.tcx.types.bool;
+        let eq_result = self.temp(bool_ty, source_info.span);
+        let eq_block = self.cfg.start_new_block();
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::Call {
+                func: Operand::Constant(Box::new(Constant {
+                    span: source_info.span,
+
+                    // FIXME(#54571): This constant comes from user input (a
+                    // constant in a pattern).  Are there forms where users can add
+                    // type annotations here?  For example, an associated constant?
+                    // Need to experiment.
+                    user_ty: None,
+
+                    literal: method,
+                })),
+                args: vec![val, expect],
+                destination: eq_result,
+                target: Some(eq_block),
+                cleanup: None,
+                from_hir_call: false,
+                fn_span: source_info.span,
+            },
+        );
+        self.diverge_from(block);
+
+        let [success_block, fail_block] = *make_target_blocks(self) else {
+            bug!("`TestKind::Eq` should have two target blocks")
+        };
+        // check the result
+        self.cfg.terminate(
+            eq_block,
+            source_info,
+            TerminatorKind::if_(self.tcx, Operand::Move(eq_result), success_block, fail_block),
+        );
+    }
+
+    /// Given that we are performing `test` against `test_place`, this job
+    /// sorts out what the status of `candidate` will be after the test. See
+    /// `test_candidates` for the usage of this function. The returned index is
+    /// the index that this candidate should be placed in the
+    /// `target_candidates` vec. The candidate may be modified to update its
+    /// `match_pairs`.
+    ///
+    /// So, for example, if this candidate is `x @ Some(P0)` and the `Test` is
+    /// a variant test, then we would modify the candidate to be `(x as
+    /// Option).0 @ P0` and return the index corresponding to the variant
+    /// `Some`.
+    ///
+    /// However, in some cases, the test may just not be relevant to candidate.
+    /// For example, suppose we are testing whether `foo.x == 22`, but in one
+    /// match arm we have `Foo { x: _, ... }`... in that case, the test for
+    /// what value `x` has has no particular relevance to this candidate. In
+    /// such cases, this function just returns None without doing anything.
+    /// This is used by the overall `match_candidates` algorithm to structure
+    /// the match as a whole. See `match_candidates` for more details.
+    ///
+    /// FIXME(#29623). In some cases, we have some tricky choices to make.  for
+    /// example, if we are testing that `x == 22`, but the candidate is `x @
+    /// 13..55`, what should we do? In the event that the test is true, we know
+    /// that the candidate applies, but in the event of false, we don't know
+    /// that it *doesn't* apply. For now, we return false, indicate that the
+    /// test does not apply to this candidate, but it might be we can get
+    /// tighter match code if we do something a bit different.
+    pub(super) fn sort_candidate<'pat>(
+        &mut self,
+        test_place: &PlaceBuilder<'tcx>,
+        test: &Test<'tcx>,
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) -> Option<usize> {
+        // Find the match_pair for this place (if any). At present,
+        // afaik, there can be at most one. (In the future, if we
+        // adopted a more general `@` operator, there might be more
+        // than one, but it'd be very unusual to have two sides that
+        // both require tests; you'd expect one side to be simplified
+        // away.)
+        let (match_pair_index, match_pair) =
+            candidate.match_pairs.iter().enumerate().find(|&(_, mp)| mp.place == *test_place)?;
+
+        match (&test.kind, &*match_pair.pattern.kind) {
+            // If we are performing a variant switch, then this
+            // informs variant patterns, but nothing else.
+            (
+                &TestKind::Switch { adt_def: tested_adt_def, .. },
+                &PatKind::Variant { adt_def, variant_index, ref subpatterns, .. },
+            ) => {
+                assert_eq!(adt_def, tested_adt_def);
+                self.candidate_after_variant_switch(
+                    match_pair_index,
+                    adt_def,
+                    variant_index,
+                    subpatterns,
+                    candidate,
+                );
+                Some(variant_index.as_usize())
+            }
+
+            (&TestKind::Switch { .. }, _) => None,
+
+            // If we are performing a switch over integers, then this informs integer
+            // equality, but nothing else.
+            //
+            // FIXME(#29623) we could use PatKind::Range to rule
+            // things out here, in some cases.
+            (
+                &TestKind::SwitchInt { switch_ty: _, ref options },
+                &PatKind::Constant { ref value },
+            ) if is_switch_ty(match_pair.pattern.ty) => {
+                let index = options.get_index_of(value).unwrap();
+                self.candidate_without_match_pair(match_pair_index, candidate);
+                Some(index)
+            }
+
+            (&TestKind::SwitchInt { switch_ty: _, ref options }, &PatKind::Range(range)) => {
+                let not_contained =
+                    self.values_not_contained_in_range(range, options).unwrap_or(false);
+
+                if not_contained {
+                    // No switch values are contained in the pattern range,
+                    // so the pattern can be matched only if this test fails.
+                    let otherwise = options.len();
+                    Some(otherwise)
+                } else {
+                    None
+                }
+            }
+
+            (&TestKind::SwitchInt { .. }, _) => None,
+
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Eq },
+                &PatKind::Slice { ref prefix, ref slice, ref suffix },
+            ) => {
+                let pat_len = (prefix.len() + suffix.len()) as u64;
+                match (test_len.cmp(&pat_len), slice) {
+                    (Ordering::Equal, &None) => {
+                        // on true, min_len = len = $actual_length,
+                        // on false, len != $actual_length
+                        self.candidate_after_slice_test(
+                            match_pair_index,
+                            candidate,
+                            prefix,
+                            slice.as_ref(),
+                            suffix,
+                        );
+                        Some(0)
+                    }
+                    (Ordering::Less, _) => {
+                        // test_len < pat_len. If $actual_len = test_len,
+                        // then $actual_len < pat_len and we don't have
+                        // enough elements.
+                        Some(1)
+                    }
+                    (Ordering::Equal | Ordering::Greater, &Some(_)) => {
+                        // This can match both if $actual_len = test_len >= pat_len,
+                        // and if $actual_len > test_len. We can't advance.
+                        None
+                    }
+                    (Ordering::Greater, &None) => {
+                        // test_len != pat_len, so if $actual_len = test_len, then
+                        // $actual_len != pat_len.
+                        Some(1)
+                    }
+                }
+            }
+
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Ge },
+                &PatKind::Slice { ref prefix, ref slice, ref suffix },
+            ) => {
+                // the test is `$actual_len >= test_len`
+                let pat_len = (prefix.len() + suffix.len()) as u64;
+                match (test_len.cmp(&pat_len), slice) {
+                    (Ordering::Equal, &Some(_)) => {
+                        // $actual_len >= test_len = pat_len,
+                        // so we can match.
+                        self.candidate_after_slice_test(
+                            match_pair_index,
+                            candidate,
+                            prefix,
+                            slice.as_ref(),
+                            suffix,
+                        );
+                        Some(0)
+                    }
+                    (Ordering::Less, _) | (Ordering::Equal, &None) => {
+                        // test_len <= pat_len. If $actual_len < test_len,
+                        // then it is also < pat_len, so the test passing is
+                        // necessary (but insufficient).
+                        Some(0)
+                    }
+                    (Ordering::Greater, &None) => {
+                        // test_len > pat_len. If $actual_len >= test_len > pat_len,
+                        // then we know we won't have a match.
+                        Some(1)
+                    }
+                    (Ordering::Greater, &Some(_)) => {
+                        // test_len < pat_len, and is therefore less
+                        // strict. This can still go both ways.
+                        None
+                    }
+                }
+            }
+
+            (&TestKind::Range(test), &PatKind::Range(pat)) => {
+                use std::cmp::Ordering::*;
+
+                if test == pat {
+                    self.candidate_without_match_pair(match_pair_index, candidate);
+                    return Some(0);
+                }
+
+                // For performance, it's important to only do the second
+                // `compare_const_vals` if necessary.
+                let no_overlap = if matches!(
+                    (compare_const_vals(self.tcx, test.hi, pat.lo, self.param_env)?, test.end),
+                    (Less, _) | (Equal, RangeEnd::Excluded) // test < pat
+                ) || matches!(
+                    (compare_const_vals(self.tcx, test.lo, pat.hi, self.param_env)?, pat.end),
+                    (Greater, _) | (Equal, RangeEnd::Excluded) // test > pat
+                ) {
+                    Some(1)
+                } else {
+                    None
+                };
+
+                // If the testing range does not overlap with pattern range,
+                // the pattern can be matched only if this test fails.
+                no_overlap
+            }
+
+            (&TestKind::Range(range), &PatKind::Constant { value }) => {
+                if let Some(false) = self.const_range_contains(range, value) {
+                    // `value` is not contained in the testing range,
+                    // so `value` can be matched only if this test fails.
+                    Some(1)
+                } else {
+                    None
+                }
+            }
+
+            (&TestKind::Range { .. }, _) => None,
+
+            (&TestKind::Eq { .. } | &TestKind::Len { .. }, _) => {
+                // The call to `self.test(&match_pair)` below is not actually used to generate any
+                // MIR. Instead, we just want to compare with `test` (the parameter of the method)
+                // to see if it is the same.
+                //
+                // However, at this point we can still encounter or-patterns that were extracted
+                // from previous calls to `sort_candidate`, so we need to manually address that
+                // case to avoid panicking in `self.test()`.
+                if let PatKind::Or { .. } = &*match_pair.pattern.kind {
+                    return None;
+                }
+
+                // These are all binary tests.
+                //
+                // FIXME(#29623) we can be more clever here
+                let pattern_test = self.test(&match_pair);
+                if pattern_test.kind == test.kind {
+                    self.candidate_without_match_pair(match_pair_index, candidate);
+                    Some(0)
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    fn candidate_without_match_pair(
+        &mut self,
+        match_pair_index: usize,
+        candidate: &mut Candidate<'_, 'tcx>,
+    ) {
+        candidate.match_pairs.remove(match_pair_index);
+    }
+
+    fn candidate_after_slice_test<'pat>(
+        &mut self,
+        match_pair_index: usize,
+        candidate: &mut Candidate<'pat, 'tcx>,
+        prefix: &'pat [Pat<'tcx>],
+        opt_slice: Option<&'pat Pat<'tcx>>,
+        suffix: &'pat [Pat<'tcx>],
+    ) {
+        let removed_place = candidate.match_pairs.remove(match_pair_index).place;
+        self.prefix_slice_suffix(
+            &mut candidate.match_pairs,
+            &removed_place,
+            prefix,
+            opt_slice,
+            suffix,
+        );
+    }
+
+    fn candidate_after_variant_switch<'pat>(
+        &mut self,
+        match_pair_index: usize,
+        adt_def: ty::AdtDef<'tcx>,
+        variant_index: VariantIdx,
+        subpatterns: &'pat [FieldPat<'tcx>],
+        candidate: &mut Candidate<'pat, 'tcx>,
+    ) {
+        let match_pair = candidate.match_pairs.remove(match_pair_index);
+
+        // So, if we have a match-pattern like `x @ Enum::Variant(P1, P2)`,
+        // we want to create a set of derived match-patterns like
+        // `(x as Variant).0 @ P1` and `(x as Variant).1 @ P1`.
+        let elem =
+            ProjectionElem::Downcast(Some(adt_def.variant(variant_index).name), variant_index);
+        let downcast_place = match_pair.place.project(elem); // `(x as Variant)`
+        let consequent_match_pairs = subpatterns.iter().map(|subpattern| {
+            // e.g., `(x as Variant).0`
+            let place = downcast_place.clone().field(subpattern.field, subpattern.pattern.ty);
+            // e.g., `(x as Variant).0 @ P1`
+            MatchPair::new(place, &subpattern.pattern)
+        });
+
+        candidate.match_pairs.extend(consequent_match_pairs);
+    }
+
+    fn error_simplifyable<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> ! {
+        span_bug!(match_pair.pattern.span, "simplifyable pattern found: {:?}", match_pair.pattern)
+    }
+
+    fn const_range_contains(
+        &self,
+        range: PatRange<'tcx>,
+        value: ConstantKind<'tcx>,
+    ) -> Option<bool> {
+        use std::cmp::Ordering::*;
+
+        // For performance, it's important to only do the second
+        // `compare_const_vals` if necessary.
+        Some(
+            matches!(compare_const_vals(self.tcx, range.lo, value, self.param_env)?, Less | Equal)
+                && matches!(
+                    (compare_const_vals(self.tcx, value, range.hi, self.param_env)?, range.end),
+                    (Less, _) | (Equal, RangeEnd::Included)
+                ),
+        )
+    }
+
+    fn values_not_contained_in_range(
+        &self,
+        range: PatRange<'tcx>,
+        options: &FxIndexMap<ConstantKind<'tcx>, u128>,
+    ) -> Option<bool> {
+        for &val in options.keys() {
+            if self.const_range_contains(range, val)? {
+                return Some(false);
+            }
+        }
+
+        Some(true)
+    }
+}
+
+impl Test<'_> {
+    pub(super) fn targets(&self) -> usize {
+        match self.kind {
+            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } => 2,
+            TestKind::Switch { adt_def, .. } => {
+                // While the switch that we generate doesn't test for all
+                // variants, we have a target for each variant and the
+                // otherwise case, and we make sure that all of the cases not
+                // specified have the same block.
+                adt_def.variants().len() + 1
+            }
+            TestKind::SwitchInt { switch_ty, ref options, .. } => {
+                if switch_ty.is_bool() {
+                    // `bool` is special cased in `perform_test` to always
+                    // branch to two blocks.
+                    2
+                } else {
+                    options.len() + 1
+                }
+            }
+        }
+    }
+}
+
+fn is_switch_ty(ty: Ty<'_>) -> bool {
+    ty.is_integral() || ty.is_char() || ty.is_bool()
+}
+
+fn trait_method<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    trait_def_id: DefId,
+    method_name: Symbol,
+    self_ty: Ty<'tcx>,
+    params: &[GenericArg<'tcx>],
+) -> ConstantKind<'tcx> {
+    let substs = tcx.mk_substs_trait(self_ty, params);
+
+    // The unhygienic comparison here is acceptable because this is only
+    // used on known traits.
+    let item = tcx
+        .associated_items(trait_def_id)
+        .filter_by_name_unhygienic(method_name)
+        .find(|item| item.kind == ty::AssocKind::Fn)
+        .expect("trait method not found");
+
+    let method_ty = tcx.bound_type_of(item.def_id);
+    let method_ty = method_ty.subst(tcx, substs);
+
+    ConstantKind::zero_sized(method_ty)
+}
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
new file mode 100644
index 000000000..9a1e98d3b
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -0,0 +1,109 @@
+use crate::build::expr::as_place::PlaceBuilder;
+use crate::build::matches::MatchPair;
+use crate::build::Builder;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty;
+use smallvec::SmallVec;
+use std::convert::TryInto;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    pub(crate) fn field_match_pairs<'pat>(
+        &mut self,
+        place: PlaceBuilder<'tcx>,
+        subpatterns: &'pat [FieldPat<'tcx>],
+    ) -> Vec<MatchPair<'pat, 'tcx>> {
+        subpatterns
+            .iter()
+            .map(|fieldpat| {
+                let place = place.clone().field(fieldpat.field, fieldpat.pattern.ty);
+                MatchPair::new(place, &fieldpat.pattern)
+            })
+            .collect()
+    }
+
+    pub(crate) fn prefix_slice_suffix<'pat>(
+        &mut self,
+        match_pairs: &mut SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
+        place: &PlaceBuilder<'tcx>,
+        prefix: &'pat [Pat<'tcx>],
+        opt_slice: Option<&'pat Pat<'tcx>>,
+        suffix: &'pat [Pat<'tcx>],
+    ) {
+        let tcx = self.tcx;
+        let (min_length, exact_size) = if let Ok(place_resolved) =
+            place.clone().try_upvars_resolved(tcx, self.typeck_results)
+        {
+            match place_resolved
+                .into_place(tcx, self.typeck_results)
+                .ty(&self.local_decls, tcx)
+                .ty
+                .kind()
+            {
+                ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
+                _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
+            }
+        } else {
+            ((prefix.len() + suffix.len()).try_into().unwrap(), false)
+        };
+
+        match_pairs.extend(prefix.iter().enumerate().map(|(idx, subpattern)| {
+            let elem =
+                ProjectionElem::ConstantIndex { offset: idx as u64, min_length, from_end: false };
+            let place = place.clone().project(elem);
+            MatchPair::new(place, subpattern)
+        }));
+
+        if let Some(subslice_pat) = opt_slice {
+            let suffix_len = suffix.len() as u64;
+            let subslice = place.clone().project(ProjectionElem::Subslice {
+                from: prefix.len() as u64,
+                to: if exact_size { min_length - suffix_len } else { suffix_len },
+                from_end: !exact_size,
+            });
+            match_pairs.push(MatchPair::new(subslice, subslice_pat));
+        }
+
+        match_pairs.extend(suffix.iter().rev().enumerate().map(|(idx, subpattern)| {
+            let end_offset = (idx + 1) as u64;
+            let elem = ProjectionElem::ConstantIndex {
+                offset: if exact_size { min_length - end_offset } else { end_offset },
+                min_length,
+                from_end: !exact_size,
+            };
+            let place = place.clone().project(elem);
+            MatchPair::new(place, subpattern)
+        }));
+    }
+
+    /// Creates a false edge to `imaginary_target` and a real edge to
+    /// real_target. If `imaginary_target` is none, or is the same as the real
+    /// target, a Goto is generated instead to simplify the generated MIR.
+    pub(crate) fn false_edges(
+        &mut self,
+        from_block: BasicBlock,
+        real_target: BasicBlock,
+        imaginary_target: Option<BasicBlock>,
+        source_info: SourceInfo,
+    ) {
+        match imaginary_target {
+            Some(target) if target != real_target => {
+                self.cfg.terminate(
+                    from_block,
+                    source_info,
+                    TerminatorKind::FalseEdge { real_target, imaginary_target: target },
+                );
+            }
+            _ => self.cfg.goto(from_block, source_info, real_target),
+        }
+    }
+}
+
+impl<'pat, 'tcx> MatchPair<'pat, 'tcx> {
+    pub(crate) fn new(
+        place: PlaceBuilder<'tcx>,
+        pattern: &'pat Pat<'tcx>,
+    ) -> MatchPair<'pat, 'tcx> {
+        MatchPair { place, pattern }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/misc.rs b/compiler/rustc_mir_build/src/build/misc.rs
new file mode 100644
index 000000000..86f466ff7
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/misc.rs
@@ -0,0 +1,75 @@
+//! Miscellaneous builder routines that are not specific to building any particular
+//! kind of thing.
+
+use crate::build::Builder;
+
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, Ty};
+use rustc_span::{Span, DUMMY_SP};
+use rustc_trait_selection::infer::InferCtxtExt;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Adds a new temporary value of type `ty` storing the result of
+    /// evaluating `expr`.
+    ///
+    /// N.B., **No cleanup is scheduled for this temporary.** You should
+    /// call `schedule_drop` once the temporary is initialized.
+    pub(crate) fn temp(&mut self, ty: Ty<'tcx>, span: Span) -> Place<'tcx> {
+        // Mark this local as internal to avoid temporaries with types not present in the
+        // user's code resulting in ICEs from the generator transform.
+        let temp = self.local_decls.push(LocalDecl::new(ty, span).internal());
+        let place = Place::from(temp);
+        debug!("temp: created temp {:?} with type {:?}", place, self.local_decls[temp].ty);
+        place
+    }
+
+    /// Convenience function for creating a literal operand, one
+    /// without any user type annotation.
+    pub(crate) fn literal_operand(
+        &mut self,
+        span: Span,
+        literal: ConstantKind<'tcx>,
+    ) -> Operand<'tcx> {
+        let constant = Box::new(Constant { span, user_ty: None, literal });
+        Operand::Constant(constant)
+    }
+
+    // Returns a zero literal operand for the appropriate type, works for
+    // bool, char and integers.
+    pub(crate) fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let literal = ConstantKind::from_bits(self.tcx, 0, ty::ParamEnv::empty().and(ty));
+
+        self.literal_operand(span, literal)
+    }
+
+    pub(crate) fn push_usize(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        value: u64,
+    ) -> Place<'tcx> {
+        let usize_ty = self.tcx.types.usize;
+        let temp = self.temp(usize_ty, source_info.span);
+        self.cfg.push_assign_constant(
+            block,
+            source_info,
+            temp,
+            Constant {
+                span: source_info.span,
+                user_ty: None,
+                literal: ConstantKind::from_usize(self.tcx, value),
+            },
+        );
+        temp
+    }
+
+    pub(crate) fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
+        let tcx = self.tcx;
+        let ty = place.ty(&self.local_decls, tcx).ty;
+        if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty, DUMMY_SP) {
+            Operand::Move(place)
+        } else {
+            Operand::Copy(place)
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
new file mode 100644
index 000000000..12b8ceede
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -0,0 +1,1171 @@
+use crate::build;
+pub(crate) use crate::build::expr::as_constant::lit_to_mir_constant;
+use crate::build::expr::as_place::PlaceBuilder;
+use crate::build::scope::DropKind;
+use crate::thir::pattern::pat_from_hir;
+use rustc_apfloat::ieee::{Double, Single};
+use rustc_apfloat::Float;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_errors::ErrorGuaranteed;
+use rustc_hir as hir;
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::lang_items::LangItem;
+use rustc_hir::{GeneratorKind, Node};
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
+use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
+use rustc_middle::middle::region;
+use rustc_middle::mir::interpret::ConstValue;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::mir::*;
+use rustc_middle::thir::{BindingMode, Expr, ExprId, LintLevel, LocalVarId, PatKind, Thir};
+use rustc_middle::ty::subst::Subst;
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable, TypeckResults};
+use rustc_span::symbol::sym;
+use rustc_span::Span;
+use rustc_span::Symbol;
+use rustc_target::spec::abi::Abi;
+
+use super::lints;
+
+pub(crate) fn mir_built<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def: ty::WithOptConstParam<LocalDefId>,
+) -> &'tcx rustc_data_structures::steal::Steal<Body<'tcx>> {
+    if let Some(def) = def.try_upgrade(tcx) {
+        return tcx.mir_built(def);
+    }
+
+    let mut body = mir_build(tcx, def);
+    if def.const_param_did.is_some() {
+        assert!(matches!(body.source.instance, ty::InstanceDef::Item(_)));
+        body.source = MirSource::from_instance(ty::InstanceDef::Item(def.to_global()));
+    }
+
+    tcx.alloc_steal_mir(body)
+}
+
+/// Construct the MIR for a given `DefId`.
+fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> {
+    let id = tcx.hir().local_def_id_to_hir_id(def.did);
+    let body_owner_kind = tcx.hir().body_owner_kind(def.did);
+    let typeck_results = tcx.typeck_opt_const_arg(def);
+
+    // Ensure unsafeck and abstract const building is ran before we steal the THIR.
+    // We can't use `ensure()` for `thir_abstract_const` as it doesn't compute the query
+    // if inputs are green. This can cause ICEs when calling `thir_abstract_const` after
+    // THIR has been stolen if we haven't computed this query yet.
+    match def {
+        ty::WithOptConstParam { did, const_param_did: Some(const_param_did) } => {
+            tcx.ensure().thir_check_unsafety_for_const_arg((did, const_param_did));
+            drop(tcx.thir_abstract_const_of_const_arg((did, const_param_did)));
+        }
+        ty::WithOptConstParam { did, const_param_did: None } => {
+            tcx.ensure().thir_check_unsafety(did);
+            drop(tcx.thir_abstract_const(did));
+        }
+    }
+
+    // Figure out what primary body this item has.
+    let (body_id, return_ty_span, span_with_body) = match tcx.hir().get(id) {
+        Node::Expr(hir::Expr {
+            kind: hir::ExprKind::Closure(hir::Closure { fn_decl, body, .. }),
+            ..
+        }) => (*body, fn_decl.output.span(), None),
+        Node::Item(hir::Item {
+            kind: hir::ItemKind::Fn(hir::FnSig { decl, .. }, _, body_id),
+            span,
+            ..
+        })
+        | Node::ImplItem(hir::ImplItem {
+            kind: hir::ImplItemKind::Fn(hir::FnSig { decl, .. }, body_id),
+            span,
+            ..
+        })
+        | Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Fn(hir::FnSig { decl, .. }, hir::TraitFn::Provided(body_id)),
+            span,
+            ..
+        }) => {
+            // Use the `Span` of the `Item/ImplItem/TraitItem` as the body span,
+            // since the def span of a function does not include the body
+            (*body_id, decl.output.span(), Some(*span))
+        }
+        Node::Item(hir::Item {
+            kind: hir::ItemKind::Static(ty, _, body_id) | hir::ItemKind::Const(ty, body_id),
+            ..
+        })
+        | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(ty, body_id), .. })
+        | Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Const(ty, Some(body_id)),
+            ..
+        }) => (*body_id, ty.span, None),
+        Node::AnonConst(hir::AnonConst { body, hir_id, .. }) => {
+            (*body, tcx.hir().span(*hir_id), None)
+        }
+
+        _ => span_bug!(tcx.hir().span(id), "can't build MIR for {:?}", def.did),
+    };
+
+    // If we don't have a specialized span for the body, just use the
+    // normal def span.
+    let span_with_body = span_with_body.unwrap_or_else(|| tcx.hir().span(id));
+
+    tcx.infer_ctxt().enter(|infcx| {
+        let body = if let Some(error_reported) = typeck_results.tainted_by_errors {
+            build::construct_error(&infcx, def, id, body_id, body_owner_kind, error_reported)
+        } else if body_owner_kind.is_fn_or_closure() {
+            // fetch the fully liberated fn signature (that is, all bound
+            // types/lifetimes replaced)
+            let fn_sig = typeck_results.liberated_fn_sigs()[id];
+            let fn_def_id = tcx.hir().local_def_id(id);
+
+            let safety = match fn_sig.unsafety {
+                hir::Unsafety::Normal => Safety::Safe,
+                hir::Unsafety::Unsafe => Safety::FnUnsafe,
+            };
+
+            let body = tcx.hir().body(body_id);
+            let (thir, expr) = tcx
+                .thir_body(def)
+                .unwrap_or_else(|_| (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0)));
+            // We ran all queries that depended on THIR at the beginning
+            // of `mir_build`, so now we can steal it
+            let thir = thir.steal();
+            let ty = tcx.type_of(fn_def_id);
+            let mut abi = fn_sig.abi;
+            let implicit_argument = match ty.kind() {
+                ty::Closure(..) => {
+                    // HACK(eddyb) Avoid having RustCall on closures,
+                    // as it adds unnecessary (and wrong) auto-tupling.
+                    abi = Abi::Rust;
+                    vec![ArgInfo(liberated_closure_env_ty(tcx, id, body_id), None, None, None)]
+                }
+                ty::Generator(..) => {
+                    let gen_ty = tcx.typeck_body(body_id).node_type(id);
+
+                    // The resume argument may be missing, in that case we need to provide it here.
+                    // It will always be `()` in this case.
+                    if body.params.is_empty() {
+                        vec![
+                            ArgInfo(gen_ty, None, None, None),
+                            ArgInfo(tcx.mk_unit(), None, None, None),
+                        ]
+                    } else {
+                        vec![ArgInfo(gen_ty, None, None, None)]
+                    }
+                }
+                _ => vec![],
+            };
+
+            let explicit_arguments = body.params.iter().enumerate().map(|(index, arg)| {
+                let owner_id = tcx.hir().body_owner(body_id);
+                let opt_ty_info;
+                let self_arg;
+                if let Some(ref fn_decl) = tcx.hir().fn_decl_by_hir_id(owner_id) {
+                    opt_ty_info = fn_decl
+                        .inputs
+                        .get(index)
+                        // Make sure that inferred closure args have no type span
+                        .and_then(|ty| if arg.pat.span != ty.span { Some(ty.span) } else { None });
+                    self_arg = if index == 0 && fn_decl.implicit_self.has_implicit_self() {
+                        match fn_decl.implicit_self {
+                            hir::ImplicitSelfKind::Imm => Some(ImplicitSelfKind::Imm),
+                            hir::ImplicitSelfKind::Mut => Some(ImplicitSelfKind::Mut),
+                            hir::ImplicitSelfKind::ImmRef => Some(ImplicitSelfKind::ImmRef),
+                            hir::ImplicitSelfKind::MutRef => Some(ImplicitSelfKind::MutRef),
+                            _ => None,
+                        }
+                    } else {
+                        None
+                    };
+                } else {
+                    opt_ty_info = None;
+                    self_arg = None;
+                }
+
+                // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
+                // (as it's created inside the body itself, not passed in from outside).
+                let ty = if fn_sig.c_variadic && index == fn_sig.inputs().len() {
+                    let va_list_did = tcx.require_lang_item(LangItem::VaList, Some(arg.span));
+
+                    tcx.bound_type_of(va_list_did).subst(tcx, &[tcx.lifetimes.re_erased.into()])
+                } else {
+                    fn_sig.inputs()[index]
+                };
+
+                ArgInfo(ty, opt_ty_info, Some(&arg), self_arg)
+            });
+
+            let arguments = implicit_argument.into_iter().chain(explicit_arguments);
+
+            let (yield_ty, return_ty) = if body.generator_kind.is_some() {
+                let gen_ty = tcx.typeck_body(body_id).node_type(id);
+                let gen_sig = match gen_ty.kind() {
+                    ty::Generator(_, gen_substs, ..) => gen_substs.as_generator().sig(),
+                    _ => span_bug!(tcx.hir().span(id), "generator w/o generator type: {:?}", ty),
+                };
+                (Some(gen_sig.yield_ty), gen_sig.return_ty)
+            } else {
+                (None, fn_sig.output())
+            };
+
+            let mut mir = build::construct_fn(
+                &thir,
+                &infcx,
+                def,
+                id,
+                arguments,
+                safety,
+                abi,
+                return_ty,
+                return_ty_span,
+                body,
+                expr,
+                span_with_body,
+            );
+            if yield_ty.is_some() {
+                mir.generator.as_mut().unwrap().yield_ty = yield_ty;
+            }
+            mir
+        } else {
+            // Get the revealed type of this const. This is *not* the adjusted
+            // type of its body, which may be a subtype of this type. For
+            // example:
+            //
+            // fn foo(_: &()) {}
+            // static X: fn(&'static ()) = foo;
+            //
+            // The adjusted type of the body of X is `for<'a> fn(&'a ())` which
+            // is not the same as the type of X. We need the type of the return
+            // place to be the type of the constant because NLL typeck will
+            // equate them.
+
+            let return_ty = typeck_results.node_type(id);
+
+            let (thir, expr) = tcx
+                .thir_body(def)
+                .unwrap_or_else(|_| (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0)));
+            // We ran all queries that depended on THIR at the beginning
+            // of `mir_build`, so now we can steal it
+            let thir = thir.steal();
+
+            build::construct_const(&thir, &infcx, expr, def, id, return_ty, return_ty_span)
+        };
+
+        lints::check(tcx, &body);
+
+        // The borrow checker will replace all the regions here with its own
+        // inference variables. There's no point having non-erased regions here.
+        // The exception is `body.user_type_annotations`, which is used unmodified
+        // by borrow checking.
+        debug_assert!(
+            !(body.local_decls.has_free_regions()
+                || body.basic_blocks().has_free_regions()
+                || body.var_debug_info.has_free_regions()
+                || body.yield_ty().has_free_regions()),
+            "Unexpected free regions in MIR: {:?}",
+            body,
+        );
+
+        body
+    })
+}
+
+///////////////////////////////////////////////////////////////////////////
+// BuildMir -- walks a crate, looking for fn items and methods to build MIR from
+
+fn liberated_closure_env_ty(
+    tcx: TyCtxt<'_>,
+    closure_expr_id: hir::HirId,
+    body_id: hir::BodyId,
+) -> Ty<'_> {
+    let closure_ty = tcx.typeck_body(body_id).node_type(closure_expr_id);
+
+    let ty::Closure(closure_def_id, closure_substs) = *closure_ty.kind() else {
+        bug!("closure expr does not have closure type: {:?}", closure_ty);
+    };
+
+    let bound_vars =
+        tcx.mk_bound_variable_kinds(std::iter::once(ty::BoundVariableKind::Region(ty::BrEnv)));
+    let br =
+        ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BrEnv };
+    let env_region = ty::ReLateBound(ty::INNERMOST, br);
+    let closure_env_ty = tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap();
+    tcx.erase_late_bound_regions(ty::Binder::bind_with_vars(closure_env_ty, bound_vars))
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum BlockFrame {
+    /// Evaluation is currently within a statement.
+    ///
+    /// Examples include:
+    /// 1. `EXPR;`
+    /// 2. `let _ = EXPR;`
+    /// 3. `let x = EXPR;`
+    Statement {
+        /// If true, then statement discards result from evaluating
+        /// the expression (such as examples 1 and 2 above).
+        ignores_expr_result: bool,
+    },
+
+    /// Evaluation is currently within the tail expression of a block.
+    ///
+    /// Example: `{ STMT_1; STMT_2; EXPR }`
+    TailExpr {
+        /// If true, then the surrounding context of the block ignores
+        /// the result of evaluating the block's tail expression.
+        ///
+        /// Example: `let _ = { STMT_1; EXPR };`
+        tail_result_is_ignored: bool,
+
+        /// `Span` of the tail expression.
+        span: Span,
+    },
+
+    /// Generic mark meaning that the block occurred as a subexpression
+    /// where the result might be used.
+    ///
+    /// Examples: `foo(EXPR)`, `match EXPR { ... }`
+    SubExpr,
+}
+
+impl BlockFrame {
+    fn is_tail_expr(&self) -> bool {
+        match *self {
+            BlockFrame::TailExpr { .. } => true,
+
+            BlockFrame::Statement { .. } | BlockFrame::SubExpr => false,
+        }
+    }
+    fn is_statement(&self) -> bool {
+        match *self {
+            BlockFrame::Statement { .. } => true,
+
+            BlockFrame::TailExpr { .. } | BlockFrame::SubExpr => false,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct BlockContext(Vec<BlockFrame>);
+
+struct Builder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    typeck_results: &'tcx TypeckResults<'tcx>,
+    region_scope_tree: &'tcx region::ScopeTree,
+    param_env: ty::ParamEnv<'tcx>,
+
+    thir: &'a Thir<'tcx>,
+    cfg: CFG<'tcx>,
+
+    def_id: DefId,
+    hir_id: hir::HirId,
+    parent_module: DefId,
+    check_overflow: bool,
+    fn_span: Span,
+    arg_count: usize,
+    generator_kind: Option<GeneratorKind>,
+
+    /// The current set of scopes, updated as we traverse;
+    /// see the `scope` module for more details.
+    scopes: scope::Scopes<'tcx>,
+
+    /// The block-context: each time we build the code within an thir::Block,
+    /// we push a frame here tracking whether we are building a statement or
+    /// if we are pushing the tail expression of the block. This is used to
+    /// embed information in generated temps about whether they were created
+    /// for a block tail expression or not.
+    ///
+    /// It would be great if we could fold this into `self.scopes`
+    /// somehow, but right now I think that is very tightly tied to
+    /// the code generation in ways that we cannot (or should not)
+    /// start just throwing new entries onto that vector in order to
+    /// distinguish the context of EXPR1 from the context of EXPR2 in
+    /// `{ STMTS; EXPR1 } + EXPR2`.
+    block_context: BlockContext,
+
+    /// The current unsafe block in scope
+    in_scope_unsafe: Safety,
+
+    /// The vector of all scopes that we have created thus far;
+    /// we track this for debuginfo later.
+    source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>,
+    source_scope: SourceScope,
+
+    /// The guard-context: each time we build the guard expression for
+    /// a match arm, we push onto this stack, and then pop when we
+    /// finish building it.
+    guard_context: Vec<GuardFrame>,
+
+    /// Maps `HirId`s of variable bindings to the `Local`s created for them.
+    /// (A match binding can have two locals; the 2nd is for the arm's guard.)
+    var_indices: FxHashMap<LocalVarId, LocalsForNode>,
+    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
+    canonical_user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>,
+    upvar_mutbls: Vec<Mutability>,
+    unit_temp: Option<Place<'tcx>>,
+
+    var_debug_info: Vec<VarDebugInfo<'tcx>>,
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    fn is_bound_var_in_guard(&self, id: LocalVarId) -> bool {
+        self.guard_context.iter().any(|frame| frame.locals.iter().any(|local| local.id == id))
+    }
+
+    fn var_local_id(&self, id: LocalVarId, for_guard: ForGuard) -> Local {
+        self.var_indices[&id].local_id(for_guard)
+    }
+}
+
+impl BlockContext {
+    fn new() -> Self {
+        BlockContext(vec![])
+    }
+    fn push(&mut self, bf: BlockFrame) {
+        self.0.push(bf);
+    }
+    fn pop(&mut self) -> Option<BlockFrame> {
+        self.0.pop()
+    }
+
+    /// Traverses the frames on the `BlockContext`, searching for either
+    /// the first block-tail expression frame with no intervening
+    /// statement frame.
+    ///
+    /// Notably, this skips over `SubExpr` frames; this method is
+    /// meant to be used in the context of understanding the
+    /// relationship of a temp (created within some complicated
+    /// expression) with its containing expression, and whether the
+    /// value of that *containing expression* (not the temp!) is
+    /// ignored.
+    fn currently_in_block_tail(&self) -> Option<BlockTailInfo> {
+        for bf in self.0.iter().rev() {
+            match bf {
+                BlockFrame::SubExpr => continue,
+                BlockFrame::Statement { .. } => break,
+                &BlockFrame::TailExpr { tail_result_is_ignored, span } => {
+                    return Some(BlockTailInfo { tail_result_is_ignored, span });
+                }
+            }
+        }
+
+        None
+    }
+
+    /// Looks at the topmost frame on the BlockContext and reports
+    /// whether its one that would discard a block tail result.
+    ///
+    /// Unlike `currently_within_ignored_tail_expression`, this does
+    /// *not* skip over `SubExpr` frames: here, we want to know
+    /// whether the block result itself is discarded.
+    fn currently_ignores_tail_results(&self) -> bool {
+        match self.0.last() {
+            // no context: conservatively assume result is read
+            None => false,
+
+            // sub-expression: block result feeds into some computation
+            Some(BlockFrame::SubExpr) => false,
+
+            // otherwise: use accumulated is_ignored state.
+            Some(
+                BlockFrame::TailExpr { tail_result_is_ignored: ignored, .. }
+                | BlockFrame::Statement { ignores_expr_result: ignored },
+            ) => *ignored,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum LocalsForNode {
+    /// In the usual case, a `HirId` for an identifier maps to at most
+    /// one `Local` declaration.
+    One(Local),
+
+    /// The exceptional case is identifiers in a match arm's pattern
+    /// that are referenced in a guard of that match arm. For these,
+    /// we have `2` Locals.
+    ///
+    /// * `for_arm_body` is the Local used in the arm body (which is
+    ///   just like the `One` case above),
+    ///
+    /// * `ref_for_guard` is the Local used in the arm's guard (which
+    ///   is a reference to a temp that is an alias of
+    ///   `for_arm_body`).
+    ForGuard { ref_for_guard: Local, for_arm_body: Local },
+}
+
+#[derive(Debug)]
+struct GuardFrameLocal {
+    id: LocalVarId,
+}
+
+impl GuardFrameLocal {
+    fn new(id: LocalVarId, _binding_mode: BindingMode) -> Self {
+        GuardFrameLocal { id }
+    }
+}
+
+#[derive(Debug)]
+struct GuardFrame {
+    /// These are the id's of names that are bound by patterns of the
+    /// arm of *this* guard.
+    ///
+    /// (Frames higher up the stack will have the id's bound in arms
+    /// further out, such as in a case like:
+    ///
+    /// match E1 {
+    ///      P1(id1) if (... (match E2 { P2(id2) if ... => B2 })) => B1,
+    /// }
+    ///
+    /// here, when building for FIXME.
+    locals: Vec<GuardFrameLocal>,
+}
+
+/// `ForGuard` indicates whether we are talking about:
+///   1. The variable for use outside of guard expressions, or
+///   2. The temp that holds reference to (1.), which is actually what the
+///      guard expressions see.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum ForGuard {
+    RefWithinGuard,
+    OutsideGuard,
+}
+
+impl LocalsForNode {
+    fn local_id(&self, for_guard: ForGuard) -> Local {
+        match (self, for_guard) {
+            (&LocalsForNode::One(local_id), ForGuard::OutsideGuard)
+            | (
+                &LocalsForNode::ForGuard { ref_for_guard: local_id, .. },
+                ForGuard::RefWithinGuard,
+            )
+            | (&LocalsForNode::ForGuard { for_arm_body: local_id, .. }, ForGuard::OutsideGuard) => {
+                local_id
+            }
+
+            (&LocalsForNode::One(_), ForGuard::RefWithinGuard) => {
+                bug!("anything with one local should never be within a guard.")
+            }
+        }
+    }
+}
+
+struct CFG<'tcx> {
+    basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+}
+
+rustc_index::newtype_index! {
+    struct ScopeId { .. }
+}
+
+#[derive(Debug)]
+enum NeedsTemporary {
+    /// Use this variant when whatever you are converting with `as_operand`
+    /// is the last thing you are converting. This means that if we introduced
+    /// an intermediate temporary, we'd only read it immediately after, so we can
+    /// also avoid it.
+    No,
+    /// For all cases where you aren't sure or that are too expensive to compute
+    /// for now. It is always safe to fall back to this.
+    Maybe,
+}
+
+///////////////////////////////////////////////////////////////////////////
+/// The `BlockAnd` "monad" packages up the new basic block along with a
+/// produced value (sometimes just unit, of course). The `unpack!`
+/// macro (and methods below) makes working with `BlockAnd` much more
+/// convenient.
+
+#[must_use = "if you don't use one of these results, you're leaving a dangling edge"]
+struct BlockAnd<T>(BasicBlock, T);
+
+trait BlockAndExtension {
+    fn and<T>(self, v: T) -> BlockAnd<T>;
+    fn unit(self) -> BlockAnd<()>;
+}
+
+impl BlockAndExtension for BasicBlock {
+    fn and<T>(self, v: T) -> BlockAnd<T> {
+        BlockAnd(self, v)
+    }
+
+    fn unit(self) -> BlockAnd<()> {
+        BlockAnd(self, ())
+    }
+}
+
+/// Update a block pointer and return the value.
+/// Use it like `let x = unpack!(block = self.foo(block, foo))`.
+macro_rules! unpack {
+    ($x:ident = $c:expr) => {{
+        let BlockAnd(b, v) = $c;
+        $x = b;
+        v
+    }};
+
+    ($c:expr) => {{
+        let BlockAnd(b, ()) = $c;
+        b
+    }};
+}
+
+///////////////////////////////////////////////////////////////////////////
+/// the main entry point for building MIR for a function
+
+struct ArgInfo<'tcx>(
+    Ty<'tcx>,
+    Option<Span>,
+    Option<&'tcx hir::Param<'tcx>>,
+    Option<ImplicitSelfKind>,
+);
+
+fn construct_fn<'tcx, A>(
+    thir: &Thir<'tcx>,
+    infcx: &InferCtxt<'_, 'tcx>,
+    fn_def: ty::WithOptConstParam<LocalDefId>,
+    fn_id: hir::HirId,
+    arguments: A,
+    safety: Safety,
+    abi: Abi,
+    return_ty: Ty<'tcx>,
+    return_ty_span: Span,
+    body: &'tcx hir::Body<'tcx>,
+    expr: ExprId,
+    span_with_body: Span,
+) -> Body<'tcx>
+where
+    A: Iterator<Item = ArgInfo<'tcx>>,
+{
+    let arguments: Vec<_> = arguments.collect();
+
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(fn_id);
+
+    let mut builder = Builder::new(
+        thir,
+        infcx,
+        fn_def,
+        fn_id,
+        span_with_body,
+        arguments.len(),
+        safety,
+        return_ty,
+        return_ty_span,
+        body.generator_kind,
+    );
+
+    let call_site_scope =
+        region::Scope { id: body.value.hir_id.local_id, data: region::ScopeData::CallSite };
+    let arg_scope =
+        region::Scope { id: body.value.hir_id.local_id, data: region::ScopeData::Arguments };
+    let source_info = builder.source_info(span);
+    let call_site_s = (call_site_scope, source_info);
+    unpack!(builder.in_scope(call_site_s, LintLevel::Inherited, |builder| {
+        let arg_scope_s = (arg_scope, source_info);
+        // Attribute epilogue to function's closing brace
+        let fn_end = span_with_body.shrink_to_hi();
+        let return_block =
+            unpack!(builder.in_breakable_scope(None, Place::return_place(), fn_end, |builder| {
+                Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
+                    builder.args_and_body(
+                        START_BLOCK,
+                        fn_def.did,
+                        &arguments,
+                        arg_scope,
+                        &thir[expr],
+                    )
+                }))
+            }));
+        let source_info = builder.source_info(fn_end);
+        builder.cfg.terminate(return_block, source_info, TerminatorKind::Return);
+        builder.build_drop_trees();
+        return_block.unit()
+    }));
+
+    let spread_arg = if abi == Abi::RustCall {
+        // RustCall pseudo-ABI untuples the last argument.
+        Some(Local::new(arguments.len()))
+    } else {
+        None
+    };
+
+    let mut body = builder.finish();
+    body.spread_arg = spread_arg;
+    body
+}
+
+fn construct_const<'a, 'tcx>(
+    thir: &'a Thir<'tcx>,
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    expr: ExprId,
+    def: ty::WithOptConstParam<LocalDefId>,
+    hir_id: hir::HirId,
+    const_ty: Ty<'tcx>,
+    const_ty_span: Span,
+) -> Body<'tcx> {
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(hir_id);
+    let mut builder = Builder::new(
+        thir,
+        infcx,
+        def,
+        hir_id,
+        span,
+        0,
+        Safety::Safe,
+        const_ty,
+        const_ty_span,
+        None,
+    );
+
+    let mut block = START_BLOCK;
+    unpack!(block = builder.expr_into_dest(Place::return_place(), block, &thir[expr]));
+
+    let source_info = builder.source_info(span);
+    builder.cfg.terminate(block, source_info, TerminatorKind::Return);
+
+    builder.build_drop_trees();
+
+    builder.finish()
+}
+
+/// Construct MIR for an item that has had errors in type checking.
+///
+/// This is required because we may still want to run MIR passes on an item
+/// with type errors, but normal MIR construction can't handle that in general.
+fn construct_error<'a, 'tcx>(
+    infcx: &'a InferCtxt<'a, 'tcx>,
+    def: ty::WithOptConstParam<LocalDefId>,
+    hir_id: hir::HirId,
+    body_id: hir::BodyId,
+    body_owner_kind: hir::BodyOwnerKind,
+    err: ErrorGuaranteed,
+) -> Body<'tcx> {
+    let tcx = infcx.tcx;
+    let span = tcx.hir().span(hir_id);
+    let ty = tcx.ty_error();
+    let generator_kind = tcx.hir().body(body_id).generator_kind;
+    let num_params = match body_owner_kind {
+        hir::BodyOwnerKind::Fn => tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len(),
+        hir::BodyOwnerKind::Closure => {
+            if generator_kind.is_some() {
+                // Generators have an implicit `self` parameter *and* a possibly
+                // implicit resume parameter.
+                2
+            } else {
+                // The implicit self parameter adds another local in MIR.
+                1 + tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len()
+            }
+        }
+        hir::BodyOwnerKind::Const => 0,
+        hir::BodyOwnerKind::Static(_) => 0,
+    };
+    let mut cfg = CFG { basic_blocks: IndexVec::new() };
+    let mut source_scopes = IndexVec::new();
+    let mut local_decls = IndexVec::from_elem_n(LocalDecl::new(ty, span), 1);
+
+    cfg.start_new_block();
+    source_scopes.push(SourceScopeData {
+        span,
+        parent_scope: None,
+        inlined: None,
+        inlined_parent_scope: None,
+        local_data: ClearCrossCrate::Set(SourceScopeLocalData {
+            lint_root: hir_id,
+            safety: Safety::Safe,
+        }),
+    });
+    let source_info = SourceInfo { span, scope: OUTERMOST_SOURCE_SCOPE };
+
+    // Some MIR passes will expect the number of parameters to match the
+    // function declaration.
+    for _ in 0..num_params {
+        local_decls.push(LocalDecl::with_source_info(ty, source_info));
+    }
+    cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable);
+
+    let mut body = Body::new(
+        MirSource::item(def.did.to_def_id()),
+        cfg.basic_blocks,
+        source_scopes,
+        local_decls,
+        IndexVec::new(),
+        num_params,
+        vec![],
+        span,
+        generator_kind,
+        Some(err),
+    );
+    body.generator.as_mut().map(|gen| gen.yield_ty = Some(ty));
+    body
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    fn new(
+        thir: &'a Thir<'tcx>,
+        infcx: &'a InferCtxt<'a, 'tcx>,
+        def: ty::WithOptConstParam<LocalDefId>,
+        hir_id: hir::HirId,
+        span: Span,
+        arg_count: usize,
+        safety: Safety,
+        return_ty: Ty<'tcx>,
+        return_span: Span,
+        generator_kind: Option<GeneratorKind>,
+    ) -> Builder<'a, 'tcx> {
+        let tcx = infcx.tcx;
+        let attrs = tcx.hir().attrs(hir_id);
+        // Some functions always have overflow checks enabled,
+        // however, they may not get codegen'd, depending on
+        // the settings for the crate they are codegened in.
+        let mut check_overflow = tcx.sess.contains_name(attrs, sym::rustc_inherit_overflow_checks);
+        // Respect -C overflow-checks.
+        check_overflow |= tcx.sess.overflow_checks();
+        // Constants always need overflow checks.
+        check_overflow |= matches!(
+            tcx.hir().body_owner_kind(def.did),
+            hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_)
+        );
+
+        let lint_level = LintLevel::Explicit(hir_id);
+        let param_env = tcx.param_env(def.did);
+        let mut builder = Builder {
+            thir,
+            tcx,
+            infcx,
+            typeck_results: tcx.typeck_opt_const_arg(def),
+            region_scope_tree: tcx.region_scope_tree(def.did),
+            param_env,
+            def_id: def.did.to_def_id(),
+            hir_id,
+            parent_module: tcx.parent_module(hir_id).to_def_id(),
+            check_overflow,
+            cfg: CFG { basic_blocks: IndexVec::new() },
+            fn_span: span,
+            arg_count,
+            generator_kind,
+            scopes: scope::Scopes::new(),
+            block_context: BlockContext::new(),
+            source_scopes: IndexVec::new(),
+            source_scope: OUTERMOST_SOURCE_SCOPE,
+            guard_context: vec![],
+            in_scope_unsafe: safety,
+            local_decls: IndexVec::from_elem_n(LocalDecl::new(return_ty, return_span), 1),
+            canonical_user_type_annotations: IndexVec::new(),
+            upvar_mutbls: vec![],
+            var_indices: Default::default(),
+            unit_temp: None,
+            var_debug_info: vec![],
+        };
+
+        assert_eq!(builder.cfg.start_new_block(), START_BLOCK);
+        assert_eq!(
+            builder.new_source_scope(span, lint_level, Some(safety)),
+            OUTERMOST_SOURCE_SCOPE
+        );
+        builder.source_scopes[OUTERMOST_SOURCE_SCOPE].parent_scope = None;
+
+        builder
+    }
+
+    fn finish(self) -> Body<'tcx> {
+        for (index, block) in self.cfg.basic_blocks.iter().enumerate() {
+            if block.terminator.is_none() {
+                span_bug!(self.fn_span, "no terminator on block {:?}", index);
+            }
+        }
+
+        Body::new(
+            MirSource::item(self.def_id),
+            self.cfg.basic_blocks,
+            self.source_scopes,
+            self.local_decls,
+            self.canonical_user_type_annotations,
+            self.arg_count,
+            self.var_debug_info,
+            self.fn_span,
+            self.generator_kind,
+            self.typeck_results.tainted_by_errors,
+        )
+    }
+
+    fn args_and_body(
+        &mut self,
+        mut block: BasicBlock,
+        fn_def_id: LocalDefId,
+        arguments: &[ArgInfo<'tcx>],
+        argument_scope: region::Scope,
+        expr: &Expr<'tcx>,
+    ) -> BlockAnd<()> {
+        // Allocate locals for the function arguments
+        for &ArgInfo(ty, _, arg_opt, _) in arguments.iter() {
+            let source_info =
+                SourceInfo::outermost(arg_opt.map_or(self.fn_span, |arg| arg.pat.span));
+            let arg_local = self.local_decls.push(LocalDecl::with_source_info(ty, source_info));
+
+            // If this is a simple binding pattern, give debuginfo a nice name.
+            if let Some(arg) = arg_opt && let Some(ident) = arg.pat.simple_ident() {
+                self.var_debug_info.push(VarDebugInfo {
+                    name: ident.name,
+                    source_info,
+                    value: VarDebugInfoContents::Place(arg_local.into()),
+                });
+            }
+        }
+
+        let tcx = self.tcx;
+        let tcx_hir = tcx.hir();
+        let hir_typeck_results = self.typeck_results;
+
+        // In analyze_closure() in upvar.rs we gathered a list of upvars used by an
+        // indexed closure and we stored in a map called closure_min_captures in TypeckResults
+        // with the closure's DefId. Here, we run through that vec of UpvarIds for
+        // the given closure and use the necessary information to create upvar
+        // debuginfo and to fill `self.upvar_mutbls`.
+        if hir_typeck_results.closure_min_captures.get(&fn_def_id).is_some() {
+            let mut closure_env_projs = vec![];
+            let mut closure_ty = self.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
+            if let ty::Ref(_, ty, _) = closure_ty.kind() {
+                closure_env_projs.push(ProjectionElem::Deref);
+                closure_ty = *ty;
+            }
+            let upvar_substs = match closure_ty.kind() {
+                ty::Closure(_, substs) => ty::UpvarSubsts::Closure(substs),
+                ty::Generator(_, substs, _) => ty::UpvarSubsts::Generator(substs),
+                _ => span_bug!(self.fn_span, "upvars with non-closure env ty {:?}", closure_ty),
+            };
+            let def_id = self.def_id.as_local().unwrap();
+            let capture_syms = tcx.symbols_for_closure_captures((def_id, fn_def_id));
+            let capture_tys = upvar_substs.upvar_tys();
+            let captures_with_tys = hir_typeck_results
+                .closure_min_captures_flattened(fn_def_id)
+                .zip(capture_tys.zip(capture_syms));
+
+            self.upvar_mutbls = captures_with_tys
+                .enumerate()
+                .map(|(i, (captured_place, (ty, sym)))| {
+                    let capture = captured_place.info.capture_kind;
+                    let var_id = match captured_place.place.base {
+                        HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
+                        _ => bug!("Expected an upvar"),
+                    };
+
+                    let mutability = captured_place.mutability;
+
+                    let mut projs = closure_env_projs.clone();
+                    projs.push(ProjectionElem::Field(Field::new(i), ty));
+                    match capture {
+                        ty::UpvarCapture::ByValue => {}
+                        ty::UpvarCapture::ByRef(..) => {
+                            projs.push(ProjectionElem::Deref);
+                        }
+                    };
+
+                    self.var_debug_info.push(VarDebugInfo {
+                        name: *sym,
+                        source_info: SourceInfo::outermost(tcx_hir.span(var_id)),
+                        value: VarDebugInfoContents::Place(Place {
+                            local: ty::CAPTURE_STRUCT_LOCAL,
+                            projection: tcx.intern_place_elems(&projs),
+                        }),
+                    });
+
+                    mutability
+                })
+                .collect();
+        }
+
+        let mut scope = None;
+        // Bind the argument patterns
+        for (index, arg_info) in arguments.iter().enumerate() {
+            // Function arguments always get the first Local indices after the return place
+            let local = Local::new(index + 1);
+            let place = Place::from(local);
+            let &ArgInfo(_, opt_ty_info, arg_opt, ref self_binding) = arg_info;
+
+            // Make sure we drop (parts of) the argument even when not matched on.
+            self.schedule_drop(
+                arg_opt.as_ref().map_or(expr.span, |arg| arg.pat.span),
+                argument_scope,
+                local,
+                DropKind::Value,
+            );
+
+            let Some(arg) = arg_opt else {
+                continue;
+            };
+            let pat = match tcx.hir().get(arg.pat.hir_id) {
+                Node::Pat(pat) => pat,
+                node => bug!("pattern became {:?}", node),
+            };
+            let pattern = pat_from_hir(tcx, self.param_env, self.typeck_results, pat);
+            let original_source_scope = self.source_scope;
+            let span = pattern.span;
+            self.set_correct_source_scope_for_arg(arg.hir_id, original_source_scope, span);
+            match *pattern.kind {
+                // Don't introduce extra copies for simple bindings
+                PatKind::Binding {
+                    mutability,
+                    var,
+                    mode: BindingMode::ByValue,
+                    subpattern: None,
+                    ..
+                } => {
+                    self.local_decls[local].mutability = mutability;
+                    self.local_decls[local].source_info.scope = self.source_scope;
+                    self.local_decls[local].local_info = if let Some(kind) = self_binding {
+                        Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(
+                            BindingForm::ImplicitSelf(*kind),
+                        ))))
+                    } else {
+                        let binding_mode = ty::BindingMode::BindByValue(mutability);
+                        Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
+                            VarBindingForm {
+                                binding_mode,
+                                opt_ty_info,
+                                opt_match_place: Some((Some(place), span)),
+                                pat_span: span,
+                            },
+                        )))))
+                    };
+                    self.var_indices.insert(var, LocalsForNode::One(local));
+                }
+                _ => {
+                    scope = self.declare_bindings(
+                        scope,
+                        expr.span,
+                        &pattern,
+                        matches::ArmHasGuard(false),
+                        Some((Some(&place), span)),
+                    );
+                    let place_builder = PlaceBuilder::from(local);
+                    unpack!(block = self.place_into_pattern(block, pattern, place_builder, false));
+                }
+            }
+            self.source_scope = original_source_scope;
+        }
+
+        // Enter the argument pattern bindings source scope, if it exists.
+        if let Some(source_scope) = scope {
+            self.source_scope = source_scope;
+        }
+
+        self.expr_into_dest(Place::return_place(), block, &expr)
+    }
+
+    fn set_correct_source_scope_for_arg(
+        &mut self,
+        arg_hir_id: hir::HirId,
+        original_source_scope: SourceScope,
+        pattern_span: Span,
+    ) {
+        let tcx = self.tcx;
+        let current_root = tcx.maybe_lint_level_root_bounded(arg_hir_id, self.hir_id);
+        let parent_root = tcx.maybe_lint_level_root_bounded(
+            self.source_scopes[original_source_scope]
+                .local_data
+                .as_ref()
+                .assert_crate_local()
+                .lint_root,
+            self.hir_id,
+        );
+        if current_root != parent_root {
+            self.source_scope =
+                self.new_source_scope(pattern_span, LintLevel::Explicit(current_root), None);
+        }
+    }
+
+    fn get_unit_temp(&mut self) -> Place<'tcx> {
+        match self.unit_temp {
+            Some(tmp) => tmp,
+            None => {
+                let ty = self.tcx.mk_unit();
+                let fn_span = self.fn_span;
+                let tmp = self.temp(ty, fn_span);
+                self.unit_temp = Some(tmp);
+                tmp
+            }
+        }
+    }
+}
+
+fn parse_float_into_constval<'tcx>(
+    num: Symbol,
+    float_ty: ty::FloatTy,
+    neg: bool,
+) -> Option<ConstValue<'tcx>> {
+    parse_float_into_scalar(num, float_ty, neg).map(ConstValue::Scalar)
+}
+
+pub(crate) fn parse_float_into_scalar(
+    num: Symbol,
+    float_ty: ty::FloatTy,
+    neg: bool,
+) -> Option<Scalar> {
+    let num = num.as_str();
+    match float_ty {
+        ty::FloatTy::F32 => {
+            let Ok(rust_f) = num.parse::<f32>() else { return None };
+            let mut f = num.parse::<Single>().unwrap_or_else(|e| {
+                panic!("apfloat::ieee::Single failed to parse `{}`: {:?}", num, e)
+            });
+
+            assert!(
+                u128::from(rust_f.to_bits()) == f.to_bits(),
+                "apfloat::ieee::Single gave different result for `{}`: \
+                 {}({:#x}) vs Rust's {}({:#x})",
+                rust_f,
+                f,
+                f.to_bits(),
+                Single::from_bits(rust_f.to_bits().into()),
+                rust_f.to_bits()
+            );
+
+            if neg {
+                f = -f;
+            }
+
+            Some(Scalar::from_f32(f))
+        }
+        ty::FloatTy::F64 => {
+            let Ok(rust_f) = num.parse::<f64>() else { return None };
+            let mut f = num.parse::<Double>().unwrap_or_else(|e| {
+                panic!("apfloat::ieee::Double failed to parse `{}`: {:?}", num, e)
+            });
+
+            assert!(
+                u128::from(rust_f.to_bits()) == f.to_bits(),
+                "apfloat::ieee::Double gave different result for `{}`: \
+                 {}({:#x}) vs Rust's {}({:#x})",
+                rust_f,
+                f,
+                f.to_bits(),
+                Double::from_bits(rust_f.to_bits().into()),
+                rust_f.to_bits()
+            );
+
+            if neg {
+                f = -f;
+            }
+
+            Some(Scalar::from_f64(f))
+        }
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Builder methods are broken up into modules, depending on what kind
+// of thing is being lowered. Note that they use the `unpack` macro
+// above extensively.
+
+mod block;
+mod cfg;
+mod expr;
+mod matches;
+mod misc;
+mod scope;
+
+pub(crate) use expr::category::Category as ExprCategory;
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
new file mode 100644
index 000000000..b2fd9f25b
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -0,0 +1,1395 @@
+/*!
+Managing the scope stack. The scopes are tied to lexical scopes, so as
+we descend the THIR, we push a scope on the stack, build its
+contents, and then pop it off. Every scope is named by a
+`region::Scope`.
+
+### SEME Regions
+
+When pushing a new [Scope], we record the current point in the graph (a
+basic block); this marks the entry to the scope. We then generate more
+stuff in the control-flow graph. Whenever the scope is exited, either
+via a `break` or `return` or just by fallthrough, that marks an exit
+from the scope. Each lexical scope thus corresponds to a single-entry,
+multiple-exit (SEME) region in the control-flow graph.
+
+For now, we record the `region::Scope` to each SEME region for later reference
+(see caveat in next paragraph). This is because destruction scopes are tied to
+them. This may change in the future so that MIR lowering determines its own
+destruction scopes.
+
+### Not so SEME Regions
+
+In the course of building matches, it sometimes happens that certain code
+(namely guards) gets executed multiple times. This means that the scope lexical
+scope may in fact correspond to multiple, disjoint SEME regions. So in fact our
+mapping is from one scope to a vector of SEME regions. Since the SEME regions
+are disjoint, the mapping is still one-to-one for the set of SEME regions that
+we're currently in.
+
+Also in matches, the scopes assigned to arms are not always even SEME regions!
+Each arm has a single region with one entry for each pattern. We manually
+manipulate the scheduled drops in this scope to avoid dropping things multiple
+times.
+
+### Drops
+
+The primary purpose for scopes is to insert drops: while building
+the contents, we also accumulate places that need to be dropped upon
+exit from each scope. This is done by calling `schedule_drop`. Once a
+drop is scheduled, whenever we branch out we will insert drops of all
+those places onto the outgoing edge. Note that we don't know the full
+set of scheduled drops up front, and so whenever we exit from the
+scope we only drop the values scheduled thus far. For example, consider
+the scope S corresponding to this loop:
+
+```
+# let cond = true;
+loop {
+    let x = ..;
+    if cond { break; }
+    let y = ..;
+}
+```
+
+When processing the `let x`, we will add one drop to the scope for
+`x`.  The break will then insert a drop for `x`. When we process `let
+y`, we will add another drop (in fact, to a subscope, but let's ignore
+that for now); any later drops would also drop `y`.
+
+### Early exit
+
+There are numerous "normal" ways to early exit a scope: `break`,
+`continue`, `return` (panics are handled separately). Whenever an
+early exit occurs, the method `break_scope` is called. It is given the
+current point in execution where the early exit occurs, as well as the
+scope you want to branch to (note that all early exits from to some
+other enclosing scope). `break_scope` will record the set of drops currently
+scheduled in a [DropTree]. Later, before `in_breakable_scope` exits, the drops
+will be added to the CFG.
+
+Panics are handled in a similar fashion, except that the drops are added to the
+MIR once the rest of the function has finished being lowered. If a terminator
+can panic, call `diverge_from(block)` with the block containing the terminator
+`block`.
+
+### Breakable scopes
+
+In addition to the normal scope stack, we track a loop scope stack
+that contains only loops and breakable blocks. It tracks where a `break`,
+`continue` or `return` should go to.
+
+*/
+
+use std::mem;
+
+use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_index::vec::IndexVec;
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::thir::{Expr, LintLevel};
+
+use rustc_span::{Span, DUMMY_SP};
+
+#[derive(Debug)]
+pub struct Scopes<'tcx> {
+    scopes: Vec<Scope>,
+
+    /// The current set of breakable scopes. See module comment for more details.
+    breakable_scopes: Vec<BreakableScope<'tcx>>,
+
+    /// The scope of the innermost if-then currently being lowered.
+    if_then_scope: Option<IfThenScope>,
+
+    /// Drops that need to be done on unwind paths. See the comment on
+    /// [DropTree] for more details.
+    unwind_drops: DropTree,
+
+    /// Drops that need to be done on paths to the `GeneratorDrop` terminator.
+    generator_drops: DropTree,
+}
+
+#[derive(Debug)]
+struct Scope {
+    /// The source scope this scope was created in.
+    source_scope: SourceScope,
+
+    /// the region span of this scope within source code.
+    region_scope: region::Scope,
+
+    /// set of places to drop when exiting this scope. This starts
+    /// out empty but grows as variables are declared during the
+    /// building process. This is a stack, so we always drop from the
+    /// end of the vector (top of the stack) first.
+    drops: Vec<DropData>,
+
+    moved_locals: Vec<Local>,
+
+    /// The drop index that will drop everything in and below this scope on an
+    /// unwind path.
+    cached_unwind_block: Option<DropIdx>,
+
+    /// The drop index that will drop everything in and below this scope on a
+    /// generator drop path.
+    cached_generator_drop_block: Option<DropIdx>,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct DropData {
+    /// The `Span` where drop obligation was incurred (typically where place was
+    /// declared)
+    source_info: SourceInfo,
+
+    /// local to drop
+    local: Local,
+
+    /// Whether this is a value Drop or a StorageDead.
+    kind: DropKind,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) enum DropKind {
+    Value,
+    Storage,
+}
+
+#[derive(Debug)]
+struct BreakableScope<'tcx> {
+    /// Region scope of the loop
+    region_scope: region::Scope,
+    /// The destination of the loop/block expression itself (i.e., where to put
+    /// the result of a `break` or `return` expression)
+    break_destination: Place<'tcx>,
+    /// Drops that happen on the `break`/`return` path.
+    break_drops: DropTree,
+    /// Drops that happen on the `continue` path.
+    continue_drops: Option<DropTree>,
+}
+
+#[derive(Debug)]
+struct IfThenScope {
+    /// The if-then scope or arm scope
+    region_scope: region::Scope,
+    /// Drops that happen on the `else` path.
+    else_drops: DropTree,
+}
+
+/// The target of an expression that breaks out of a scope
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum BreakableTarget {
+    Continue(region::Scope),
+    Break(region::Scope),
+    Return,
+}
+
+rustc_index::newtype_index! {
+    struct DropIdx { .. }
+}
+
+const ROOT_NODE: DropIdx = DropIdx::from_u32(0);
+
+/// A tree of drops that we have deferred lowering. It's used for:
+///
+/// * Drops on unwind paths
+/// * Drops on generator drop paths (when a suspended generator is dropped)
+/// * Drops on return and loop exit paths
+/// * Drops on the else path in an `if let` chain
+///
+/// Once no more nodes could be added to the tree, we lower it to MIR in one go
+/// in `build_mir`.
+#[derive(Debug)]
+struct DropTree {
+    /// Drops in the tree.
+    drops: IndexVec<DropIdx, (DropData, DropIdx)>,
+    /// Map for finding the inverse of the `next_drop` relation:
+    ///
+    /// `previous_drops[(drops[i].1, drops[i].0.local, drops[i].0.kind)] == i`
+    previous_drops: FxHashMap<(DropIdx, Local, DropKind), DropIdx>,
+    /// Edges into the `DropTree` that need to be added once it's lowered.
+    entry_points: Vec<(DropIdx, BasicBlock)>,
+}
+
+impl Scope {
+    /// Whether there's anything to do for the cleanup path, that is,
+    /// when unwinding through this scope. This includes destructors,
+    /// but not StorageDead statements, which don't get emitted at all
+    /// for unwinding, for several reasons:
+    ///  * clang doesn't emit llvm.lifetime.end for C++ unwinding
+    ///  * LLVM's memory dependency analysis can't handle it atm
+    ///  * polluting the cleanup MIR with StorageDead creates
+    ///    landing pads even though there's no actual destructors
+    ///  * freeing up stack space has no effect during unwinding
+    /// Note that for generators we do emit StorageDeads, for the
+    /// use of optimizations in the MIR generator transform.
+    fn needs_cleanup(&self) -> bool {
+        self.drops.iter().any(|drop| match drop.kind {
+            DropKind::Value => true,
+            DropKind::Storage => false,
+        })
+    }
+
+    fn invalidate_cache(&mut self) {
+        self.cached_unwind_block = None;
+        self.cached_generator_drop_block = None;
+    }
+}
+
+/// A trait that determined how [DropTree] creates its blocks and
+/// links to any entry nodes.
+trait DropTreeBuilder<'tcx> {
+    /// Create a new block for the tree. This should call either
+    /// `cfg.start_new_block()` or `cfg.start_new_cleanup_block()`.
+    fn make_block(cfg: &mut CFG<'tcx>) -> BasicBlock;
+
+    /// Links a block outside the drop tree, `from`, to the block `to` inside
+    /// the drop tree.
+    fn add_entry(cfg: &mut CFG<'tcx>, from: BasicBlock, to: BasicBlock);
+}
+
+impl DropTree {
+    fn new() -> Self {
+        // The root node of the tree doesn't represent a drop, but instead
+        // represents the block in the tree that should be jumped to once all
+        // of the required drops have been performed.
+        let fake_source_info = SourceInfo::outermost(DUMMY_SP);
+        let fake_data =
+            DropData { source_info: fake_source_info, local: Local::MAX, kind: DropKind::Storage };
+        let drop_idx = DropIdx::MAX;
+        let drops = IndexVec::from_elem_n((fake_data, drop_idx), 1);
+        Self { drops, entry_points: Vec::new(), previous_drops: FxHashMap::default() }
+    }
+
+    fn add_drop(&mut self, drop: DropData, next: DropIdx) -> DropIdx {
+        let drops = &mut self.drops;
+        *self
+            .previous_drops
+            .entry((next, drop.local, drop.kind))
+            .or_insert_with(|| drops.push((drop, next)))
+    }
+
+    fn add_entry(&mut self, from: BasicBlock, to: DropIdx) {
+        debug_assert!(to < self.drops.next_index());
+        self.entry_points.push((to, from));
+    }
+
+    /// Builds the MIR for a given drop tree.
+    ///
+    /// `blocks` should have the same length as `self.drops`, and may have its
+    /// first value set to some already existing block.
+    fn build_mir<'tcx, T: DropTreeBuilder<'tcx>>(
+        &mut self,
+        cfg: &mut CFG<'tcx>,
+        blocks: &mut IndexVec<DropIdx, Option<BasicBlock>>,
+    ) {
+        debug!("DropTree::build_mir(drops = {:#?})", self);
+        assert_eq!(blocks.len(), self.drops.len());
+
+        self.assign_blocks::<T>(cfg, blocks);
+        self.link_blocks(cfg, blocks)
+    }
+
+    /// Assign blocks for all of the drops in the drop tree that need them.
+    fn assign_blocks<'tcx, T: DropTreeBuilder<'tcx>>(
+        &mut self,
+        cfg: &mut CFG<'tcx>,
+        blocks: &mut IndexVec<DropIdx, Option<BasicBlock>>,
+    ) {
+        // StorageDead statements can share blocks with each other and also with
+        // a Drop terminator. We iterate through the drops to find which drops
+        // need their own block.
+        #[derive(Clone, Copy)]
+        enum Block {
+            // This drop is unreachable
+            None,
+            // This drop is only reachable through the `StorageDead` with the
+            // specified index.
+            Shares(DropIdx),
+            // This drop has more than one way of being reached, or it is
+            // branched to from outside the tree, or its predecessor is a
+            // `Value` drop.
+            Own,
+        }
+
+        let mut needs_block = IndexVec::from_elem(Block::None, &self.drops);
+        if blocks[ROOT_NODE].is_some() {
+            // In some cases (such as drops for `continue`) the root node
+            // already has a block. In this case, make sure that we don't
+            // override it.
+            needs_block[ROOT_NODE] = Block::Own;
+        }
+
+        // Sort so that we only need to check the last value.
+        let entry_points = &mut self.entry_points;
+        entry_points.sort();
+
+        for (drop_idx, drop_data) in self.drops.iter_enumerated().rev() {
+            if entry_points.last().map_or(false, |entry_point| entry_point.0 == drop_idx) {
+                let block = *blocks[drop_idx].get_or_insert_with(|| T::make_block(cfg));
+                needs_block[drop_idx] = Block::Own;
+                while entry_points.last().map_or(false, |entry_point| entry_point.0 == drop_idx) {
+                    let entry_block = entry_points.pop().unwrap().1;
+                    T::add_entry(cfg, entry_block, block);
+                }
+            }
+            match needs_block[drop_idx] {
+                Block::None => continue,
+                Block::Own => {
+                    blocks[drop_idx].get_or_insert_with(|| T::make_block(cfg));
+                }
+                Block::Shares(pred) => {
+                    blocks[drop_idx] = blocks[pred];
+                }
+            }
+            if let DropKind::Value = drop_data.0.kind {
+                needs_block[drop_data.1] = Block::Own;
+            } else if drop_idx != ROOT_NODE {
+                match &mut needs_block[drop_data.1] {
+                    pred @ Block::None => *pred = Block::Shares(drop_idx),
+                    pred @ Block::Shares(_) => *pred = Block::Own,
+                    Block::Own => (),
+                }
+            }
+        }
+
+        debug!("assign_blocks: blocks = {:#?}", blocks);
+        assert!(entry_points.is_empty());
+    }
+
+    fn link_blocks<'tcx>(
+        &self,
+        cfg: &mut CFG<'tcx>,
+        blocks: &IndexVec<DropIdx, Option<BasicBlock>>,
+    ) {
+        for (drop_idx, drop_data) in self.drops.iter_enumerated().rev() {
+            let Some(block) = blocks[drop_idx] else { continue };
+            match drop_data.0.kind {
+                DropKind::Value => {
+                    let terminator = TerminatorKind::Drop {
+                        target: blocks[drop_data.1].unwrap(),
+                        // The caller will handle this if needed.
+                        unwind: None,
+                        place: drop_data.0.local.into(),
+                    };
+                    cfg.terminate(block, drop_data.0.source_info, terminator);
+                }
+                // Root nodes don't correspond to a drop.
+                DropKind::Storage if drop_idx == ROOT_NODE => {}
+                DropKind::Storage => {
+                    let stmt = Statement {
+                        source_info: drop_data.0.source_info,
+                        kind: StatementKind::StorageDead(drop_data.0.local),
+                    };
+                    cfg.push(block, stmt);
+                    let target = blocks[drop_data.1].unwrap();
+                    if target != block {
+                        // Diagnostics don't use this `Span` but debuginfo
+                        // might. Since we don't want breakpoints to be placed
+                        // here, especially when this is on an unwind path, we
+                        // use `DUMMY_SP`.
+                        let source_info = SourceInfo { span: DUMMY_SP, ..drop_data.0.source_info };
+                        let terminator = TerminatorKind::Goto { target };
+                        cfg.terminate(block, source_info, terminator);
+                    }
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx> Scopes<'tcx> {
+    pub(crate) fn new() -> Self {
+        Self {
+            scopes: Vec::new(),
+            breakable_scopes: Vec::new(),
+            if_then_scope: None,
+            unwind_drops: DropTree::new(),
+            generator_drops: DropTree::new(),
+        }
+    }
+
+    fn push_scope(&mut self, region_scope: (region::Scope, SourceInfo), vis_scope: SourceScope) {
+        debug!("push_scope({:?})", region_scope);
+        self.scopes.push(Scope {
+            source_scope: vis_scope,
+            region_scope: region_scope.0,
+            drops: vec![],
+            moved_locals: vec![],
+            cached_unwind_block: None,
+            cached_generator_drop_block: None,
+        });
+    }
+
+    fn pop_scope(&mut self, region_scope: (region::Scope, SourceInfo)) -> Scope {
+        let scope = self.scopes.pop().unwrap();
+        assert_eq!(scope.region_scope, region_scope.0);
+        scope
+    }
+
+    fn scope_index(&self, region_scope: region::Scope, span: Span) -> usize {
+        self.scopes
+            .iter()
+            .rposition(|scope| scope.region_scope == region_scope)
+            .unwrap_or_else(|| span_bug!(span, "region_scope {:?} does not enclose", region_scope))
+    }
+
+    /// Returns the topmost active scope, which is known to be alive until
+    /// the next scope expression.
+    fn topmost(&self) -> region::Scope {
+        self.scopes.last().expect("topmost_scope: no scopes present").region_scope
+    }
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    // Adding and removing scopes
+    // ==========================
+    //  Start a breakable scope, which tracks where `continue`, `break` and
+    //  `return` should branch to.
+    pub(crate) fn in_breakable_scope<F>(
+        &mut self,
+        loop_block: Option<BasicBlock>,
+        break_destination: Place<'tcx>,
+        span: Span,
+        f: F,
+    ) -> BlockAnd<()>
+    where
+        F: FnOnce(&mut Builder<'a, 'tcx>) -> Option<BlockAnd<()>>,
+    {
+        let region_scope = self.scopes.topmost();
+        let scope = BreakableScope {
+            region_scope,
+            break_destination,
+            break_drops: DropTree::new(),
+            continue_drops: loop_block.map(|_| DropTree::new()),
+        };
+        self.scopes.breakable_scopes.push(scope);
+        let normal_exit_block = f(self);
+        let breakable_scope = self.scopes.breakable_scopes.pop().unwrap();
+        assert!(breakable_scope.region_scope == region_scope);
+        let break_block = self.build_exit_tree(breakable_scope.break_drops, None);
+        if let Some(drops) = breakable_scope.continue_drops {
+            self.build_exit_tree(drops, loop_block);
+        }
+        match (normal_exit_block, break_block) {
+            (Some(block), None) | (None, Some(block)) => block,
+            (None, None) => self.cfg.start_new_block().unit(),
+            (Some(normal_block), Some(exit_block)) => {
+                let target = self.cfg.start_new_block();
+                let source_info = self.source_info(span);
+                self.cfg.terminate(
+                    unpack!(normal_block),
+                    source_info,
+                    TerminatorKind::Goto { target },
+                );
+                self.cfg.terminate(
+                    unpack!(exit_block),
+                    source_info,
+                    TerminatorKind::Goto { target },
+                );
+                target.unit()
+            }
+        }
+    }
+
+    /// Start an if-then scope which tracks drop for `if` expressions and `if`
+    /// guards.
+    ///
+    /// For an if-let chain:
+    ///
+    /// if let Some(x) = a && let Some(y) = b && let Some(z) = c { ... }
+    ///
+    /// There are three possible ways the condition can be false and we may have
+    /// to drop `x`, `x` and `y`, or neither depending on which binding fails.
+    /// To handle this correctly we use a `DropTree` in a similar way to a
+    /// `loop` expression and 'break' out on all of the 'else' paths.
+    ///
+    /// Notes:
+    /// - We don't need to keep a stack of scopes in the `Builder` because the
+    ///   'else' paths will only leave the innermost scope.
+    /// - This is also used for match guards.
+    pub(crate) fn in_if_then_scope<F>(
+        &mut self,
+        region_scope: region::Scope,
+        f: F,
+    ) -> (BasicBlock, BasicBlock)
+    where
+        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<()>,
+    {
+        let scope = IfThenScope { region_scope, else_drops: DropTree::new() };
+        let previous_scope = mem::replace(&mut self.scopes.if_then_scope, Some(scope));
+
+        let then_block = unpack!(f(self));
+
+        let if_then_scope = mem::replace(&mut self.scopes.if_then_scope, previous_scope).unwrap();
+        assert!(if_then_scope.region_scope == region_scope);
+
+        let else_block = self
+            .build_exit_tree(if_then_scope.else_drops, None)
+            .map_or_else(|| self.cfg.start_new_block(), |else_block_and| unpack!(else_block_and));
+
+        (then_block, else_block)
+    }
+
+    pub(crate) fn in_opt_scope<F, R>(
+        &mut self,
+        opt_scope: Option<(region::Scope, SourceInfo)>,
+        f: F,
+    ) -> BlockAnd<R>
+    where
+        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<R>,
+    {
+        debug!("in_opt_scope(opt_scope={:?})", opt_scope);
+        if let Some(region_scope) = opt_scope {
+            self.push_scope(region_scope);
+        }
+        let mut block;
+        let rv = unpack!(block = f(self));
+        if let Some(region_scope) = opt_scope {
+            unpack!(block = self.pop_scope(region_scope, block));
+        }
+        debug!("in_scope: exiting opt_scope={:?} block={:?}", opt_scope, block);
+        block.and(rv)
+    }
+
+    /// Convenience wrapper that pushes a scope and then executes `f`
+    /// to build its contents, popping the scope afterwards.
+    pub(crate) fn in_scope<F, R>(
+        &mut self,
+        region_scope: (region::Scope, SourceInfo),
+        lint_level: LintLevel,
+        f: F,
+    ) -> BlockAnd<R>
+    where
+        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<R>,
+    {
+        debug!("in_scope(region_scope={:?})", region_scope);
+        let source_scope = self.source_scope;
+        let tcx = self.tcx;
+        if let LintLevel::Explicit(current_hir_id) = lint_level {
+            // Use `maybe_lint_level_root_bounded` with `root_lint_level` as a bound
+            // to avoid adding Hir dependencies on our parents.
+            // We estimate the true lint roots here to avoid creating a lot of source scopes.
+
+            let parent_root = tcx.maybe_lint_level_root_bounded(
+                self.source_scopes[source_scope].local_data.as_ref().assert_crate_local().lint_root,
+                self.hir_id,
+            );
+            let current_root = tcx.maybe_lint_level_root_bounded(current_hir_id, self.hir_id);
+
+            if parent_root != current_root {
+                self.source_scope = self.new_source_scope(
+                    region_scope.1.span,
+                    LintLevel::Explicit(current_root),
+                    None,
+                );
+            }
+        }
+        self.push_scope(region_scope);
+        let mut block;
+        let rv = unpack!(block = f(self));
+        unpack!(block = self.pop_scope(region_scope, block));
+        self.source_scope = source_scope;
+        debug!("in_scope: exiting region_scope={:?} block={:?}", region_scope, block);
+        block.and(rv)
+    }
+
+    /// Push a scope onto the stack. You can then build code in this
+    /// scope and call `pop_scope` afterwards. Note that these two
+    /// calls must be paired; using `in_scope` as a convenience
+    /// wrapper maybe preferable.
+    pub(crate) fn push_scope(&mut self, region_scope: (region::Scope, SourceInfo)) {
+        self.scopes.push_scope(region_scope, self.source_scope);
+    }
+
+    /// Pops a scope, which should have region scope `region_scope`,
+    /// adding any drops onto the end of `block` that are needed.
+    /// This must match 1-to-1 with `push_scope`.
+    pub(crate) fn pop_scope(
+        &mut self,
+        region_scope: (region::Scope, SourceInfo),
+        mut block: BasicBlock,
+    ) -> BlockAnd<()> {
+        debug!("pop_scope({:?}, {:?})", region_scope, block);
+
+        block = self.leave_top_scope(block);
+
+        self.scopes.pop_scope(region_scope);
+
+        block.unit()
+    }
+
+    /// Sets up the drops for breaking from `block` to `target`.
+    pub(crate) fn break_scope(
+        &mut self,
+        mut block: BasicBlock,
+        value: Option<&Expr<'tcx>>,
+        target: BreakableTarget,
+        source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let span = source_info.span;
+
+        let get_scope_index = |scope: region::Scope| {
+            // find the loop-scope by its `region::Scope`.
+            self.scopes
+                .breakable_scopes
+                .iter()
+                .rposition(|breakable_scope| breakable_scope.region_scope == scope)
+                .unwrap_or_else(|| span_bug!(span, "no enclosing breakable scope found"))
+        };
+        let (break_index, destination) = match target {
+            BreakableTarget::Return => {
+                let scope = &self.scopes.breakable_scopes[0];
+                if scope.break_destination != Place::return_place() {
+                    span_bug!(span, "`return` in item with no return scope");
+                }
+                (0, Some(scope.break_destination))
+            }
+            BreakableTarget::Break(scope) => {
+                let break_index = get_scope_index(scope);
+                let scope = &self.scopes.breakable_scopes[break_index];
+                (break_index, Some(scope.break_destination))
+            }
+            BreakableTarget::Continue(scope) => {
+                let break_index = get_scope_index(scope);
+                (break_index, None)
+            }
+        };
+
+        if let Some(destination) = destination {
+            if let Some(value) = value {
+                debug!("stmt_expr Break val block_context.push(SubExpr)");
+                self.block_context.push(BlockFrame::SubExpr);
+                unpack!(block = self.expr_into_dest(destination, block, value));
+                self.block_context.pop();
+            } else {
+                self.cfg.push_assign_unit(block, source_info, destination, self.tcx)
+            }
+        } else {
+            assert!(value.is_none(), "`return` and `break` should have a destination");
+            if self.tcx.sess.instrument_coverage() {
+                // Unlike `break` and `return`, which push an `Assign` statement to MIR, from which
+                // a Coverage code region can be generated, `continue` needs no `Assign`; but
+                // without one, the `InstrumentCoverage` MIR pass cannot generate a code region for
+                // `continue`. Coverage will be missing unless we add a dummy `Assign` to MIR.
+                self.add_dummy_assignment(span, block, source_info);
+            }
+        }
+
+        let region_scope = self.scopes.breakable_scopes[break_index].region_scope;
+        let scope_index = self.scopes.scope_index(region_scope, span);
+        let drops = if destination.is_some() {
+            &mut self.scopes.breakable_scopes[break_index].break_drops
+        } else {
+            self.scopes.breakable_scopes[break_index].continue_drops.as_mut().unwrap()
+        };
+        let mut drop_idx = ROOT_NODE;
+        for scope in &self.scopes.scopes[scope_index + 1..] {
+            for drop in &scope.drops {
+                drop_idx = drops.add_drop(*drop, drop_idx);
+            }
+        }
+        drops.add_entry(block, drop_idx);
+
+        // `build_drop_trees` doesn't have access to our source_info, so we
+        // create a dummy terminator now. `TerminatorKind::Resume` is used
+        // because MIR type checking will panic if it hasn't been overwritten.
+        self.cfg.terminate(block, source_info, TerminatorKind::Resume);
+
+        self.cfg.start_new_block().unit()
+    }
+
+    pub(crate) fn break_for_else(
+        &mut self,
+        block: BasicBlock,
+        target: region::Scope,
+        source_info: SourceInfo,
+    ) {
+        let scope_index = self.scopes.scope_index(target, source_info.span);
+        let if_then_scope = self
+            .scopes
+            .if_then_scope
+            .as_mut()
+            .unwrap_or_else(|| span_bug!(source_info.span, "no if-then scope found"));
+
+        assert_eq!(if_then_scope.region_scope, target, "breaking to incorrect scope");
+
+        let mut drop_idx = ROOT_NODE;
+        let drops = &mut if_then_scope.else_drops;
+        for scope in &self.scopes.scopes[scope_index + 1..] {
+            for drop in &scope.drops {
+                drop_idx = drops.add_drop(*drop, drop_idx);
+            }
+        }
+        drops.add_entry(block, drop_idx);
+
+        // `build_drop_trees` doesn't have access to our source_info, so we
+        // create a dummy terminator now. `TerminatorKind::Resume` is used
+        // because MIR type checking will panic if it hasn't been overwritten.
+        self.cfg.terminate(block, source_info, TerminatorKind::Resume);
+    }
+
+    // Add a dummy `Assign` statement to the CFG, with the span for the source code's `continue`
+    // statement.
+    fn add_dummy_assignment(&mut self, span: Span, block: BasicBlock, source_info: SourceInfo) {
+        let local_decl = LocalDecl::new(self.tcx.mk_unit(), span).internal();
+        let temp_place = Place::from(self.local_decls.push(local_decl));
+        self.cfg.push_assign_unit(block, source_info, temp_place, self.tcx);
+    }
+
+    fn leave_top_scope(&mut self, block: BasicBlock) -> BasicBlock {
+        // If we are emitting a `drop` statement, we need to have the cached
+        // diverge cleanup pads ready in case that drop panics.
+        let needs_cleanup = self.scopes.scopes.last().map_or(false, |scope| scope.needs_cleanup());
+        let is_generator = self.generator_kind.is_some();
+        let unwind_to = if needs_cleanup { self.diverge_cleanup() } else { DropIdx::MAX };
+
+        let scope = self.scopes.scopes.last().expect("leave_top_scope called with no scopes");
+        unpack!(build_scope_drops(
+            &mut self.cfg,
+            &mut self.scopes.unwind_drops,
+            scope,
+            block,
+            unwind_to,
+            is_generator && needs_cleanup,
+            self.arg_count,
+        ))
+    }
+
+    /// Creates a new source scope, nested in the current one.
+    pub(crate) fn new_source_scope(
+        &mut self,
+        span: Span,
+        lint_level: LintLevel,
+        safety: Option<Safety>,
+    ) -> SourceScope {
+        let parent = self.source_scope;
+        debug!(
+            "new_source_scope({:?}, {:?}, {:?}) - parent({:?})={:?}",
+            span,
+            lint_level,
+            safety,
+            parent,
+            self.source_scopes.get(parent)
+        );
+        let scope_local_data = SourceScopeLocalData {
+            lint_root: if let LintLevel::Explicit(lint_root) = lint_level {
+                lint_root
+            } else {
+                self.source_scopes[parent].local_data.as_ref().assert_crate_local().lint_root
+            },
+            safety: safety.unwrap_or_else(|| {
+                self.source_scopes[parent].local_data.as_ref().assert_crate_local().safety
+            }),
+        };
+        self.source_scopes.push(SourceScopeData {
+            span,
+            parent_scope: Some(parent),
+            inlined: None,
+            inlined_parent_scope: None,
+            local_data: ClearCrossCrate::Set(scope_local_data),
+        })
+    }
+
+    /// Given a span and the current source scope, make a SourceInfo.
+    pub(crate) fn source_info(&self, span: Span) -> SourceInfo {
+        SourceInfo { span, scope: self.source_scope }
+    }
+
+    // Finding scopes
+    // ==============
+    /// Returns the scope that we should use as the lifetime of an
+    /// operand. Basically, an operand must live until it is consumed.
+    /// This is similar to, but not quite the same as, the temporary
+    /// scope (which can be larger or smaller).
+    ///
+    /// Consider:
+    /// ```ignore (illustrative)
+    /// let x = foo(bar(X, Y));
+    /// ```
+    /// We wish to pop the storage for X and Y after `bar()` is
+    /// called, not after the whole `let` is completed.
+    ///
+    /// As another example, if the second argument diverges:
+    /// ```ignore (illustrative)
+    /// foo(Box::new(2), panic!())
+    /// ```
+    /// We would allocate the box but then free it on the unwinding
+    /// path; we would also emit a free on the 'success' path from
+    /// panic, but that will turn out to be removed as dead-code.
+    pub(crate) fn local_scope(&self) -> region::Scope {
+        self.scopes.topmost()
+    }
+
+    // Scheduling drops
+    // ================
+    pub(crate) fn schedule_drop_storage_and_value(
+        &mut self,
+        span: Span,
+        region_scope: region::Scope,
+        local: Local,
+    ) {
+        self.schedule_drop(span, region_scope, local, DropKind::Storage);
+        self.schedule_drop(span, region_scope, local, DropKind::Value);
+    }
+
+    /// Indicates that `place` should be dropped on exit from `region_scope`.
+    ///
+    /// When called with `DropKind::Storage`, `place` shouldn't be the return
+    /// place, or a function parameter.
+    pub(crate) fn schedule_drop(
+        &mut self,
+        span: Span,
+        region_scope: region::Scope,
+        local: Local,
+        drop_kind: DropKind,
+    ) {
+        let needs_drop = match drop_kind {
+            DropKind::Value => {
+                if !self.local_decls[local].ty.needs_drop(self.tcx, self.param_env) {
+                    return;
+                }
+                true
+            }
+            DropKind::Storage => {
+                if local.index() <= self.arg_count {
+                    span_bug!(
+                        span,
+                        "`schedule_drop` called with local {:?} and arg_count {}",
+                        local,
+                        self.arg_count,
+                    )
+                }
+                false
+            }
+        };
+
+        // When building drops, we try to cache chains of drops to reduce the
+        // number of `DropTree::add_drop` calls. This, however, means that
+        // whenever we add a drop into a scope which already had some entries
+        // in the drop tree built (and thus, cached) for it, we must invalidate
+        // all caches which might branch into the scope which had a drop just
+        // added to it. This is necessary, because otherwise some other code
+        // might use the cache to branch into already built chain of drops,
+        // essentially ignoring the newly added drop.
+        //
+        // For example consider there’s two scopes with a drop in each. These
+        // are built and thus the caches are filled:
+        //
+        // +--------------------------------------------------------+
+        // | +---------------------------------+                    |
+        // | | +--------+     +-------------+  |  +---------------+ |
+        // | | | return | <-+ | drop(outer) | <-+ |  drop(middle) | |
+        // | | +--------+     +-------------+  |  +---------------+ |
+        // | +------------|outer_scope cache|--+                    |
+        // +------------------------------|middle_scope cache|------+
+        //
+        // Now, a new, inner-most scope is added along with a new drop into
+        // both inner-most and outer-most scopes:
+        //
+        // +------------------------------------------------------------+
+        // | +----------------------------------+                       |
+        // | | +--------+      +-------------+  |   +---------------+   | +-------------+
+        // | | | return | <+   | drop(new)   | <-+  |  drop(middle) | <--+| drop(inner) |
+        // | | +--------+  |   | drop(outer) |  |   +---------------+   | +-------------+
+        // | |             +-+ +-------------+  |                       |
+        // | +---|invalid outer_scope cache|----+                       |
+        // +----=----------------|invalid middle_scope cache|-----------+
+        //
+        // If, when adding `drop(new)` we do not invalidate the cached blocks for both
+        // outer_scope and middle_scope, then, when building drops for the inner (right-most)
+        // scope, the old, cached blocks, without `drop(new)` will get used, producing the
+        // wrong results.
+        //
+        // Note that this code iterates scopes from the inner-most to the outer-most,
+        // invalidating caches of each scope visited. This way bare minimum of the
+        // caches gets invalidated. i.e., if a new drop is added into the middle scope, the
+        // cache of outer scope stays intact.
+        //
+        // Since we only cache drops for the unwind path and the generator drop
+        // path, we only need to invalidate the cache for drops that happen on
+        // the unwind or generator drop paths. This means that for
+        // non-generators we don't need to invalidate caches for `DropKind::Storage`.
+        let invalidate_caches = needs_drop || self.generator_kind.is_some();
+        for scope in self.scopes.scopes.iter_mut().rev() {
+            if invalidate_caches {
+                scope.invalidate_cache();
+            }
+
+            if scope.region_scope == region_scope {
+                let region_scope_span = region_scope.span(self.tcx, &self.region_scope_tree);
+                // Attribute scope exit drops to scope's closing brace.
+                let scope_end = self.tcx.sess.source_map().end_point(region_scope_span);
+
+                scope.drops.push(DropData {
+                    source_info: SourceInfo { span: scope_end, scope: scope.source_scope },
+                    local,
+                    kind: drop_kind,
+                });
+
+                return;
+            }
+        }
+
+        span_bug!(span, "region scope {:?} not in scope to drop {:?}", region_scope, local);
+    }
+
+    /// Indicates that the "local operand" stored in `local` is
+    /// *moved* at some point during execution (see `local_scope` for
+    /// more information about what a "local operand" is -- in short,
+    /// it's an intermediate operand created as part of preparing some
+    /// MIR instruction). We use this information to suppress
+    /// redundant drops on the non-unwind paths. This results in less
+    /// MIR, but also avoids spurious borrow check errors
+    /// (c.f. #64391).
+    ///
+    /// Example: when compiling the call to `foo` here:
+    ///
+    /// ```ignore (illustrative)
+    /// foo(bar(), ...)
+    /// ```
+    ///
+    /// we would evaluate `bar()` to an operand `_X`. We would also
+    /// schedule `_X` to be dropped when the expression scope for
+    /// `foo(bar())` is exited. This is relevant, for example, if the
+    /// later arguments should unwind (it would ensure that `_X` gets
+    /// dropped). However, if no unwind occurs, then `_X` will be
+    /// unconditionally consumed by the `call`:
+    ///
+    /// ```ignore (illustrative)
+    /// bb {
+    ///   ...
+    ///   _R = CALL(foo, _X, ...)
+    /// }
+    /// ```
+    ///
+    /// However, `_X` is still registered to be dropped, and so if we
+    /// do nothing else, we would generate a `DROP(_X)` that occurs
+    /// after the call. This will later be optimized out by the
+    /// drop-elaboration code, but in the meantime it can lead to
+    /// spurious borrow-check errors -- the problem, ironically, is
+    /// not the `DROP(_X)` itself, but the (spurious) unwind pathways
+    /// that it creates. See #64391 for an example.
+    pub(crate) fn record_operands_moved(&mut self, operands: &[Operand<'tcx>]) {
+        let local_scope = self.local_scope();
+        let scope = self.scopes.scopes.last_mut().unwrap();
+
+        assert_eq!(scope.region_scope, local_scope, "local scope is not the topmost scope!",);
+
+        // look for moves of a local variable, like `MOVE(_X)`
+        let locals_moved = operands.iter().flat_map(|operand| match operand {
+            Operand::Copy(_) | Operand::Constant(_) => None,
+            Operand::Move(place) => place.as_local(),
+        });
+
+        for local in locals_moved {
+            // check if we have a Drop for this operand and -- if so
+            // -- add it to the list of moved operands. Note that this
+            // local might not have been an operand created for this
+            // call, it could come from other places too.
+            if scope.drops.iter().any(|drop| drop.local == local && drop.kind == DropKind::Value) {
+                scope.moved_locals.push(local);
+            }
+        }
+    }
+
+    // Other
+    // =====
+    /// Returns the [DropIdx] for the innermost drop if the function unwound at
+    /// this point. The `DropIdx` will be created if it doesn't already exist.
+    fn diverge_cleanup(&mut self) -> DropIdx {
+        let is_generator = self.generator_kind.is_some();
+        let (uncached_scope, mut cached_drop) = self
+            .scopes
+            .scopes
+            .iter()
+            .enumerate()
+            .rev()
+            .find_map(|(scope_idx, scope)| {
+                scope.cached_unwind_block.map(|cached_block| (scope_idx + 1, cached_block))
+            })
+            .unwrap_or((0, ROOT_NODE));
+
+        for scope in &mut self.scopes.scopes[uncached_scope..] {
+            for drop in &scope.drops {
+                if is_generator || drop.kind == DropKind::Value {
+                    cached_drop = self.scopes.unwind_drops.add_drop(*drop, cached_drop);
+                }
+            }
+            scope.cached_unwind_block = Some(cached_drop);
+        }
+
+        cached_drop
+    }
+
+    /// Prepares to create a path that performs all required cleanup for a
+    /// terminator that can unwind at the given basic block.
+    ///
+    /// This path terminates in Resume. The path isn't created until after all
+    /// of the non-unwind paths in this item have been lowered.
+    pub(crate) fn diverge_from(&mut self, start: BasicBlock) {
+        debug_assert!(
+            matches!(
+                self.cfg.block_data(start).terminator().kind,
+                TerminatorKind::Assert { .. }
+                    | TerminatorKind::Call { .. }
+                    | TerminatorKind::Drop { .. }
+                    | TerminatorKind::DropAndReplace { .. }
+                    | TerminatorKind::FalseUnwind { .. }
+                    | TerminatorKind::InlineAsm { .. }
+            ),
+            "diverge_from called on block with terminator that cannot unwind."
+        );
+
+        let next_drop = self.diverge_cleanup();
+        self.scopes.unwind_drops.add_entry(start, next_drop);
+    }
+
+    /// Sets up a path that performs all required cleanup for dropping a
+    /// generator, starting from the given block that ends in
+    /// [TerminatorKind::Yield].
+    ///
+    /// This path terminates in GeneratorDrop.
+    pub(crate) fn generator_drop_cleanup(&mut self, yield_block: BasicBlock) {
+        debug_assert!(
+            matches!(
+                self.cfg.block_data(yield_block).terminator().kind,
+                TerminatorKind::Yield { .. }
+            ),
+            "generator_drop_cleanup called on block with non-yield terminator."
+        );
+        let (uncached_scope, mut cached_drop) = self
+            .scopes
+            .scopes
+            .iter()
+            .enumerate()
+            .rev()
+            .find_map(|(scope_idx, scope)| {
+                scope.cached_generator_drop_block.map(|cached_block| (scope_idx + 1, cached_block))
+            })
+            .unwrap_or((0, ROOT_NODE));
+
+        for scope in &mut self.scopes.scopes[uncached_scope..] {
+            for drop in &scope.drops {
+                cached_drop = self.scopes.generator_drops.add_drop(*drop, cached_drop);
+            }
+            scope.cached_generator_drop_block = Some(cached_drop);
+        }
+
+        self.scopes.generator_drops.add_entry(yield_block, cached_drop);
+    }
+
+    /// Utility function for *non*-scope code to build their own drops
+    pub(crate) fn build_drop_and_replace(
+        &mut self,
+        block: BasicBlock,
+        span: Span,
+        place: Place<'tcx>,
+        value: Operand<'tcx>,
+    ) -> BlockAnd<()> {
+        let source_info = self.source_info(span);
+        let next_target = self.cfg.start_new_block();
+
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::DropAndReplace { place, value, target: next_target, unwind: None },
+        );
+        self.diverge_from(block);
+
+        next_target.unit()
+    }
+
+    /// Creates an `Assert` terminator and return the success block.
+    /// If the boolean condition operand is not the expected value,
+    /// a runtime panic will be caused with the given message.
+    pub(crate) fn assert(
+        &mut self,
+        block: BasicBlock,
+        cond: Operand<'tcx>,
+        expected: bool,
+        msg: AssertMessage<'tcx>,
+        span: Span,
+    ) -> BasicBlock {
+        let source_info = self.source_info(span);
+        let success_block = self.cfg.start_new_block();
+
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::Assert { cond, expected, msg, target: success_block, cleanup: None },
+        );
+        self.diverge_from(block);
+
+        success_block
+    }
+
+    /// Unschedules any drops in the top scope.
+    ///
+    /// This is only needed for `match` arm scopes, because they have one
+    /// entrance per pattern, but only one exit.
+    pub(crate) fn clear_top_scope(&mut self, region_scope: region::Scope) {
+        let top_scope = self.scopes.scopes.last_mut().unwrap();
+
+        assert_eq!(top_scope.region_scope, region_scope);
+
+        top_scope.drops.clear();
+        top_scope.invalidate_cache();
+    }
+}
+
+/// Builds drops for `pop_scope` and `leave_top_scope`.
+fn build_scope_drops<'tcx>(
+    cfg: &mut CFG<'tcx>,
+    unwind_drops: &mut DropTree,
+    scope: &Scope,
+    mut block: BasicBlock,
+    mut unwind_to: DropIdx,
+    storage_dead_on_unwind: bool,
+    arg_count: usize,
+) -> BlockAnd<()> {
+    debug!("build_scope_drops({:?} -> {:?})", block, scope);
+
+    // Build up the drops in evaluation order. The end result will
+    // look like:
+    //
+    // [SDs, drops[n]] --..> [SDs, drop[1]] -> [SDs, drop[0]] -> [[SDs]]
+    //               |                    |                 |
+    //               :                    |                 |
+    //                                    V                 V
+    // [drop[n]] -...-> [drop[1]] ------> [drop[0]] ------> [last_unwind_to]
+    //
+    // The horizontal arrows represent the execution path when the drops return
+    // successfully. The downwards arrows represent the execution path when the
+    // drops panic (panicking while unwinding will abort, so there's no need for
+    // another set of arrows).
+    //
+    // For generators, we unwind from a drop on a local to its StorageDead
+    // statement. For other functions we don't worry about StorageDead. The
+    // drops for the unwind path should have already been generated by
+    // `diverge_cleanup_gen`.
+
+    for drop_data in scope.drops.iter().rev() {
+        let source_info = drop_data.source_info;
+        let local = drop_data.local;
+
+        match drop_data.kind {
+            DropKind::Value => {
+                // `unwind_to` should drop the value that we're about to
+                // schedule. If dropping this value panics, then we continue
+                // with the *next* value on the unwind path.
+                debug_assert_eq!(unwind_drops.drops[unwind_to].0.local, drop_data.local);
+                debug_assert_eq!(unwind_drops.drops[unwind_to].0.kind, drop_data.kind);
+                unwind_to = unwind_drops.drops[unwind_to].1;
+
+                // If the operand has been moved, and we are not on an unwind
+                // path, then don't generate the drop. (We only take this into
+                // account for non-unwind paths so as not to disturb the
+                // caching mechanism.)
+                if scope.moved_locals.iter().any(|&o| o == local) {
+                    continue;
+                }
+
+                unwind_drops.add_entry(block, unwind_to);
+
+                let next = cfg.start_new_block();
+                cfg.terminate(
+                    block,
+                    source_info,
+                    TerminatorKind::Drop { place: local.into(), target: next, unwind: None },
+                );
+                block = next;
+            }
+            DropKind::Storage => {
+                if storage_dead_on_unwind {
+                    debug_assert_eq!(unwind_drops.drops[unwind_to].0.local, drop_data.local);
+                    debug_assert_eq!(unwind_drops.drops[unwind_to].0.kind, drop_data.kind);
+                    unwind_to = unwind_drops.drops[unwind_to].1;
+                }
+                // Only temps and vars need their storage dead.
+                assert!(local.index() > arg_count);
+                cfg.push(block, Statement { source_info, kind: StatementKind::StorageDead(local) });
+            }
+        }
+    }
+    block.unit()
+}
+
+impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
+    /// Build a drop tree for a breakable scope.
+    ///
+    /// If `continue_block` is `Some`, then the tree is for `continue` inside a
+    /// loop. Otherwise this is for `break` or `return`.
+    fn build_exit_tree(
+        &mut self,
+        mut drops: DropTree,
+        continue_block: Option<BasicBlock>,
+    ) -> Option<BlockAnd<()>> {
+        let mut blocks = IndexVec::from_elem(None, &drops.drops);
+        blocks[ROOT_NODE] = continue_block;
+
+        drops.build_mir::<ExitScopes>(&mut self.cfg, &mut blocks);
+
+        // Link the exit drop tree to unwind drop tree.
+        if drops.drops.iter().any(|(drop, _)| drop.kind == DropKind::Value) {
+            let unwind_target = self.diverge_cleanup();
+            let mut unwind_indices = IndexVec::from_elem_n(unwind_target, 1);
+            for (drop_idx, drop_data) in drops.drops.iter_enumerated().skip(1) {
+                match drop_data.0.kind {
+                    DropKind::Storage => {
+                        if self.generator_kind.is_some() {
+                            let unwind_drop = self
+                                .scopes
+                                .unwind_drops
+                                .add_drop(drop_data.0, unwind_indices[drop_data.1]);
+                            unwind_indices.push(unwind_drop);
+                        } else {
+                            unwind_indices.push(unwind_indices[drop_data.1]);
+                        }
+                    }
+                    DropKind::Value => {
+                        let unwind_drop = self
+                            .scopes
+                            .unwind_drops
+                            .add_drop(drop_data.0, unwind_indices[drop_data.1]);
+                        self.scopes
+                            .unwind_drops
+                            .add_entry(blocks[drop_idx].unwrap(), unwind_indices[drop_data.1]);
+                        unwind_indices.push(unwind_drop);
+                    }
+                }
+            }
+        }
+        blocks[ROOT_NODE].map(BasicBlock::unit)
+    }
+
+    /// Build the unwind and generator drop trees.
+    pub(crate) fn build_drop_trees(&mut self) {
+        if self.generator_kind.is_some() {
+            self.build_generator_drop_trees();
+        } else {
+            Self::build_unwind_tree(
+                &mut self.cfg,
+                &mut self.scopes.unwind_drops,
+                self.fn_span,
+                &mut None,
+            );
+        }
+    }
+
+    fn build_generator_drop_trees(&mut self) {
+        // Build the drop tree for dropping the generator while it's suspended.
+        let drops = &mut self.scopes.generator_drops;
+        let cfg = &mut self.cfg;
+        let fn_span = self.fn_span;
+        let mut blocks = IndexVec::from_elem(None, &drops.drops);
+        drops.build_mir::<GeneratorDrop>(cfg, &mut blocks);
+        if let Some(root_block) = blocks[ROOT_NODE] {
+            cfg.terminate(
+                root_block,
+                SourceInfo::outermost(fn_span),
+                TerminatorKind::GeneratorDrop,
+            );
+        }
+
+        // Build the drop tree for unwinding in the normal control flow paths.
+        let resume_block = &mut None;
+        let unwind_drops = &mut self.scopes.unwind_drops;
+        Self::build_unwind_tree(cfg, unwind_drops, fn_span, resume_block);
+
+        // Build the drop tree for unwinding when dropping a suspended
+        // generator.
+        //
+        // This is a different tree to the standard unwind paths here to
+        // prevent drop elaboration from creating drop flags that would have
+        // to be captured by the generator. I'm not sure how important this
+        // optimization is, but it is here.
+        for (drop_idx, drop_data) in drops.drops.iter_enumerated() {
+            if let DropKind::Value = drop_data.0.kind {
+                debug_assert!(drop_data.1 < drops.drops.next_index());
+                drops.entry_points.push((drop_data.1, blocks[drop_idx].unwrap()));
+            }
+        }
+        Self::build_unwind_tree(cfg, drops, fn_span, resume_block);
+    }
+
+    fn build_unwind_tree(
+        cfg: &mut CFG<'tcx>,
+        drops: &mut DropTree,
+        fn_span: Span,
+        resume_block: &mut Option<BasicBlock>,
+    ) {
+        let mut blocks = IndexVec::from_elem(None, &drops.drops);
+        blocks[ROOT_NODE] = *resume_block;
+        drops.build_mir::<Unwind>(cfg, &mut blocks);
+        if let (None, Some(resume)) = (*resume_block, blocks[ROOT_NODE]) {
+            cfg.terminate(resume, SourceInfo::outermost(fn_span), TerminatorKind::Resume);
+
+            *resume_block = blocks[ROOT_NODE];
+        }
+    }
+}
+
+// DropTreeBuilder implementations.
+
+struct ExitScopes;
+
+impl<'tcx> DropTreeBuilder<'tcx> for ExitScopes {
+    fn make_block(cfg: &mut CFG<'tcx>) -> BasicBlock {
+        cfg.start_new_block()
+    }
+    fn add_entry(cfg: &mut CFG<'tcx>, from: BasicBlock, to: BasicBlock) {
+        cfg.block_data_mut(from).terminator_mut().kind = TerminatorKind::Goto { target: to };
+    }
+}
+
+struct GeneratorDrop;
+
+impl<'tcx> DropTreeBuilder<'tcx> for GeneratorDrop {
+    fn make_block(cfg: &mut CFG<'tcx>) -> BasicBlock {
+        cfg.start_new_block()
+    }
+    fn add_entry(cfg: &mut CFG<'tcx>, from: BasicBlock, to: BasicBlock) {
+        let term = cfg.block_data_mut(from).terminator_mut();
+        if let TerminatorKind::Yield { ref mut drop, .. } = term.kind {
+            *drop = Some(to);
+        } else {
+            span_bug!(
+                term.source_info.span,
+                "cannot enter generator drop tree from {:?}",
+                term.kind
+            )
+        }
+    }
+}
+
+struct Unwind;
+
+impl<'tcx> DropTreeBuilder<'tcx> for Unwind {
+    fn make_block(cfg: &mut CFG<'tcx>) -> BasicBlock {
+        cfg.start_new_cleanup_block()
+    }
+    fn add_entry(cfg: &mut CFG<'tcx>, from: BasicBlock, to: BasicBlock) {
+        let term = &mut cfg.block_data_mut(from).terminator_mut();
+        match &mut term.kind {
+            TerminatorKind::Drop { unwind, .. }
+            | TerminatorKind::DropAndReplace { unwind, .. }
+            | TerminatorKind::FalseUnwind { unwind, .. }
+            | TerminatorKind::Call { cleanup: unwind, .. }
+            | TerminatorKind::Assert { cleanup: unwind, .. }
+            | TerminatorKind::InlineAsm { cleanup: unwind, .. } => {
+                *unwind = Some(to);
+            }
+            TerminatorKind::Goto { .. }
+            | TerminatorKind::SwitchInt { .. }
+            | TerminatorKind::Resume
+            | TerminatorKind::Abort
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::Yield { .. }
+            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::FalseEdge { .. } => {
+                span_bug!(term.source_info.span, "cannot unwind from {:?}", term.kind)
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
new file mode 100644
index 000000000..864caf0ba
--- /dev/null
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -0,0 +1,680 @@
+use crate::build::ExprCategory;
+use rustc_middle::thir::visit::{self, Visitor};
+
+use rustc_errors::struct_span_err;
+use rustc_hir as hir;
+use rustc_middle::mir::BorrowKind;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
+use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
+use rustc_session::lint::Level;
+use rustc_span::def_id::{DefId, LocalDefId};
+use rustc_span::symbol::Symbol;
+use rustc_span::Span;
+
+use std::borrow::Cow;
+use std::ops::Bound;
+
+struct UnsafetyVisitor<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    thir: &'a Thir<'tcx>,
+    /// The `HirId` of the current scope, which would be the `HirId`
+    /// of the current HIR node, modulo adjustments. Used for lint levels.
+    hir_context: hir::HirId,
+    /// The current "safety context". This notably tracks whether we are in an
+    /// `unsafe` block, and whether it has been used.
+    safety_context: SafetyContext,
+    body_unsafety: BodyUnsafety,
+    /// The `#[target_feature]` attributes of the body. Used for checking
+    /// calls to functions with `#[target_feature]` (RFC 2396).
+    body_target_features: &'tcx [Symbol],
+    /// When inside the LHS of an assignment to a field, this is the type
+    /// of the LHS and the span of the assignment expression.
+    assignment_info: Option<(Ty<'tcx>, Span)>,
+    in_union_destructure: bool,
+    param_env: ParamEnv<'tcx>,
+    inside_adt: bool,
+}
+
+impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
+    fn in_safety_context(&mut self, safety_context: SafetyContext, f: impl FnOnce(&mut Self)) {
+        if let (
+            SafetyContext::UnsafeBlock { span: enclosing_span, .. },
+            SafetyContext::UnsafeBlock { span: block_span, hir_id, .. },
+        ) = (self.safety_context, safety_context)
+        {
+            self.warn_unused_unsafe(
+                hir_id,
+                block_span,
+                Some((self.tcx.sess.source_map().guess_head_span(enclosing_span), "block")),
+            );
+            f(self);
+        } else {
+            let prev_context = self.safety_context;
+            self.safety_context = safety_context;
+
+            f(self);
+
+            if let SafetyContext::UnsafeBlock { used: false, span, hir_id } = self.safety_context {
+                self.warn_unused_unsafe(
+                    hir_id,
+                    span,
+                    if self.unsafe_op_in_unsafe_fn_allowed() {
+                        self.body_unsafety.unsafe_fn_sig_span().map(|span| (span, "fn"))
+                    } else {
+                        None
+                    },
+                );
+            }
+            self.safety_context = prev_context;
+        }
+    }
+
+    fn requires_unsafe(&mut self, span: Span, kind: UnsafeOpKind) {
+        let unsafe_op_in_unsafe_fn_allowed = self.unsafe_op_in_unsafe_fn_allowed();
+        match self.safety_context {
+            SafetyContext::BuiltinUnsafeBlock => {}
+            SafetyContext::UnsafeBlock { ref mut used, .. } => {
+                if !self.body_unsafety.is_unsafe() || !unsafe_op_in_unsafe_fn_allowed {
+                    // Mark this block as useful
+                    *used = true;
+                }
+            }
+            SafetyContext::UnsafeFn if unsafe_op_in_unsafe_fn_allowed => {}
+            SafetyContext::UnsafeFn => {
+                let (description, note) = kind.description_and_note(self.tcx);
+                // unsafe_op_in_unsafe_fn is disallowed
+                self.tcx.struct_span_lint_hir(
+                    UNSAFE_OP_IN_UNSAFE_FN,
+                    self.hir_context,
+                    span,
+                    |lint| {
+                        lint.build(&format!(
+                            "{} is unsafe and requires unsafe block (error E0133)",
+                            description,
+                        ))
+                        .span_label(span, kind.simple_description())
+                        .note(note)
+                        .emit();
+                    },
+                )
+            }
+            SafetyContext::Safe => {
+                let (description, note) = kind.description_and_note(self.tcx);
+                let fn_sugg = if unsafe_op_in_unsafe_fn_allowed { " function or" } else { "" };
+                struct_span_err!(
+                    self.tcx.sess,
+                    span,
+                    E0133,
+                    "{} is unsafe and requires unsafe{} block",
+                    description,
+                    fn_sugg,
+                )
+                .span_label(span, kind.simple_description())
+                .note(note)
+                .emit();
+            }
+        }
+    }
+
+    fn warn_unused_unsafe(
+        &self,
+        hir_id: hir::HirId,
+        block_span: Span,
+        enclosing_unsafe: Option<(Span, &'static str)>,
+    ) {
+        let block_span = self.tcx.sess.source_map().guess_head_span(block_span);
+        self.tcx.struct_span_lint_hir(UNUSED_UNSAFE, hir_id, block_span, |lint| {
+            let msg = "unnecessary `unsafe` block";
+            let mut db = lint.build(msg);
+            db.span_label(block_span, msg);
+            if let Some((span, kind)) = enclosing_unsafe {
+                db.span_label(span, format!("because it's nested under this `unsafe` {}", kind));
+            }
+            db.emit();
+        });
+    }
+
+    /// Whether the `unsafe_op_in_unsafe_fn` lint is `allow`ed at the current HIR node.
+    fn unsafe_op_in_unsafe_fn_allowed(&self) -> bool {
+        self.tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, self.hir_context).0 == Level::Allow
+    }
+}
+
+// Searches for accesses to layout constrained fields.
+struct LayoutConstrainedPlaceVisitor<'a, 'tcx> {
+    found: bool,
+    thir: &'a Thir<'tcx>,
+    tcx: TyCtxt<'tcx>,
+}
+
+impl<'a, 'tcx> LayoutConstrainedPlaceVisitor<'a, 'tcx> {
+    fn new(thir: &'a Thir<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
+        Self { found: false, thir, tcx }
+    }
+}
+
+impl<'a, 'tcx> Visitor<'a, 'tcx> for LayoutConstrainedPlaceVisitor<'a, 'tcx> {
+    fn thir(&self) -> &'a Thir<'tcx> {
+        self.thir
+    }
+
+    fn visit_expr(&mut self, expr: &Expr<'tcx>) {
+        match expr.kind {
+            ExprKind::Field { lhs, .. } => {
+                if let ty::Adt(adt_def, _) = self.thir[lhs].ty.kind() {
+                    if (Bound::Unbounded, Bound::Unbounded)
+                        != self.tcx.layout_scalar_valid_range(adt_def.did())
+                    {
+                        self.found = true;
+                    }
+                }
+                visit::walk_expr(self, expr);
+            }
+
+            // Keep walking through the expression as long as we stay in the same
+            // place, i.e. the expression is a place expression and not a dereference
+            // (since dereferencing something leads us to a different place).
+            ExprKind::Deref { .. } => {}
+            ref kind if ExprCategory::of(kind).map_or(true, |cat| cat == ExprCategory::Place) => {
+                visit::walk_expr(self, expr);
+            }
+
+            _ => {}
+        }
+    }
+}
+
+impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
+    fn thir(&self) -> &'a Thir<'tcx> {
+        &self.thir
+    }
+
+    fn visit_block(&mut self, block: &Block) {
+        match block.safety_mode {
+            // compiler-generated unsafe code should not count towards the usefulness of
+            // an outer unsafe block
+            BlockSafety::BuiltinUnsafe => {
+                self.in_safety_context(SafetyContext::BuiltinUnsafeBlock, |this| {
+                    visit::walk_block(this, block)
+                });
+            }
+            BlockSafety::ExplicitUnsafe(hir_id) => {
+                self.in_safety_context(
+                    SafetyContext::UnsafeBlock { span: block.span, hir_id, used: false },
+                    |this| visit::walk_block(this, block),
+                );
+            }
+            BlockSafety::Safe => {
+                visit::walk_block(self, block);
+            }
+        }
+    }
+
+    fn visit_pat(&mut self, pat: &Pat<'tcx>) {
+        if self.in_union_destructure {
+            match *pat.kind {
+                // binding to a variable allows getting stuff out of variable
+                PatKind::Binding { .. }
+                // match is conditional on having this value
+                | PatKind::Constant { .. }
+                | PatKind::Variant { .. }
+                | PatKind::Leaf { .. }
+                | PatKind::Deref { .. }
+                | PatKind::Range { .. }
+                | PatKind::Slice { .. }
+                | PatKind::Array { .. } => {
+                    self.requires_unsafe(pat.span, AccessToUnionField);
+                    return; // we can return here since this already requires unsafe
+                }
+                // wildcard doesn't take anything
+                PatKind::Wild |
+                // these just wrap other patterns
+                PatKind::Or { .. } |
+                PatKind::AscribeUserType { .. } => {}
+            }
+        };
+
+        match &*pat.kind {
+            PatKind::Leaf { .. } => {
+                if let ty::Adt(adt_def, ..) = pat.ty.kind() {
+                    if adt_def.is_union() {
+                        let old_in_union_destructure =
+                            std::mem::replace(&mut self.in_union_destructure, true);
+                        visit::walk_pat(self, pat);
+                        self.in_union_destructure = old_in_union_destructure;
+                    } else if (Bound::Unbounded, Bound::Unbounded)
+                        != self.tcx.layout_scalar_valid_range(adt_def.did())
+                    {
+                        let old_inside_adt = std::mem::replace(&mut self.inside_adt, true);
+                        visit::walk_pat(self, pat);
+                        self.inside_adt = old_inside_adt;
+                    } else {
+                        visit::walk_pat(self, pat);
+                    }
+                } else {
+                    visit::walk_pat(self, pat);
+                }
+            }
+            PatKind::Binding { mode: BindingMode::ByRef(borrow_kind), ty, .. } => {
+                if self.inside_adt {
+                    let ty::Ref(_, ty, _) = ty.kind() else {
+                        span_bug!(
+                            pat.span,
+                            "BindingMode::ByRef in pattern, but found non-reference type {}",
+                            ty
+                        );
+                    };
+                    match borrow_kind {
+                        BorrowKind::Shallow | BorrowKind::Shared | BorrowKind::Unique => {
+                            if !ty.is_freeze(self.tcx.at(pat.span), self.param_env) {
+                                self.requires_unsafe(pat.span, BorrowOfLayoutConstrainedField);
+                            }
+                        }
+                        BorrowKind::Mut { .. } => {
+                            self.requires_unsafe(pat.span, MutationOfLayoutConstrainedField);
+                        }
+                    }
+                }
+                visit::walk_pat(self, pat);
+            }
+            PatKind::Deref { .. } => {
+                let old_inside_adt = std::mem::replace(&mut self.inside_adt, false);
+                visit::walk_pat(self, pat);
+                self.inside_adt = old_inside_adt;
+            }
+            _ => {
+                visit::walk_pat(self, pat);
+            }
+        }
+    }
+
+    fn visit_expr(&mut self, expr: &Expr<'tcx>) {
+        // could we be in the LHS of an assignment to a field?
+        match expr.kind {
+            ExprKind::Field { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::Scope { .. }
+            | ExprKind::Cast { .. } => {}
+
+            ExprKind::AddressOf { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Array { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. }
+            | ExprKind::Pointer { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::ThreadLocalRef { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::If { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Use { .. } => {
+                // We don't need to save the old value and restore it
+                // because all the place expressions can't have more
+                // than one child.
+                self.assignment_info = None;
+            }
+        };
+        match expr.kind {
+            ExprKind::Scope { value, lint_level: LintLevel::Explicit(hir_id), region_scope: _ } => {
+                let prev_id = self.hir_context;
+                self.hir_context = hir_id;
+                self.visit_expr(&self.thir[value]);
+                self.hir_context = prev_id;
+                return; // don't visit the whole expression
+            }
+            ExprKind::Call { fun, ty: _, args: _, from_hir_call: _, fn_span: _ } => {
+                if self.thir[fun].ty.fn_sig(self.tcx).unsafety() == hir::Unsafety::Unsafe {
+                    let func_id = if let ty::FnDef(func_id, _) = self.thir[fun].ty.kind() {
+                        Some(*func_id)
+                    } else {
+                        None
+                    };
+                    self.requires_unsafe(expr.span, CallToUnsafeFunction(func_id));
+                } else if let &ty::FnDef(func_did, _) = self.thir[fun].ty.kind() {
+                    // If the called function has target features the calling function hasn't,
+                    // the call requires `unsafe`. Don't check this on wasm
+                    // targets, though. For more information on wasm see the
+                    // is_like_wasm check in typeck/src/collect.rs
+                    if !self.tcx.sess.target.options.is_like_wasm
+                        && !self
+                            .tcx
+                            .codegen_fn_attrs(func_did)
+                            .target_features
+                            .iter()
+                            .all(|feature| self.body_target_features.contains(feature))
+                    {
+                        self.requires_unsafe(expr.span, CallToFunctionWith(func_did));
+                    }
+                }
+            }
+            ExprKind::Deref { arg } => {
+                if let ExprKind::StaticRef { def_id, .. } = self.thir[arg].kind {
+                    if self.tcx.is_mutable_static(def_id) {
+                        self.requires_unsafe(expr.span, UseOfMutableStatic);
+                    } else if self.tcx.is_foreign_item(def_id) {
+                        self.requires_unsafe(expr.span, UseOfExternStatic);
+                    }
+                } else if self.thir[arg].ty.is_unsafe_ptr() {
+                    self.requires_unsafe(expr.span, DerefOfRawPointer);
+                }
+            }
+            ExprKind::InlineAsm { .. } => {
+                self.requires_unsafe(expr.span, UseOfInlineAssembly);
+            }
+            ExprKind::Adt(box Adt {
+                adt_def,
+                variant_index: _,
+                substs: _,
+                user_ty: _,
+                fields: _,
+                base: _,
+            }) => match self.tcx.layout_scalar_valid_range(adt_def.did()) {
+                (Bound::Unbounded, Bound::Unbounded) => {}
+                _ => self.requires_unsafe(expr.span, InitializingTypeWith),
+            },
+            ExprKind::Closure {
+                closure_id,
+                substs: _,
+                upvars: _,
+                movability: _,
+                fake_reads: _,
+            } => {
+                let closure_def = if let Some((did, const_param_id)) =
+                    ty::WithOptConstParam::try_lookup(closure_id, self.tcx)
+                {
+                    ty::WithOptConstParam { did, const_param_did: Some(const_param_id) }
+                } else {
+                    ty::WithOptConstParam::unknown(closure_id)
+                };
+                let (closure_thir, expr) = self.tcx.thir_body(closure_def).unwrap_or_else(|_| {
+                    (self.tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0))
+                });
+                let closure_thir = &closure_thir.borrow();
+                let hir_context = self.tcx.hir().local_def_id_to_hir_id(closure_id);
+                let mut closure_visitor =
+                    UnsafetyVisitor { thir: closure_thir, hir_context, ..*self };
+                closure_visitor.visit_expr(&closure_thir[expr]);
+                // Unsafe blocks can be used in closures, make sure to take it into account
+                self.safety_context = closure_visitor.safety_context;
+            }
+            ExprKind::Field { lhs, .. } => {
+                let lhs = &self.thir[lhs];
+                if let ty::Adt(adt_def, _) = lhs.ty.kind() && adt_def.is_union() {
+                    if let Some((assigned_ty, assignment_span)) = self.assignment_info {
+                        if assigned_ty.needs_drop(self.tcx, self.param_env) {
+                            // This would be unsafe, but should be outright impossible since we reject such unions.
+                            self.tcx.sess.delay_span_bug(assignment_span, format!("union fields that need dropping should be impossible: {assigned_ty}"));
+                        }
+                    } else {
+                        self.requires_unsafe(expr.span, AccessToUnionField);
+                    }
+                }
+            }
+            ExprKind::Assign { lhs, rhs } | ExprKind::AssignOp { lhs, rhs, .. } => {
+                let lhs = &self.thir[lhs];
+                // First, check whether we are mutating a layout constrained field
+                let mut visitor = LayoutConstrainedPlaceVisitor::new(self.thir, self.tcx);
+                visit::walk_expr(&mut visitor, lhs);
+                if visitor.found {
+                    self.requires_unsafe(expr.span, MutationOfLayoutConstrainedField);
+                }
+
+                // Second, check for accesses to union fields
+                // don't have any special handling for AssignOp since it causes a read *and* write to lhs
+                if matches!(expr.kind, ExprKind::Assign { .. }) {
+                    self.assignment_info = Some((lhs.ty, expr.span));
+                    visit::walk_expr(self, lhs);
+                    self.assignment_info = None;
+                    visit::walk_expr(self, &self.thir()[rhs]);
+                    return; // we have already visited everything by now
+                }
+            }
+            ExprKind::Borrow { borrow_kind, arg } => {
+                let mut visitor = LayoutConstrainedPlaceVisitor::new(self.thir, self.tcx);
+                visit::walk_expr(&mut visitor, expr);
+                if visitor.found {
+                    match borrow_kind {
+                        BorrowKind::Shallow | BorrowKind::Shared | BorrowKind::Unique
+                            if !self.thir[arg]
+                                .ty
+                                .is_freeze(self.tcx.at(self.thir[arg].span), self.param_env) =>
+                        {
+                            self.requires_unsafe(expr.span, BorrowOfLayoutConstrainedField)
+                        }
+                        BorrowKind::Mut { .. } => {
+                            self.requires_unsafe(expr.span, MutationOfLayoutConstrainedField)
+                        }
+                        BorrowKind::Shallow | BorrowKind::Shared | BorrowKind::Unique => {}
+                    }
+                }
+            }
+            ExprKind::Let { expr: expr_id, .. } => {
+                let let_expr = &self.thir[expr_id];
+                if let ty::Adt(adt_def, _) = let_expr.ty.kind() && adt_def.is_union() {
+                    self.requires_unsafe(expr.span, AccessToUnionField);
+                }
+            }
+            _ => {}
+        }
+        visit::walk_expr(self, expr);
+    }
+}
+
+#[derive(Clone, Copy)]
+enum SafetyContext {
+    Safe,
+    BuiltinUnsafeBlock,
+    UnsafeFn,
+    UnsafeBlock { span: Span, hir_id: hir::HirId, used: bool },
+}
+
+#[derive(Clone, Copy)]
+enum BodyUnsafety {
+    /// The body is not unsafe.
+    Safe,
+    /// The body is an unsafe function. The span points to
+    /// the signature of the function.
+    Unsafe(Span),
+}
+
+impl BodyUnsafety {
+    /// Returns whether the body is unsafe.
+    fn is_unsafe(&self) -> bool {
+        matches!(self, BodyUnsafety::Unsafe(_))
+    }
+
+    /// If the body is unsafe, returns the `Span` of its signature.
+    fn unsafe_fn_sig_span(self) -> Option<Span> {
+        match self {
+            BodyUnsafety::Unsafe(span) => Some(span),
+            BodyUnsafety::Safe => None,
+        }
+    }
+}
+
+#[derive(Clone, Copy, PartialEq)]
+enum UnsafeOpKind {
+    CallToUnsafeFunction(Option<DefId>),
+    UseOfInlineAssembly,
+    InitializingTypeWith,
+    UseOfMutableStatic,
+    UseOfExternStatic,
+    DerefOfRawPointer,
+    AccessToUnionField,
+    MutationOfLayoutConstrainedField,
+    BorrowOfLayoutConstrainedField,
+    CallToFunctionWith(DefId),
+}
+
+use UnsafeOpKind::*;
+
+impl UnsafeOpKind {
+    pub fn simple_description(&self) -> &'static str {
+        match self {
+            CallToUnsafeFunction(..) => "call to unsafe function",
+            UseOfInlineAssembly => "use of inline assembly",
+            InitializingTypeWith => "initializing type with `rustc_layout_scalar_valid_range` attr",
+            UseOfMutableStatic => "use of mutable static",
+            UseOfExternStatic => "use of extern static",
+            DerefOfRawPointer => "dereference of raw pointer",
+            AccessToUnionField => "access to union field",
+            MutationOfLayoutConstrainedField => "mutation of layout constrained field",
+            BorrowOfLayoutConstrainedField => {
+                "borrow of layout constrained field with interior mutability"
+            }
+            CallToFunctionWith(..) => "call to function with `#[target_feature]`",
+        }
+    }
+
+    pub fn description_and_note(&self, tcx: TyCtxt<'_>) -> (Cow<'static, str>, &'static str) {
+        match self {
+            CallToUnsafeFunction(did) => (
+                if let Some(did) = did {
+                    Cow::from(format!("call to unsafe function `{}`", tcx.def_path_str(*did)))
+                } else {
+                    Cow::Borrowed(self.simple_description())
+                },
+                "consult the function's documentation for information on how to avoid undefined \
+                 behavior",
+            ),
+            UseOfInlineAssembly => (
+                Cow::Borrowed(self.simple_description()),
+                "inline assembly is entirely unchecked and can cause undefined behavior",
+            ),
+            InitializingTypeWith => (
+                Cow::Borrowed(self.simple_description()),
+                "initializing a layout restricted type's field with a value outside the valid \
+                 range is undefined behavior",
+            ),
+            UseOfMutableStatic => (
+                Cow::Borrowed(self.simple_description()),
+                "mutable statics can be mutated by multiple threads: aliasing violations or data \
+                 races will cause undefined behavior",
+            ),
+            UseOfExternStatic => (
+                Cow::Borrowed(self.simple_description()),
+                "extern statics are not controlled by the Rust type system: invalid data, \
+                 aliasing violations or data races will cause undefined behavior",
+            ),
+            DerefOfRawPointer => (
+                Cow::Borrowed(self.simple_description()),
+                "raw pointers may be null, dangling or unaligned; they can violate aliasing rules \
+                 and cause data races: all of these are undefined behavior",
+            ),
+            AccessToUnionField => (
+                Cow::Borrowed(self.simple_description()),
+                "the field may not be properly initialized: using uninitialized data will cause \
+                 undefined behavior",
+            ),
+            MutationOfLayoutConstrainedField => (
+                Cow::Borrowed(self.simple_description()),
+                "mutating layout constrained fields cannot statically be checked for valid values",
+            ),
+            BorrowOfLayoutConstrainedField => (
+                Cow::Borrowed(self.simple_description()),
+                "references to fields of layout constrained fields lose the constraints. Coupled \
+                 with interior mutability, the field can be changed to invalid values",
+            ),
+            CallToFunctionWith(did) => (
+                Cow::from(format!(
+                    "call to function `{}` with `#[target_feature]`",
+                    tcx.def_path_str(*did)
+                )),
+                "can only be called if the required target features are available",
+            ),
+        }
+    }
+}
+
+pub fn check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>) {
+    // THIR unsafeck is gated under `-Z thir-unsafeck`
+    if !tcx.sess.opts.unstable_opts.thir_unsafeck {
+        return;
+    }
+
+    // Closures are handled by their owner, if it has a body
+    if tcx.is_closure(def.did.to_def_id()) {
+        let hir = tcx.hir();
+        let owner = hir.enclosing_body_owner(hir.local_def_id_to_hir_id(def.did));
+        tcx.ensure().thir_check_unsafety(owner);
+        return;
+    }
+
+    let Ok((thir, expr)) = tcx.thir_body(def) else {
+        return
+    };
+    let thir = &thir.borrow();
+    // If `thir` is empty, a type error occurred, skip this body.
+    if thir.exprs.is_empty() {
+        return;
+    }
+
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
+    let body_unsafety = tcx.hir().fn_sig_by_hir_id(hir_id).map_or(BodyUnsafety::Safe, |fn_sig| {
+        if fn_sig.header.unsafety == hir::Unsafety::Unsafe {
+            BodyUnsafety::Unsafe(fn_sig.span)
+        } else {
+            BodyUnsafety::Safe
+        }
+    });
+    let body_target_features = &tcx.body_codegen_attrs(def.did.to_def_id()).target_features;
+    let safety_context =
+        if body_unsafety.is_unsafe() { SafetyContext::UnsafeFn } else { SafetyContext::Safe };
+    let mut visitor = UnsafetyVisitor {
+        tcx,
+        thir,
+        safety_context,
+        hir_context: hir_id,
+        body_unsafety,
+        body_target_features,
+        assignment_info: None,
+        in_union_destructure: false,
+        param_env: tcx.param_env(def.did),
+        inside_adt: false,
+    };
+    visitor.visit_expr(&thir[expr]);
+}
+
+pub(crate) fn thir_check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) {
+    if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
+        tcx.thir_check_unsafety_for_const_arg(def)
+    } else {
+        check_unsafety(tcx, ty::WithOptConstParam::unknown(def_id))
+    }
+}
+
+pub(crate) fn thir_check_unsafety_for_const_arg<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    (did, param_did): (LocalDefId, DefId),
+) {
+    check_unsafety(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) })
+}
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
new file mode 100644
index 000000000..11cd2a9aa
--- /dev/null
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -0,0 +1,35 @@
+//! Construction of MIR from HIR.
+//!
+//! This crate also contains the match exhaustiveness and usefulness checking.
+#![allow(rustc::potential_query_instability)]
+#![feature(box_patterns)]
+#![feature(control_flow_enum)]
+#![feature(if_let_guard)]
+#![feature(let_chains)]
+#![feature(let_else)]
+#![feature(min_specialization)]
+#![feature(once_cell)]
+#![recursion_limit = "256"]
+
+#[macro_use]
+extern crate tracing;
+#[macro_use]
+extern crate rustc_middle;
+
+mod build;
+mod check_unsafety;
+mod lints;
+pub mod thir;
+
+use rustc_middle::ty::query::Providers;
+
+pub fn provide(providers: &mut Providers) {
+    providers.check_match = thir::pattern::check_match;
+    providers.lit_to_const = thir::constant::lit_to_const;
+    providers.lit_to_mir_constant = build::lit_to_mir_constant;
+    providers.mir_built = build::mir_built;
+    providers.thir_check_unsafety = check_unsafety::thir_check_unsafety;
+    providers.thir_check_unsafety_for_const_arg = check_unsafety::thir_check_unsafety_for_const_arg;
+    providers.thir_body = thir::cx::thir_body;
+    providers.thir_tree = thir::cx::thir_tree;
+}
diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
new file mode 100644
index 000000000..54d549fd6
--- /dev/null
+++ b/compiler/rustc_mir_build/src/lints.rs
@@ -0,0 +1,166 @@
+use rustc_data_structures::graph::iterate::{
+    NodeStatus, TriColorDepthFirstSearch, TriColorVisitor,
+};
+use rustc_hir::def::DefKind;
+use rustc_middle::mir::{BasicBlock, BasicBlocks, Body, Operand, TerminatorKind};
+use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
+use rustc_middle::ty::{self, Instance, TyCtxt};
+use rustc_session::lint::builtin::UNCONDITIONAL_RECURSION;
+use rustc_span::Span;
+use std::ops::ControlFlow;
+
+pub(crate) fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
+    let def_id = body.source.def_id().expect_local();
+
+    if let DefKind::Fn | DefKind::AssocFn = tcx.def_kind(def_id) {
+        // If this is trait/impl method, extract the trait's substs.
+        let trait_substs = match tcx.trait_of_item(def_id.to_def_id()) {
+            Some(trait_def_id) => {
+                let trait_substs_count = tcx.generics_of(trait_def_id).count();
+                &InternalSubsts::identity_for_item(tcx, def_id.to_def_id())[..trait_substs_count]
+            }
+            _ => &[],
+        };
+
+        let mut vis = Search { tcx, body, reachable_recursive_calls: vec![], trait_substs };
+        if let Some(NonRecursive) =
+            TriColorDepthFirstSearch::new(&body.basic_blocks).run_from_start(&mut vis)
+        {
+            return;
+        }
+        if vis.reachable_recursive_calls.is_empty() {
+            return;
+        }
+
+        vis.reachable_recursive_calls.sort();
+
+        let sp = tcx.def_span(def_id);
+        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        tcx.struct_span_lint_hir(UNCONDITIONAL_RECURSION, hir_id, sp, |lint| {
+            let mut db = lint.build("function cannot return without recursing");
+            db.span_label(sp, "cannot return without recursing");
+            // offer some help to the programmer.
+            for call_span in vis.reachable_recursive_calls {
+                db.span_label(call_span, "recursive call site");
+            }
+            db.help("a `loop` may express intention better if this is on purpose");
+            db.emit();
+        });
+    }
+}
+
+struct NonRecursive;
+
+struct Search<'mir, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    body: &'mir Body<'tcx>,
+    trait_substs: &'tcx [GenericArg<'tcx>],
+
+    reachable_recursive_calls: Vec<Span>,
+}
+
+impl<'mir, 'tcx> Search<'mir, 'tcx> {
+    /// Returns `true` if `func` refers to the function we are searching in.
+    fn is_recursive_call(&self, func: &Operand<'tcx>, args: &[Operand<'tcx>]) -> bool {
+        let Search { tcx, body, trait_substs, .. } = *self;
+        // Resolving function type to a specific instance that is being called is expensive.  To
+        // avoid the cost we check the number of arguments first, which is sufficient to reject
+        // most of calls as non-recursive.
+        if args.len() != body.arg_count {
+            return false;
+        }
+        let caller = body.source.def_id();
+        let param_env = tcx.param_env(caller);
+
+        let func_ty = func.ty(body, tcx);
+        if let ty::FnDef(callee, substs) = *func_ty.kind() {
+            let normalized_substs = tcx.normalize_erasing_regions(param_env, substs);
+            let (callee, call_substs) = if let Ok(Some(instance)) =
+                Instance::resolve(tcx, param_env, callee, normalized_substs)
+            {
+                (instance.def_id(), instance.substs)
+            } else {
+                (callee, normalized_substs)
+            };
+
+            // FIXME(#57965): Make this work across function boundaries
+
+            // If this is a trait fn, the substs on the trait have to match, or we might be
+            // calling into an entirely different method (for example, a call from the default
+            // method in the trait to `<A as Trait<B>>::method`, where `A` and/or `B` are
+            // specific types).
+            return callee == caller && &call_substs[..trait_substs.len()] == trait_substs;
+        }
+
+        false
+    }
+}
+
+impl<'mir, 'tcx> TriColorVisitor<BasicBlocks<'tcx>> for Search<'mir, 'tcx> {
+    type BreakVal = NonRecursive;
+
+    fn node_examined(
+        &mut self,
+        bb: BasicBlock,
+        prior_status: Option<NodeStatus>,
+    ) -> ControlFlow<Self::BreakVal> {
+        // Back-edge in the CFG (loop).
+        if let Some(NodeStatus::Visited) = prior_status {
+            return ControlFlow::Break(NonRecursive);
+        }
+
+        match self.body[bb].terminator().kind {
+            // These terminators return control flow to the caller.
+            TerminatorKind::Abort
+            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::Resume
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive),
+
+            // A diverging InlineAsm is treated as non-recursing
+            TerminatorKind::InlineAsm { destination, .. } => {
+                if destination.is_some() {
+                    ControlFlow::CONTINUE
+                } else {
+                    ControlFlow::Break(NonRecursive)
+                }
+            }
+
+            // These do not.
+            TerminatorKind::Assert { .. }
+            | TerminatorKind::Call { .. }
+            | TerminatorKind::Drop { .. }
+            | TerminatorKind::DropAndReplace { .. }
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::FalseUnwind { .. }
+            | TerminatorKind::Goto { .. }
+            | TerminatorKind::SwitchInt { .. } => ControlFlow::CONTINUE,
+        }
+    }
+
+    fn node_settled(&mut self, bb: BasicBlock) -> ControlFlow<Self::BreakVal> {
+        // When we examine a node for the last time, remember it if it is a recursive call.
+        let terminator = self.body[bb].terminator();
+        if let TerminatorKind::Call { func, args, .. } = &terminator.kind {
+            if self.is_recursive_call(func, args) {
+                self.reachable_recursive_calls.push(terminator.source_info.span);
+            }
+        }
+
+        ControlFlow::CONTINUE
+    }
+
+    fn ignore_edge(&mut self, bb: BasicBlock, target: BasicBlock) -> bool {
+        let terminator = self.body[bb].terminator();
+        if terminator.unwind() == Some(&Some(target)) && terminator.successors().count() > 1 {
+            return true;
+        }
+        // Don't traverse successors of recursive calls or false CFG edges.
+        match self.body[bb].terminator().kind {
+            TerminatorKind::Call { ref func, ref args, .. } => self.is_recursive_call(func, args),
+            TerminatorKind::FalseEdge { imaginary_target, .. } => imaginary_target == target,
+            _ => false,
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs
new file mode 100644
index 000000000..a7e4403a2
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/constant.rs
@@ -0,0 +1,52 @@
+use rustc_ast as ast;
+use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
+use rustc_middle::ty::{self, ParamEnv, ScalarInt, TyCtxt};
+
+pub(crate) fn lit_to_const<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    lit_input: LitToConstInput<'tcx>,
+) -> Result<ty::Const<'tcx>, LitToConstError> {
+    let LitToConstInput { lit, ty, neg } = lit_input;
+
+    let trunc = |n| {
+        let param_ty = ParamEnv::reveal_all().and(ty);
+        let width = tcx.layout_of(param_ty).map_err(|_| LitToConstError::Reported)?.size;
+        trace!("trunc {} with size {} and shift {}", n, width.bits(), 128 - width.bits());
+        let result = width.truncate(n);
+        trace!("trunc result: {}", result);
+
+        Ok(ScalarInt::try_from_uint(result, width)
+            .unwrap_or_else(|| bug!("expected to create ScalarInt from uint {:?}", result)))
+    };
+
+    let valtree = match (lit, &ty.kind()) {
+        (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
+            let str_bytes = s.as_str().as_bytes();
+            ty::ValTree::from_raw_bytes(tcx, str_bytes)
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _))
+            if matches!(inner_ty.kind(), ty::Slice(_)) =>
+        {
+            let bytes = data as &[u8];
+            ty::ValTree::from_raw_bytes(tcx, bytes)
+        }
+        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
+            let bytes = data as &[u8];
+            ty::ValTree::from_raw_bytes(tcx, bytes)
+        }
+        (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
+            ty::ValTree::from_scalar_int((*n).into())
+        }
+        (ast::LitKind::Int(n, _), ty::Uint(_)) | (ast::LitKind::Int(n, _), ty::Int(_)) => {
+            let scalar_int =
+                trunc(if neg { (*n as i128).overflowing_neg().0 as u128 } else { *n })?;
+            ty::ValTree::from_scalar_int(scalar_int)
+        }
+        (ast::LitKind::Bool(b), ty::Bool) => ty::ValTree::from_scalar_int((*b).into()),
+        (ast::LitKind::Char(c), ty::Char) => ty::ValTree::from_scalar_int((*c).into()),
+        (ast::LitKind::Err(_), _) => return Err(LitToConstError::Reported),
+        _ => return Err(LitToConstError::TypeError),
+    };
+
+    Ok(ty::Const::from_value(tcx, valtree, ty))
+}
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
new file mode 100644
index 000000000..dccaa61ed
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -0,0 +1,126 @@
+use crate::thir::cx::Cx;
+
+use rustc_hir as hir;
+use rustc_middle::middle::region;
+use rustc_middle::thir::*;
+use rustc_middle::ty;
+
+use rustc_index::vec::Idx;
+use rustc_middle::ty::CanonicalUserTypeAnnotation;
+
+impl<'tcx> Cx<'tcx> {
+    pub(crate) fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> Block {
+        // We have to eagerly lower the "spine" of the statements
+        // in order to get the lexical scoping correctly.
+        let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts);
+        let opt_destruction_scope =
+            self.region_scope_tree.opt_destruction_scope(block.hir_id.local_id);
+        Block {
+            targeted_by_break: block.targeted_by_break,
+            region_scope: region::Scope {
+                id: block.hir_id.local_id,
+                data: region::ScopeData::Node,
+            },
+            opt_destruction_scope,
+            span: block.span,
+            stmts,
+            expr: block.expr.map(|expr| self.mirror_expr(expr)),
+            safety_mode: match block.rules {
+                hir::BlockCheckMode::DefaultBlock => BlockSafety::Safe,
+                hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::CompilerGenerated) => {
+                    BlockSafety::BuiltinUnsafe
+                }
+                hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::UserProvided) => {
+                    BlockSafety::ExplicitUnsafe(block.hir_id)
+                }
+            },
+        }
+    }
+
+    fn mirror_stmts(
+        &mut self,
+        block_id: hir::ItemLocalId,
+        stmts: &'tcx [hir::Stmt<'tcx>],
+    ) -> Box<[StmtId]> {
+        stmts
+            .iter()
+            .enumerate()
+            .filter_map(|(index, stmt)| {
+                let hir_id = stmt.hir_id;
+                let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id);
+                match stmt.kind {
+                    hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
+                        let stmt = Stmt {
+                            kind: StmtKind::Expr {
+                                scope: region::Scope {
+                                    id: hir_id.local_id,
+                                    data: region::ScopeData::Node,
+                                },
+                                expr: self.mirror_expr(expr),
+                            },
+                            opt_destruction_scope: opt_dxn_ext,
+                        };
+                        Some(self.thir.stmts.push(stmt))
+                    }
+                    hir::StmtKind::Item(..) => {
+                        // ignore for purposes of the MIR
+                        None
+                    }
+                    hir::StmtKind::Local(ref local) => {
+                        let remainder_scope = region::Scope {
+                            id: block_id,
+                            data: region::ScopeData::Remainder(region::FirstStatementIndex::new(
+                                index,
+                            )),
+                        };
+
+                        let else_block = local.els.map(|els| self.mirror_block(els));
+
+                        let mut pattern = self.pattern_from_hir(local.pat);
+                        debug!(?pattern);
+
+                        if let Some(ty) = &local.ty {
+                            if let Some(&user_ty) =
+                                self.typeck_results.user_provided_types().get(ty.hir_id)
+                            {
+                                debug!("mirror_stmts: user_ty={:?}", user_ty);
+                                let annotation = CanonicalUserTypeAnnotation {
+                                    user_ty,
+                                    span: ty.span,
+                                    inferred_ty: self.typeck_results.node_type(ty.hir_id),
+                                };
+                                pattern = Pat {
+                                    ty: pattern.ty,
+                                    span: pattern.span,
+                                    kind: Box::new(PatKind::AscribeUserType {
+                                        ascription: Ascription {
+                                            annotation,
+                                            variance: ty::Variance::Covariant,
+                                        },
+                                        subpattern: pattern,
+                                    }),
+                                };
+                            }
+                        }
+
+                        let stmt = Stmt {
+                            kind: StmtKind::Let {
+                                remainder_scope,
+                                init_scope: region::Scope {
+                                    id: hir_id.local_id,
+                                    data: region::ScopeData::Node,
+                                },
+                                pattern,
+                                initializer: local.init.map(|init| self.mirror_expr(init)),
+                                else_block,
+                                lint_level: LintLevel::Explicit(local.hir_id),
+                            },
+                            opt_destruction_scope: opt_dxn_ext,
+                        };
+                        Some(self.thir.stmts.push(stmt))
+                    }
+                }
+            })
+            .collect()
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
new file mode 100644
index 000000000..985601712
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -0,0 +1,1117 @@
+use crate::thir::cx::region::Scope;
+use crate::thir::cx::Cx;
+use crate::thir::util::UserAnnotatedTyHelpers;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir as hir;
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
+use rustc_index::vec::Idx;
+use rustc_middle::hir::place::Place as HirPlace;
+use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
+use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
+use rustc_middle::middle::region;
+use rustc_middle::mir::{self, BinOp, BorrowKind, Field, UnOp};
+use rustc_middle::thir::*;
+use rustc_middle::ty::adjustment::{
+    Adjust, Adjustment, AutoBorrow, AutoBorrowMutability, PointerCast,
+};
+use rustc_middle::ty::subst::{InternalSubsts, SubstsRef};
+use rustc_middle::ty::{
+    self, AdtKind, InlineConstSubsts, InlineConstSubstsParts, ScalarInt, Ty, UpvarSubsts, UserType,
+};
+use rustc_span::def_id::DefId;
+use rustc_span::Span;
+use rustc_target::abi::VariantIdx;
+
+impl<'tcx> Cx<'tcx> {
+    pub(crate) fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> ExprId {
+        // `mirror_expr` is recursing very deep. Make sure the stack doesn't overflow.
+        ensure_sufficient_stack(|| self.mirror_expr_inner(expr))
+    }
+
+    pub(crate) fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> Box<[ExprId]> {
+        exprs.iter().map(|expr| self.mirror_expr_inner(expr)).collect()
+    }
+
+    #[instrument(level = "trace", skip(self, hir_expr))]
+    pub(super) fn mirror_expr_inner(&mut self, hir_expr: &'tcx hir::Expr<'tcx>) -> ExprId {
+        let temp_lifetime =
+            self.rvalue_scopes.temporary_scope(self.region_scope_tree, hir_expr.hir_id.local_id);
+        let expr_scope =
+            region::Scope { id: hir_expr.hir_id.local_id, data: region::ScopeData::Node };
+
+        trace!(?hir_expr.hir_id, ?hir_expr.span);
+
+        let mut expr = self.make_mirror_unadjusted(hir_expr);
+
+        let adjustment_span = match self.adjustment_span {
+            Some((hir_id, span)) if hir_id == hir_expr.hir_id => Some(span),
+            _ => None,
+        };
+
+        // Now apply adjustments, if any.
+        for adjustment in self.typeck_results.expr_adjustments(hir_expr) {
+            trace!(?expr, ?adjustment);
+            let span = expr.span;
+            expr =
+                self.apply_adjustment(hir_expr, expr, adjustment, adjustment_span.unwrap_or(span));
+        }
+
+        // Next, wrap this up in the expr's scope.
+        expr = Expr {
+            temp_lifetime,
+            ty: expr.ty,
+            span: hir_expr.span,
+            kind: ExprKind::Scope {
+                region_scope: expr_scope,
+                value: self.thir.exprs.push(expr),
+                lint_level: LintLevel::Explicit(hir_expr.hir_id),
+            },
+        };
+
+        // Finally, create a destruction scope, if any.
+        if let Some(region_scope) =
+            self.region_scope_tree.opt_destruction_scope(hir_expr.hir_id.local_id)
+        {
+            expr = Expr {
+                temp_lifetime,
+                ty: expr.ty,
+                span: hir_expr.span,
+                kind: ExprKind::Scope {
+                    region_scope,
+                    value: self.thir.exprs.push(expr),
+                    lint_level: LintLevel::Inherited,
+                },
+            };
+        }
+
+        // OK, all done!
+        self.thir.exprs.push(expr)
+    }
+
+    fn apply_adjustment(
+        &mut self,
+        hir_expr: &'tcx hir::Expr<'tcx>,
+        mut expr: Expr<'tcx>,
+        adjustment: &Adjustment<'tcx>,
+        mut span: Span,
+    ) -> Expr<'tcx> {
+        let Expr { temp_lifetime, .. } = expr;
+
+        // Adjust the span from the block, to the last expression of the
+        // block. This is a better span when returning a mutable reference
+        // with too short a lifetime. The error message will use the span
+        // from the assignment to the return place, which should only point
+        // at the returned value, not the entire function body.
+        //
+        // fn return_short_lived<'a>(x: &'a mut i32) -> &'static mut i32 {
+        //      x
+        //   // ^ error message points at this expression.
+        // }
+        let mut adjust_span = |expr: &mut Expr<'tcx>| {
+            if let ExprKind::Block { body } = &expr.kind {
+                if let Some(last_expr) = body.expr {
+                    span = self.thir[last_expr].span;
+                    expr.span = span;
+                }
+            }
+        };
+
+        let kind = match adjustment.kind {
+            Adjust::Pointer(PointerCast::Unsize) => {
+                adjust_span(&mut expr);
+                ExprKind::Pointer { cast: PointerCast::Unsize, source: self.thir.exprs.push(expr) }
+            }
+            Adjust::Pointer(cast) => ExprKind::Pointer { cast, source: self.thir.exprs.push(expr) },
+            Adjust::NeverToAny => ExprKind::NeverToAny { source: self.thir.exprs.push(expr) },
+            Adjust::Deref(None) => {
+                adjust_span(&mut expr);
+                ExprKind::Deref { arg: self.thir.exprs.push(expr) }
+            }
+            Adjust::Deref(Some(deref)) => {
+                // We don't need to do call adjust_span here since
+                // deref coercions always start with a built-in deref.
+                let call = deref.method_call(self.tcx(), expr.ty);
+
+                expr = Expr {
+                    temp_lifetime,
+                    ty: self
+                        .tcx
+                        .mk_ref(deref.region, ty::TypeAndMut { ty: expr.ty, mutbl: deref.mutbl }),
+                    span,
+                    kind: ExprKind::Borrow {
+                        borrow_kind: deref.mutbl.to_borrow_kind(),
+                        arg: self.thir.exprs.push(expr),
+                    },
+                };
+
+                let expr = Box::new([self.thir.exprs.push(expr)]);
+
+                self.overloaded_place(hir_expr, adjustment.target, Some(call), expr, deref.span)
+            }
+            Adjust::Borrow(AutoBorrow::Ref(_, m)) => ExprKind::Borrow {
+                borrow_kind: m.to_borrow_kind(),
+                arg: self.thir.exprs.push(expr),
+            },
+            Adjust::Borrow(AutoBorrow::RawPtr(mutability)) => {
+                ExprKind::AddressOf { mutability, arg: self.thir.exprs.push(expr) }
+            }
+        };
+
+        Expr { temp_lifetime, ty: adjustment.target, span, kind }
+    }
+
+    /// Lowers a cast expression.
+    ///
+    /// Dealing with user type annotations is left to the caller.
+    fn mirror_expr_cast(
+        &mut self,
+        source: &'tcx hir::Expr<'tcx>,
+        temp_lifetime: Option<Scope>,
+        span: Span,
+    ) -> ExprKind<'tcx> {
+        let tcx = self.tcx;
+
+        // Check to see if this cast is a "coercion cast", where the cast is actually done
+        // using a coercion (or is a no-op).
+        if self.typeck_results().is_coercion_cast(source.hir_id) {
+            // Convert the lexpr to a vexpr.
+            ExprKind::Use { source: self.mirror_expr(source) }
+        } else if self.typeck_results().expr_ty(source).is_region_ptr() {
+            // Special cased so that we can type check that the element
+            // type of the source matches the pointed to type of the
+            // destination.
+            ExprKind::Pointer {
+                source: self.mirror_expr(source),
+                cast: PointerCast::ArrayToPointer,
+            }
+        } else {
+            // check whether this is casting an enum variant discriminant
+            // to prevent cycles, we refer to the discriminant initializer
+            // which is always an integer and thus doesn't need to know the
+            // enum's layout (or its tag type) to compute it during const eval
+            // Example:
+            // enum Foo {
+            //     A,
+            //     B = A as isize + 4,
+            // }
+            // The correct solution would be to add symbolic computations to miri,
+            // so we wouldn't have to compute and store the actual value
+
+            let hir::ExprKind::Path(ref qpath) = source.kind else {
+                return ExprKind::Cast { source: self.mirror_expr(source)};
+            };
+
+            let res = self.typeck_results().qpath_res(qpath, source.hir_id);
+            let ty = self.typeck_results().node_type(source.hir_id);
+            let ty::Adt(adt_def, substs) = ty.kind() else {
+                return ExprKind::Cast { source: self.mirror_expr(source)};
+            };
+
+            let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Const), variant_ctor_id) = res else {
+                return ExprKind::Cast { source: self.mirror_expr(source)};
+            };
+
+            let idx = adt_def.variant_index_with_ctor_id(variant_ctor_id);
+            let (discr_did, discr_offset) = adt_def.discriminant_def_for_variant(idx);
+
+            use rustc_middle::ty::util::IntTypeExt;
+            let ty = adt_def.repr().discr_type();
+            let discr_ty = ty.to_ty(tcx);
+
+            let param_env_ty = self.param_env.and(discr_ty);
+            let size = tcx
+                .layout_of(param_env_ty)
+                .unwrap_or_else(|e| {
+                    panic!("could not compute layout for {:?}: {:?}", param_env_ty, e)
+                })
+                .size;
+
+            let lit = ScalarInt::try_from_uint(discr_offset as u128, size).unwrap();
+            let kind = ExprKind::NonHirLiteral { lit, user_ty: None };
+            let offset = self.thir.exprs.push(Expr { temp_lifetime, ty: discr_ty, span, kind });
+
+            let source = match discr_did {
+                // in case we are offsetting from a computed discriminant
+                // and not the beginning of discriminants (which is always `0`)
+                Some(did) => {
+                    let kind = ExprKind::NamedConst { def_id: did, substs, user_ty: None };
+                    let lhs =
+                        self.thir.exprs.push(Expr { temp_lifetime, ty: discr_ty, span, kind });
+                    let bin = ExprKind::Binary { op: BinOp::Add, lhs, rhs: offset };
+                    self.thir.exprs.push(Expr {
+                        temp_lifetime,
+                        ty: discr_ty,
+                        span: span,
+                        kind: bin,
+                    })
+                }
+                None => offset,
+            };
+
+            ExprKind::Cast { source }
+        }
+    }
+
+    fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'tcx> {
+        let tcx = self.tcx;
+        let expr_ty = self.typeck_results().expr_ty(expr);
+        let expr_span = expr.span;
+        let temp_lifetime =
+            self.rvalue_scopes.temporary_scope(self.region_scope_tree, expr.hir_id.local_id);
+
+        let kind = match expr.kind {
+            // Here comes the interesting stuff:
+            hir::ExprKind::MethodCall(segment, ref args, fn_span) => {
+                // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
+                let expr = self.method_callee(expr, segment.ident.span, None);
+                // When we apply adjustments to the receiver, use the span of
+                // the overall method call for better diagnostics. args[0]
+                // is guaranteed to exist, since a method call always has a receiver.
+                let old_adjustment_span = self.adjustment_span.replace((args[0].hir_id, expr_span));
+                tracing::info!("Using method span: {:?}", expr.span);
+                let args = self.mirror_exprs(args);
+                self.adjustment_span = old_adjustment_span;
+                ExprKind::Call {
+                    ty: expr.ty,
+                    fun: self.thir.exprs.push(expr),
+                    args,
+                    from_hir_call: true,
+                    fn_span,
+                }
+            }
+
+            hir::ExprKind::Call(ref fun, ref args) => {
+                if self.typeck_results().is_method_call(expr) {
+                    // The callee is something implementing Fn, FnMut, or FnOnce.
+                    // Find the actual method implementation being called and
+                    // build the appropriate UFCS call expression with the
+                    // callee-object as expr parameter.
+
+                    // rewrite f(u, v) into FnOnce::call_once(f, (u, v))
+
+                    let method = self.method_callee(expr, fun.span, None);
+
+                    let arg_tys = args.iter().map(|e| self.typeck_results().expr_ty_adjusted(e));
+                    let tupled_args = Expr {
+                        ty: tcx.mk_tup(arg_tys),
+                        temp_lifetime,
+                        span: expr.span,
+                        kind: ExprKind::Tuple { fields: self.mirror_exprs(args) },
+                    };
+                    let tupled_args = self.thir.exprs.push(tupled_args);
+
+                    ExprKind::Call {
+                        ty: method.ty,
+                        fun: self.thir.exprs.push(method),
+                        args: Box::new([self.mirror_expr(fun), tupled_args]),
+                        from_hir_call: true,
+                        fn_span: expr.span,
+                    }
+                } else {
+                    let adt_data =
+                        if let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = fun.kind {
+                            // Tuple-like ADTs are represented as ExprKind::Call. We convert them here.
+                            expr_ty.ty_adt_def().and_then(|adt_def| match path.res {
+                                Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
+                                    Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
+                                }
+                                Res::SelfCtor(..) => Some((adt_def, VariantIdx::new(0))),
+                                _ => None,
+                            })
+                        } else {
+                            None
+                        };
+                    if let Some((adt_def, index)) = adt_data {
+                        let substs = self.typeck_results().node_substs(fun.hir_id);
+                        let user_provided_types = self.typeck_results().user_provided_types();
+                        let user_ty =
+                            user_provided_types.get(fun.hir_id).copied().map(|mut u_ty| {
+                                if let UserType::TypeOf(ref mut did, _) = &mut u_ty.value {
+                                    *did = adt_def.did();
+                                }
+                                u_ty
+                            });
+                        debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty);
+
+                        let field_refs = args
+                            .iter()
+                            .enumerate()
+                            .map(|(idx, e)| FieldExpr {
+                                name: Field::new(idx),
+                                expr: self.mirror_expr(e),
+                            })
+                            .collect();
+                        ExprKind::Adt(Box::new(Adt {
+                            adt_def,
+                            substs,
+                            variant_index: index,
+                            fields: field_refs,
+                            user_ty,
+                            base: None,
+                        }))
+                    } else {
+                        ExprKind::Call {
+                            ty: self.typeck_results().node_type(fun.hir_id),
+                            fun: self.mirror_expr(fun),
+                            args: self.mirror_exprs(args),
+                            from_hir_call: true,
+                            fn_span: expr.span,
+                        }
+                    }
+                }
+            }
+
+            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, ref arg) => {
+                ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: self.mirror_expr(arg) }
+            }
+
+            hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, ref arg) => {
+                ExprKind::AddressOf { mutability, arg: self.mirror_expr(arg) }
+            }
+
+            hir::ExprKind::Block(ref blk, _) => ExprKind::Block { body: self.mirror_block(blk) },
+
+            hir::ExprKind::Assign(ref lhs, ref rhs, _) => {
+                ExprKind::Assign { lhs: self.mirror_expr(lhs), rhs: self.mirror_expr(rhs) }
+            }
+
+            hir::ExprKind::AssignOp(op, ref lhs, ref rhs) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr(lhs);
+                    let rhs = self.mirror_expr(rhs);
+                    self.overloaded_operator(expr, Box::new([lhs, rhs]))
+                } else {
+                    ExprKind::AssignOp {
+                        op: bin_op(op.node),
+                        lhs: self.mirror_expr(lhs),
+                        rhs: self.mirror_expr(rhs),
+                    }
+                }
+            }
+
+            hir::ExprKind::Lit(ref lit) => ExprKind::Literal { lit, neg: false },
+
+            hir::ExprKind::Binary(op, ref lhs, ref rhs) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr(lhs);
+                    let rhs = self.mirror_expr(rhs);
+                    self.overloaded_operator(expr, Box::new([lhs, rhs]))
+                } else {
+                    // FIXME overflow
+                    match op.node {
+                        hir::BinOpKind::And => ExprKind::LogicalOp {
+                            op: LogicalOp::And,
+                            lhs: self.mirror_expr(lhs),
+                            rhs: self.mirror_expr(rhs),
+                        },
+                        hir::BinOpKind::Or => ExprKind::LogicalOp {
+                            op: LogicalOp::Or,
+                            lhs: self.mirror_expr(lhs),
+                            rhs: self.mirror_expr(rhs),
+                        },
+                        _ => {
+                            let op = bin_op(op.node);
+                            ExprKind::Binary {
+                                op,
+                                lhs: self.mirror_expr(lhs),
+                                rhs: self.mirror_expr(rhs),
+                            }
+                        }
+                    }
+                }
+            }
+
+            hir::ExprKind::Index(ref lhs, ref index) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let lhs = self.mirror_expr(lhs);
+                    let index = self.mirror_expr(index);
+                    self.overloaded_place(expr, expr_ty, None, Box::new([lhs, index]), expr.span)
+                } else {
+                    ExprKind::Index { lhs: self.mirror_expr(lhs), index: self.mirror_expr(index) }
+                }
+            }
+
+            hir::ExprKind::Unary(hir::UnOp::Deref, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr(arg);
+                    self.overloaded_place(expr, expr_ty, None, Box::new([arg]), expr.span)
+                } else {
+                    ExprKind::Deref { arg: self.mirror_expr(arg) }
+                }
+            }
+
+            hir::ExprKind::Unary(hir::UnOp::Not, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr(arg);
+                    self.overloaded_operator(expr, Box::new([arg]))
+                } else {
+                    ExprKind::Unary { op: UnOp::Not, arg: self.mirror_expr(arg) }
+                }
+            }
+
+            hir::ExprKind::Unary(hir::UnOp::Neg, ref arg) => {
+                if self.typeck_results().is_method_call(expr) {
+                    let arg = self.mirror_expr(arg);
+                    self.overloaded_operator(expr, Box::new([arg]))
+                } else if let hir::ExprKind::Lit(ref lit) = arg.kind {
+                    ExprKind::Literal { lit, neg: true }
+                } else {
+                    ExprKind::Unary { op: UnOp::Neg, arg: self.mirror_expr(arg) }
+                }
+            }
+
+            hir::ExprKind::Struct(ref qpath, ref fields, ref base) => match expr_ty.kind() {
+                ty::Adt(adt, substs) => match adt.adt_kind() {
+                    AdtKind::Struct | AdtKind::Union => {
+                        let user_provided_types = self.typeck_results().user_provided_types();
+                        let user_ty = user_provided_types.get(expr.hir_id).copied();
+                        debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
+                        ExprKind::Adt(Box::new(Adt {
+                            adt_def: *adt,
+                            variant_index: VariantIdx::new(0),
+                            substs,
+                            user_ty,
+                            fields: self.field_refs(fields),
+                            base: base.as_ref().map(|base| FruInfo {
+                                base: self.mirror_expr(base),
+                                field_types: self.typeck_results().fru_field_types()[expr.hir_id]
+                                    .iter()
+                                    .copied()
+                                    .collect(),
+                            }),
+                        }))
+                    }
+                    AdtKind::Enum => {
+                        let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
+                        match res {
+                            Res::Def(DefKind::Variant, variant_id) => {
+                                assert!(base.is_none());
+
+                                let index = adt.variant_index_with_id(variant_id);
+                                let user_provided_types =
+                                    self.typeck_results().user_provided_types();
+                                let user_ty = user_provided_types.get(expr.hir_id).copied();
+                                debug!("make_mirror_unadjusted: (variant) user_ty={:?}", user_ty);
+                                ExprKind::Adt(Box::new(Adt {
+                                    adt_def: *adt,
+                                    variant_index: index,
+                                    substs,
+                                    user_ty,
+                                    fields: self.field_refs(fields),
+                                    base: None,
+                                }))
+                            }
+                            _ => {
+                                span_bug!(expr.span, "unexpected res: {:?}", res);
+                            }
+                        }
+                    }
+                },
+                _ => {
+                    span_bug!(expr.span, "unexpected type for struct literal: {:?}", expr_ty);
+                }
+            },
+
+            hir::ExprKind::Closure { .. } => {
+                let closure_ty = self.typeck_results().expr_ty(expr);
+                let (def_id, substs, movability) = match *closure_ty.kind() {
+                    ty::Closure(def_id, substs) => (def_id, UpvarSubsts::Closure(substs), None),
+                    ty::Generator(def_id, substs, movability) => {
+                        (def_id, UpvarSubsts::Generator(substs), Some(movability))
+                    }
+                    _ => {
+                        span_bug!(expr.span, "closure expr w/o closure type: {:?}", closure_ty);
+                    }
+                };
+                let def_id = def_id.expect_local();
+
+                let upvars = self
+                    .typeck_results
+                    .closure_min_captures_flattened(def_id)
+                    .zip(substs.upvar_tys())
+                    .map(|(captured_place, ty)| {
+                        let upvars = self.capture_upvar(expr, captured_place, ty);
+                        self.thir.exprs.push(upvars)
+                    })
+                    .collect();
+
+                // Convert the closure fake reads, if any, from hir `Place` to ExprRef
+                let fake_reads = match self.typeck_results.closure_fake_reads.get(&def_id) {
+                    Some(fake_reads) => fake_reads
+                        .iter()
+                        .map(|(place, cause, hir_id)| {
+                            let expr = self.convert_captured_hir_place(expr, place.clone());
+                            (self.thir.exprs.push(expr), *cause, *hir_id)
+                        })
+                        .collect(),
+                    None => Vec::new(),
+                };
+
+                ExprKind::Closure { closure_id: def_id, substs, upvars, movability, fake_reads }
+            }
+
+            hir::ExprKind::Path(ref qpath) => {
+                let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
+                self.convert_path_expr(expr, res)
+            }
+
+            hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm {
+                template: asm.template,
+                operands: asm
+                    .operands
+                    .iter()
+                    .map(|(op, _op_sp)| match *op {
+                        hir::InlineAsmOperand::In { reg, ref expr } => {
+                            InlineAsmOperand::In { reg, expr: self.mirror_expr(expr) }
+                        }
+                        hir::InlineAsmOperand::Out { reg, late, ref expr } => {
+                            InlineAsmOperand::Out {
+                                reg,
+                                late,
+                                expr: expr.as_ref().map(|expr| self.mirror_expr(expr)),
+                            }
+                        }
+                        hir::InlineAsmOperand::InOut { reg, late, ref expr } => {
+                            InlineAsmOperand::InOut { reg, late, expr: self.mirror_expr(expr) }
+                        }
+                        hir::InlineAsmOperand::SplitInOut {
+                            reg,
+                            late,
+                            ref in_expr,
+                            ref out_expr,
+                        } => InlineAsmOperand::SplitInOut {
+                            reg,
+                            late,
+                            in_expr: self.mirror_expr(in_expr),
+                            out_expr: out_expr.as_ref().map(|expr| self.mirror_expr(expr)),
+                        },
+                        hir::InlineAsmOperand::Const { ref anon_const } => {
+                            let anon_const_def_id = tcx.hir().local_def_id(anon_const.hir_id);
+                            let value = mir::ConstantKind::from_anon_const(
+                                tcx,
+                                anon_const_def_id,
+                                self.param_env,
+                            );
+                            let span = tcx.hir().span(anon_const.hir_id);
+
+                            InlineAsmOperand::Const { value, span }
+                        }
+                        hir::InlineAsmOperand::SymFn { ref anon_const } => {
+                            let anon_const_def_id = tcx.hir().local_def_id(anon_const.hir_id);
+                            let value = mir::ConstantKind::from_anon_const(
+                                tcx,
+                                anon_const_def_id,
+                                self.param_env,
+                            );
+                            let span = tcx.hir().span(anon_const.hir_id);
+
+                            InlineAsmOperand::SymFn { value, span }
+                        }
+                        hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
+                            InlineAsmOperand::SymStatic { def_id }
+                        }
+                    })
+                    .collect(),
+                options: asm.options,
+                line_spans: asm.line_spans,
+            },
+
+            hir::ExprKind::ConstBlock(ref anon_const) => {
+                let ty = self.typeck_results().node_type(anon_const.hir_id);
+                let did = tcx.hir().local_def_id(anon_const.hir_id).to_def_id();
+                let typeck_root_def_id = tcx.typeck_root_def_id(did);
+                let parent_substs =
+                    tcx.erase_regions(InternalSubsts::identity_for_item(tcx, typeck_root_def_id));
+                let substs =
+                    InlineConstSubsts::new(tcx, InlineConstSubstsParts { parent_substs, ty })
+                        .substs;
+
+                ExprKind::ConstBlock { did, substs }
+            }
+            // Now comes the rote stuff:
+            hir::ExprKind::Repeat(ref v, _) => {
+                let ty = self.typeck_results().expr_ty(expr);
+                let ty::Array(_, count) = ty.kind() else {
+                    span_bug!(expr.span, "unexpected repeat expr ty: {:?}", ty);
+                };
+
+                ExprKind::Repeat { value: self.mirror_expr(v), count: *count }
+            }
+            hir::ExprKind::Ret(ref v) => {
+                ExprKind::Return { value: v.as_ref().map(|v| self.mirror_expr(v)) }
+            }
+            hir::ExprKind::Break(dest, ref value) => match dest.target_id {
+                Ok(target_id) => ExprKind::Break {
+                    label: region::Scope { id: target_id.local_id, data: region::ScopeData::Node },
+                    value: value.as_ref().map(|value| self.mirror_expr(value)),
+                },
+                Err(err) => bug!("invalid loop id for break: {}", err),
+            },
+            hir::ExprKind::Continue(dest) => match dest.target_id {
+                Ok(loop_id) => ExprKind::Continue {
+                    label: region::Scope { id: loop_id.local_id, data: region::ScopeData::Node },
+                },
+                Err(err) => bug!("invalid loop id for continue: {}", err),
+            },
+            hir::ExprKind::Let(let_expr) => ExprKind::Let {
+                expr: self.mirror_expr(let_expr.init),
+                pat: self.pattern_from_hir(let_expr.pat),
+            },
+            hir::ExprKind::If(cond, then, else_opt) => ExprKind::If {
+                if_then_scope: region::Scope {
+                    id: then.hir_id.local_id,
+                    data: region::ScopeData::IfThen,
+                },
+                cond: self.mirror_expr(cond),
+                then: self.mirror_expr(then),
+                else_opt: else_opt.map(|el| self.mirror_expr(el)),
+            },
+            hir::ExprKind::Match(ref discr, ref arms, _) => ExprKind::Match {
+                scrutinee: self.mirror_expr(discr),
+                arms: arms.iter().map(|a| self.convert_arm(a)).collect(),
+            },
+            hir::ExprKind::Loop(ref body, ..) => {
+                let block_ty = self.typeck_results().node_type(body.hir_id);
+                let temp_lifetime = self
+                    .rvalue_scopes
+                    .temporary_scope(self.region_scope_tree, body.hir_id.local_id);
+                let block = self.mirror_block(body);
+                let body = self.thir.exprs.push(Expr {
+                    ty: block_ty,
+                    temp_lifetime,
+                    span: block.span,
+                    kind: ExprKind::Block { body: block },
+                });
+                ExprKind::Loop { body }
+            }
+            hir::ExprKind::Field(ref source, ..) => ExprKind::Field {
+                lhs: self.mirror_expr(source),
+                variant_index: VariantIdx::new(0),
+                name: Field::new(tcx.field_index(expr.hir_id, self.typeck_results)),
+            },
+            hir::ExprKind::Cast(ref source, ref cast_ty) => {
+                // Check for a user-given type annotation on this `cast`
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_ty = user_provided_types.get(cast_ty.hir_id);
+
+                debug!(
+                    "cast({:?}) has ty w/ hir_id {:?} and user provided ty {:?}",
+                    expr, cast_ty.hir_id, user_ty,
+                );
+
+                let cast = self.mirror_expr_cast(*source, temp_lifetime, expr.span);
+
+                if let Some(user_ty) = user_ty {
+                    // NOTE: Creating a new Expr and wrapping a Cast inside of it may be
+                    //       inefficient, revisit this when performance becomes an issue.
+                    let cast_expr = self.thir.exprs.push(Expr {
+                        temp_lifetime,
+                        ty: expr_ty,
+                        span: expr.span,
+                        kind: cast,
+                    });
+                    debug!("make_mirror_unadjusted: (cast) user_ty={:?}", user_ty);
+
+                    ExprKind::ValueTypeAscription { source: cast_expr, user_ty: Some(*user_ty) }
+                } else {
+                    cast
+                }
+            }
+            hir::ExprKind::Type(ref source, ref ty) => {
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_ty = user_provided_types.get(ty.hir_id).copied();
+                debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
+                let mirrored = self.mirror_expr(source);
+                if source.is_syntactic_place_expr() {
+                    ExprKind::PlaceTypeAscription { source: mirrored, user_ty }
+                } else {
+                    ExprKind::ValueTypeAscription { source: mirrored, user_ty }
+                }
+            }
+            hir::ExprKind::DropTemps(ref source) => {
+                ExprKind::Use { source: self.mirror_expr(source) }
+            }
+            hir::ExprKind::Box(ref value) => ExprKind::Box { value: self.mirror_expr(value) },
+            hir::ExprKind::Array(ref fields) => {
+                ExprKind::Array { fields: self.mirror_exprs(fields) }
+            }
+            hir::ExprKind::Tup(ref fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },
+
+            hir::ExprKind::Yield(ref v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
+            hir::ExprKind::Err => unreachable!(),
+        };
+
+        Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
+    }
+
+    fn user_substs_applied_to_res(
+        &mut self,
+        hir_id: hir::HirId,
+        res: Res,
+    ) -> Option<ty::CanonicalUserType<'tcx>> {
+        debug!("user_substs_applied_to_res: res={:?}", res);
+        let user_provided_type = match res {
+            // A reference to something callable -- e.g., a fn, method, or
+            // a tuple-struct or tuple-variant. This has the type of a
+            // `Fn` but with the user-given substitutions.
+            Res::Def(DefKind::Fn, _)
+            | Res::Def(DefKind::AssocFn, _)
+            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
+            | Res::Def(DefKind::Const, _)
+            | Res::Def(DefKind::AssocConst, _) => {
+                self.typeck_results().user_provided_types().get(hir_id).copied()
+            }
+
+            // A unit struct/variant which is used as a value (e.g.,
+            // `None`). This has the type of the enum/struct that defines
+            // this variant -- but with the substitutions given by the
+            // user.
+            Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => {
+                self.user_substs_applied_to_ty_of_hir_id(hir_id)
+            }
+
+            // `Self` is used in expression as a tuple struct constructor or a unit struct constructor
+            Res::SelfCtor(_) => self.user_substs_applied_to_ty_of_hir_id(hir_id),
+
+            _ => bug!("user_substs_applied_to_res: unexpected res {:?} at {:?}", res, hir_id),
+        };
+        debug!("user_substs_applied_to_res: user_provided_type={:?}", user_provided_type);
+        user_provided_type
+    }
+
+    fn method_callee(
+        &mut self,
+        expr: &hir::Expr<'_>,
+        span: Span,
+        overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
+    ) -> Expr<'tcx> {
+        let temp_lifetime =
+            self.rvalue_scopes.temporary_scope(self.region_scope_tree, expr.hir_id.local_id);
+        let (def_id, substs, user_ty) = match overloaded_callee {
+            Some((def_id, substs)) => (def_id, substs, None),
+            None => {
+                let (kind, def_id) =
+                    self.typeck_results().type_dependent_def(expr.hir_id).unwrap_or_else(|| {
+                        span_bug!(expr.span, "no type-dependent def for method callee")
+                    });
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, Res::Def(kind, def_id));
+                debug!("method_callee: user_ty={:?}", user_ty);
+                (def_id, self.typeck_results().node_substs(expr.hir_id), user_ty)
+            }
+        };
+        let ty = self.tcx().mk_fn_def(def_id, substs);
+        Expr { temp_lifetime, ty, span, kind: ExprKind::ZstLiteral { user_ty } }
+    }
+
+    fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> ArmId {
+        let arm = Arm {
+            pattern: self.pattern_from_hir(&arm.pat),
+            guard: arm.guard.as_ref().map(|g| match g {
+                hir::Guard::If(ref e) => Guard::If(self.mirror_expr(e)),
+                hir::Guard::IfLet(ref l) => {
+                    Guard::IfLet(self.pattern_from_hir(l.pat), self.mirror_expr(l.init))
+                }
+            }),
+            body: self.mirror_expr(arm.body),
+            lint_level: LintLevel::Explicit(arm.hir_id),
+            scope: region::Scope { id: arm.hir_id.local_id, data: region::ScopeData::Node },
+            span: arm.span,
+        };
+        self.thir.arms.push(arm)
+    }
+
+    fn convert_path_expr(&mut self, expr: &'tcx hir::Expr<'tcx>, res: Res) -> ExprKind<'tcx> {
+        let substs = self.typeck_results().node_substs(expr.hir_id);
+        match res {
+            // A regular function, constructor function or a constant.
+            Res::Def(DefKind::Fn, _)
+            | Res::Def(DefKind::AssocFn, _)
+            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
+            | Res::SelfCtor(_) => {
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
+                ExprKind::ZstLiteral { user_ty }
+            }
+
+            Res::Def(DefKind::ConstParam, def_id) => {
+                let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+                let item_id = self.tcx.hir().get_parent_node(hir_id);
+                let item_def_id = self.tcx.hir().local_def_id(item_id);
+                let generics = self.tcx.generics_of(item_def_id);
+                let index = generics.param_def_id_to_index[&def_id];
+                let name = self.tcx.hir().name(hir_id);
+                let param = ty::ParamConst::new(index, name);
+
+                ExprKind::ConstParam { param, def_id }
+            }
+
+            Res::Def(DefKind::Const, def_id) | Res::Def(DefKind::AssocConst, def_id) => {
+                let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
+                ExprKind::NamedConst { def_id, substs, user_ty: user_ty }
+            }
+
+            Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id) => {
+                let user_provided_types = self.typeck_results.user_provided_types();
+                let user_provided_type = user_provided_types.get(expr.hir_id).copied();
+                debug!("convert_path_expr: user_provided_type={:?}", user_provided_type);
+                let ty = self.typeck_results().node_type(expr.hir_id);
+                match ty.kind() {
+                    // A unit struct/variant which is used as a value.
+                    // We return a completely different ExprKind here to account for this special case.
+                    ty::Adt(adt_def, substs) => ExprKind::Adt(Box::new(Adt {
+                        adt_def: *adt_def,
+                        variant_index: adt_def.variant_index_with_ctor_id(def_id),
+                        substs,
+                        user_ty: user_provided_type,
+                        fields: Box::new([]),
+                        base: None,
+                    })),
+                    _ => bug!("unexpected ty: {:?}", ty),
+                }
+            }
+
+            // We encode uses of statics as a `*&STATIC` where the `&STATIC` part is
+            // a constant reference (or constant raw pointer for `static mut`) in MIR
+            Res::Def(DefKind::Static(_), id) => {
+                let ty = self.tcx.static_ptr_ty(id);
+                let temp_lifetime = self
+                    .rvalue_scopes
+                    .temporary_scope(self.region_scope_tree, expr.hir_id.local_id);
+                let kind = if self.tcx.is_thread_local_static(id) {
+                    ExprKind::ThreadLocalRef(id)
+                } else {
+                    let alloc_id = self.tcx.create_static_alloc(id);
+                    ExprKind::StaticRef { alloc_id, ty, def_id: id }
+                };
+                ExprKind::Deref {
+                    arg: self.thir.exprs.push(Expr { ty, temp_lifetime, span: expr.span, kind }),
+                }
+            }
+
+            Res::Local(var_hir_id) => self.convert_var(var_hir_id),
+
+            _ => span_bug!(expr.span, "res `{:?}` not yet implemented", res),
+        }
+    }
+
+    fn convert_var(&mut self, var_hir_id: hir::HirId) -> ExprKind<'tcx> {
+        // We want upvars here not captures.
+        // Captures will be handled in MIR.
+        let is_upvar = self
+            .tcx
+            .upvars_mentioned(self.body_owner)
+            .map_or(false, |upvars| upvars.contains_key(&var_hir_id));
+
+        debug!(
+            "convert_var({:?}): is_upvar={}, body_owner={:?}",
+            var_hir_id, is_upvar, self.body_owner
+        );
+
+        if is_upvar {
+            ExprKind::UpvarRef {
+                closure_def_id: self.body_owner,
+                var_hir_id: LocalVarId(var_hir_id),
+            }
+        } else {
+            ExprKind::VarRef { id: LocalVarId(var_hir_id) }
+        }
+    }
+
+    fn overloaded_operator(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+        args: Box<[ExprId]>,
+    ) -> ExprKind<'tcx> {
+        let fun = self.method_callee(expr, expr.span, None);
+        let fun = self.thir.exprs.push(fun);
+        ExprKind::Call {
+            ty: self.thir[fun].ty,
+            fun,
+            args,
+            from_hir_call: false,
+            fn_span: expr.span,
+        }
+    }
+
+    fn overloaded_place(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+        place_ty: Ty<'tcx>,
+        overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>,
+        args: Box<[ExprId]>,
+        span: Span,
+    ) -> ExprKind<'tcx> {
+        // For an overloaded *x or x[y] expression of type T, the method
+        // call returns an &T and we must add the deref so that the types
+        // line up (this is because `*x` and `x[y]` represent places):
+
+        // Reconstruct the output assuming it's a reference with the
+        // same region and mutability as the receiver. This holds for
+        // `Deref(Mut)::Deref(_mut)` and `Index(Mut)::index(_mut)`.
+        let ty::Ref(region, _, mutbl) = *self.thir[args[0]].ty.kind() else {
+            span_bug!(span, "overloaded_place: receiver is not a reference");
+        };
+        let ref_ty = self.tcx.mk_ref(region, ty::TypeAndMut { ty: place_ty, mutbl });
+
+        // construct the complete expression `foo()` for the overloaded call,
+        // which will yield the &T type
+        let temp_lifetime =
+            self.rvalue_scopes.temporary_scope(self.region_scope_tree, expr.hir_id.local_id);
+        let fun = self.method_callee(expr, span, overloaded_callee);
+        let fun = self.thir.exprs.push(fun);
+        let fun_ty = self.thir[fun].ty;
+        let ref_expr = self.thir.exprs.push(Expr {
+            temp_lifetime,
+            ty: ref_ty,
+            span,
+            kind: ExprKind::Call { ty: fun_ty, fun, args, from_hir_call: false, fn_span: span },
+        });
+
+        // construct and return a deref wrapper `*foo()`
+        ExprKind::Deref { arg: ref_expr }
+    }
+
+    fn convert_captured_hir_place(
+        &mut self,
+        closure_expr: &'tcx hir::Expr<'tcx>,
+        place: HirPlace<'tcx>,
+    ) -> Expr<'tcx> {
+        let temp_lifetime = self
+            .rvalue_scopes
+            .temporary_scope(self.region_scope_tree, closure_expr.hir_id.local_id);
+        let var_ty = place.base_ty;
+
+        // The result of capture analysis in `rustc_typeck/check/upvar.rs`represents a captured path
+        // as it's seen for use within the closure and not at the time of closure creation.
+        //
+        // That is we see expect to see it start from a captured upvar and not something that is local
+        // to the closure's parent.
+        let var_hir_id = match place.base {
+            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
+            base => bug!("Expected an upvar, found {:?}", base),
+        };
+
+        let mut captured_place_expr = Expr {
+            temp_lifetime,
+            ty: var_ty,
+            span: closure_expr.span,
+            kind: self.convert_var(var_hir_id),
+        };
+
+        for proj in place.projections.iter() {
+            let kind = match proj.kind {
+                HirProjectionKind::Deref => {
+                    ExprKind::Deref { arg: self.thir.exprs.push(captured_place_expr) }
+                }
+                HirProjectionKind::Field(field, variant_index) => ExprKind::Field {
+                    lhs: self.thir.exprs.push(captured_place_expr),
+                    variant_index,
+                    name: Field::new(field as usize),
+                },
+                HirProjectionKind::Index | HirProjectionKind::Subslice => {
+                    // We don't capture these projections, so we can ignore them here
+                    continue;
+                }
+            };
+
+            captured_place_expr =
+                Expr { temp_lifetime, ty: proj.ty, span: closure_expr.span, kind };
+        }
+
+        captured_place_expr
+    }
+
+    fn capture_upvar(
+        &mut self,
+        closure_expr: &'tcx hir::Expr<'tcx>,
+        captured_place: &'tcx ty::CapturedPlace<'tcx>,
+        upvar_ty: Ty<'tcx>,
+    ) -> Expr<'tcx> {
+        let upvar_capture = captured_place.info.capture_kind;
+        let captured_place_expr =
+            self.convert_captured_hir_place(closure_expr, captured_place.place.clone());
+        let temp_lifetime = self
+            .rvalue_scopes
+            .temporary_scope(self.region_scope_tree, closure_expr.hir_id.local_id);
+
+        match upvar_capture {
+            ty::UpvarCapture::ByValue => captured_place_expr,
+            ty::UpvarCapture::ByRef(upvar_borrow) => {
+                let borrow_kind = match upvar_borrow {
+                    ty::BorrowKind::ImmBorrow => BorrowKind::Shared,
+                    ty::BorrowKind::UniqueImmBorrow => BorrowKind::Unique,
+                    ty::BorrowKind::MutBorrow => BorrowKind::Mut { allow_two_phase_borrow: false },
+                };
+                Expr {
+                    temp_lifetime,
+                    ty: upvar_ty,
+                    span: closure_expr.span,
+                    kind: ExprKind::Borrow {
+                        borrow_kind,
+                        arg: self.thir.exprs.push(captured_place_expr),
+                    },
+                }
+            }
+        }
+    }
+
+    /// Converts a list of named fields (i.e., for struct-like struct/enum ADTs) into FieldExpr.
+    fn field_refs(&mut self, fields: &'tcx [hir::ExprField<'tcx>]) -> Box<[FieldExpr]> {
+        fields
+            .iter()
+            .map(|field| FieldExpr {
+                name: Field::new(self.tcx.field_index(field.hir_id, self.typeck_results)),
+                expr: self.mirror_expr(field.expr),
+            })
+            .collect()
+    }
+}
+
+trait ToBorrowKind {
+    fn to_borrow_kind(&self) -> BorrowKind;
+}
+
+impl ToBorrowKind for AutoBorrowMutability {
+    fn to_borrow_kind(&self) -> BorrowKind {
+        use rustc_middle::ty::adjustment::AllowTwoPhase;
+        match *self {
+            AutoBorrowMutability::Mut { allow_two_phase_borrow } => BorrowKind::Mut {
+                allow_two_phase_borrow: match allow_two_phase_borrow {
+                    AllowTwoPhase::Yes => true,
+                    AllowTwoPhase::No => false,
+                },
+            },
+            AutoBorrowMutability::Not => BorrowKind::Shared,
+        }
+    }
+}
+
+impl ToBorrowKind for hir::Mutability {
+    fn to_borrow_kind(&self) -> BorrowKind {
+        match *self {
+            hir::Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
+            hir::Mutability::Not => BorrowKind::Shared,
+        }
+    }
+}
+
+fn bin_op(op: hir::BinOpKind) -> BinOp {
+    match op {
+        hir::BinOpKind::Add => BinOp::Add,
+        hir::BinOpKind::Sub => BinOp::Sub,
+        hir::BinOpKind::Mul => BinOp::Mul,
+        hir::BinOpKind::Div => BinOp::Div,
+        hir::BinOpKind::Rem => BinOp::Rem,
+        hir::BinOpKind::BitXor => BinOp::BitXor,
+        hir::BinOpKind::BitAnd => BinOp::BitAnd,
+        hir::BinOpKind::BitOr => BinOp::BitOr,
+        hir::BinOpKind::Shl => BinOp::Shl,
+        hir::BinOpKind::Shr => BinOp::Shr,
+        hir::BinOpKind::Eq => BinOp::Eq,
+        hir::BinOpKind::Lt => BinOp::Lt,
+        hir::BinOpKind::Le => BinOp::Le,
+        hir::BinOpKind::Ne => BinOp::Ne,
+        hir::BinOpKind::Ge => BinOp::Ge,
+        hir::BinOpKind::Gt => BinOp::Gt,
+        _ => bug!("no equivalent for ast binop {:?}", op),
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
new file mode 100644
index 000000000..f7351a4ca
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -0,0 +1,101 @@
+//! This module contains the functionality to convert from the wacky tcx data
+//! structures into the THIR. The `builder` is generally ignorant of the tcx,
+//! etc., and instead goes through the `Cx` for most of its work.
+
+use crate::thir::pattern::pat_from_hir;
+use crate::thir::util::UserAnnotatedTyHelpers;
+
+use rustc_data_structures::steal::Steal;
+use rustc_errors::ErrorGuaranteed;
+use rustc_hir as hir;
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::HirId;
+use rustc_hir::Node;
+use rustc_middle::middle::region;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, RvalueScopes, TyCtxt};
+use rustc_span::Span;
+
+pub(crate) fn thir_body<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    owner_def: ty::WithOptConstParam<LocalDefId>,
+) -> Result<(&'tcx Steal<Thir<'tcx>>, ExprId), ErrorGuaranteed> {
+    let hir = tcx.hir();
+    let body = hir.body(hir.body_owned_by(owner_def.did));
+    let mut cx = Cx::new(tcx, owner_def);
+    if let Some(reported) = cx.typeck_results.tainted_by_errors {
+        return Err(reported);
+    }
+    let expr = cx.mirror_expr(&body.value);
+    Ok((tcx.alloc_steal_thir(cx.thir), expr))
+}
+
+pub(crate) fn thir_tree<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    owner_def: ty::WithOptConstParam<LocalDefId>,
+) -> String {
+    match thir_body(tcx, owner_def) {
+        Ok((thir, _)) => format!("{:#?}", thir.steal()),
+        Err(_) => "error".into(),
+    }
+}
+
+struct Cx<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    thir: Thir<'tcx>,
+
+    pub(crate) param_env: ty::ParamEnv<'tcx>,
+
+    pub(crate) region_scope_tree: &'tcx region::ScopeTree,
+    pub(crate) typeck_results: &'tcx ty::TypeckResults<'tcx>,
+    pub(crate) rvalue_scopes: &'tcx RvalueScopes,
+
+    /// When applying adjustments to the expression
+    /// with the given `HirId`, use the given `Span`,
+    /// instead of the usual span. This is used to
+    /// assign the span of an overall method call
+    /// (e.g. `my_val.foo()`) to the adjustment expressions
+    /// for the receiver.
+    adjustment_span: Option<(HirId, Span)>,
+
+    /// The `DefId` of the owner of this body.
+    body_owner: DefId,
+}
+
+impl<'tcx> Cx<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>) -> Cx<'tcx> {
+        let typeck_results = tcx.typeck_opt_const_arg(def);
+        Cx {
+            tcx,
+            thir: Thir::new(),
+            param_env: tcx.param_env(def.did),
+            region_scope_tree: tcx.region_scope_tree(def.did),
+            typeck_results,
+            rvalue_scopes: &typeck_results.rvalue_scopes,
+            body_owner: def.did.to_def_id(),
+            adjustment_span: None,
+        }
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    pub(crate) fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Pat<'tcx> {
+        let p = match self.tcx.hir().get(p.hir_id) {
+            Node::Pat(p) => p,
+            node => bug!("pattern became {:?}", node),
+        };
+        pat_from_hir(self.tcx, self.param_env, self.typeck_results(), p)
+    }
+}
+
+impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn typeck_results(&self) -> &ty::TypeckResults<'tcx> {
+        self.typeck_results
+    }
+}
+
+mod block;
+mod expr;
diff --git a/compiler/rustc_mir_build/src/thir/mod.rs b/compiler/rustc_mir_build/src/thir/mod.rs
new file mode 100644
index 000000000..e0e6ac266
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/mod.rs
@@ -0,0 +1,13 @@
+//! The MIR is built from some typed high-level IR
+//! (THIR). This section defines the THIR along with a trait for
+//! accessing it. The intention is to allow MIR construction to be
+//! unit-tested and separated from the Rust source and compiler data
+//! structures.
+
+pub(crate) mod constant;
+
+pub(crate) mod cx;
+
+pub(crate) mod pattern;
+
+mod util;
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
new file mode 100644
index 000000000..063c07647
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -0,0 +1,1162 @@
+use super::deconstruct_pat::{Constructor, DeconstructedPat};
+use super::usefulness::{
+    compute_match_usefulness, MatchArm, MatchCheckCtxt, Reachability, UsefulnessReport,
+};
+use super::{PatCtxt, PatternError};
+
+use rustc_arena::TypedArena;
+use rustc_ast::Mutability;
+use rustc_errors::{
+    error_code, pluralize, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder,
+    ErrorGuaranteed, MultiSpan,
+};
+use rustc_hir as hir;
+use rustc_hir::def::*;
+use rustc_hir::def_id::DefId;
+use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::{HirId, Pat};
+use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
+use rustc_session::lint::builtin::{
+    BINDINGS_WITH_VARIANT_NAME, IRREFUTABLE_LET_PATTERNS, UNREACHABLE_PATTERNS,
+};
+use rustc_session::Session;
+use rustc_span::source_map::Spanned;
+use rustc_span::{BytePos, Span};
+
+pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: DefId) {
+    let body_id = match def_id.as_local() {
+        None => return,
+        Some(def_id) => tcx.hir().body_owned_by(def_id),
+    };
+
+    let pattern_arena = TypedArena::default();
+    let mut visitor = MatchVisitor {
+        tcx,
+        typeck_results: tcx.typeck_body(body_id),
+        param_env: tcx.param_env(def_id),
+        pattern_arena: &pattern_arena,
+    };
+    visitor.visit_body(tcx.hir().body(body_id));
+}
+
+fn create_e0004(
+    sess: &Session,
+    sp: Span,
+    error_message: String,
+) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+    struct_span_err!(sess, sp, E0004, "{}", &error_message)
+}
+
+#[derive(PartialEq)]
+enum RefutableFlag {
+    Irrefutable,
+    Refutable,
+}
+use RefutableFlag::*;
+
+struct MatchVisitor<'a, 'p, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+}
+
+impl<'tcx> Visitor<'tcx> for MatchVisitor<'_, '_, 'tcx> {
+    fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
+        intravisit::walk_expr(self, ex);
+        match &ex.kind {
+            hir::ExprKind::Match(scrut, arms, source) => {
+                self.check_match(scrut, arms, *source, ex.span)
+            }
+            hir::ExprKind::Let(hir::Let { pat, init, span, .. }) => {
+                self.check_let(pat, init, *span)
+            }
+            _ => {}
+        }
+    }
+
+    fn visit_local(&mut self, loc: &'tcx hir::Local<'tcx>) {
+        intravisit::walk_local(self, loc);
+        let els = loc.els;
+        if let Some(init) = loc.init && els.is_some() {
+            self.check_let(&loc.pat, init, loc.span);
+        }
+
+        let (msg, sp) = match loc.source {
+            hir::LocalSource::Normal => ("local binding", Some(loc.span)),
+            hir::LocalSource::AsyncFn => ("async fn binding", None),
+            hir::LocalSource::AwaitDesugar => ("`await` future binding", None),
+            hir::LocalSource::AssignDesugar(_) => ("destructuring assignment binding", None),
+        };
+        if els.is_none() {
+            self.check_irrefutable(&loc.pat, msg, sp);
+        }
+    }
+
+    fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
+        intravisit::walk_param(self, param);
+        self.check_irrefutable(&param.pat, "function argument", None);
+    }
+}
+
+impl PatCtxt<'_, '_> {
+    fn report_inlining_errors(&self) {
+        for error in &self.errors {
+            match *error {
+                PatternError::StaticInPattern(span) => {
+                    self.span_e0158(span, "statics cannot be referenced in patterns")
+                }
+                PatternError::AssocConstInPattern(span) => {
+                    self.span_e0158(span, "associated consts cannot be referenced in patterns")
+                }
+                PatternError::ConstParamInPattern(span) => {
+                    self.span_e0158(span, "const parameters cannot be referenced in patterns")
+                }
+                PatternError::NonConstPath(span) => {
+                    rustc_middle::mir::interpret::struct_error(
+                        self.tcx.at(span),
+                        "runtime values cannot be referenced in patterns",
+                    )
+                    .emit();
+                }
+            }
+        }
+    }
+
+    fn span_e0158(&self, span: Span, text: &str) {
+        struct_span_err!(self.tcx.sess, span, E0158, "{}", text).emit();
+    }
+}
+
+impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
+    fn check_patterns(&self, pat: &Pat<'_>, rf: RefutableFlag) {
+        pat.walk_always(|pat| check_borrow_conflicts_in_at_patterns(self, pat));
+        check_for_bindings_named_same_as_variants(self, pat, rf);
+    }
+
+    fn lower_pattern(
+        &self,
+        cx: &mut MatchCheckCtxt<'p, 'tcx>,
+        pat: &'tcx hir::Pat<'tcx>,
+        have_errors: &mut bool,
+    ) -> &'p DeconstructedPat<'p, 'tcx> {
+        let mut patcx = PatCtxt::new(self.tcx, self.param_env, self.typeck_results);
+        patcx.include_lint_checks();
+        let pattern = patcx.lower_pattern(pat);
+        let pattern: &_ = cx.pattern_arena.alloc(DeconstructedPat::from_pat(cx, &pattern));
+        if !patcx.errors.is_empty() {
+            *have_errors = true;
+            patcx.report_inlining_errors();
+        }
+        pattern
+    }
+
+    fn new_cx(&self, hir_id: HirId) -> MatchCheckCtxt<'p, 'tcx> {
+        MatchCheckCtxt {
+            tcx: self.tcx,
+            param_env: self.param_env,
+            module: self.tcx.parent_module(hir_id).to_def_id(),
+            pattern_arena: &self.pattern_arena,
+        }
+    }
+
+    fn check_let(&mut self, pat: &'tcx hir::Pat<'tcx>, scrutinee: &hir::Expr<'_>, span: Span) {
+        self.check_patterns(pat, Refutable);
+        let mut cx = self.new_cx(scrutinee.hir_id);
+        let tpat = self.lower_pattern(&mut cx, pat, &mut false);
+        self.check_let_reachability(&mut cx, pat.hir_id, tpat, span);
+    }
+
+    fn check_match(
+        &mut self,
+        scrut: &hir::Expr<'_>,
+        hir_arms: &'tcx [hir::Arm<'tcx>],
+        source: hir::MatchSource,
+        expr_span: Span,
+    ) {
+        let mut cx = self.new_cx(scrut.hir_id);
+
+        for arm in hir_arms {
+            // Check the arm for some things unrelated to exhaustiveness.
+            self.check_patterns(&arm.pat, Refutable);
+            if let Some(hir::Guard::IfLet(ref let_expr)) = arm.guard {
+                self.check_patterns(let_expr.pat, Refutable);
+                let tpat = self.lower_pattern(&mut cx, let_expr.pat, &mut false);
+                self.check_let_reachability(&mut cx, let_expr.pat.hir_id, tpat, tpat.span());
+            }
+        }
+
+        let mut have_errors = false;
+
+        let arms: Vec<_> = hir_arms
+            .iter()
+            .map(|hir::Arm { pat, guard, .. }| MatchArm {
+                pat: self.lower_pattern(&mut cx, pat, &mut have_errors),
+                hir_id: pat.hir_id,
+                has_guard: guard.is_some(),
+            })
+            .collect();
+
+        // Bail out early if lowering failed.
+        if have_errors {
+            return;
+        }
+
+        let scrut_ty = self.typeck_results.expr_ty_adjusted(scrut);
+        let report = compute_match_usefulness(&cx, &arms, scrut.hir_id, scrut_ty);
+
+        match source {
+            // Don't report arm reachability of desugared `match $iter.into_iter() { iter => .. }`
+            // when the iterator is an uninhabited type. unreachable_code will trigger instead.
+            hir::MatchSource::ForLoopDesugar if arms.len() == 1 => {}
+            hir::MatchSource::ForLoopDesugar | hir::MatchSource::Normal => {
+                report_arm_reachability(&cx, &report)
+            }
+            // Unreachable patterns in try and await expressions occur when one of
+            // the arms are an uninhabited type. Which is OK.
+            hir::MatchSource::AwaitDesugar | hir::MatchSource::TryDesugar => {}
+        }
+
+        // Check if the match is exhaustive.
+        let witnesses = report.non_exhaustiveness_witnesses;
+        if !witnesses.is_empty() {
+            if source == hir::MatchSource::ForLoopDesugar && hir_arms.len() == 2 {
+                // the for loop pattern is not irrefutable
+                let pat = hir_arms[1].pat.for_loop_some().unwrap();
+                self.check_irrefutable(pat, "`for` loop binding", None);
+            } else {
+                non_exhaustive_match(&cx, scrut_ty, scrut.span, witnesses, hir_arms, expr_span);
+            }
+        }
+    }
+
+    fn check_let_reachability(
+        &mut self,
+        cx: &mut MatchCheckCtxt<'p, 'tcx>,
+        pat_id: HirId,
+        pat: &'p DeconstructedPat<'p, 'tcx>,
+        span: Span,
+    ) {
+        if self.check_let_chain(cx, pat_id) {
+            return;
+        }
+
+        if is_let_irrefutable(cx, pat_id, pat) {
+            irrefutable_let_pattern(cx.tcx, pat_id, span);
+        }
+    }
+
+    fn check_let_chain(&mut self, cx: &mut MatchCheckCtxt<'p, 'tcx>, pat_id: HirId) -> bool {
+        let hir = self.tcx.hir();
+        let parent = hir.get_parent_node(pat_id);
+
+        // First, figure out if the given pattern is part of a let chain,
+        // and if so, obtain the top node of the chain.
+        let mut top = parent;
+        let mut part_of_chain = false;
+        loop {
+            let new_top = hir.get_parent_node(top);
+            if let hir::Node::Expr(
+                hir::Expr {
+                    kind: hir::ExprKind::Binary(Spanned { node: hir::BinOpKind::And, .. }, lhs, rhs),
+                    ..
+                },
+                ..,
+            ) = hir.get(new_top)
+            {
+                // If this isn't the first iteration, we need to check
+                // if there is a let expr before us in the chain, so
+                // that we avoid doubly checking the let chain.
+
+                // The way a chain of &&s is encoded is ((let ... && let ...) && let ...) && let ...
+                // as && is left-to-right associative. Thus, we need to check rhs.
+                if part_of_chain && matches!(rhs.kind, hir::ExprKind::Let(..)) {
+                    return true;
+                }
+                // If there is a let at the lhs, and we provide the rhs, we don't do any checking either.
+                if !part_of_chain && matches!(lhs.kind, hir::ExprKind::Let(..)) && rhs.hir_id == top
+                {
+                    return true;
+                }
+            } else {
+                // We've reached the top.
+                break;
+            }
+
+            // Since this function is called within a let context, it is reasonable to assume that any parent
+            // `&&` infers a let chain
+            part_of_chain = true;
+            top = new_top;
+        }
+        if !part_of_chain {
+            return false;
+        }
+
+        // Second, obtain the refutabilities of all exprs in the chain,
+        // and record chain members that aren't let exprs.
+        let mut chain_refutabilities = Vec::new();
+        let hir::Node::Expr(top_expr) = hir.get(top) else {
+            // We ensure right above that it's an Expr
+            unreachable!()
+        };
+        let mut cur_expr = top_expr;
+        loop {
+            let mut add = |expr: &hir::Expr<'tcx>| {
+                let refutability = match expr.kind {
+                    hir::ExprKind::Let(hir::Let { pat, init, span, .. }) => {
+                        let mut ncx = self.new_cx(init.hir_id);
+                        let tpat = self.lower_pattern(&mut ncx, pat, &mut false);
+
+                        let refutable = !is_let_irrefutable(&mut ncx, pat.hir_id, tpat);
+                        Some((*span, refutable))
+                    }
+                    _ => None,
+                };
+                chain_refutabilities.push(refutability);
+            };
+            if let hir::Expr {
+                kind: hir::ExprKind::Binary(Spanned { node: hir::BinOpKind::And, .. }, lhs, rhs),
+                ..
+            } = cur_expr
+            {
+                add(rhs);
+                cur_expr = lhs;
+            } else {
+                add(cur_expr);
+                break;
+            }
+        }
+        chain_refutabilities.reverse();
+
+        // Third, emit the actual warnings.
+
+        if chain_refutabilities.iter().all(|r| matches!(*r, Some((_, false)))) {
+            // The entire chain is made up of irrefutable `let` statements
+            let let_source = let_source_parent(self.tcx, top, None);
+            irrefutable_let_patterns(
+                cx.tcx,
+                top,
+                let_source,
+                chain_refutabilities.len(),
+                top_expr.span,
+            );
+            return true;
+        }
+        let lint_affix = |affix: &[Option<(Span, bool)>], kind, suggestion| {
+            let span_start = affix[0].unwrap().0;
+            let span_end = affix.last().unwrap().unwrap().0;
+            let span = span_start.to(span_end);
+            let cnt = affix.len();
+            cx.tcx.struct_span_lint_hir(IRREFUTABLE_LET_PATTERNS, top, span, |lint| {
+                let s = pluralize!(cnt);
+                let mut diag = lint.build(&format!("{kind} irrefutable pattern{s} in let chain"));
+                diag.note(&format!(
+                    "{these} pattern{s} will always match",
+                    these = pluralize!("this", cnt),
+                ));
+                diag.help(&format!(
+                    "consider moving {} {suggestion}",
+                    if cnt > 1 { "them" } else { "it" }
+                ));
+                diag.emit()
+            });
+        };
+        if let Some(until) = chain_refutabilities.iter().position(|r| !matches!(*r, Some((_, false)))) && until > 0 {
+            // The chain has a non-zero prefix of irrefutable `let` statements.
+
+            // Check if the let source is while, for there is no alternative place to put a prefix,
+            // and we shouldn't lint.
+            let let_source = let_source_parent(self.tcx, top, None);
+            if !matches!(let_source, LetSource::WhileLet) {
+                // Emit the lint
+                let prefix = &chain_refutabilities[..until];
+                lint_affix(prefix, "leading", "outside of the construct");
+            }
+        }
+        if let Some(from) = chain_refutabilities.iter().rposition(|r| !matches!(*r, Some((_, false)))) && from != (chain_refutabilities.len() - 1) {
+            // The chain has a non-empty suffix of irrefutable `let` statements
+            let suffix = &chain_refutabilities[from + 1..];
+            lint_affix(suffix, "trailing", "into the body");
+        }
+        true
+    }
+
+    fn check_irrefutable(&self, pat: &'tcx Pat<'tcx>, origin: &str, sp: Option<Span>) {
+        let mut cx = self.new_cx(pat.hir_id);
+
+        let pattern = self.lower_pattern(&mut cx, pat, &mut false);
+        let pattern_ty = pattern.ty();
+        let arms = vec![MatchArm { pat: pattern, hir_id: pat.hir_id, has_guard: false }];
+        let report = compute_match_usefulness(&cx, &arms, pat.hir_id, pattern_ty);
+
+        // Note: we ignore whether the pattern is unreachable (i.e. whether the type is empty). We
+        // only care about exhaustiveness here.
+        let witnesses = report.non_exhaustiveness_witnesses;
+        if witnesses.is_empty() {
+            // The pattern is irrefutable.
+            self.check_patterns(pat, Irrefutable);
+            return;
+        }
+
+        let joined_patterns = joined_uncovered_patterns(&cx, &witnesses);
+
+        let mut bindings = vec![];
+
+        let mut err = struct_span_err!(
+            self.tcx.sess,
+            pat.span,
+            E0005,
+            "refutable pattern in {}: {} not covered",
+            origin,
+            joined_patterns
+        );
+        let suggest_if_let = match &pat.kind {
+            hir::PatKind::Path(hir::QPath::Resolved(None, path))
+                if path.segments.len() == 1 && path.segments[0].args.is_none() =>
+            {
+                const_not_var(&mut err, cx.tcx, pat, path);
+                false
+            }
+            _ => {
+                pat.walk(&mut |pat: &hir::Pat<'_>| {
+                    match pat.kind {
+                        hir::PatKind::Binding(_, _, ident, _) => {
+                            bindings.push(ident);
+                        }
+                        _ => {}
+                    }
+                    true
+                });
+
+                err.span_label(pat.span, pattern_not_covered_label(&witnesses, &joined_patterns));
+                true
+            }
+        };
+
+        if let (Some(span), true) = (sp, suggest_if_let) {
+            err.note(
+                "`let` bindings require an \"irrefutable pattern\", like a `struct` or \
+                 an `enum` with only one variant",
+            );
+            if self.tcx.sess.source_map().is_span_accessible(span) {
+                let semi_span = span.shrink_to_hi().with_lo(span.hi() - BytePos(1));
+                let start_span = span.shrink_to_lo();
+                let end_span = semi_span.shrink_to_lo();
+                err.multipart_suggestion(
+                    &format!(
+                        "you might want to use `if let` to ignore the variant{} that {} matched",
+                        pluralize!(witnesses.len()),
+                        match witnesses.len() {
+                            1 => "isn't",
+                            _ => "aren't",
+                        },
+                    ),
+                    vec![
+                        match &bindings[..] {
+                            [] => (start_span, "if ".to_string()),
+                            [binding] => (start_span, format!("let {} = if ", binding)),
+                            bindings => (
+                                start_span,
+                                format!(
+                                    "let ({}) = if ",
+                                    bindings
+                                        .iter()
+                                        .map(|ident| ident.to_string())
+                                        .collect::<Vec<_>>()
+                                        .join(", ")
+                                ),
+                            ),
+                        },
+                        match &bindings[..] {
+                            [] => (semi_span, " { todo!() }".to_string()),
+                            [binding] => {
+                                (end_span, format!(" {{ {} }} else {{ todo!() }}", binding))
+                            }
+                            bindings => (
+                                end_span,
+                                format!(
+                                    " {{ ({}) }} else {{ todo!() }}",
+                                    bindings
+                                        .iter()
+                                        .map(|ident| ident.to_string())
+                                        .collect::<Vec<_>>()
+                                        .join(", ")
+                                ),
+                            ),
+                        },
+                    ],
+                    Applicability::HasPlaceholders,
+                );
+                if !bindings.is_empty() && cx.tcx.sess.is_nightly_build() {
+                    err.span_suggestion_verbose(
+                        semi_span.shrink_to_lo(),
+                        &format!(
+                            "alternatively, on nightly, you might want to use \
+                             `#![feature(let_else)]` to handle the variant{} that {} matched",
+                            pluralize!(witnesses.len()),
+                            match witnesses.len() {
+                                1 => "isn't",
+                                _ => "aren't",
+                            },
+                        ),
+                        " else { todo!() }".to_string(),
+                        Applicability::HasPlaceholders,
+                    );
+                }
+            }
+            err.note(
+                "for more information, visit \
+                 https://doc.rust-lang.org/book/ch18-02-refutability.html",
+            );
+        }
+
+        adt_defined_here(&cx, &mut err, pattern_ty, &witnesses);
+        err.note(&format!("the matched value is of type `{}`", pattern_ty));
+        err.emit();
+    }
+}
+
+/// A path pattern was interpreted as a constant, not a new variable.
+/// This caused an irrefutable match failure in e.g. `let`.
+fn const_not_var(err: &mut Diagnostic, tcx: TyCtxt<'_>, pat: &Pat<'_>, path: &hir::Path<'_>) {
+    let descr = path.res.descr();
+    err.span_label(
+        pat.span,
+        format!("interpreted as {} {} pattern, not a new variable", path.res.article(), descr,),
+    );
+
+    err.span_suggestion(
+        pat.span,
+        "introduce a variable instead",
+        format!("{}_var", path.segments[0].ident).to_lowercase(),
+        // Cannot use `MachineApplicable` as it's not really *always* correct
+        // because there may be such an identifier in scope or the user maybe
+        // really wanted to match against the constant. This is quite unlikely however.
+        Applicability::MaybeIncorrect,
+    );
+
+    if let Some(span) = tcx.hir().res_span(path.res) {
+        err.span_label(span, format!("{} defined here", descr));
+    }
+}
+
+fn check_for_bindings_named_same_as_variants(
+    cx: &MatchVisitor<'_, '_, '_>,
+    pat: &Pat<'_>,
+    rf: RefutableFlag,
+) {
+    pat.walk_always(|p| {
+        if let hir::PatKind::Binding(_, _, ident, None) = p.kind
+            && let Some(ty::BindByValue(hir::Mutability::Not)) =
+                cx.typeck_results.extract_binding_mode(cx.tcx.sess, p.hir_id, p.span)
+            && let pat_ty = cx.typeck_results.pat_ty(p).peel_refs()
+            && let ty::Adt(edef, _) = pat_ty.kind()
+            && edef.is_enum()
+            && edef.variants().iter().any(|variant| {
+                variant.ident(cx.tcx) == ident && variant.ctor_kind == CtorKind::Const
+            })
+        {
+            let variant_count = edef.variants().len();
+            cx.tcx.struct_span_lint_hir(
+                BINDINGS_WITH_VARIANT_NAME,
+                p.hir_id,
+                p.span,
+                |lint| {
+                    let ty_path = cx.tcx.def_path_str(edef.did());
+                    let mut err = lint.build(&format!(
+                        "pattern binding `{}` is named the same as one \
+                         of the variants of the type `{}`",
+                        ident, ty_path
+                    ));
+                    err.code(error_code!(E0170));
+                    // If this is an irrefutable pattern, and there's > 1 variant,
+                    // then we can't actually match on this. Applying the below
+                    // suggestion would produce code that breaks on `check_irrefutable`.
+                    if rf == Refutable || variant_count == 1 {
+                        err.span_suggestion(
+                            p.span,
+                            "to match on the variant, qualify the path",
+                            format!("{}::{}", ty_path, ident),
+                            Applicability::MachineApplicable,
+                        );
+                    }
+                    err.emit();
+                },
+            )
+        }
+    });
+}
+
+/// Checks for common cases of "catchall" patterns that may not be intended as such.
+fn pat_is_catchall(pat: &DeconstructedPat<'_, '_>) -> bool {
+    use Constructor::*;
+    match pat.ctor() {
+        Wildcard => true,
+        Single => pat.iter_fields().all(|pat| pat_is_catchall(pat)),
+        _ => false,
+    }
+}
+
+fn unreachable_pattern(tcx: TyCtxt<'_>, span: Span, id: HirId, catchall: Option<Span>) {
+    tcx.struct_span_lint_hir(UNREACHABLE_PATTERNS, id, span, |lint| {
+        let mut err = lint.build("unreachable pattern");
+        if let Some(catchall) = catchall {
+            // We had a catchall pattern, hint at that.
+            err.span_label(span, "unreachable pattern");
+            err.span_label(catchall, "matches any value");
+        }
+        err.emit();
+    });
+}
+
+fn irrefutable_let_pattern(tcx: TyCtxt<'_>, id: HirId, span: Span) {
+    let source = let_source(tcx, id);
+    irrefutable_let_patterns(tcx, id, source, 1, span);
+}
+
+fn irrefutable_let_patterns(
+    tcx: TyCtxt<'_>,
+    id: HirId,
+    source: LetSource,
+    count: usize,
+    span: Span,
+) {
+    macro_rules! emit_diag {
+        (
+            $lint:expr,
+            $source_name:expr,
+            $note_sufix:expr,
+            $help_sufix:expr
+        ) => {{
+            let s = pluralize!(count);
+            let these = pluralize!("this", count);
+            let mut diag = $lint.build(&format!("irrefutable {} pattern{s}", $source_name));
+            diag.note(&format!("{these} pattern{s} will always match, so the {}", $note_sufix));
+            diag.help(concat!("consider ", $help_sufix));
+            diag.emit()
+        }};
+    }
+
+    let span = match source {
+        LetSource::LetElse(span) => span,
+        _ => span,
+    };
+    tcx.struct_span_lint_hir(IRREFUTABLE_LET_PATTERNS, id, span, |lint| match source {
+        LetSource::GenericLet => {
+            emit_diag!(lint, "`let`", "`let` is useless", "removing `let`");
+        }
+        LetSource::IfLet => {
+            emit_diag!(
+                lint,
+                "`if let`",
+                "`if let` is useless",
+                "replacing the `if let` with a `let`"
+            );
+        }
+        LetSource::IfLetGuard => {
+            emit_diag!(
+                lint,
+                "`if let` guard",
+                "guard is useless",
+                "removing the guard and adding a `let` inside the match arm"
+            );
+        }
+        LetSource::LetElse(..) => {
+            emit_diag!(
+                lint,
+                "`let...else`",
+                "`else` clause is useless",
+                "removing the `else` clause"
+            );
+        }
+        LetSource::WhileLet => {
+            emit_diag!(
+                lint,
+                "`while let`",
+                "loop will never exit",
+                "instead using a `loop { ... }` with a `let` inside it"
+            );
+        }
+    });
+}
+
+fn is_let_irrefutable<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    pat_id: HirId,
+    pat: &'p DeconstructedPat<'p, 'tcx>,
+) -> bool {
+    let arms = [MatchArm { pat, hir_id: pat_id, has_guard: false }];
+    let report = compute_match_usefulness(&cx, &arms, pat_id, pat.ty());
+
+    // Report if the pattern is unreachable, which can only occur when the type is uninhabited.
+    // This also reports unreachable sub-patterns though, so we can't just replace it with an
+    // `is_uninhabited` check.
+    report_arm_reachability(&cx, &report);
+
+    // If the list of witnesses is empty, the match is exhaustive,
+    // i.e. the `if let` pattern is irrefutable.
+    report.non_exhaustiveness_witnesses.is_empty()
+}
+
+/// Report unreachable arms, if any.
+fn report_arm_reachability<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    report: &UsefulnessReport<'p, 'tcx>,
+) {
+    use Reachability::*;
+    let mut catchall = None;
+    for (arm, is_useful) in report.arm_usefulness.iter() {
+        match is_useful {
+            Unreachable => unreachable_pattern(cx.tcx, arm.pat.span(), arm.hir_id, catchall),
+            Reachable(unreachables) if unreachables.is_empty() => {}
+            // The arm is reachable, but contains unreachable subpatterns (from or-patterns).
+            Reachable(unreachables) => {
+                let mut unreachables = unreachables.clone();
+                // Emit lints in the order in which they occur in the file.
+                unreachables.sort_unstable();
+                for span in unreachables {
+                    unreachable_pattern(cx.tcx, span, arm.hir_id, None);
+                }
+            }
+        }
+        if !arm.has_guard && catchall.is_none() && pat_is_catchall(arm.pat) {
+            catchall = Some(arm.pat.span());
+        }
+    }
+}
+
+/// Report that a match is not exhaustive.
+fn non_exhaustive_match<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    scrut_ty: Ty<'tcx>,
+    sp: Span,
+    witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
+    arms: &[hir::Arm<'tcx>],
+    expr_span: Span,
+) {
+    let is_empty_match = arms.is_empty();
+    let non_empty_enum = match scrut_ty.kind() {
+        ty::Adt(def, _) => def.is_enum() && !def.variants().is_empty(),
+        _ => false,
+    };
+    // In the case of an empty match, replace the '`_` not covered' diagnostic with something more
+    // informative.
+    let mut err;
+    let pattern;
+    let mut patterns_len = 0;
+    if is_empty_match && !non_empty_enum {
+        err = create_e0004(
+            cx.tcx.sess,
+            sp,
+            format!("non-exhaustive patterns: type `{}` is non-empty", scrut_ty),
+        );
+        pattern = "_".to_string();
+    } else {
+        let joined_patterns = joined_uncovered_patterns(cx, &witnesses);
+        err = create_e0004(
+            cx.tcx.sess,
+            sp,
+            format!("non-exhaustive patterns: {} not covered", joined_patterns),
+        );
+        err.span_label(sp, pattern_not_covered_label(&witnesses, &joined_patterns));
+        patterns_len = witnesses.len();
+        pattern = if witnesses.len() < 4 {
+            witnesses
+                .iter()
+                .map(|witness| witness.to_pat(cx).to_string())
+                .collect::<Vec<String>>()
+                .join(" | ")
+        } else {
+            "_".to_string()
+        };
+    };
+
+    let is_variant_list_non_exhaustive = match scrut_ty.kind() {
+        ty::Adt(def, _) if def.is_variant_list_non_exhaustive() && !def.did().is_local() => true,
+        _ => false,
+    };
+
+    adt_defined_here(cx, &mut err, scrut_ty, &witnesses);
+    err.note(&format!(
+        "the matched value is of type `{}`{}",
+        scrut_ty,
+        if is_variant_list_non_exhaustive { ", which is marked as non-exhaustive" } else { "" }
+    ));
+    if (scrut_ty == cx.tcx.types.usize || scrut_ty == cx.tcx.types.isize)
+        && !is_empty_match
+        && witnesses.len() == 1
+        && matches!(witnesses[0].ctor(), Constructor::NonExhaustive)
+    {
+        err.note(&format!(
+            "`{}` does not have a fixed maximum value, so a wildcard `_` is necessary to match \
+             exhaustively",
+            scrut_ty,
+        ));
+        if cx.tcx.sess.is_nightly_build() {
+            err.help(&format!(
+                "add `#![feature(precise_pointer_size_matching)]` to the crate attributes to \
+                 enable precise `{}` matching",
+                scrut_ty,
+            ));
+        }
+    }
+    if let ty::Ref(_, sub_ty, _) = scrut_ty.kind() {
+        if cx.tcx.is_ty_uninhabited_from(cx.module, *sub_ty, cx.param_env) {
+            err.note("references are always considered inhabited");
+        }
+    }
+
+    let mut suggestion = None;
+    let sm = cx.tcx.sess.source_map();
+    match arms {
+        [] if sp.eq_ctxt(expr_span) => {
+            // Get the span for the empty match body `{}`.
+            let (indentation, more) = if let Some(snippet) = sm.indentation_before(sp) {
+                (format!("\n{}", snippet), "    ")
+            } else {
+                (" ".to_string(), "")
+            };
+            suggestion = Some((
+                sp.shrink_to_hi().with_hi(expr_span.hi()),
+                format!(
+                    " {{{indentation}{more}{pattern} => todo!(),{indentation}}}",
+                    indentation = indentation,
+                    more = more,
+                    pattern = pattern,
+                ),
+            ));
+        }
+        [only] => {
+            let (pre_indentation, is_multiline) = if let Some(snippet) = sm.indentation_before(only.span)
+                && let Ok(with_trailing) = sm.span_extend_while(only.span, |c| c.is_whitespace() || c == ',')
+                && sm.is_multiline(with_trailing)
+            {
+                (format!("\n{}", snippet), true)
+            } else {
+                (" ".to_string(), false)
+            };
+            let comma = if matches!(only.body.kind, hir::ExprKind::Block(..))
+                && only.span.eq_ctxt(only.body.span)
+                && is_multiline
+            {
+                ""
+            } else {
+                ","
+            };
+            suggestion = Some((
+                only.span.shrink_to_hi(),
+                format!("{}{}{} => todo!()", comma, pre_indentation, pattern),
+            ));
+        }
+        [.., prev, last] if prev.span.eq_ctxt(last.span) => {
+            if let Ok(snippet) = sm.span_to_snippet(prev.span.between(last.span)) {
+                let comma = if matches!(last.body.kind, hir::ExprKind::Block(..))
+                    && last.span.eq_ctxt(last.body.span)
+                {
+                    ""
+                } else {
+                    ","
+                };
+                suggestion = Some((
+                    last.span.shrink_to_hi(),
+                    format!(
+                        "{}{}{} => todo!()",
+                        comma,
+                        snippet.strip_prefix(',').unwrap_or(&snippet),
+                        pattern
+                    ),
+                ));
+            }
+        }
+        _ => {}
+    }
+
+    let msg = format!(
+        "ensure that all possible cases are being handled by adding a match arm with a wildcard \
+         pattern{}{}",
+        if patterns_len > 1 && patterns_len < 4 && suggestion.is_some() {
+            ", a match arm with multiple or-patterns"
+        } else {
+            // we are either not suggesting anything, or suggesting `_`
+            ""
+        },
+        match patterns_len {
+            // non-exhaustive enum case
+            0 if suggestion.is_some() => " as shown",
+            0 => "",
+            1 if suggestion.is_some() => " or an explicit pattern as shown",
+            1 => " or an explicit pattern",
+            _ if suggestion.is_some() => " as shown, or multiple match arms",
+            _ => " or multiple match arms",
+        },
+    );
+    if let Some((span, sugg)) = suggestion {
+        err.span_suggestion_verbose(span, &msg, sugg, Applicability::HasPlaceholders);
+    } else {
+        err.help(&msg);
+    }
+    err.emit();
+}
+
+pub(crate) fn joined_uncovered_patterns<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    witnesses: &[DeconstructedPat<'p, 'tcx>],
+) -> String {
+    const LIMIT: usize = 3;
+    let pat_to_str = |pat: &DeconstructedPat<'p, 'tcx>| pat.to_pat(cx).to_string();
+    match witnesses {
+        [] => bug!(),
+        [witness] => format!("`{}`", witness.to_pat(cx)),
+        [head @ .., tail] if head.len() < LIMIT => {
+            let head: Vec<_> = head.iter().map(pat_to_str).collect();
+            format!("`{}` and `{}`", head.join("`, `"), tail.to_pat(cx))
+        }
+        _ => {
+            let (head, tail) = witnesses.split_at(LIMIT);
+            let head: Vec<_> = head.iter().map(pat_to_str).collect();
+            format!("`{}` and {} more", head.join("`, `"), tail.len())
+        }
+    }
+}
+
+pub(crate) fn pattern_not_covered_label(
+    witnesses: &[DeconstructedPat<'_, '_>],
+    joined_patterns: &str,
+) -> String {
+    format!("pattern{} {} not covered", rustc_errors::pluralize!(witnesses.len()), joined_patterns)
+}
+
+/// Point at the definition of non-covered `enum` variants.
+fn adt_defined_here<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    err: &mut Diagnostic,
+    ty: Ty<'tcx>,
+    witnesses: &[DeconstructedPat<'p, 'tcx>],
+) {
+    let ty = ty.peel_refs();
+    if let ty::Adt(def, _) = ty.kind() {
+        let mut spans = vec![];
+        if witnesses.len() < 5 {
+            for sp in maybe_point_at_variant(cx, *def, witnesses.iter()) {
+                spans.push(sp);
+            }
+        }
+        let def_span = cx
+            .tcx
+            .hir()
+            .get_if_local(def.did())
+            .and_then(|node| node.ident())
+            .map(|ident| ident.span)
+            .unwrap_or_else(|| cx.tcx.def_span(def.did()));
+        let mut span: MultiSpan =
+            if spans.is_empty() { def_span.into() } else { spans.clone().into() };
+
+        span.push_span_label(def_span, "");
+        for pat in spans {
+            span.push_span_label(pat, "not covered");
+        }
+        err.span_note(span, &format!("`{}` defined here", ty));
+    }
+}
+
+fn maybe_point_at_variant<'a, 'p: 'a, 'tcx: 'a>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    def: AdtDef<'tcx>,
+    patterns: impl Iterator<Item = &'a DeconstructedPat<'p, 'tcx>>,
+) -> Vec<Span> {
+    use Constructor::*;
+    let mut covered = vec![];
+    for pattern in patterns {
+        if let Variant(variant_index) = pattern.ctor() {
+            if let ty::Adt(this_def, _) = pattern.ty().kind() && this_def.did() != def.did() {
+                continue;
+            }
+            let sp = def.variant(*variant_index).ident(cx.tcx).span;
+            if covered.contains(&sp) {
+                // Don't point at variants that have already been covered due to other patterns to avoid
+                // visual clutter.
+                continue;
+            }
+            covered.push(sp);
+        }
+        covered.extend(maybe_point_at_variant(cx, def, pattern.iter_fields()));
+    }
+    covered
+}
+
+/// Check if a by-value binding is by-value. That is, check if the binding's type is not `Copy`.
+fn is_binding_by_move(cx: &MatchVisitor<'_, '_, '_>, hir_id: HirId, span: Span) -> bool {
+    !cx.typeck_results.node_type(hir_id).is_copy_modulo_regions(cx.tcx.at(span), cx.param_env)
+}
+
+/// Check that there are no borrow or move conflicts in `binding @ subpat` patterns.
+///
+/// For example, this would reject:
+/// - `ref x @ Some(ref mut y)`,
+/// - `ref mut x @ Some(ref y)`,
+/// - `ref mut x @ Some(ref mut y)`,
+/// - `ref mut? x @ Some(y)`, and
+/// - `x @ Some(ref mut? y)`.
+///
+/// This analysis is *not* subsumed by NLL.
+fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pat<'_>) {
+    // Extract `sub` in `binding @ sub`.
+    let (name, sub) = match &pat.kind {
+        hir::PatKind::Binding(.., name, Some(sub)) => (*name, sub),
+        _ => return,
+    };
+    let binding_span = pat.span.with_hi(name.span.hi());
+
+    let typeck_results = cx.typeck_results;
+    let sess = cx.tcx.sess;
+
+    // Get the binding move, extract the mutability if by-ref.
+    let mut_outer = match typeck_results.extract_binding_mode(sess, pat.hir_id, pat.span) {
+        Some(ty::BindByValue(_)) if is_binding_by_move(cx, pat.hir_id, pat.span) => {
+            // We have `x @ pat` where `x` is by-move. Reject all borrows in `pat`.
+            let mut conflicts_ref = Vec::new();
+            sub.each_binding(|_, hir_id, span, _| {
+                match typeck_results.extract_binding_mode(sess, hir_id, span) {
+                    Some(ty::BindByValue(_)) | None => {}
+                    Some(ty::BindByReference(_)) => conflicts_ref.push(span),
+                }
+            });
+            if !conflicts_ref.is_empty() {
+                let occurs_because = format!(
+                    "move occurs because `{}` has type `{}` which does not implement the `Copy` trait",
+                    name,
+                    typeck_results.node_type(pat.hir_id),
+                );
+                sess.struct_span_err(pat.span, "borrow of moved value")
+                    .span_label(binding_span, format!("value moved into `{}` here", name))
+                    .span_label(binding_span, occurs_because)
+                    .span_labels(conflicts_ref, "value borrowed here after move")
+                    .emit();
+            }
+            return;
+        }
+        Some(ty::BindByValue(_)) | None => return,
+        Some(ty::BindByReference(m)) => m,
+    };
+
+    // We now have `ref $mut_outer binding @ sub` (semantically).
+    // Recurse into each binding in `sub` and find mutability or move conflicts.
+    let mut conflicts_move = Vec::new();
+    let mut conflicts_mut_mut = Vec::new();
+    let mut conflicts_mut_ref = Vec::new();
+    sub.each_binding(|_, hir_id, span, name| {
+        match typeck_results.extract_binding_mode(sess, hir_id, span) {
+            Some(ty::BindByReference(mut_inner)) => match (mut_outer, mut_inner) {
+                (Mutability::Not, Mutability::Not) => {} // Both sides are `ref`.
+                (Mutability::Mut, Mutability::Mut) => conflicts_mut_mut.push((span, name)), // 2x `ref mut`.
+                _ => conflicts_mut_ref.push((span, name)), // `ref` + `ref mut` in either direction.
+            },
+            Some(ty::BindByValue(_)) if is_binding_by_move(cx, hir_id, span) => {
+                conflicts_move.push((span, name)) // `ref mut?` + by-move conflict.
+            }
+            Some(ty::BindByValue(_)) | None => {} // `ref mut?` + by-copy is fine.
+        }
+    });
+
+    // Report errors if any.
+    if !conflicts_mut_mut.is_empty() {
+        // Report mutability conflicts for e.g. `ref mut x @ Some(ref mut y)`.
+        let mut err = sess
+            .struct_span_err(pat.span, "cannot borrow value as mutable more than once at a time");
+        err.span_label(binding_span, format!("first mutable borrow, by `{}`, occurs here", name));
+        for (span, name) in conflicts_mut_mut {
+            err.span_label(span, format!("another mutable borrow, by `{}`, occurs here", name));
+        }
+        for (span, name) in conflicts_mut_ref {
+            err.span_label(span, format!("also borrowed as immutable, by `{}`, here", name));
+        }
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("also moved into `{}` here", name));
+        }
+        err.emit();
+    } else if !conflicts_mut_ref.is_empty() {
+        // Report mutability conflicts for e.g. `ref x @ Some(ref mut y)` or the converse.
+        let (primary, also) = match mut_outer {
+            Mutability::Mut => ("mutable", "immutable"),
+            Mutability::Not => ("immutable", "mutable"),
+        };
+        let msg =
+            format!("cannot borrow value as {} because it is also borrowed as {}", also, primary);
+        let mut err = sess.struct_span_err(pat.span, &msg);
+        err.span_label(binding_span, format!("{} borrow, by `{}`, occurs here", primary, name));
+        for (span, name) in conflicts_mut_ref {
+            err.span_label(span, format!("{} borrow, by `{}`, occurs here", also, name));
+        }
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("also moved into `{}` here", name));
+        }
+        err.emit();
+    } else if !conflicts_move.is_empty() {
+        // Report by-ref and by-move conflicts, e.g. `ref x @ y`.
+        let mut err =
+            sess.struct_span_err(pat.span, "cannot move out of value because it is borrowed");
+        err.span_label(binding_span, format!("value borrowed, by `{}`, here", name));
+        for (span, name) in conflicts_move {
+            err.span_label(span, format!("value moved into `{}` here", name));
+        }
+        err.emit();
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum LetSource {
+    GenericLet,
+    IfLet,
+    IfLetGuard,
+    LetElse(Span),
+    WhileLet,
+}
+
+fn let_source(tcx: TyCtxt<'_>, pat_id: HirId) -> LetSource {
+    let hir = tcx.hir();
+
+    let parent = hir.get_parent_node(pat_id);
+    let_source_parent(tcx, parent, Some(pat_id))
+}
+
+fn let_source_parent(tcx: TyCtxt<'_>, parent: HirId, pat_id: Option<HirId>) -> LetSource {
+    let hir = tcx.hir();
+
+    let parent_node = hir.get(parent);
+
+    match parent_node {
+        hir::Node::Arm(hir::Arm {
+            guard: Some(hir::Guard::IfLet(&hir::Let { pat: hir::Pat { hir_id, .. }, .. })),
+            ..
+        }) if Some(*hir_id) == pat_id => {
+            return LetSource::IfLetGuard;
+        }
+        _ => {}
+    }
+
+    let parent_parent = hir.get_parent_node(parent);
+    let parent_parent_node = hir.get(parent_parent);
+    if let hir::Node::Stmt(hir::Stmt { kind: hir::StmtKind::Local(_), span, .. }) =
+        parent_parent_node
+    {
+        return LetSource::LetElse(*span);
+    }
+
+    let parent_parent_parent = hir.get_parent_node(parent_parent);
+    let parent_parent_parent_parent = hir.get_parent_node(parent_parent_parent);
+    let parent_parent_parent_parent_node = hir.get(parent_parent_parent_parent);
+
+    if let hir::Node::Expr(hir::Expr {
+        kind: hir::ExprKind::Loop(_, _, hir::LoopSource::While, _),
+        ..
+    }) = parent_parent_parent_parent_node
+    {
+        return LetSource::WhileLet;
+    }
+
+    if let hir::Node::Expr(hir::Expr { kind: hir::ExprKind::If(..), .. }) = parent_parent_node {
+        return LetSource::IfLet;
+    }
+
+    LetSource::GenericLet
+}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
new file mode 100644
index 000000000..d6dd0f017
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -0,0 +1,601 @@
+use rustc_hir as hir;
+use rustc_index::vec::Idx;
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
+use rustc_middle::mir::{self, Field};
+use rustc_middle::thir::{FieldPat, Pat, PatKind};
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
+use rustc_session::lint;
+use rustc_span::Span;
+use rustc_trait_selection::traits::predicate_for_trait_def;
+use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
+use rustc_trait_selection::traits::{self, ObligationCause, PredicateObligation};
+
+use std::cell::Cell;
+
+use super::PatCtxt;
+
+impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
+    /// Converts an evaluated constant to a pattern (if possible).
+    /// This means aggregate values (like structs and enums) are converted
+    /// to a pattern that matches the value (as if you'd compared via structural equality).
+    #[instrument(level = "debug", skip(self))]
+    pub(super) fn const_to_pat(
+        &self,
+        cv: mir::ConstantKind<'tcx>,
+        id: hir::HirId,
+        span: Span,
+        mir_structural_match_violation: bool,
+    ) -> Pat<'tcx> {
+        let pat = self.tcx.infer_ctxt().enter(|infcx| {
+            let mut convert = ConstToPat::new(self, id, span, infcx);
+            convert.to_pat(cv, mir_structural_match_violation)
+        });
+
+        debug!(?pat);
+        pat
+    }
+}
+
+struct ConstToPat<'a, 'tcx> {
+    id: hir::HirId,
+    span: Span,
+    param_env: ty::ParamEnv<'tcx>,
+
+    // This tracks if we emitted some hard error for a given const value, so that
+    // we will not subsequently issue an irrelevant lint for the same const
+    // value.
+    saw_const_match_error: Cell<bool>,
+
+    // This tracks if we emitted some diagnostic for a given const value, so that
+    // we will not subsequently issue an irrelevant lint for the same const
+    // value.
+    saw_const_match_lint: Cell<bool>,
+
+    // For backcompat we need to keep allowing non-structurally-eq types behind references.
+    // See also all the `cant-hide-behind` tests.
+    behind_reference: Cell<bool>,
+
+    // inference context used for checking `T: Structural` bounds.
+    infcx: InferCtxt<'a, 'tcx>,
+
+    include_lint_checks: bool,
+
+    treat_byte_string_as_slice: bool,
+}
+
+mod fallback_to_const_ref {
+    #[derive(Debug)]
+    /// This error type signals that we encountered a non-struct-eq situation behind a reference.
+    /// We bubble this up in order to get back to the reference destructuring and make that emit
+    /// a const pattern instead of a deref pattern. This allows us to simply call `PartialEq::eq`
+    /// on such patterns (since that function takes a reference) and not have to jump through any
+    /// hoops to get a reference to the value.
+    pub(super) struct FallbackToConstRef(());
+
+    pub(super) fn fallback_to_const_ref<'a, 'tcx>(
+        c2p: &super::ConstToPat<'a, 'tcx>,
+    ) -> FallbackToConstRef {
+        assert!(c2p.behind_reference.get());
+        FallbackToConstRef(())
+    }
+}
+use fallback_to_const_ref::{fallback_to_const_ref, FallbackToConstRef};
+
+impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
+    fn new(
+        pat_ctxt: &PatCtxt<'_, 'tcx>,
+        id: hir::HirId,
+        span: Span,
+        infcx: InferCtxt<'a, 'tcx>,
+    ) -> Self {
+        trace!(?pat_ctxt.typeck_results.hir_owner);
+        ConstToPat {
+            id,
+            span,
+            infcx,
+            param_env: pat_ctxt.param_env,
+            include_lint_checks: pat_ctxt.include_lint_checks,
+            saw_const_match_error: Cell::new(false),
+            saw_const_match_lint: Cell::new(false),
+            behind_reference: Cell::new(false),
+            treat_byte_string_as_slice: pat_ctxt
+                .typeck_results
+                .treat_byte_string_as_slice
+                .contains(&id.local_id),
+        }
+    }
+
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
+    }
+
+    fn adt_derive_msg(&self, adt_def: AdtDef<'tcx>) -> String {
+        let path = self.tcx().def_path_str(adt_def.did());
+        format!(
+            "to use a constant of type `{}` in a pattern, \
+            `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+            path, path,
+        )
+    }
+
+    fn search_for_structural_match_violation(&self, ty: Ty<'tcx>) -> Option<String> {
+        traits::search_for_structural_match_violation(self.span, self.tcx(), ty).map(|non_sm_ty| {
+            with_no_trimmed_paths!(match non_sm_ty.kind() {
+                ty::Adt(adt, _) => self.adt_derive_msg(*adt),
+                ty::Dynamic(..) => {
+                    "trait objects cannot be used in patterns".to_string()
+                }
+                ty::Opaque(..) => {
+                    "opaque types cannot be used in patterns".to_string()
+                }
+                ty::Closure(..) => {
+                    "closures cannot be used in patterns".to_string()
+                }
+                ty::Generator(..) | ty::GeneratorWitness(..) => {
+                    "generators cannot be used in patterns".to_string()
+                }
+                ty::Float(..) => {
+                    "floating-point numbers cannot be used in patterns".to_string()
+                }
+                ty::FnPtr(..) => {
+                    "function pointers cannot be used in patterns".to_string()
+                }
+                ty::RawPtr(..) => {
+                    "raw pointers cannot be used in patterns".to_string()
+                }
+                _ => {
+                    bug!("use of a value of `{non_sm_ty}` inside a pattern")
+                }
+            })
+        })
+    }
+
+    fn type_marked_structural(&self, ty: Ty<'tcx>) -> bool {
+        ty.is_structural_eq_shallow(self.infcx.tcx)
+    }
+
+    fn to_pat(
+        &mut self,
+        cv: mir::ConstantKind<'tcx>,
+        mir_structural_match_violation: bool,
+    ) -> Pat<'tcx> {
+        trace!(self.treat_byte_string_as_slice);
+        // This method is just a wrapper handling a validity check; the heavy lifting is
+        // performed by the recursive `recur` method, which is not meant to be
+        // invoked except by this method.
+        //
+        // once indirect_structural_match is a full fledged error, this
+        // level of indirection can be eliminated
+
+        let inlined_const_as_pat = self.recur(cv, mir_structural_match_violation).unwrap();
+
+        if self.include_lint_checks && !self.saw_const_match_error.get() {
+            // If we were able to successfully convert the const to some pat,
+            // double-check that all types in the const implement `Structural`.
+
+            let structural = self.search_for_structural_match_violation(cv.ty());
+            debug!(
+                "search_for_structural_match_violation cv.ty: {:?} returned: {:?}",
+                cv.ty(),
+                structural
+            );
+
+            // This can occur because const qualification treats all associated constants as
+            // opaque, whereas `search_for_structural_match_violation` tries to monomorphize them
+            // before it runs.
+            //
+            // FIXME(#73448): Find a way to bring const qualification into parity with
+            // `search_for_structural_match_violation`.
+            if structural.is_none() && mir_structural_match_violation {
+                warn!("MIR const-checker found novel structural match violation. See #73448.");
+                return inlined_const_as_pat;
+            }
+
+            if let Some(msg) = structural {
+                if !self.type_may_have_partial_eq_impl(cv.ty()) {
+                    // span_fatal avoids ICE from resolution of non-existent method (rare case).
+                    self.tcx().sess.span_fatal(self.span, &msg);
+                } else if mir_structural_match_violation && !self.saw_const_match_lint.get() {
+                    self.tcx().struct_span_lint_hir(
+                        lint::builtin::INDIRECT_STRUCTURAL_MATCH,
+                        self.id,
+                        self.span,
+                        |lint| {
+                            lint.build(&msg).emit();
+                        },
+                    );
+                } else {
+                    debug!(
+                        "`search_for_structural_match_violation` found one, but `CustomEq` was \
+                          not in the qualifs for that `const`"
+                    );
+                }
+            }
+        }
+
+        inlined_const_as_pat
+    }
+
+    fn type_may_have_partial_eq_impl(&self, ty: Ty<'tcx>) -> bool {
+        // double-check there even *is* a semantic `PartialEq` to dispatch to.
+        //
+        // (If there isn't, then we can safely issue a hard
+        // error, because that's never worked, due to compiler
+        // using `PartialEq::eq` in this scenario in the past.)
+        let partial_eq_trait_id =
+            self.tcx().require_lang_item(hir::LangItem::PartialEq, Some(self.span));
+        let obligation: PredicateObligation<'_> = predicate_for_trait_def(
+            self.tcx(),
+            self.param_env,
+            ObligationCause::misc(self.span, self.id),
+            partial_eq_trait_id,
+            0,
+            ty,
+            &[],
+        );
+        // FIXME: should this call a `predicate_must_hold` variant instead?
+
+        let has_impl = self.infcx.predicate_may_hold(&obligation);
+
+        // Note: To fix rust-lang/rust#65466, we could just remove this type
+        // walk hack for function pointers, and unconditionally error
+        // if `PartialEq` is not implemented. However, that breaks stable
+        // code at the moment, because types like `for <'a> fn(&'a ())` do
+        // not *yet* implement `PartialEq`. So for now we leave this here.
+        has_impl
+            || ty.walk().any(|t| match t.unpack() {
+                ty::subst::GenericArgKind::Lifetime(_) => false,
+                ty::subst::GenericArgKind::Type(t) => t.is_fn_ptr(),
+                ty::subst::GenericArgKind::Const(_) => false,
+            })
+    }
+
+    fn field_pats(
+        &self,
+        vals: impl Iterator<Item = mir::ConstantKind<'tcx>>,
+    ) -> Result<Vec<FieldPat<'tcx>>, FallbackToConstRef> {
+        vals.enumerate()
+            .map(|(idx, val)| {
+                let field = Field::new(idx);
+                Ok(FieldPat { field, pattern: self.recur(val, false)? })
+            })
+            .collect()
+    }
+
+    // Recursive helper for `to_pat`; invoke that (instead of calling this directly).
+    #[instrument(skip(self), level = "debug")]
+    fn recur(
+        &self,
+        cv: mir::ConstantKind<'tcx>,
+        mir_structural_match_violation: bool,
+    ) -> Result<Pat<'tcx>, FallbackToConstRef> {
+        let id = self.id;
+        let span = self.span;
+        let tcx = self.tcx();
+        let param_env = self.param_env;
+
+        let kind = match cv.ty().kind() {
+            ty::Float(_) => {
+                if self.include_lint_checks {
+                    tcx.struct_span_lint_hir(
+                        lint::builtin::ILLEGAL_FLOATING_POINT_LITERAL_PATTERN,
+                        id,
+                        span,
+                        |lint| {
+                            lint.build("floating-point types cannot be used in patterns").emit();
+                        },
+                    );
+                }
+                PatKind::Constant { value: cv }
+            }
+            ty::Adt(adt_def, _) if adt_def.is_union() => {
+                // Matching on union fields is unsafe, we can't hide it in constants
+                self.saw_const_match_error.set(true);
+                let msg = "cannot use unions in constant patterns";
+                if self.include_lint_checks {
+                    tcx.sess.span_err(span, msg);
+                } else {
+                    tcx.sess.delay_span_bug(span, msg);
+                }
+                PatKind::Wild
+            }
+            ty::Adt(..)
+                if !self.type_may_have_partial_eq_impl(cv.ty())
+                    // FIXME(#73448): Find a way to bring const qualification into parity with
+                    // `search_for_structural_match_violation` and then remove this condition.
+                    && self.search_for_structural_match_violation(cv.ty()).is_some() =>
+            {
+                // Obtain the actual type that isn't annotated. If we just looked at `cv.ty` we
+                // could get `Option<NonStructEq>`, even though `Option` is annotated with derive.
+                let msg = self.search_for_structural_match_violation(cv.ty()).unwrap();
+                self.saw_const_match_error.set(true);
+                if self.include_lint_checks {
+                    tcx.sess.span_err(self.span, &msg);
+                } else {
+                    tcx.sess.delay_span_bug(self.span, &msg);
+                }
+                PatKind::Wild
+            }
+            // If the type is not structurally comparable, just emit the constant directly,
+            // causing the pattern match code to treat it opaquely.
+            // FIXME: This code doesn't emit errors itself, the caller emits the errors.
+            // So instead of specific errors, you just get blanket errors about the whole
+            // const type. See
+            // https://github.com/rust-lang/rust/pull/70743#discussion_r404701963 for
+            // details.
+            // Backwards compatibility hack because we can't cause hard errors on these
+            // types, so we compare them via `PartialEq::eq` at runtime.
+            ty::Adt(..) if !self.type_marked_structural(cv.ty()) && self.behind_reference.get() => {
+                if self.include_lint_checks
+                    && !self.saw_const_match_error.get()
+                    && !self.saw_const_match_lint.get()
+                {
+                    self.saw_const_match_lint.set(true);
+                    tcx.struct_span_lint_hir(
+                        lint::builtin::INDIRECT_STRUCTURAL_MATCH,
+                        id,
+                        span,
+                        |lint| {
+                            let msg = format!(
+                                "to use a constant of type `{}` in a pattern, \
+                                 `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+                                cv.ty(),
+                                cv.ty(),
+                            );
+                            lint.build(&msg).emit();
+                        },
+                    );
+                }
+                // Since we are behind a reference, we can just bubble the error up so we get a
+                // constant at reference type, making it easy to let the fallback call
+                // `PartialEq::eq` on it.
+                return Err(fallback_to_const_ref(self));
+            }
+            ty::Adt(adt_def, _) if !self.type_marked_structural(cv.ty()) => {
+                debug!(
+                    "adt_def {:?} has !type_marked_structural for cv.ty: {:?}",
+                    adt_def,
+                    cv.ty()
+                );
+                let path = tcx.def_path_str(adt_def.did());
+                let msg = format!(
+                    "to use a constant of type `{}` in a pattern, \
+                     `{}` must be annotated with `#[derive(PartialEq, Eq)]`",
+                    path, path,
+                );
+                self.saw_const_match_error.set(true);
+                if self.include_lint_checks {
+                    tcx.sess.span_err(span, &msg);
+                } else {
+                    tcx.sess.delay_span_bug(span, &msg);
+                }
+                PatKind::Wild
+            }
+            ty::Adt(adt_def, substs) if adt_def.is_enum() => {
+                let destructured = tcx.destructure_mir_constant(param_env, cv);
+
+                PatKind::Variant {
+                    adt_def: *adt_def,
+                    substs,
+                    variant_index: destructured
+                        .variant
+                        .expect("destructed const of adt without variant id"),
+                    subpatterns: self.field_pats(destructured.fields.iter().copied())?,
+                }
+            }
+            ty::Tuple(_) | ty::Adt(_, _) => {
+                let destructured = tcx.destructure_mir_constant(param_env, cv);
+                PatKind::Leaf { subpatterns: self.field_pats(destructured.fields.iter().copied())? }
+            }
+            ty::Array(..) => PatKind::Array {
+                prefix: tcx
+                    .destructure_mir_constant(param_env, cv)
+                    .fields
+                    .iter()
+                    .map(|val| self.recur(*val, false))
+                    .collect::<Result<_, _>>()?,
+                slice: None,
+                suffix: Vec::new(),
+            },
+            ty::Ref(_, pointee_ty, ..) => match *pointee_ty.kind() {
+                // These are not allowed and will error elsewhere anyway.
+                ty::Dynamic(..) => {
+                    self.saw_const_match_error.set(true);
+                    let msg = format!("`{}` cannot be used in patterns", cv.ty());
+                    if self.include_lint_checks {
+                        tcx.sess.span_err(span, &msg);
+                    } else {
+                        tcx.sess.delay_span_bug(span, &msg);
+                    }
+                    PatKind::Wild
+                }
+                // `&str` is represented as `ConstValue::Slice`, let's keep using this
+                // optimization for now.
+                ty::Str => PatKind::Constant { value: cv },
+                // `b"foo"` produces a `&[u8; 3]`, but you can't use constants of array type when
+                // matching against references, you can only use byte string literals.
+                // The typechecker has a special case for byte string literals, by treating them
+                // as slices. This means we turn `&[T; N]` constants into slice patterns, which
+                // has no negative effects on pattern matching, even if we're actually matching on
+                // arrays.
+                ty::Array(..) if !self.treat_byte_string_as_slice => {
+                    let old = self.behind_reference.replace(true);
+                    let array = tcx.deref_mir_constant(self.param_env.and(cv));
+                    let val = PatKind::Deref {
+                        subpattern: Pat {
+                            kind: Box::new(PatKind::Array {
+                                prefix: tcx
+                                    .destructure_mir_constant(param_env, array)
+                                    .fields
+                                    .iter()
+                                    .map(|val| self.recur(*val, false))
+                                    .collect::<Result<_, _>>()?,
+                                slice: None,
+                                suffix: vec![],
+                            }),
+                            span,
+                            ty: *pointee_ty,
+                        },
+                    };
+                    self.behind_reference.set(old);
+                    val
+                }
+                ty::Array(elem_ty, _) |
+                // Cannot merge this with the catch all branch below, because the `const_deref`
+                // changes the type from slice to array, we need to keep the original type in the
+                // pattern.
+                ty::Slice(elem_ty) => {
+                    let old = self.behind_reference.replace(true);
+                    let array = tcx.deref_mir_constant(self.param_env.and(cv));
+                    let val = PatKind::Deref {
+                        subpattern: Pat {
+                            kind: Box::new(PatKind::Slice {
+                                prefix: tcx
+                                    .destructure_mir_constant(param_env, array)
+                                    .fields
+                                    .iter()
+                                    .map(|val| self.recur(*val, false))
+                                    .collect::<Result<_, _>>()?,
+                                slice: None,
+                                suffix: vec![],
+                            }),
+                            span,
+                            ty: tcx.mk_slice(elem_ty),
+                        },
+                    };
+                    self.behind_reference.set(old);
+                    val
+                }
+                // Backwards compatibility hack: support references to non-structural types.
+                // We'll lower
+                // this pattern to a `PartialEq::eq` comparison and `PartialEq::eq` takes a
+                // reference. This makes the rest of the matching logic simpler as it doesn't have
+                // to figure out how to get a reference again.
+                ty::Adt(adt_def, _) if !self.type_marked_structural(*pointee_ty) => {
+                    if self.behind_reference.get() {
+                        if self.include_lint_checks
+                            && !self.saw_const_match_error.get()
+                            && !self.saw_const_match_lint.get()
+                        {
+                            self.saw_const_match_lint.set(true);
+                            let msg = self.adt_derive_msg(adt_def);
+                            self.tcx().struct_span_lint_hir(
+                                lint::builtin::INDIRECT_STRUCTURAL_MATCH,
+                                self.id,
+                                self.span,
+                                |lint| {lint.build(&msg).emit();},
+                            );
+                        }
+                        PatKind::Constant { value: cv }
+                    } else {
+                        if !self.saw_const_match_error.get() {
+                            self.saw_const_match_error.set(true);
+                            let msg = self.adt_derive_msg(adt_def);
+                            if self.include_lint_checks {
+                                tcx.sess.span_err(span, &msg);
+                            } else {
+                                tcx.sess.delay_span_bug(span, &msg);
+                            }
+                        }
+                        PatKind::Wild
+                    }
+                }
+                // All other references are converted into deref patterns and then recursively
+                // convert the dereferenced constant to a pattern that is the sub-pattern of the
+                // deref pattern.
+                _ => {
+                    if !pointee_ty.is_sized(tcx.at(span), param_env) {
+                        // `tcx.deref_mir_constant()` below will ICE with an unsized type
+                        // (except slices, which are handled in a separate arm above).
+                        let msg = format!("cannot use unsized non-slice type `{}` in constant patterns", pointee_ty);
+                        if self.include_lint_checks {
+                            tcx.sess.span_err(span, &msg);
+                        } else {
+                            tcx.sess.delay_span_bug(span, &msg);
+                        }
+                        PatKind::Wild
+                    } else {
+                        let old = self.behind_reference.replace(true);
+                        // In case there are structural-match violations somewhere in this subpattern,
+                        // we fall back to a const pattern. If we do not do this, we may end up with
+                        // a !structural-match constant that is not of reference type, which makes it
+                        // very hard to invoke `PartialEq::eq` on it as a fallback.
+                        let val = match self.recur(tcx.deref_mir_constant(self.param_env.and(cv)), false) {
+                            Ok(subpattern) => PatKind::Deref { subpattern },
+                            Err(_) => PatKind::Constant { value: cv },
+                        };
+                        self.behind_reference.set(old);
+                        val
+                    }
+                }
+            },
+            ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::FnDef(..) => {
+                PatKind::Constant { value: cv }
+            }
+            ty::RawPtr(pointee) if pointee.ty.is_sized(tcx.at(span), param_env) => {
+                PatKind::Constant { value: cv }
+            }
+            // FIXME: these can have very surprising behaviour where optimization levels or other
+            // compilation choices change the runtime behaviour of the match.
+            // See https://github.com/rust-lang/rust/issues/70861 for examples.
+            ty::FnPtr(..) | ty::RawPtr(..) => {
+                if self.include_lint_checks
+                    && !self.saw_const_match_error.get()
+                    && !self.saw_const_match_lint.get()
+                {
+                    self.saw_const_match_lint.set(true);
+                    let msg = "function pointers and unsized pointers in patterns behave \
+                        unpredictably and should not be relied upon. \
+                        See https://github.com/rust-lang/rust/issues/70861 for details.";
+                    tcx.struct_span_lint_hir(
+                        lint::builtin::POINTER_STRUCTURAL_MATCH,
+                        id,
+                        span,
+                        |lint| {
+                            lint.build(msg).emit();
+                        },
+                    );
+                }
+                PatKind::Constant { value: cv }
+            }
+            _ => {
+                self.saw_const_match_error.set(true);
+                let msg = format!("`{}` cannot be used in patterns", cv.ty());
+                if self.include_lint_checks {
+                    tcx.sess.span_err(span, &msg);
+                } else {
+                    tcx.sess.delay_span_bug(span, &msg);
+                }
+                PatKind::Wild
+            }
+        };
+
+        if self.include_lint_checks
+            && !self.saw_const_match_error.get()
+            && !self.saw_const_match_lint.get()
+            && mir_structural_match_violation
+            // FIXME(#73448): Find a way to bring const qualification into parity with
+            // `search_for_structural_match_violation` and then remove this condition.
+            && self.search_for_structural_match_violation(cv.ty()).is_some()
+        {
+            self.saw_const_match_lint.set(true);
+            // Obtain the actual type that isn't annotated. If we just looked at `cv.ty` we
+            // could get `Option<NonStructEq>`, even though `Option` is annotated with derive.
+            let msg = self.search_for_structural_match_violation(cv.ty()).unwrap().replace(
+                "in a pattern,",
+                "in a pattern, the constant's initializer must be trivial or",
+            );
+            tcx.struct_span_lint_hir(
+                lint::builtin::NONTRIVIAL_STRUCTURAL_MATCH,
+                id,
+                span,
+                |lint| {
+                    lint.build(&msg).emit();
+                },
+            );
+        }
+
+        Ok(Pat { span, ty: cv.ty(), kind: Box::new(kind) })
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
new file mode 100644
index 000000000..8d6f8efb6
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -0,0 +1,1711 @@
+//! [`super::usefulness`] explains most of what is happening in this file. As explained there,
+//! values and patterns are made from constructors applied to fields. This file defines a
+//! `Constructor` enum, a `Fields` struct, and various operations to manipulate them and convert
+//! them from/to patterns.
+//!
+//! There's one idea that is not detailed in [`super::usefulness`] because the details are not
+//! needed there: _constructor splitting_.
+//!
+//! # Constructor splitting
+//!
+//! The idea is as follows: given a constructor `c` and a matrix, we want to specialize in turn
+//! with all the value constructors that are covered by `c`, and compute usefulness for each.
+//! Instead of listing all those constructors (which is intractable), we group those value
+//! constructors together as much as possible. Example:
+//!
+//! ```compile_fail,E0004
+//! match (0, false) {
+//!     (0 ..=100, true) => {} // `p_1`
+//!     (50..=150, false) => {} // `p_2`
+//!     (0 ..=200, _) => {} // `q`
+//! }
+//! ```
+//!
+//! The naive approach would try all numbers in the range `0..=200`. But we can be a lot more
+//! clever: `0` and `1` for example will match the exact same rows, and return equivalent
+//! witnesses. In fact all of `0..50` would. We can thus restrict our exploration to 4
+//! constructors: `0..50`, `50..=100`, `101..=150` and `151..=200`. That is enough and infinitely
+//! more tractable.
+//!
+//! We capture this idea in a function `split(p_1 ... p_n, c)` which returns a list of constructors
+//! `c'` covered by `c`. Given such a `c'`, we require that all value ctors `c''` covered by `c'`
+//! return an equivalent set of witnesses after specializing and computing usefulness.
+//! In the example above, witnesses for specializing by `c''` covered by `0..50` will only differ
+//! in their first element.
+//!
+//! We usually also ask that the `c'` together cover all of the original `c`. However we allow
+//! skipping some constructors as long as it doesn't change whether the resulting list of witnesses
+//! is empty of not. We use this in the wildcard `_` case.
+//!
+//! Splitting is implemented in the [`Constructor::split`] function. We don't do splitting for
+//! or-patterns; instead we just try the alternatives one-by-one. For details on splitting
+//! wildcards, see [`SplitWildcard`]; for integer ranges, see [`SplitIntRange`]; for slices, see
+//! [`SplitVarLenSlice`].
+
+use self::Constructor::*;
+use self::SliceKind::*;
+
+use super::compare_const_vals;
+use super::usefulness::{MatchCheckCtxt, PatCtxt};
+
+use rustc_data_structures::captures::Captures;
+use rustc_index::vec::Idx;
+
+use rustc_hir::{HirId, RangeEnd};
+use rustc_middle::mir::{self, Field};
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
+use rustc_middle::{middle::stability::EvalResult, mir::interpret::ConstValue};
+use rustc_session::lint;
+use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi::{Integer, Size, VariantIdx};
+
+use smallvec::{smallvec, SmallVec};
+use std::cell::Cell;
+use std::cmp::{self, max, min, Ordering};
+use std::fmt;
+use std::iter::{once, IntoIterator};
+use std::ops::RangeInclusive;
+
+/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
+fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
+    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
+        if let PatKind::Or { pats } = pat.kind.as_ref() {
+            for pat in pats {
+                expand(pat, vec);
+            }
+        } else {
+            vec.push(pat)
+        }
+    }
+
+    let mut pats = Vec::new();
+    expand(pat, &mut pats);
+    pats
+}
+
+/// An inclusive interval, used for precise integer exhaustiveness checking.
+/// `IntRange`s always store a contiguous range. This means that values are
+/// encoded such that `0` encodes the minimum value for the integer,
+/// regardless of the signedness.
+/// For example, the pattern `-128..=127i8` is encoded as `0..=255`.
+/// This makes comparisons and arithmetic on interval endpoints much more
+/// straightforward. See `signed_bias` for details.
+///
+/// `IntRange` is never used to encode an empty range or a "range" that wraps
+/// around the (offset) space: i.e., `range.lo <= range.hi`.
+#[derive(Clone, PartialEq, Eq)]
+pub(super) struct IntRange {
+    range: RangeInclusive<u128>,
+    /// Keeps the bias used for encoding the range. It depends on the type of the range and
+    /// possibly the pointer size of the current architecture. The algorithm ensures we never
+    /// compare `IntRange`s with different types/architectures.
+    bias: u128,
+}
+
+impl IntRange {
+    #[inline]
+    fn is_integral(ty: Ty<'_>) -> bool {
+        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_) | ty::Bool)
+    }
+
+    fn is_singleton(&self) -> bool {
+        self.range.start() == self.range.end()
+    }
+
+    fn boundaries(&self) -> (u128, u128) {
+        (*self.range.start(), *self.range.end())
+    }
+
+    #[inline]
+    fn integral_size_and_signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> Option<(Size, u128)> {
+        match *ty.kind() {
+            ty::Bool => Some((Size::from_bytes(1), 0)),
+            ty::Char => Some((Size::from_bytes(4), 0)),
+            ty::Int(ity) => {
+                let size = Integer::from_int_ty(&tcx, ity).size();
+                Some((size, 1u128 << (size.bits() as u128 - 1)))
+            }
+            ty::Uint(uty) => Some((Integer::from_uint_ty(&tcx, uty).size(), 0)),
+            _ => None,
+        }
+    }
+
+    #[inline]
+    fn from_constant<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+        value: mir::ConstantKind<'tcx>,
+    ) -> Option<IntRange> {
+        let ty = value.ty();
+        if let Some((target_size, bias)) = Self::integral_size_and_signed_bias(tcx, ty) {
+            let val = (|| {
+                match value {
+                    mir::ConstantKind::Val(ConstValue::Scalar(scalar), _) => {
+                        // For this specific pattern we can skip a lot of effort and go
+                        // straight to the result, after doing a bit of checking. (We
+                        // could remove this branch and just fall through, which
+                        // is more general but much slower.)
+                        if let Ok(Ok(bits)) = scalar.to_bits_or_ptr_internal(target_size) {
+                            return Some(bits);
+                        } else {
+                            return None;
+                        }
+                    }
+                    mir::ConstantKind::Ty(c) => match c.kind() {
+                        ty::ConstKind::Value(_) => bug!(
+                            "encountered ConstValue in mir::ConstantKind::Ty, whereas this is expected to be in ConstantKind::Val"
+                        ),
+                        _ => {}
+                    },
+                    _ => {}
+                }
+
+                // This is a more general form of the previous case.
+                value.try_eval_bits(tcx, param_env, ty)
+            })()?;
+            let val = val ^ bias;
+            Some(IntRange { range: val..=val, bias })
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn from_range<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        lo: u128,
+        hi: u128,
+        ty: Ty<'tcx>,
+        end: &RangeEnd,
+    ) -> Option<IntRange> {
+        if Self::is_integral(ty) {
+            // Perform a shift if the underlying types are signed,
+            // which makes the interval arithmetic simpler.
+            let bias = IntRange::signed_bias(tcx, ty);
+            let (lo, hi) = (lo ^ bias, hi ^ bias);
+            let offset = (*end == RangeEnd::Excluded) as u128;
+            if lo > hi || (lo == hi && *end == RangeEnd::Excluded) {
+                // This should have been caught earlier by E0030.
+                bug!("malformed range pattern: {}..={}", lo, (hi - offset));
+            }
+            Some(IntRange { range: lo..=(hi - offset), bias })
+        } else {
+            None
+        }
+    }
+
+    // The return value of `signed_bias` should be XORed with an endpoint to encode/decode it.
+    fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
+        match *ty.kind() {
+            ty::Int(ity) => {
+                let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
+                1u128 << (bits - 1)
+            }
+            _ => 0,
+        }
+    }
+
+    fn is_subrange(&self, other: &Self) -> bool {
+        other.range.start() <= self.range.start() && self.range.end() <= other.range.end()
+    }
+
+    fn intersection(&self, other: &Self) -> Option<Self> {
+        let (lo, hi) = self.boundaries();
+        let (other_lo, other_hi) = other.boundaries();
+        if lo <= other_hi && other_lo <= hi {
+            Some(IntRange { range: max(lo, other_lo)..=min(hi, other_hi), bias: self.bias })
+        } else {
+            None
+        }
+    }
+
+    fn suspicious_intersection(&self, other: &Self) -> bool {
+        // `false` in the following cases:
+        // 1     ----      // 1  ----------   // 1 ----        // 1       ----
+        // 2  ----------   // 2     ----      // 2       ----  // 2 ----
+        //
+        // The following are currently `false`, but could be `true` in the future (#64007):
+        // 1 ---------       // 1     ---------
+        // 2     ----------  // 2 ----------
+        //
+        // `true` in the following cases:
+        // 1 -------          // 1       -------
+        // 2       --------   // 2 -------
+        let (lo, hi) = self.boundaries();
+        let (other_lo, other_hi) = other.boundaries();
+        (lo == other_hi || hi == other_lo) && !self.is_singleton() && !other.is_singleton()
+    }
+
+    /// Only used for displaying the range properly.
+    fn to_pat<'tcx>(&self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Pat<'tcx> {
+        let (lo, hi) = self.boundaries();
+
+        let bias = self.bias;
+        let (lo, hi) = (lo ^ bias, hi ^ bias);
+
+        let env = ty::ParamEnv::empty().and(ty);
+        let lo_const = mir::ConstantKind::from_bits(tcx, lo, env);
+        let hi_const = mir::ConstantKind::from_bits(tcx, hi, env);
+
+        let kind = if lo == hi {
+            PatKind::Constant { value: lo_const }
+        } else {
+            PatKind::Range(PatRange { lo: lo_const, hi: hi_const, end: RangeEnd::Included })
+        };
+
+        Pat { ty, span: DUMMY_SP, kind: Box::new(kind) }
+    }
+
+    /// Lint on likely incorrect range patterns (#63987)
+    pub(super) fn lint_overlapping_range_endpoints<'a, 'p: 'a, 'tcx: 'a>(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        pats: impl Iterator<Item = &'a DeconstructedPat<'p, 'tcx>>,
+        column_count: usize,
+        hir_id: HirId,
+    ) {
+        if self.is_singleton() {
+            return;
+        }
+
+        if column_count != 1 {
+            // FIXME: for now, only check for overlapping ranges on simple range
+            // patterns. Otherwise with the current logic the following is detected
+            // as overlapping:
+            // ```
+            // match (0u8, true) {
+            //   (0 ..= 125, false) => {}
+            //   (125 ..= 255, true) => {}
+            //   _ => {}
+            // }
+            // ```
+            return;
+        }
+
+        let overlaps: Vec<_> = pats
+            .filter_map(|pat| Some((pat.ctor().as_int_range()?, pat.span())))
+            .filter(|(range, _)| self.suspicious_intersection(range))
+            .map(|(range, span)| (self.intersection(&range).unwrap(), span))
+            .collect();
+
+        if !overlaps.is_empty() {
+            pcx.cx.tcx.struct_span_lint_hir(
+                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
+                hir_id,
+                pcx.span,
+                |lint| {
+                    let mut err = lint.build("multiple patterns overlap on their endpoints");
+                    for (int_range, span) in overlaps {
+                        err.span_label(
+                            span,
+                            &format!(
+                                "this range overlaps on `{}`...",
+                                int_range.to_pat(pcx.cx.tcx, pcx.ty)
+                            ),
+                        );
+                    }
+                    err.span_label(pcx.span, "... with this range");
+                    err.note("you likely meant to write mutually exclusive ranges");
+                    err.emit();
+                },
+            );
+        }
+    }
+
+    /// See `Constructor::is_covered_by`
+    fn is_covered_by(&self, other: &Self) -> bool {
+        if self.intersection(other).is_some() {
+            // Constructor splitting should ensure that all intersections we encounter are actually
+            // inclusions.
+            assert!(self.is_subrange(other));
+            true
+        } else {
+            false
+        }
+    }
+}
+
+/// Note: this is often not what we want: e.g. `false` is converted into the range `0..=0` and
+/// would be displayed as such. To render properly, convert to a pattern first.
+impl fmt::Debug for IntRange {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let (lo, hi) = self.boundaries();
+        let bias = self.bias;
+        let (lo, hi) = (lo ^ bias, hi ^ bias);
+        write!(f, "{}", lo)?;
+        write!(f, "{}", RangeEnd::Included)?;
+        write!(f, "{}", hi)
+    }
+}
+
+/// Represents a border between 2 integers. Because the intervals spanning borders must be able to
+/// cover every integer, we need to be able to represent 2^128 + 1 such borders.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum IntBorder {
+    JustBefore(u128),
+    AfterMax,
+}
+
+/// A range of integers that is partitioned into disjoint subranges. This does constructor
+/// splitting for integer ranges as explained at the top of the file.
+///
+/// This is fed multiple ranges, and returns an output that covers the input, but is split so that
+/// the only intersections between an output range and a seen range are inclusions. No output range
+/// straddles the boundary of one of the inputs.
+///
+/// The following input:
+/// ```text
+///   |-------------------------| // `self`
+/// |------|  |----------|   |----|
+///    |-------| |-------|
+/// ```
+/// would be iterated over as follows:
+/// ```text
+///   ||---|--||-|---|---|---|--|
+/// ```
+#[derive(Debug, Clone)]
+struct SplitIntRange {
+    /// The range we are splitting
+    range: IntRange,
+    /// The borders of ranges we have seen. They are all contained within `range`. This is kept
+    /// sorted.
+    borders: Vec<IntBorder>,
+}
+
+impl SplitIntRange {
+    fn new(range: IntRange) -> Self {
+        SplitIntRange { range, borders: Vec::new() }
+    }
+
+    /// Internal use
+    fn to_borders(r: IntRange) -> [IntBorder; 2] {
+        use IntBorder::*;
+        let (lo, hi) = r.boundaries();
+        let lo = JustBefore(lo);
+        let hi = match hi.checked_add(1) {
+            Some(m) => JustBefore(m),
+            None => AfterMax,
+        };
+        [lo, hi]
+    }
+
+    /// Add ranges relative to which we split.
+    fn split(&mut self, ranges: impl Iterator<Item = IntRange>) {
+        let this_range = &self.range;
+        let included_ranges = ranges.filter_map(|r| this_range.intersection(&r));
+        let included_borders = included_ranges.flat_map(|r| {
+            let borders = Self::to_borders(r);
+            once(borders[0]).chain(once(borders[1]))
+        });
+        self.borders.extend(included_borders);
+        self.borders.sort_unstable();
+    }
+
+    /// Iterate over the contained ranges.
+    fn iter<'a>(&'a self) -> impl Iterator<Item = IntRange> + Captures<'a> {
+        use IntBorder::*;
+
+        let self_range = Self::to_borders(self.range.clone());
+        // Start with the start of the range.
+        let mut prev_border = self_range[0];
+        self.borders
+            .iter()
+            .copied()
+            // End with the end of the range.
+            .chain(once(self_range[1]))
+            // List pairs of adjacent borders.
+            .map(move |border| {
+                let ret = (prev_border, border);
+                prev_border = border;
+                ret
+            })
+            // Skip duplicates.
+            .filter(|(prev_border, border)| prev_border != border)
+            // Finally, convert to ranges.
+            .map(move |(prev_border, border)| {
+                let range = match (prev_border, border) {
+                    (JustBefore(n), JustBefore(m)) if n < m => n..=(m - 1),
+                    (JustBefore(n), AfterMax) => n..=u128::MAX,
+                    _ => unreachable!(), // Ruled out by the sorting and filtering we did
+                };
+                IntRange { range, bias: self.range.bias }
+            })
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum SliceKind {
+    /// Patterns of length `n` (`[x, y]`).
+    FixedLen(usize),
+    /// Patterns using the `..` notation (`[x, .., y]`).
+    /// Captures any array constructor of `length >= i + j`.
+    /// In the case where `array_len` is `Some(_)`,
+    /// this indicates that we only care about the first `i` and the last `j` values of the array,
+    /// and everything in between is a wildcard `_`.
+    VarLen(usize, usize),
+}
+
+impl SliceKind {
+    fn arity(self) -> usize {
+        match self {
+            FixedLen(length) => length,
+            VarLen(prefix, suffix) => prefix + suffix,
+        }
+    }
+
+    /// Whether this pattern includes patterns of length `other_len`.
+    fn covers_length(self, other_len: usize) -> bool {
+        match self {
+            FixedLen(len) => len == other_len,
+            VarLen(prefix, suffix) => prefix + suffix <= other_len,
+        }
+    }
+}
+
+/// A constructor for array and slice patterns.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(super) struct Slice {
+    /// `None` if the matched value is a slice, `Some(n)` if it is an array of size `n`.
+    array_len: Option<usize>,
+    /// The kind of pattern it is: fixed-length `[x, y]` or variable length `[x, .., y]`.
+    kind: SliceKind,
+}
+
+impl Slice {
+    fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
+        let kind = match (array_len, kind) {
+            // If the middle `..` is empty, we effectively have a fixed-length pattern.
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix >= len => FixedLen(len),
+            _ => kind,
+        };
+        Slice { array_len, kind }
+    }
+
+    fn arity(self) -> usize {
+        self.kind.arity()
+    }
+
+    /// See `Constructor::is_covered_by`
+    fn is_covered_by(self, other: Self) -> bool {
+        other.kind.covers_length(self.arity())
+    }
+}
+
+/// This computes constructor splitting for variable-length slices, as explained at the top of the
+/// file.
+///
+/// A slice pattern `[x, .., y]` behaves like the infinite or-pattern `[x, y] | [x, _, y] | [x, _,
+/// _, y] | ...`. The corresponding value constructors are fixed-length array constructors above a
+/// given minimum length. We obviously can't list this infinitude of constructors. Thankfully,
+/// it turns out that for each finite set of slice patterns, all sufficiently large array lengths
+/// are equivalent.
+///
+/// Let's look at an example, where we are trying to split the last pattern:
+/// ```
+/// # fn foo(x: &[bool]) {
+/// match x {
+///     [true, true, ..] => {}
+///     [.., false, false] => {}
+///     [..] => {}
+/// }
+/// # }
+/// ```
+/// Here are the results of specialization for the first few lengths:
+/// ```
+/// # fn foo(x: &[bool]) { match x {
+/// // length 0
+/// [] => {}
+/// // length 1
+/// [_] => {}
+/// // length 2
+/// [true, true] => {}
+/// [false, false] => {}
+/// [_, _] => {}
+/// // length 3
+/// [true, true,  _    ] => {}
+/// [_,    false, false] => {}
+/// [_,    _,     _    ] => {}
+/// // length 4
+/// [true, true, _,     _    ] => {}
+/// [_,    _,    false, false] => {}
+/// [_,    _,    _,     _    ] => {}
+/// // length 5
+/// [true, true, _, _,     _    ] => {}
+/// [_,    _,    _, false, false] => {}
+/// [_,    _,    _, _,     _    ] => {}
+/// # _ => {}
+/// # }}
+/// ```
+///
+/// If we went above length 5, we would simply be inserting more columns full of wildcards in the
+/// middle. This means that the set of witnesses for length `l >= 5` if equivalent to the set for
+/// any other `l' >= 5`: simply add or remove wildcards in the middle to convert between them.
+///
+/// This applies to any set of slice patterns: there will be a length `L` above which all lengths
+/// behave the same. This is exactly what we need for constructor splitting. Therefore a
+/// variable-length slice can be split into a variable-length slice of minimal length `L`, and many
+/// fixed-length slices of lengths `< L`.
+///
+/// For each variable-length pattern `p` with a prefix of length `plₚ` and suffix of length `slₚ`,
+/// only the first `plₚ` and the last `slₚ` elements are examined. Therefore, as long as `L` is
+/// positive (to avoid concerns about empty types), all elements after the maximum prefix length
+/// and before the maximum suffix length are not examined by any variable-length pattern, and
+/// therefore can be added/removed without affecting them - creating equivalent patterns from any
+/// sufficiently-large length.
+///
+/// Of course, if fixed-length patterns exist, we must be sure that our length is large enough to
+/// miss them all, so we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))`
+///
+/// `max_slice` below will be made to have arity `L`.
+#[derive(Debug)]
+struct SplitVarLenSlice {
+    /// If the type is an array, this is its size.
+    array_len: Option<usize>,
+    /// The arity of the input slice.
+    arity: usize,
+    /// The smallest slice bigger than any slice seen. `max_slice.arity()` is the length `L`
+    /// described above.
+    max_slice: SliceKind,
+}
+
+impl SplitVarLenSlice {
+    fn new(prefix: usize, suffix: usize, array_len: Option<usize>) -> Self {
+        SplitVarLenSlice { array_len, arity: prefix + suffix, max_slice: VarLen(prefix, suffix) }
+    }
+
+    /// Pass a set of slices relative to which to split this one.
+    fn split(&mut self, slices: impl Iterator<Item = SliceKind>) {
+        let VarLen(max_prefix_len, max_suffix_len) = &mut self.max_slice else {
+            // No need to split
+            return;
+        };
+        // We grow `self.max_slice` to be larger than all slices encountered, as described above.
+        // For diagnostics, we keep the prefix and suffix lengths separate, but grow them so that
+        // `L = max_prefix_len + max_suffix_len`.
+        let mut max_fixed_len = 0;
+        for slice in slices {
+            match slice {
+                FixedLen(len) => {
+                    max_fixed_len = cmp::max(max_fixed_len, len);
+                }
+                VarLen(prefix, suffix) => {
+                    *max_prefix_len = cmp::max(*max_prefix_len, prefix);
+                    *max_suffix_len = cmp::max(*max_suffix_len, suffix);
+                }
+            }
+        }
+        // We want `L = max(L, max_fixed_len + 1)`, modulo the fact that we keep prefix and
+        // suffix separate.
+        if max_fixed_len + 1 >= *max_prefix_len + *max_suffix_len {
+            // The subtraction can't overflow thanks to the above check.
+            // The new `max_prefix_len` is larger than its previous value.
+            *max_prefix_len = max_fixed_len + 1 - *max_suffix_len;
+        }
+
+        // We cap the arity of `max_slice` at the array size.
+        match self.array_len {
+            Some(len) if self.max_slice.arity() >= len => self.max_slice = FixedLen(len),
+            _ => {}
+        }
+    }
+
+    /// Iterate over the partition of this slice.
+    fn iter<'a>(&'a self) -> impl Iterator<Item = Slice> + Captures<'a> {
+        let smaller_lengths = match self.array_len {
+            // The only admissible fixed-length slice is one of the array size. Whether `max_slice`
+            // is fixed-length or variable-length, it will be the only relevant slice to output
+            // here.
+            Some(_) => 0..0, // empty range
+            // We cover all arities in the range `(self.arity..infinity)`. We split that range into
+            // two: lengths smaller than `max_slice.arity()` are treated independently as
+            // fixed-lengths slices, and lengths above are captured by `max_slice`.
+            None => self.arity..self.max_slice.arity(),
+        };
+        smaller_lengths
+            .map(FixedLen)
+            .chain(once(self.max_slice))
+            .map(move |kind| Slice::new(self.array_len, kind))
+    }
+}
+
+/// A value can be decomposed into a constructor applied to some fields. This struct represents
+/// the constructor. See also `Fields`.
+///
+/// `pat_constructor` retrieves the constructor corresponding to a pattern.
+/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a
+/// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and
+/// `Fields`.
+#[derive(Clone, Debug, PartialEq)]
+pub(super) enum Constructor<'tcx> {
+    /// The constructor for patterns that have a single constructor, like tuples, struct patterns
+    /// and fixed-length arrays.
+    Single,
+    /// Enum variants.
+    Variant(VariantIdx),
+    /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
+    IntRange(IntRange),
+    /// Ranges of floating-point literal values (`2.0..=5.2`).
+    FloatRange(mir::ConstantKind<'tcx>, mir::ConstantKind<'tcx>, RangeEnd),
+    /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
+    Str(mir::ConstantKind<'tcx>),
+    /// Array and slice patterns.
+    Slice(Slice),
+    /// Constants that must not be matched structurally. They are treated as black
+    /// boxes for the purposes of exhaustiveness: we must not inspect them, and they
+    /// don't count towards making a match exhaustive.
+    Opaque,
+    /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
+    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
+    NonExhaustive,
+    /// Stands for constructors that are not seen in the matrix, as explained in the documentation
+    /// for [`SplitWildcard`]. The carried `bool` is used for the `non_exhaustive_omitted_patterns`
+    /// lint.
+    Missing { nonexhaustive_enum_missing_real_variants: bool },
+    /// Wildcard pattern.
+    Wildcard,
+    /// Or-pattern.
+    Or,
+}
+
+impl<'tcx> Constructor<'tcx> {
+    pub(super) fn is_wildcard(&self) -> bool {
+        matches!(self, Wildcard)
+    }
+
+    pub(super) fn is_non_exhaustive(&self) -> bool {
+        matches!(self, NonExhaustive)
+    }
+
+    fn as_int_range(&self) -> Option<&IntRange> {
+        match self {
+            IntRange(range) => Some(range),
+            _ => None,
+        }
+    }
+
+    fn as_slice(&self) -> Option<Slice> {
+        match self {
+            Slice(slice) => Some(*slice),
+            _ => None,
+        }
+    }
+
+    /// Checks if the `Constructor` is a variant and `TyCtxt::eval_stability` returns
+    /// `EvalResult::Deny { .. }`.
+    ///
+    /// This means that the variant has a stdlib unstable feature marking it.
+    pub(super) fn is_unstable_variant(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> bool {
+        if let Constructor::Variant(idx) = self && let ty::Adt(adt, _) = pcx.ty.kind() {
+            let variant_def_id = adt.variant(*idx).def_id;
+            // Filter variants that depend on a disabled unstable feature.
+            return matches!(
+                pcx.cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
+                EvalResult::Deny { .. }
+            );
+        }
+        false
+    }
+
+    /// Checks if the `Constructor` is a `Constructor::Variant` with a `#[doc(hidden)]`
+    /// attribute from a type not local to the current crate.
+    pub(super) fn is_doc_hidden_variant(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> bool {
+        if let Constructor::Variant(idx) = self && let ty::Adt(adt, _) = pcx.ty.kind() {
+            let variant_def_id = adt.variants()[*idx].def_id;
+            return pcx.cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
+        }
+        false
+    }
+
+    fn variant_index_for_adt(&self, adt: ty::AdtDef<'tcx>) -> VariantIdx {
+        match *self {
+            Variant(idx) => idx,
+            Single => {
+                assert!(!adt.is_enum());
+                VariantIdx::new(0)
+            }
+            _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
+        }
+    }
+
+    /// The number of fields for this constructor. This must be kept in sync with
+    /// `Fields::wildcards`.
+    pub(super) fn arity(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> usize {
+        match self {
+            Single | Variant(_) => match pcx.ty.kind() {
+                ty::Tuple(fs) => fs.len(),
+                ty::Ref(..) => 1,
+                ty::Adt(adt, ..) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        1
+                    } else {
+                        let variant = &adt.variant(self.variant_index_for_adt(*adt));
+                        Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant).count()
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
+            },
+            Slice(slice) => slice.arity(),
+            Str(..)
+            | FloatRange(..)
+            | IntRange(..)
+            | NonExhaustive
+            | Opaque
+            | Missing { .. }
+            | Wildcard => 0,
+            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
+        }
+    }
+
+    /// Some constructors (namely `Wildcard`, `IntRange` and `Slice`) actually stand for a set of actual
+    /// constructors (like variants, integers or fixed-sized slices). When specializing for these
+    /// constructors, we want to be specialising for the actual underlying constructors.
+    /// Naively, we would simply return the list of constructors they correspond to. We instead are
+    /// more clever: if there are constructors that we know will behave the same wrt the current
+    /// matrix, we keep them grouped. For example, all slices of a sufficiently large length
+    /// will either be all useful or all non-useful with a given matrix.
+    ///
+    /// See the branches for details on how the splitting is done.
+    ///
+    /// This function may discard some irrelevant constructors if this preserves behavior and
+    /// diagnostics. Eg. for the `_` case, we ignore the constructors already present in the
+    /// matrix, unless all of them are.
+    pub(super) fn split<'a>(
+        &self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
+    ) -> SmallVec<[Self; 1]>
+    where
+        'tcx: 'a,
+    {
+        match self {
+            Wildcard => {
+                let mut split_wildcard = SplitWildcard::new(pcx);
+                split_wildcard.split(pcx, ctors);
+                split_wildcard.into_ctors(pcx)
+            }
+            // Fast-track if the range is trivial. In particular, we don't do the overlapping
+            // ranges check.
+            IntRange(ctor_range) if !ctor_range.is_singleton() => {
+                let mut split_range = SplitIntRange::new(ctor_range.clone());
+                let int_ranges = ctors.filter_map(|ctor| ctor.as_int_range());
+                split_range.split(int_ranges.cloned());
+                split_range.iter().map(IntRange).collect()
+            }
+            &Slice(Slice { kind: VarLen(self_prefix, self_suffix), array_len }) => {
+                let mut split_self = SplitVarLenSlice::new(self_prefix, self_suffix, array_len);
+                let slices = ctors.filter_map(|c| c.as_slice()).map(|s| s.kind);
+                split_self.split(slices);
+                split_self.iter().map(Slice).collect()
+            }
+            // Any other constructor can be used unchanged.
+            _ => smallvec![self.clone()],
+        }
+    }
+
+    /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
+    /// For the simple cases, this is simply checking for equality. For the "grouped" constructors,
+    /// this checks for inclusion.
+    // We inline because this has a single call site in `Matrix::specialize_constructor`.
+    #[inline]
+    pub(super) fn is_covered_by<'p>(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool {
+        // This must be kept in sync with `is_covered_by_any`.
+        match (self, other) {
+            // Wildcards cover anything
+            (_, Wildcard) => true,
+            // The missing ctors are not covered by anything in the matrix except wildcards.
+            (Missing { .. } | Wildcard, _) => false,
+
+            (Single, Single) => true,
+            (Variant(self_id), Variant(other_id)) => self_id == other_id,
+
+            (IntRange(self_range), IntRange(other_range)) => self_range.is_covered_by(other_range),
+            (
+                FloatRange(self_from, self_to, self_end),
+                FloatRange(other_from, other_to, other_end),
+            ) => {
+                match (
+                    compare_const_vals(pcx.cx.tcx, *self_to, *other_to, pcx.cx.param_env),
+                    compare_const_vals(pcx.cx.tcx, *self_from, *other_from, pcx.cx.param_env),
+                ) {
+                    (Some(to), Some(from)) => {
+                        (from == Ordering::Greater || from == Ordering::Equal)
+                            && (to == Ordering::Less
+                                || (other_end == self_end && to == Ordering::Equal))
+                    }
+                    _ => false,
+                }
+            }
+            (Str(self_val), Str(other_val)) => {
+                // FIXME Once valtrees are available we can directly use the bytes
+                // in the `Str` variant of the valtree for the comparison here.
+                self_val == other_val
+            }
+            (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice),
+
+            // We are trying to inspect an opaque constant. Thus we skip the row.
+            (Opaque, _) | (_, Opaque) => false,
+            // Only a wildcard pattern can match the special extra constructor.
+            (NonExhaustive, _) => false,
+
+            _ => span_bug!(
+                pcx.span,
+                "trying to compare incompatible constructors {:?} and {:?}",
+                self,
+                other
+            ),
+        }
+    }
+
+    /// Faster version of `is_covered_by` when applied to many constructors. `used_ctors` is
+    /// assumed to be built from `matrix.head_ctors()` with wildcards filtered out, and `self` is
+    /// assumed to have been split from a wildcard.
+    fn is_covered_by_any<'p>(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        used_ctors: &[Constructor<'tcx>],
+    ) -> bool {
+        if used_ctors.is_empty() {
+            return false;
+        }
+
+        // This must be kept in sync with `is_covered_by`.
+        match self {
+            // If `self` is `Single`, `used_ctors` cannot contain anything else than `Single`s.
+            Single => !used_ctors.is_empty(),
+            Variant(vid) => used_ctors.iter().any(|c| matches!(c, Variant(i) if i == vid)),
+            IntRange(range) => used_ctors
+                .iter()
+                .filter_map(|c| c.as_int_range())
+                .any(|other| range.is_covered_by(other)),
+            Slice(slice) => used_ctors
+                .iter()
+                .filter_map(|c| c.as_slice())
+                .any(|other| slice.is_covered_by(other)),
+            // This constructor is never covered by anything else
+            NonExhaustive => false,
+            Str(..) | FloatRange(..) | Opaque | Missing { .. } | Wildcard | Or => {
+                span_bug!(pcx.span, "found unexpected ctor in all_ctors: {:?}", self)
+            }
+        }
+    }
+}
+
+/// A wildcard constructor that we split relative to the constructors in the matrix, as explained
+/// at the top of the file.
+///
+/// A constructor that is not present in the matrix rows will only be covered by the rows that have
+/// wildcards. Thus we can group all of those constructors together; we call them "missing
+/// constructors". Splitting a wildcard would therefore list all present constructors individually
+/// (or grouped if they are integers or slices), and then all missing constructors together as a
+/// group.
+///
+/// However we can go further: since any constructor will match the wildcard rows, and having more
+/// rows can only reduce the amount of usefulness witnesses, we can skip the present constructors
+/// and only try the missing ones.
+/// This will not preserve the whole list of witnesses, but will preserve whether the list is empty
+/// or not. In fact this is quite natural from the point of view of diagnostics too. This is done
+/// in `to_ctors`: in some cases we only return `Missing`.
+#[derive(Debug)]
+pub(super) struct SplitWildcard<'tcx> {
+    /// Constructors seen in the matrix.
+    matrix_ctors: Vec<Constructor<'tcx>>,
+    /// All the constructors for this type
+    all_ctors: SmallVec<[Constructor<'tcx>; 1]>,
+}
+
+impl<'tcx> SplitWildcard<'tcx> {
+    pub(super) fn new<'p>(pcx: &PatCtxt<'_, 'p, 'tcx>) -> Self {
+        debug!("SplitWildcard::new({:?})", pcx.ty);
+        let cx = pcx.cx;
+        let make_range = |start, end| {
+            IntRange(
+                // `unwrap()` is ok because we know the type is an integer.
+                IntRange::from_range(cx.tcx, start, end, pcx.ty, &RangeEnd::Included).unwrap(),
+            )
+        };
+        // This determines the set of all possible constructors for the type `pcx.ty`. For numbers,
+        // arrays and slices we use ranges and variable-length slices when appropriate.
+        //
+        // If the `exhaustive_patterns` feature is enabled, we make sure to omit constructors that
+        // are statically impossible. E.g., for `Option<!>`, we do not include `Some(_)` in the
+        // returned list of constructors.
+        // Invariant: this is empty if and only if the type is uninhabited (as determined by
+        // `cx.is_uninhabited()`).
+        let all_ctors = match pcx.ty.kind() {
+            ty::Bool => smallvec![make_range(0, 1)],
+            ty::Array(sub_ty, len) if len.try_eval_usize(cx.tcx, cx.param_env).is_some() => {
+                let len = len.eval_usize(cx.tcx, cx.param_env) as usize;
+                if len != 0 && cx.is_uninhabited(*sub_ty) {
+                    smallvec![]
+                } else {
+                    smallvec![Slice(Slice::new(Some(len), VarLen(0, 0)))]
+                }
+            }
+            // Treat arrays of a constant but unknown length like slices.
+            ty::Array(sub_ty, _) | ty::Slice(sub_ty) => {
+                let kind = if cx.is_uninhabited(*sub_ty) { FixedLen(0) } else { VarLen(0, 0) };
+                smallvec![Slice(Slice::new(None, kind))]
+            }
+            ty::Adt(def, substs) if def.is_enum() => {
+                // If the enum is declared as `#[non_exhaustive]`, we treat it as if it had an
+                // additional "unknown" constructor.
+                // There is no point in enumerating all possible variants, because the user can't
+                // actually match against them all themselves. So we always return only the fictitious
+                // constructor.
+                // E.g., in an example like:
+                //
+                // ```
+                //     let err: io::ErrorKind = ...;
+                //     match err {
+                //         io::ErrorKind::NotFound => {},
+                //     }
+                // ```
+                //
+                // we don't want to show every possible IO error, but instead have only `_` as the
+                // witness.
+                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(pcx.ty);
+
+                let is_exhaustive_pat_feature = cx.tcx.features().exhaustive_patterns;
+
+                // If `exhaustive_patterns` is disabled and our scrutinee is an empty enum, we treat it
+                // as though it had an "unknown" constructor to avoid exposing its emptiness. The
+                // exception is if the pattern is at the top level, because we want empty matches to be
+                // considered exhaustive.
+                let is_secretly_empty =
+                    def.variants().is_empty() && !is_exhaustive_pat_feature && !pcx.is_top_level;
+
+                let mut ctors: SmallVec<[_; 1]> = def
+                    .variants()
+                    .iter_enumerated()
+                    .filter(|(_, v)| {
+                        // If `exhaustive_patterns` is enabled, we exclude variants known to be
+                        // uninhabited.
+                        let is_uninhabited = is_exhaustive_pat_feature
+                            && v.uninhabited_from(cx.tcx, substs, def.adt_kind(), cx.param_env)
+                                .contains(cx.tcx, cx.module);
+                        !is_uninhabited
+                    })
+                    .map(|(idx, _)| Variant(idx))
+                    .collect();
+
+                if is_secretly_empty || is_declared_nonexhaustive {
+                    ctors.push(NonExhaustive);
+                }
+                ctors
+            }
+            ty::Char => {
+                smallvec![
+                    // The valid Unicode Scalar Value ranges.
+                    make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
+                    make_range('\u{E000}' as u128, '\u{10FFFF}' as u128),
+                ]
+            }
+            ty::Int(_) | ty::Uint(_)
+                if pcx.ty.is_ptr_sized_integral()
+                    && !cx.tcx.features().precise_pointer_size_matching =>
+            {
+                // `usize`/`isize` are not allowed to be matched exhaustively unless the
+                // `precise_pointer_size_matching` feature is enabled. So we treat those types like
+                // `#[non_exhaustive]` enums by returning a special unmatchable constructor.
+                smallvec![NonExhaustive]
+            }
+            &ty::Int(ity) => {
+                let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
+                let min = 1u128 << (bits - 1);
+                let max = min - 1;
+                smallvec![make_range(min, max)]
+            }
+            &ty::Uint(uty) => {
+                let size = Integer::from_uint_ty(&cx.tcx, uty).size();
+                let max = size.truncate(u128::MAX);
+                smallvec![make_range(0, max)]
+            }
+            // If `exhaustive_patterns` is disabled and our scrutinee is the never type, we cannot
+            // expose its emptiness. The exception is if the pattern is at the top level, because we
+            // want empty matches to be considered exhaustive.
+            ty::Never if !cx.tcx.features().exhaustive_patterns && !pcx.is_top_level => {
+                smallvec![NonExhaustive]
+            }
+            ty::Never => smallvec![],
+            _ if cx.is_uninhabited(pcx.ty) => smallvec![],
+            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => smallvec![Single],
+            // This type is one for which we cannot list constructors, like `str` or `f64`.
+            _ => smallvec![NonExhaustive],
+        };
+
+        SplitWildcard { matrix_ctors: Vec::new(), all_ctors }
+    }
+
+    /// Pass a set of constructors relative to which to split this one. Don't call twice, it won't
+    /// do what you want.
+    pub(super) fn split<'a>(
+        &mut self,
+        pcx: &PatCtxt<'_, '_, 'tcx>,
+        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
+    ) where
+        'tcx: 'a,
+    {
+        // Since `all_ctors` never contains wildcards, this won't recurse further.
+        self.all_ctors =
+            self.all_ctors.iter().flat_map(|ctor| ctor.split(pcx, ctors.clone())).collect();
+        self.matrix_ctors = ctors.filter(|c| !c.is_wildcard()).cloned().collect();
+    }
+
+    /// Whether there are any value constructors for this type that are not present in the matrix.
+    fn any_missing(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> bool {
+        self.iter_missing(pcx).next().is_some()
+    }
+
+    /// Iterate over the constructors for this type that are not present in the matrix.
+    pub(super) fn iter_missing<'a, 'p>(
+        &'a self,
+        pcx: &'a PatCtxt<'a, 'p, 'tcx>,
+    ) -> impl Iterator<Item = &'a Constructor<'tcx>> + Captures<'p> {
+        self.all_ctors.iter().filter(move |ctor| !ctor.is_covered_by_any(pcx, &self.matrix_ctors))
+    }
+
+    /// Return the set of constructors resulting from splitting the wildcard. As explained at the
+    /// top of the file, if any constructors are missing we can ignore the present ones.
+    fn into_ctors(self, pcx: &PatCtxt<'_, '_, 'tcx>) -> SmallVec<[Constructor<'tcx>; 1]> {
+        if self.any_missing(pcx) {
+            // Some constructors are missing, thus we can specialize with the special `Missing`
+            // constructor, which stands for those constructors that are not seen in the matrix,
+            // and matches the same rows as any of them (namely the wildcard rows). See the top of
+            // the file for details.
+            // However, when all constructors are missing we can also specialize with the full
+            // `Wildcard` constructor. The difference will depend on what we want in diagnostics.
+
+            // If some constructors are missing, we typically want to report those constructors,
+            // e.g.:
+            // ```
+            //     enum Direction { N, S, E, W }
+            //     let Direction::N = ...;
+            // ```
+            // we can report 3 witnesses: `S`, `E`, and `W`.
+            //
+            // However, if the user didn't actually specify a constructor
+            // in this arm, e.g., in
+            // ```
+            //     let x: (Direction, Direction, bool) = ...;
+            //     let (_, _, false) = x;
+            // ```
+            // we don't want to show all 16 possible witnesses `(<direction-1>, <direction-2>,
+            // true)` - we are satisfied with `(_, _, true)`. So if all constructors are missing we
+            // prefer to report just a wildcard `_`.
+            //
+            // The exception is: if we are at the top-level, for example in an empty match, we
+            // sometimes prefer reporting the list of constructors instead of just `_`.
+            let report_when_all_missing = pcx.is_top_level && !IntRange::is_integral(pcx.ty);
+            let ctor = if !self.matrix_ctors.is_empty() || report_when_all_missing {
+                if pcx.is_non_exhaustive {
+                    Missing {
+                        nonexhaustive_enum_missing_real_variants: self
+                            .iter_missing(pcx)
+                            .any(|c| !(c.is_non_exhaustive() || c.is_unstable_variant(pcx))),
+                    }
+                } else {
+                    Missing { nonexhaustive_enum_missing_real_variants: false }
+                }
+            } else {
+                Wildcard
+            };
+            return smallvec![ctor];
+        }
+
+        // All the constructors are present in the matrix, so we just go through them all.
+        self.all_ctors
+    }
+}
+
+/// A value can be decomposed into a constructor applied to some fields. This struct represents
+/// those fields, generalized to allow patterns in each field. See also `Constructor`.
+///
+/// This is constructed for a constructor using [`Fields::wildcards()`]. The idea is that
+/// [`Fields::wildcards()`] constructs a list of fields where all entries are wildcards, and then
+/// given a pattern we fill some of the fields with its subpatterns.
+/// In the following example `Fields::wildcards` returns `[_, _, _, _]`. Then in
+/// `extract_pattern_arguments` we fill some of the entries, and the result is
+/// `[Some(0), _, _, _]`.
+/// ```compile_fail,E0004
+/// # fn foo() -> [Option<u8>; 4] { [None; 4] }
+/// let x: [Option<u8>; 4] = foo();
+/// match x {
+///     [Some(0), ..] => {}
+/// }
+/// ```
+///
+/// Note that the number of fields of a constructor may not match the fields declared in the
+/// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
+/// because the code mustn't observe that it is uninhabited. In that case that field is not
+/// included in `fields`. For that reason, when you have a `mir::Field` you must use
+/// `index_with_declared_idx`.
+#[derive(Debug, Clone, Copy)]
+pub(super) struct Fields<'p, 'tcx> {
+    fields: &'p [DeconstructedPat<'p, 'tcx>],
+}
+
+impl<'p, 'tcx> Fields<'p, 'tcx> {
+    fn empty() -> Self {
+        Fields { fields: &[] }
+    }
+
+    fn singleton(cx: &MatchCheckCtxt<'p, 'tcx>, field: DeconstructedPat<'p, 'tcx>) -> Self {
+        let field: &_ = cx.pattern_arena.alloc(field);
+        Fields { fields: std::slice::from_ref(field) }
+    }
+
+    pub(super) fn from_iter(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        fields: impl IntoIterator<Item = DeconstructedPat<'p, 'tcx>>,
+    ) -> Self {
+        let fields: &[_] = cx.pattern_arena.alloc_from_iter(fields);
+        Fields { fields }
+    }
+
+    fn wildcards_from_tys(
+        cx: &MatchCheckCtxt<'p, 'tcx>,
+        tys: impl IntoIterator<Item = Ty<'tcx>>,
+    ) -> Self {
+        Fields::from_iter(cx, tys.into_iter().map(DeconstructedPat::wildcard))
+    }
+
+    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
+    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
+    // This lists the fields we keep along with their types.
+    fn list_variant_nonhidden_fields<'a>(
+        cx: &'a MatchCheckCtxt<'p, 'tcx>,
+        ty: Ty<'tcx>,
+        variant: &'a VariantDef,
+    ) -> impl Iterator<Item = (Field, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
+        let ty::Adt(adt, substs) = ty.kind() else { bug!() };
+        // Whether we must not match the fields of this variant exhaustively.
+        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
+
+        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
+            let ty = field.ty(cx.tcx, substs);
+            // `field.ty()` doesn't normalize after substituting.
+            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
+            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
+            let is_uninhabited = cx.is_uninhabited(ty);
+
+            if is_uninhabited && (!is_visible || is_non_exhaustive) {
+                None
+            } else {
+                Some((Field::new(i), ty))
+            }
+        })
+    }
+
+    /// Creates a new list of wildcard fields for a given constructor. The result must have a
+    /// length of `constructor.arity()`.
+    #[instrument(level = "trace")]
+    pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
+        let ret = match constructor {
+            Single | Variant(_) => match pcx.ty.kind() {
+                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter()),
+                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty)),
+                ty::Adt(adt, substs) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        Fields::wildcards_from_tys(pcx.cx, once(substs.type_at(0)))
+                    } else {
+                        let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
+                        let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
+                            .map(|(_, ty)| ty);
+                        Fields::wildcards_from_tys(pcx.cx, tys)
+                    }
+                }
+                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
+            },
+            Slice(slice) => match *pcx.ty.kind() {
+                ty::Slice(ty) | ty::Array(ty, _) => {
+                    let arity = slice.arity();
+                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty))
+                }
+                _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
+            },
+            Str(..)
+            | FloatRange(..)
+            | IntRange(..)
+            | NonExhaustive
+            | Opaque
+            | Missing { .. }
+            | Wildcard => Fields::empty(),
+            Or => {
+                bug!("called `Fields::wildcards` on an `Or` ctor")
+            }
+        };
+        debug!(?ret);
+        ret
+    }
+
+    /// Returns the list of patterns.
+    pub(super) fn iter_patterns<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.fields.iter()
+    }
+}
+
+/// Values and patterns can be represented as a constructor applied to some fields. This represents
+/// a pattern in this form.
+/// This also keeps track of whether the pattern has been found reachable during analysis. For this
+/// reason we should be careful not to clone patterns for which we care about that. Use
+/// `clone_and_forget_reachability` if you're sure.
+pub(crate) struct DeconstructedPat<'p, 'tcx> {
+    ctor: Constructor<'tcx>,
+    fields: Fields<'p, 'tcx>,
+    ty: Ty<'tcx>,
+    span: Span,
+    reachable: Cell<bool>,
+}
+
+impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
+    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
+        Self::new(Wildcard, Fields::empty(), ty, DUMMY_SP)
+    }
+
+    pub(super) fn new(
+        ctor: Constructor<'tcx>,
+        fields: Fields<'p, 'tcx>,
+        ty: Ty<'tcx>,
+        span: Span,
+    ) -> Self {
+        DeconstructedPat { ctor, fields, ty, span, reachable: Cell::new(false) }
+    }
+
+    /// Construct a pattern that matches everything that starts with this constructor.
+    /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
+    /// `Some(_)`.
+    pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: Constructor<'tcx>) -> Self {
+        let fields = Fields::wildcards(pcx, &ctor);
+        DeconstructedPat::new(ctor, fields, pcx.ty, DUMMY_SP)
+    }
+
+    /// Clone this value. This method emphasizes that cloning loses reachability information and
+    /// should be done carefully.
+    pub(super) fn clone_and_forget_reachability(&self) -> Self {
+        DeconstructedPat::new(self.ctor.clone(), self.fields, self.ty, self.span)
+    }
+
+    pub(crate) fn from_pat(cx: &MatchCheckCtxt<'p, 'tcx>, pat: &Pat<'tcx>) -> Self {
+        let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
+        let ctor;
+        let fields;
+        match pat.kind.as_ref() {
+            PatKind::AscribeUserType { subpattern, .. } => return mkpat(subpattern),
+            PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
+            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
+                ctor = Wildcard;
+                fields = Fields::empty();
+            }
+            PatKind::Deref { subpattern } => {
+                ctor = Single;
+                fields = Fields::singleton(cx, mkpat(subpattern));
+            }
+            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
+                match pat.ty.kind() {
+                    ty::Tuple(fs) => {
+                        ctor = Single;
+                        let mut wilds: SmallVec<[_; 2]> =
+                            fs.iter().map(DeconstructedPat::wildcard).collect();
+                        for pat in subpatterns {
+                            wilds[pat.field.index()] = mkpat(&pat.pattern);
+                        }
+                        fields = Fields::from_iter(cx, wilds);
+                    }
+                    ty::Adt(adt, substs) if adt.is_box() => {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
+                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
+                        // ignore other fields than the first one. This will trigger an error later
+                        // anyway.
+                        // See https://github.com/rust-lang/rust/issues/82772 ,
+                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
+                        // The problem is that we can't know from the type whether we'll match
+                        // normally or through box-patterns. We'll have to figure out a proper
+                        // solution when we introduce generalized deref patterns. Also need to
+                        // prevent mixing of those two options.
+                        let pat = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
+                        let pat = if let Some(pat) = pat {
+                            mkpat(&pat.pattern)
+                        } else {
+                            DeconstructedPat::wildcard(substs.type_at(0))
+                        };
+                        ctor = Single;
+                        fields = Fields::singleton(cx, pat);
+                    }
+                    ty::Adt(adt, _) => {
+                        ctor = match pat.kind.as_ref() {
+                            PatKind::Leaf { .. } => Single,
+                            PatKind::Variant { variant_index, .. } => Variant(*variant_index),
+                            _ => bug!(),
+                        };
+                        let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
+                        // For each field in the variant, we store the relevant index into `self.fields` if any.
+                        let mut field_id_to_id: Vec<Option<usize>> =
+                            (0..variant.fields.len()).map(|_| None).collect();
+                        let tys = Fields::list_variant_nonhidden_fields(cx, pat.ty, variant)
+                            .enumerate()
+                            .map(|(i, (field, ty))| {
+                                field_id_to_id[field.index()] = Some(i);
+                                ty
+                            });
+                        let mut wilds: SmallVec<[_; 2]> =
+                            tys.map(DeconstructedPat::wildcard).collect();
+                        for pat in subpatterns {
+                            if let Some(i) = field_id_to_id[pat.field.index()] {
+                                wilds[i] = mkpat(&pat.pattern);
+                            }
+                        }
+                        fields = Fields::from_iter(cx, wilds);
+                    }
+                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
+                }
+            }
+            PatKind::Constant { value } => {
+                if let Some(int_range) = IntRange::from_constant(cx.tcx, cx.param_env, *value) {
+                    ctor = IntRange(int_range);
+                    fields = Fields::empty();
+                } else {
+                    match pat.ty.kind() {
+                        ty::Float(_) => {
+                            ctor = FloatRange(*value, *value, RangeEnd::Included);
+                            fields = Fields::empty();
+                        }
+                        ty::Ref(_, t, _) if t.is_str() => {
+                            // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
+                            // with other `Deref` patterns. This could have been done in `const_to_pat`,
+                            // but that causes issues with the rest of the matching code.
+                            // So here, the constructor for a `"foo"` pattern is `&` (represented by
+                            // `Single`), and has one field. That field has constructor `Str(value)` and no
+                            // fields.
+                            // Note: `t` is `str`, not `&str`.
+                            let subpattern =
+                                DeconstructedPat::new(Str(*value), Fields::empty(), *t, pat.span);
+                            ctor = Single;
+                            fields = Fields::singleton(cx, subpattern)
+                        }
+                        // All constants that can be structurally matched have already been expanded
+                        // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
+                        // opaque.
+                        _ => {
+                            ctor = Opaque;
+                            fields = Fields::empty();
+                        }
+                    }
+                }
+            }
+            &PatKind::Range(PatRange { lo, hi, end }) => {
+                let ty = lo.ty();
+                ctor = if let Some(int_range) = IntRange::from_range(
+                    cx.tcx,
+                    lo.eval_bits(cx.tcx, cx.param_env, lo.ty()),
+                    hi.eval_bits(cx.tcx, cx.param_env, hi.ty()),
+                    ty,
+                    &end,
+                ) {
+                    IntRange(int_range)
+                } else {
+                    FloatRange(lo, hi, end)
+                };
+                fields = Fields::empty();
+            }
+            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
+                let array_len = match pat.ty.kind() {
+                    ty::Array(_, length) => Some(length.eval_usize(cx.tcx, cx.param_env) as usize),
+                    ty::Slice(_) => None,
+                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
+                };
+                let kind = if slice.is_some() {
+                    VarLen(prefix.len(), suffix.len())
+                } else {
+                    FixedLen(prefix.len() + suffix.len())
+                };
+                ctor = Slice(Slice::new(array_len, kind));
+                fields = Fields::from_iter(cx, prefix.iter().chain(suffix).map(mkpat));
+            }
+            PatKind::Or { .. } => {
+                ctor = Or;
+                let pats = expand_or_pat(pat);
+                fields = Fields::from_iter(cx, pats.into_iter().map(mkpat));
+            }
+        }
+        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
+    }
+
+    pub(crate) fn to_pat(&self, cx: &MatchCheckCtxt<'p, 'tcx>) -> Pat<'tcx> {
+        let is_wildcard = |pat: &Pat<'_>| {
+            matches!(*pat.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild)
+        };
+        let mut subpatterns = self.iter_fields().map(|p| p.to_pat(cx));
+        let pat = match &self.ctor {
+            Single | Variant(_) => match self.ty.kind() {
+                ty::Tuple(..) => PatKind::Leaf {
+                    subpatterns: subpatterns
+                        .enumerate()
+                        .map(|(i, p)| FieldPat { field: Field::new(i), pattern: p })
+                        .collect(),
+                },
+                ty::Adt(adt_def, _) if adt_def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
+                }
+                ty::Adt(adt_def, substs) => {
+                    let variant_index = self.ctor.variant_index_for_adt(*adt_def);
+                    let variant = &adt_def.variant(variant_index);
+                    let subpatterns = Fields::list_variant_nonhidden_fields(cx, self.ty, variant)
+                        .zip(subpatterns)
+                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
+                        .collect();
+
+                    if adt_def.is_enum() {
+                        PatKind::Variant { adt_def: *adt_def, substs, variant_index, subpatterns }
+                    } else {
+                        PatKind::Leaf { subpatterns }
+                    }
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to reconstruct the correct constant pattern here. However a string
+                // literal pattern will never be reported as a non-exhaustiveness witness, so we
+                // ignore this issue.
+                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
+                _ => bug!("unexpected ctor for type {:?} {:?}", self.ctor, self.ty),
+            },
+            Slice(slice) => {
+                match slice.kind {
+                    FixedLen(_) => PatKind::Slice {
+                        prefix: subpatterns.collect(),
+                        slice: None,
+                        suffix: vec![],
+                    },
+                    VarLen(prefix, _) => {
+                        let mut subpatterns = subpatterns.peekable();
+                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
+                        if slice.array_len.is_some() {
+                            // Improves diagnostics a bit: if the type is a known-size array, instead
+                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
+                            // This is incorrect if the size is not known, since `[_, ..]` captures
+                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
+                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
+                                prefix.pop();
+                            }
+                            while subpatterns.peek().is_some()
+                                && is_wildcard(subpatterns.peek().unwrap())
+                            {
+                                subpatterns.next();
+                            }
+                        }
+                        let suffix: Vec<_> = subpatterns.collect();
+                        let wild = Pat::wildcard_from_ty(self.ty);
+                        PatKind::Slice { prefix, slice: Some(wild), suffix }
+                    }
+                }
+            }
+            &Str(value) => PatKind::Constant { value },
+            &FloatRange(lo, hi, end) => PatKind::Range(PatRange { lo, hi, end }),
+            IntRange(range) => return range.to_pat(cx.tcx, self.ty),
+            Wildcard | NonExhaustive => PatKind::Wild,
+            Missing { .. } => bug!(
+                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
+                `Missing` should have been processed in `apply_constructors`"
+            ),
+            Opaque | Or => {
+                bug!("can't convert to pattern: {:?}", self)
+            }
+        };
+
+        Pat { ty: self.ty, span: DUMMY_SP, kind: Box::new(pat) }
+    }
+
+    pub(super) fn is_or_pat(&self) -> bool {
+        matches!(self.ctor, Or)
+    }
+
+    pub(super) fn ctor(&self) -> &Constructor<'tcx> {
+        &self.ctor
+    }
+    pub(super) fn ty(&self) -> Ty<'tcx> {
+        self.ty
+    }
+    pub(super) fn span(&self) -> Span {
+        self.span
+    }
+
+    pub(super) fn iter_fields<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
+        self.fields.iter_patterns()
+    }
+
+    /// Specialize this pattern with a constructor.
+    /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
+    pub(super) fn specialize<'a>(
+        &'a self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        other_ctor: &Constructor<'tcx>,
+    ) -> SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]> {
+        match (&self.ctor, other_ctor) {
+            (Wildcard, _) => {
+                // We return a wildcard for each field of `other_ctor`.
+                Fields::wildcards(pcx, other_ctor).iter_patterns().collect()
+            }
+            (Slice(self_slice), Slice(other_slice))
+                if self_slice.arity() != other_slice.arity() =>
+            {
+                // The only tricky case: two slices of different arity. Since `self_slice` covers
+                // `other_slice`, `self_slice` must be `VarLen`, i.e. of the form
+                // `[prefix, .., suffix]`. Moreover `other_slice` is guaranteed to have a larger
+                // arity. So we fill the middle part with enough wildcards to reach the length of
+                // the new, larger slice.
+                match self_slice.kind {
+                    FixedLen(_) => bug!("{:?} doesn't cover {:?}", self_slice, other_slice),
+                    VarLen(prefix, suffix) => {
+                        let (ty::Slice(inner_ty) | ty::Array(inner_ty, _)) = *self.ty.kind() else {
+                            bug!("bad slice pattern {:?} {:?}", self.ctor, self.ty);
+                        };
+                        let prefix = &self.fields.fields[..prefix];
+                        let suffix = &self.fields.fields[self_slice.arity() - suffix..];
+                        let wildcard: &_ =
+                            pcx.cx.pattern_arena.alloc(DeconstructedPat::wildcard(inner_ty));
+                        let extra_wildcards = other_slice.arity() - self_slice.arity();
+                        let extra_wildcards = (0..extra_wildcards).map(|_| wildcard);
+                        prefix.iter().chain(extra_wildcards).chain(suffix).collect()
+                    }
+                }
+            }
+            _ => self.fields.iter_patterns().collect(),
+        }
+    }
+
+    /// We keep track for each pattern if it was ever reachable during the analysis. This is used
+    /// with `unreachable_spans` to report unreachable subpatterns arising from or patterns.
+    pub(super) fn set_reachable(&self) {
+        self.reachable.set(true)
+    }
+    pub(super) fn is_reachable(&self) -> bool {
+        self.reachable.get()
+    }
+
+    /// Report the spans of subpatterns that were not reachable, if any.
+    pub(super) fn unreachable_spans(&self) -> Vec<Span> {
+        let mut spans = Vec::new();
+        self.collect_unreachable_spans(&mut spans);
+        spans
+    }
+
+    fn collect_unreachable_spans(&self, spans: &mut Vec<Span>) {
+        // We don't look at subpatterns if we already reported the whole pattern as unreachable.
+        if !self.is_reachable() {
+            spans.push(self.span);
+        } else {
+            for p in self.iter_fields() {
+                p.collect_unreachable_spans(spans);
+            }
+        }
+    }
+}
+
+/// This is mostly copied from the `Pat` impl. This is best effort and not good enough for a
+/// `Display` impl.
+impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Printing lists is a chore.
+        let mut first = true;
+        let mut start_or_continue = |s| {
+            if first {
+                first = false;
+                ""
+            } else {
+                s
+            }
+        };
+        let mut start_or_comma = || start_or_continue(", ");
+
+        match &self.ctor {
+            Single | Variant(_) => match self.ty.kind() {
+                ty::Adt(def, _) if def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    let subpattern = self.iter_fields().next().unwrap();
+                    write!(f, "box {:?}", subpattern)
+                }
+                ty::Adt(..) | ty::Tuple(..) => {
+                    let variant = match self.ty.kind() {
+                        ty::Adt(adt, _) => Some(adt.variant(self.ctor.variant_index_for_adt(*adt))),
+                        ty::Tuple(_) => None,
+                        _ => unreachable!(),
+                    };
+
+                    if let Some(variant) = variant {
+                        write!(f, "{}", variant.name)?;
+                    }
+
+                    // Without `cx`, we can't know which field corresponds to which, so we can't
+                    // get the names of the fields. Instead we just display everything as a tuple
+                    // struct, which should be good enough.
+                    write!(f, "(")?;
+                    for p in self.iter_fields() {
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "{:?}", p)?;
+                    }
+                    write!(f, ")")
+                }
+                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+                // be careful to detect strings here. However a string literal pattern will never
+                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
+                ty::Ref(_, _, mutbl) => {
+                    let subpattern = self.iter_fields().next().unwrap();
+                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
+                }
+                _ => write!(f, "_"),
+            },
+            Slice(slice) => {
+                let mut subpatterns = self.fields.iter_patterns();
+                write!(f, "[")?;
+                match slice.kind {
+                    FixedLen(_) => {
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                    VarLen(prefix_len, _) => {
+                        for p in subpatterns.by_ref().take(prefix_len) {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "..")?;
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                }
+                write!(f, "]")
+            }
+            &FloatRange(lo, hi, end) => {
+                write!(f, "{}", lo)?;
+                write!(f, "{}", end)?;
+                write!(f, "{}", hi)
+            }
+            IntRange(range) => write!(f, "{:?}", range), // Best-effort, will render e.g. `false` as `0..=0`
+            Wildcard | Missing { .. } | NonExhaustive => write!(f, "_ : {:?}", self.ty),
+            Or => {
+                for pat in self.iter_fields() {
+                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
+                }
+                Ok(())
+            }
+            Str(value) => write!(f, "{}", value),
+            Opaque => write!(f, "<constant pattern>"),
+        }
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
new file mode 100644
index 000000000..a13748a2d
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -0,0 +1,802 @@
+//! Validation of patterns/matches.
+
+mod check_match;
+mod const_to_pat;
+mod deconstruct_pat;
+mod usefulness;
+
+pub(crate) use self::check_match::check_match;
+
+use crate::thir::util::UserAnnotatedTyHelpers;
+
+use rustc_errors::struct_span_err;
+use rustc_hir as hir;
+use rustc_hir::def::{CtorOf, DefKind, Res};
+use rustc_hir::pat_util::EnumerateAndAdjustIterator;
+use rustc_hir::RangeEnd;
+use rustc_index::vec::Idx;
+use rustc_middle::mir::interpret::{
+    ConstValue, ErrorHandled, LitToConstError, LitToConstInput, Scalar,
+};
+use rustc_middle::mir::{self, UserTypeProjection};
+use rustc_middle::mir::{BorrowKind, Field, Mutability};
+use rustc_middle::thir::{Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange};
+use rustc_middle::ty::subst::{GenericArg, SubstsRef};
+use rustc_middle::ty::CanonicalUserTypeAnnotation;
+use rustc_middle::ty::{self, AdtDef, ConstKind, DefIdTree, Region, Ty, TyCtxt, UserType};
+use rustc_span::{Span, Symbol};
+
+use std::cmp::Ordering;
+
+#[derive(Clone, Debug)]
+pub(crate) enum PatternError {
+    AssocConstInPattern(Span),
+    ConstParamInPattern(Span),
+    StaticInPattern(Span),
+    NonConstPath(Span),
+}
+
+pub(crate) struct PatCtxt<'a, 'tcx> {
+    pub(crate) tcx: TyCtxt<'tcx>,
+    pub(crate) param_env: ty::ParamEnv<'tcx>,
+    pub(crate) typeck_results: &'a ty::TypeckResults<'tcx>,
+    pub(crate) errors: Vec<PatternError>,
+    include_lint_checks: bool,
+}
+
+pub(crate) fn pat_from_hir<'a, 'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    pat: &'tcx hir::Pat<'tcx>,
+) -> Pat<'tcx> {
+    let mut pcx = PatCtxt::new(tcx, param_env, typeck_results);
+    let result = pcx.lower_pattern(pat);
+    if !pcx.errors.is_empty() {
+        let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors);
+        tcx.sess.delay_span_bug(pat.span, &msg);
+    }
+    debug!("pat_from_hir({:?}) = {:?}", pat, result);
+    result
+}
+
+impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
+    pub(crate) fn new(
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+        typeck_results: &'a ty::TypeckResults<'tcx>,
+    ) -> Self {
+        PatCtxt { tcx, param_env, typeck_results, errors: vec![], include_lint_checks: false }
+    }
+
+    pub(crate) fn include_lint_checks(&mut self) -> &mut Self {
+        self.include_lint_checks = true;
+        self
+    }
+
+    pub(crate) fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
+        // When implicit dereferences have been inserted in this pattern, the unadjusted lowered
+        // pattern has the type that results *after* dereferencing. For example, in this code:
+        //
+        // ```
+        // match &&Some(0i32) {
+        //     Some(n) => { ... },
+        //     _ => { ... },
+        // }
+        // ```
+        //
+        // the type assigned to `Some(n)` in `unadjusted_pat` would be `Option<i32>` (this is
+        // determined in rustc_typeck::check::match). The adjustments would be
+        //
+        // `vec![&&Option<i32>, &Option<i32>]`.
+        //
+        // Applying the adjustments, we want to instead output `&&Some(n)` (as a THIR pattern). So
+        // we wrap the unadjusted pattern in `PatKind::Deref` repeatedly, consuming the
+        // adjustments in *reverse order* (last-in-first-out, so that the last `Deref` inserted
+        // gets the least-dereferenced type).
+        let unadjusted_pat = self.lower_pattern_unadjusted(pat);
+        self.typeck_results.pat_adjustments().get(pat.hir_id).unwrap_or(&vec![]).iter().rev().fold(
+            unadjusted_pat,
+            |pat, ref_ty| {
+                debug!("{:?}: wrapping pattern with type {:?}", pat, ref_ty);
+                Pat {
+                    span: pat.span,
+                    ty: *ref_ty,
+                    kind: Box::new(PatKind::Deref { subpattern: pat }),
+                }
+            },
+        )
+    }
+
+    fn lower_range_expr(
+        &mut self,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) -> (PatKind<'tcx>, Option<Ascription<'tcx>>) {
+        match self.lower_lit(expr) {
+            PatKind::AscribeUserType { ascription, subpattern: Pat { kind: box kind, .. } } => {
+                (kind, Some(ascription))
+            }
+            kind => (kind, None),
+        }
+    }
+
+    fn lower_pattern_range(
+        &mut self,
+        ty: Ty<'tcx>,
+        lo: mir::ConstantKind<'tcx>,
+        hi: mir::ConstantKind<'tcx>,
+        end: RangeEnd,
+        span: Span,
+    ) -> PatKind<'tcx> {
+        assert_eq!(lo.ty(), ty);
+        assert_eq!(hi.ty(), ty);
+        let cmp = compare_const_vals(self.tcx, lo, hi, self.param_env);
+        match (end, cmp) {
+            // `x..y` where `x < y`.
+            // Non-empty because the range includes at least `x`.
+            (RangeEnd::Excluded, Some(Ordering::Less)) => PatKind::Range(PatRange { lo, hi, end }),
+            // `x..y` where `x >= y`. The range is empty => error.
+            (RangeEnd::Excluded, _) => {
+                struct_span_err!(
+                    self.tcx.sess,
+                    span,
+                    E0579,
+                    "lower range bound must be less than upper"
+                )
+                .emit();
+                PatKind::Wild
+            }
+            // `x..=y` where `x == y`.
+            (RangeEnd::Included, Some(Ordering::Equal)) => PatKind::Constant { value: lo },
+            // `x..=y` where `x < y`.
+            (RangeEnd::Included, Some(Ordering::Less)) => PatKind::Range(PatRange { lo, hi, end }),
+            // `x..=y` where `x > y` hence the range is empty => error.
+            (RangeEnd::Included, _) => {
+                let mut err = struct_span_err!(
+                    self.tcx.sess,
+                    span,
+                    E0030,
+                    "lower range bound must be less than or equal to upper"
+                );
+                err.span_label(span, "lower bound larger than upper bound");
+                if self.tcx.sess.teach(&err.get_code().unwrap()) {
+                    err.note(
+                        "When matching against a range, the compiler \
+                              verifies that the range is non-empty. Range \
+                              patterns include both end-points, so this is \
+                              equivalent to requiring the start of the range \
+                              to be less than or equal to the end of the range.",
+                    );
+                }
+                err.emit();
+                PatKind::Wild
+            }
+        }
+    }
+
+    fn normalize_range_pattern_ends(
+        &self,
+        ty: Ty<'tcx>,
+        lo: Option<&PatKind<'tcx>>,
+        hi: Option<&PatKind<'tcx>>,
+    ) -> Option<(mir::ConstantKind<'tcx>, mir::ConstantKind<'tcx>)> {
+        match (lo, hi) {
+            (Some(PatKind::Constant { value: lo }), Some(PatKind::Constant { value: hi })) => {
+                Some((*lo, *hi))
+            }
+            (Some(PatKind::Constant { value: lo }), None) => {
+                let hi = ty.numeric_max_val(self.tcx)?;
+                Some((*lo, mir::ConstantKind::from_const(hi, self.tcx)))
+            }
+            (None, Some(PatKind::Constant { value: hi })) => {
+                let lo = ty.numeric_min_val(self.tcx)?;
+                Some((mir::ConstantKind::from_const(lo, self.tcx), *hi))
+            }
+            _ => None,
+        }
+    }
+
+    fn lower_pattern_unadjusted(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
+        let mut ty = self.typeck_results.node_type(pat.hir_id);
+
+        let kind = match pat.kind {
+            hir::PatKind::Wild => PatKind::Wild,
+
+            hir::PatKind::Lit(value) => self.lower_lit(value),
+
+            hir::PatKind::Range(ref lo_expr, ref hi_expr, end) => {
+                let (lo_expr, hi_expr) = (lo_expr.as_deref(), hi_expr.as_deref());
+                let lo_span = lo_expr.map_or(pat.span, |e| e.span);
+                let lo = lo_expr.map(|e| self.lower_range_expr(e));
+                let hi = hi_expr.map(|e| self.lower_range_expr(e));
+
+                let (lp, hp) = (lo.as_ref().map(|x| &x.0), hi.as_ref().map(|x| &x.0));
+                let mut kind = match self.normalize_range_pattern_ends(ty, lp, hp) {
+                    Some((lc, hc)) => self.lower_pattern_range(ty, lc, hc, end, lo_span),
+                    None => {
+                        let msg = &format!(
+                            "found bad range pattern `{:?}` outside of error recovery",
+                            (&lo, &hi),
+                        );
+                        self.tcx.sess.delay_span_bug(pat.span, msg);
+                        PatKind::Wild
+                    }
+                };
+
+                // If we are handling a range with associated constants (e.g.
+                // `Foo::<'a>::A..=Foo::B`), we need to put the ascriptions for the associated
+                // constants somewhere. Have them on the range pattern.
+                for end in &[lo, hi] {
+                    if let Some((_, Some(ascription))) = end {
+                        let subpattern = Pat { span: pat.span, ty, kind: Box::new(kind) };
+                        kind =
+                            PatKind::AscribeUserType { ascription: ascription.clone(), subpattern };
+                    }
+                }
+
+                kind
+            }
+
+            hir::PatKind::Path(ref qpath) => {
+                return self.lower_path(qpath, pat.hir_id, pat.span);
+            }
+
+            hir::PatKind::Ref(ref subpattern, _) | hir::PatKind::Box(ref subpattern) => {
+                PatKind::Deref { subpattern: self.lower_pattern(subpattern) }
+            }
+
+            hir::PatKind::Slice(ref prefix, ref slice, ref suffix) => {
+                self.slice_or_array_pattern(pat.span, ty, prefix, slice, suffix)
+            }
+
+            hir::PatKind::Tuple(ref pats, ddpos) => {
+                let ty::Tuple(ref tys) = ty.kind() else {
+                    span_bug!(pat.span, "unexpected type for tuple pattern: {:?}", ty);
+                };
+                let subpatterns = self.lower_tuple_subpats(pats, tys.len(), ddpos);
+                PatKind::Leaf { subpatterns }
+            }
+
+            hir::PatKind::Binding(_, id, ident, ref sub) => {
+                let bm = *self
+                    .typeck_results
+                    .pat_binding_modes()
+                    .get(pat.hir_id)
+                    .expect("missing binding mode");
+                let (mutability, mode) = match bm {
+                    ty::BindByValue(mutbl) => (mutbl, BindingMode::ByValue),
+                    ty::BindByReference(hir::Mutability::Mut) => (
+                        Mutability::Not,
+                        BindingMode::ByRef(BorrowKind::Mut { allow_two_phase_borrow: false }),
+                    ),
+                    ty::BindByReference(hir::Mutability::Not) => {
+                        (Mutability::Not, BindingMode::ByRef(BorrowKind::Shared))
+                    }
+                };
+
+                // A ref x pattern is the same node used for x, and as such it has
+                // x's type, which is &T, where we want T (the type being matched).
+                let var_ty = ty;
+                if let ty::BindByReference(_) = bm {
+                    if let ty::Ref(_, rty, _) = ty.kind() {
+                        ty = *rty;
+                    } else {
+                        bug!("`ref {}` has wrong type {}", ident, ty);
+                    }
+                };
+
+                PatKind::Binding {
+                    mutability,
+                    mode,
+                    name: ident.name,
+                    var: LocalVarId(id),
+                    ty: var_ty,
+                    subpattern: self.lower_opt_pattern(sub),
+                    is_primary: id == pat.hir_id,
+                }
+            }
+
+            hir::PatKind::TupleStruct(ref qpath, ref pats, ddpos) => {
+                let res = self.typeck_results.qpath_res(qpath, pat.hir_id);
+                let ty::Adt(adt_def, _) = ty.kind() else {
+                    span_bug!(pat.span, "tuple struct pattern not applied to an ADT {:?}", ty);
+                };
+                let variant_def = adt_def.variant_of_res(res);
+                let subpatterns = self.lower_tuple_subpats(pats, variant_def.fields.len(), ddpos);
+                self.lower_variant_or_leaf(res, pat.hir_id, pat.span, ty, subpatterns)
+            }
+
+            hir::PatKind::Struct(ref qpath, ref fields, _) => {
+                let res = self.typeck_results.qpath_res(qpath, pat.hir_id);
+                let subpatterns = fields
+                    .iter()
+                    .map(|field| FieldPat {
+                        field: Field::new(self.tcx.field_index(field.hir_id, self.typeck_results)),
+                        pattern: self.lower_pattern(&field.pat),
+                    })
+                    .collect();
+
+                self.lower_variant_or_leaf(res, pat.hir_id, pat.span, ty, subpatterns)
+            }
+
+            hir::PatKind::Or(ref pats) => PatKind::Or { pats: self.lower_patterns(pats) },
+        };
+
+        Pat { span: pat.span, ty, kind: Box::new(kind) }
+    }
+
+    fn lower_tuple_subpats(
+        &mut self,
+        pats: &'tcx [hir::Pat<'tcx>],
+        expected_len: usize,
+        gap_pos: Option<usize>,
+    ) -> Vec<FieldPat<'tcx>> {
+        pats.iter()
+            .enumerate_and_adjust(expected_len, gap_pos)
+            .map(|(i, subpattern)| FieldPat {
+                field: Field::new(i),
+                pattern: self.lower_pattern(subpattern),
+            })
+            .collect()
+    }
+
+    fn lower_patterns(&mut self, pats: &'tcx [hir::Pat<'tcx>]) -> Vec<Pat<'tcx>> {
+        pats.iter().map(|p| self.lower_pattern(p)).collect()
+    }
+
+    fn lower_opt_pattern(&mut self, pat: &'tcx Option<&'tcx hir::Pat<'tcx>>) -> Option<Pat<'tcx>> {
+        pat.as_ref().map(|p| self.lower_pattern(p))
+    }
+
+    fn slice_or_array_pattern(
+        &mut self,
+        span: Span,
+        ty: Ty<'tcx>,
+        prefix: &'tcx [hir::Pat<'tcx>],
+        slice: &'tcx Option<&'tcx hir::Pat<'tcx>>,
+        suffix: &'tcx [hir::Pat<'tcx>],
+    ) -> PatKind<'tcx> {
+        let prefix = self.lower_patterns(prefix);
+        let slice = self.lower_opt_pattern(slice);
+        let suffix = self.lower_patterns(suffix);
+        match ty.kind() {
+            // Matching a slice, `[T]`.
+            ty::Slice(..) => PatKind::Slice { prefix, slice, suffix },
+            // Fixed-length array, `[T; len]`.
+            ty::Array(_, len) => {
+                let len = len.eval_usize(self.tcx, self.param_env);
+                assert!(len >= prefix.len() as u64 + suffix.len() as u64);
+                PatKind::Array { prefix, slice, suffix }
+            }
+            _ => span_bug!(span, "bad slice pattern type {:?}", ty),
+        }
+    }
+
+    fn lower_variant_or_leaf(
+        &mut self,
+        res: Res,
+        hir_id: hir::HirId,
+        span: Span,
+        ty: Ty<'tcx>,
+        subpatterns: Vec<FieldPat<'tcx>>,
+    ) -> PatKind<'tcx> {
+        let res = match res {
+            Res::Def(DefKind::Ctor(CtorOf::Variant, ..), variant_ctor_id) => {
+                let variant_id = self.tcx.parent(variant_ctor_id);
+                Res::Def(DefKind::Variant, variant_id)
+            }
+            res => res,
+        };
+
+        let mut kind = match res {
+            Res::Def(DefKind::Variant, variant_id) => {
+                let enum_id = self.tcx.parent(variant_id);
+                let adt_def = self.tcx.adt_def(enum_id);
+                if adt_def.is_enum() {
+                    let substs = match ty.kind() {
+                        ty::Adt(_, substs) | ty::FnDef(_, substs) => substs,
+                        ty::Error(_) => {
+                            // Avoid ICE (#50585)
+                            return PatKind::Wild;
+                        }
+                        _ => bug!("inappropriate type for def: {:?}", ty),
+                    };
+                    PatKind::Variant {
+                        adt_def,
+                        substs,
+                        variant_index: adt_def.variant_index_with_id(variant_id),
+                        subpatterns,
+                    }
+                } else {
+                    PatKind::Leaf { subpatterns }
+                }
+            }
+
+            Res::Def(
+                DefKind::Struct
+                | DefKind::Ctor(CtorOf::Struct, ..)
+                | DefKind::Union
+                | DefKind::TyAlias
+                | DefKind::AssocTy,
+                _,
+            )
+            | Res::SelfTy { .. }
+            | Res::SelfCtor(..) => PatKind::Leaf { subpatterns },
+            _ => {
+                let pattern_error = match res {
+                    Res::Def(DefKind::ConstParam, _) => PatternError::ConstParamInPattern(span),
+                    Res::Def(DefKind::Static(_), _) => PatternError::StaticInPattern(span),
+                    _ => PatternError::NonConstPath(span),
+                };
+                self.errors.push(pattern_error);
+                PatKind::Wild
+            }
+        };
+
+        if let Some(user_ty) = self.user_substs_applied_to_ty_of_hir_id(hir_id) {
+            debug!("lower_variant_or_leaf: kind={:?} user_ty={:?} span={:?}", kind, user_ty, span);
+            let annotation = CanonicalUserTypeAnnotation {
+                user_ty,
+                span,
+                inferred_ty: self.typeck_results.node_type(hir_id),
+            };
+            kind = PatKind::AscribeUserType {
+                subpattern: Pat { span, ty, kind: Box::new(kind) },
+                ascription: Ascription { annotation, variance: ty::Variance::Covariant },
+            };
+        }
+
+        kind
+    }
+
+    /// Takes a HIR Path. If the path is a constant, evaluates it and feeds
+    /// it to `const_to_pat`. Any other path (like enum variants without fields)
+    /// is converted to the corresponding pattern via `lower_variant_or_leaf`.
+    #[instrument(skip(self), level = "debug")]
+    fn lower_path(&mut self, qpath: &hir::QPath<'_>, id: hir::HirId, span: Span) -> Pat<'tcx> {
+        let ty = self.typeck_results.node_type(id);
+        let res = self.typeck_results.qpath_res(qpath, id);
+
+        let pat_from_kind = |kind| Pat { span, ty, kind: Box::new(kind) };
+
+        let (def_id, is_associated_const) = match res {
+            Res::Def(DefKind::Const, def_id) => (def_id, false),
+            Res::Def(DefKind::AssocConst, def_id) => (def_id, true),
+
+            _ => return pat_from_kind(self.lower_variant_or_leaf(res, id, span, ty, vec![])),
+        };
+
+        // Use `Reveal::All` here because patterns are always monomorphic even if their function
+        // isn't.
+        let param_env_reveal_all = self.param_env.with_reveal_all_normalized(self.tcx);
+        let substs = self.typeck_results.node_substs(id);
+        let instance = match ty::Instance::resolve(self.tcx, param_env_reveal_all, def_id, substs) {
+            Ok(Some(i)) => i,
+            Ok(None) => {
+                // It should be assoc consts if there's no error but we cannot resolve it.
+                debug_assert!(is_associated_const);
+
+                self.errors.push(PatternError::AssocConstInPattern(span));
+
+                return pat_from_kind(PatKind::Wild);
+            }
+
+            Err(_) => {
+                self.tcx.sess.span_err(span, "could not evaluate constant pattern");
+                return pat_from_kind(PatKind::Wild);
+            }
+        };
+
+        // `mir_const_qualif` must be called with the `DefId` of the item where the const is
+        // defined, not where it is declared. The difference is significant for associated
+        // constants.
+        let mir_structural_match_violation = self.tcx.mir_const_qualif(instance.def_id()).custom_eq;
+        debug!("mir_structural_match_violation({:?}) -> {}", qpath, mir_structural_match_violation);
+
+        match self.tcx.const_eval_instance(param_env_reveal_all, instance, Some(span)) {
+            Ok(literal) => {
+                let const_ = mir::ConstantKind::Val(literal, ty);
+                let pattern = self.const_to_pat(const_, id, span, mir_structural_match_violation);
+
+                if !is_associated_const {
+                    return pattern;
+                }
+
+                let user_provided_types = self.typeck_results().user_provided_types();
+                if let Some(&user_ty) = user_provided_types.get(id) {
+                    let annotation = CanonicalUserTypeAnnotation {
+                        user_ty,
+                        span,
+                        inferred_ty: self.typeck_results().node_type(id),
+                    };
+                    Pat {
+                        span,
+                        kind: Box::new(PatKind::AscribeUserType {
+                            subpattern: pattern,
+                            ascription: Ascription {
+                                annotation,
+                                /// Note that use `Contravariant` here. See the
+                                /// `variance` field documentation for details.
+                                variance: ty::Variance::Contravariant,
+                            },
+                        }),
+                        ty: const_.ty(),
+                    }
+                } else {
+                    pattern
+                }
+            }
+            Err(ErrorHandled::TooGeneric) => {
+                // While `Reported | Linted` cases will have diagnostics emitted already
+                // it is not true for TooGeneric case, so we need to give user more information.
+                self.tcx.sess.span_err(span, "constant pattern depends on a generic parameter");
+                pat_from_kind(PatKind::Wild)
+            }
+            Err(_) => {
+                self.tcx.sess.span_err(span, "could not evaluate constant pattern");
+                pat_from_kind(PatKind::Wild)
+            }
+        }
+    }
+
+    /// Converts inline const patterns.
+    fn lower_inline_const(
+        &mut self,
+        anon_const: &'tcx hir::AnonConst,
+        id: hir::HirId,
+        span: Span,
+    ) -> PatKind<'tcx> {
+        let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id);
+        let value = mir::ConstantKind::from_inline_const(self.tcx, anon_const_def_id);
+
+        // Evaluate early like we do in `lower_path`.
+        let value = value.eval(self.tcx, self.param_env);
+
+        match value {
+            mir::ConstantKind::Ty(c) => {
+                match c.kind() {
+                    ConstKind::Param(_) => {
+                        self.errors.push(PatternError::ConstParamInPattern(span));
+                        return PatKind::Wild;
+                    }
+                    ConstKind::Unevaluated(_) => {
+                        // If we land here it means the const can't be evaluated because it's `TooGeneric`.
+                        self.tcx
+                            .sess
+                            .span_err(span, "constant pattern depends on a generic parameter");
+                        return PatKind::Wild;
+                    }
+                    _ => bug!("Expected either ConstKind::Param or ConstKind::Unevaluated"),
+                }
+            }
+            mir::ConstantKind::Val(_, _) => *self.const_to_pat(value, id, span, false).kind,
+        }
+    }
+
+    /// Converts literals, paths and negation of literals to patterns.
+    /// The special case for negation exists to allow things like `-128_i8`
+    /// which would overflow if we tried to evaluate `128_i8` and then negate
+    /// afterwards.
+    fn lower_lit(&mut self, expr: &'tcx hir::Expr<'tcx>) -> PatKind<'tcx> {
+        let (lit, neg) = match expr.kind {
+            hir::ExprKind::Path(ref qpath) => {
+                return *self.lower_path(qpath, expr.hir_id, expr.span).kind;
+            }
+            hir::ExprKind::ConstBlock(ref anon_const) => {
+                return self.lower_inline_const(anon_const, expr.hir_id, expr.span);
+            }
+            hir::ExprKind::Lit(ref lit) => (lit, false),
+            hir::ExprKind::Unary(hir::UnOp::Neg, ref expr) => {
+                let hir::ExprKind::Lit(ref lit) = expr.kind else {
+                    span_bug!(expr.span, "not a literal: {:?}", expr);
+                };
+                (lit, true)
+            }
+            _ => span_bug!(expr.span, "not a literal: {:?}", expr),
+        };
+
+        let lit_input =
+            LitToConstInput { lit: &lit.node, ty: self.typeck_results.expr_ty(expr), neg };
+        match self.tcx.at(expr.span).lit_to_mir_constant(lit_input) {
+            Ok(constant) => *self.const_to_pat(constant, expr.hir_id, lit.span, false).kind,
+            Err(LitToConstError::Reported) => PatKind::Wild,
+            Err(LitToConstError::TypeError) => bug!("lower_lit: had type error"),
+        }
+    }
+}
+
+impl<'tcx> UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn typeck_results(&self) -> &ty::TypeckResults<'tcx> {
+        self.typeck_results
+    }
+}
+
+pub(crate) trait PatternFoldable<'tcx>: Sized {
+    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        self.super_fold_with(folder)
+    }
+
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self;
+}
+
+pub(crate) trait PatternFolder<'tcx>: Sized {
+    fn fold_pattern(&mut self, pattern: &Pat<'tcx>) -> Pat<'tcx> {
+        pattern.super_fold_with(self)
+    }
+
+    fn fold_pattern_kind(&mut self, kind: &PatKind<'tcx>) -> PatKind<'tcx> {
+        kind.super_fold_with(self)
+    }
+}
+
+impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Box<T> {
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        let content: T = (**self).fold_with(folder);
+        Box::new(content)
+    }
+}
+
+impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Vec<T> {
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        self.iter().map(|t| t.fold_with(folder)).collect()
+    }
+}
+
+impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Option<T> {
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        self.as_ref().map(|t| t.fold_with(folder))
+    }
+}
+
+macro_rules! ClonePatternFoldableImpls {
+    (<$lt_tcx:tt> $($ty:ty),+) => {
+        $(
+            impl<$lt_tcx> PatternFoldable<$lt_tcx> for $ty {
+                fn super_fold_with<F: PatternFolder<$lt_tcx>>(&self, _: &mut F) -> Self {
+                    Clone::clone(self)
+                }
+            }
+        )+
+    }
+}
+
+ClonePatternFoldableImpls! { <'tcx>
+    Span, Field, Mutability, Symbol, LocalVarId, usize, ty::Const<'tcx>,
+    Region<'tcx>, Ty<'tcx>, BindingMode, AdtDef<'tcx>,
+    SubstsRef<'tcx>, &'tcx GenericArg<'tcx>, UserType<'tcx>,
+    UserTypeProjection, CanonicalUserTypeAnnotation<'tcx>
+}
+
+impl<'tcx> PatternFoldable<'tcx> for FieldPat<'tcx> {
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        FieldPat { field: self.field.fold_with(folder), pattern: self.pattern.fold_with(folder) }
+    }
+}
+
+impl<'tcx> PatternFoldable<'tcx> for Pat<'tcx> {
+    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        folder.fold_pattern(self)
+    }
+
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        Pat {
+            ty: self.ty.fold_with(folder),
+            span: self.span.fold_with(folder),
+            kind: self.kind.fold_with(folder),
+        }
+    }
+}
+
+impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
+    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        folder.fold_pattern_kind(self)
+    }
+
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        match *self {
+            PatKind::Wild => PatKind::Wild,
+            PatKind::AscribeUserType {
+                ref subpattern,
+                ascription: Ascription { ref annotation, variance },
+            } => PatKind::AscribeUserType {
+                subpattern: subpattern.fold_with(folder),
+                ascription: Ascription { annotation: annotation.fold_with(folder), variance },
+            },
+            PatKind::Binding { mutability, name, mode, var, ty, ref subpattern, is_primary } => {
+                PatKind::Binding {
+                    mutability: mutability.fold_with(folder),
+                    name: name.fold_with(folder),
+                    mode: mode.fold_with(folder),
+                    var: var.fold_with(folder),
+                    ty: ty.fold_with(folder),
+                    subpattern: subpattern.fold_with(folder),
+                    is_primary,
+                }
+            }
+            PatKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
+                PatKind::Variant {
+                    adt_def: adt_def.fold_with(folder),
+                    substs: substs.fold_with(folder),
+                    variant_index,
+                    subpatterns: subpatterns.fold_with(folder),
+                }
+            }
+            PatKind::Leaf { ref subpatterns } => {
+                PatKind::Leaf { subpatterns: subpatterns.fold_with(folder) }
+            }
+            PatKind::Deref { ref subpattern } => {
+                PatKind::Deref { subpattern: subpattern.fold_with(folder) }
+            }
+            PatKind::Constant { value } => PatKind::Constant { value },
+            PatKind::Range(range) => PatKind::Range(range),
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => PatKind::Slice {
+                prefix: prefix.fold_with(folder),
+                slice: slice.fold_with(folder),
+                suffix: suffix.fold_with(folder),
+            },
+            PatKind::Array { ref prefix, ref slice, ref suffix } => PatKind::Array {
+                prefix: prefix.fold_with(folder),
+                slice: slice.fold_with(folder),
+                suffix: suffix.fold_with(folder),
+            },
+            PatKind::Or { ref pats } => PatKind::Or { pats: pats.fold_with(folder) },
+        }
+    }
+}
+
+#[instrument(skip(tcx), level = "debug")]
+pub(crate) fn compare_const_vals<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    a: mir::ConstantKind<'tcx>,
+    b: mir::ConstantKind<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+) -> Option<Ordering> {
+    assert_eq!(a.ty(), b.ty());
+
+    let ty = a.ty();
+
+    // This code is hot when compiling matches with many ranges. So we
+    // special-case extraction of evaluated scalars for speed, for types where
+    // raw data comparisons are appropriate. E.g. `unicode-normalization` has
+    // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared
+    // in this way.
+    match ty.kind() {
+        ty::Float(_) | ty::Int(_) => {} // require special handling, see below
+        _ => match (a, b) {
+            (
+                mir::ConstantKind::Val(ConstValue::Scalar(Scalar::Int(a)), _a_ty),
+                mir::ConstantKind::Val(ConstValue::Scalar(Scalar::Int(b)), _b_ty),
+            ) => return Some(a.cmp(&b)),
+            _ => {}
+        },
+    }
+
+    let a = a.eval_bits(tcx, param_env, ty);
+    let b = b.eval_bits(tcx, param_env, ty);
+
+    use rustc_apfloat::Float;
+    match *ty.kind() {
+        ty::Float(ty::FloatTy::F32) => {
+            let a = rustc_apfloat::ieee::Single::from_bits(a);
+            let b = rustc_apfloat::ieee::Single::from_bits(b);
+            a.partial_cmp(&b)
+        }
+        ty::Float(ty::FloatTy::F64) => {
+            let a = rustc_apfloat::ieee::Double::from_bits(a);
+            let b = rustc_apfloat::ieee::Double::from_bits(b);
+            a.partial_cmp(&b)
+        }
+        ty::Int(ity) => {
+            use rustc_middle::ty::layout::IntegerExt;
+            let size = rustc_target::abi::Integer::from_int_ty(&tcx, ity).size();
+            let a = size.sign_extend(a);
+            let b = size.sign_extend(b);
+            Some((a as i128).cmp(&(b as i128)))
+        }
+        _ => Some(a.cmp(&b)),
+    }
+}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
new file mode 100644
index 000000000..0a660ef30
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -0,0 +1,978 @@
+//! Note: tests specific to this file can be found in:
+//!
+//!   - `ui/pattern/usefulness`
+//!   - `ui/or-patterns`
+//!   - `ui/consts/const_in_pattern`
+//!   - `ui/rfc-2008-non-exhaustive`
+//!   - `ui/half-open-range-patterns`
+//!   - probably many others
+//!
+//! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
+//! reason not to, for example if they depend on a particular feature like `or_patterns`.
+//!
+//! -----
+//!
+//! This file includes the logic for exhaustiveness and reachability checking for pattern-matching.
+//! Specifically, given a list of patterns for a type, we can tell whether:
+//! (a) each pattern is reachable (reachability)
+//! (b) the patterns cover every possible value for the type (exhaustiveness)
+//!
+//! The algorithm implemented here is a modified version of the one described in [this
+//! paper](http://moscova.inria.fr/~maranget/papers/warn/index.html). We have however generalized
+//! it to accommodate the variety of patterns that Rust supports. We thus explain our version here,
+//! without being as rigorous.
+//!
+//!
+//! # Summary
+//!
+//! The core of the algorithm is the notion of "usefulness". A pattern `q` is said to be *useful*
+//! relative to another pattern `p` of the same type if there is a value that is matched by `q` and
+//! not matched by `p`. This generalizes to many `p`s: `q` is useful w.r.t. a list of patterns
+//! `p_1 .. p_n` if there is a value that is matched by `q` and by none of the `p_i`. We write
+//! `usefulness(p_1 .. p_n, q)` for a function that returns a list of such values. The aim of this
+//! file is to compute it efficiently.
+//!
+//! This is enough to compute reachability: a pattern in a `match` expression is reachable iff it
+//! is useful w.r.t. the patterns above it:
+//! ```rust
+//! # fn foo(x: Option<i32>) {
+//! match x {
+//!     Some(_) => {},
+//!     None => {},    // reachable: `None` is matched by this but not the branch above
+//!     Some(0) => {}, // unreachable: all the values this matches are already matched by
+//!                    // `Some(_)` above
+//! }
+//! # }
+//! ```
+//!
+//! This is also enough to compute exhaustiveness: a match is exhaustive iff the wildcard `_`
+//! pattern is _not_ useful w.r.t. the patterns in the match. The values returned by `usefulness`
+//! are used to tell the user which values are missing.
+//! ```compile_fail,E0004
+//! # fn foo(x: Option<i32>) {
+//! match x {
+//!     Some(0) => {},
+//!     None => {},
+//!     // not exhaustive: `_` is useful because it matches `Some(1)`
+//! }
+//! # }
+//! ```
+//!
+//! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
+//! reachability for each match branch and exhaustiveness for the whole match.
+//!
+//!
+//! # Constructors and fields
+//!
+//! Note: we will often abbreviate "constructor" as "ctor".
+//!
+//! The idea that powers everything that is done in this file is the following: a (matchable)
+//! value is made from a constructor applied to a number of subvalues. Examples of constructors are
+//! `Some`, `None`, `(,)` (the 2-tuple constructor), `Foo {..}` (the constructor for a struct
+//! `Foo`), and `2` (the constructor for the number `2`). This is natural when we think of
+//! pattern-matching, and this is the basis for what follows.
+//!
+//! Some of the ctors listed above might feel weird: `None` and `2` don't take any arguments.
+//! That's ok: those are ctors that take a list of 0 arguments; they are the simplest case of
+//! ctors. We treat `2` as a ctor because `u64` and other number types behave exactly like a huge
+//! `enum`, with one variant for each number. This allows us to see any matchable value as made up
+//! from a tree of ctors, each having a set number of children. For example: `Foo { bar: None,
+//! baz: Ok(0) }` is made from 4 different ctors, namely `Foo{..}`, `None`, `Ok` and `0`.
+//!
+//! This idea can be extended to patterns: they are also made from constructors applied to fields.
+//! A pattern for a given type is allowed to use all the ctors for values of that type (which we
+//! call "value constructors"), but there are also pattern-only ctors. The most important one is
+//! the wildcard (`_`), and the others are integer ranges (`0..=10`), variable-length slices (`[x,
+//! ..]`), and or-patterns (`Ok(0) | Err(_)`). Examples of valid patterns are `42`, `Some(_)`, `Foo
+//! { bar: Some(0) | None, baz: _ }`. Note that a binder in a pattern (e.g. `Some(x)`) matches the
+//! same values as a wildcard (e.g. `Some(_)`), so we treat both as wildcards.
+//!
+//! From this deconstruction we can compute whether a given value matches a given pattern; we
+//! simply look at ctors one at a time. Given a pattern `p` and a value `v`, we want to compute
+//! `matches!(v, p)`. It's mostly straightforward: we compare the head ctors and when they match
+//! we compare their fields recursively. A few representative examples:
+//!
+//! - `matches!(v, _) := true`
+//! - `matches!((v0,  v1), (p0,  p1)) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Foo { bar: v0, baz: v1 }, Foo { bar: p0, baz: p1 }) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)`
+//! - `matches!(Ok(v0), Err(p0)) := false` (incompatible variants)
+//! - `matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)`
+//! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
+//! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
+//! - `matches!(v, p0 | p1) := matches!(v, p0) || matches!(v, p1)`
+//!
+//! Constructors, fields and relevant operations are defined in the [`super::deconstruct_pat`] module.
+//!
+//! Note: this constructors/fields distinction may not straightforwardly apply to every Rust type.
+//! For example a value of type `Rc<u64>` can't be deconstructed that way, and `&str` has an
+//! infinitude of constructors. There are also subtleties with visibility of fields and
+//! uninhabitedness and various other things. The constructors idea can be extended to handle most
+//! of these subtleties though; caveats are documented where relevant throughout the code.
+//!
+//! Whether constructors cover each other is computed by [`Constructor::is_covered_by`].
+//!
+//!
+//! # Specialization
+//!
+//! Recall that we wish to compute `usefulness(p_1 .. p_n, q)`: given a list of patterns `p_1 ..
+//! p_n` and a pattern `q`, all of the same type, we want to find a list of values (called
+//! "witnesses") that are matched by `q` and by none of the `p_i`. We obviously don't just
+//! enumerate all possible values. From the discussion above we see that we can proceed
+//! ctor-by-ctor: for each value ctor of the given type, we ask "is there a value that starts with
+//! this constructor and matches `q` and none of the `p_i`?". As we saw above, there's a lot we can
+//! say from knowing only the first constructor of our candidate value.
+//!
+//! Let's take the following example:
+//! ```compile_fail,E0004
+//! # enum Enum { Variant1(()), Variant2(Option<bool>, u32)}
+//! # fn foo(x: Enum) {
+//! match x {
+//!     Enum::Variant1(_) => {} // `p1`
+//!     Enum::Variant2(None, 0) => {} // `p2`
+//!     Enum::Variant2(Some(_), 0) => {} // `q`
+//! }
+//! # }
+//! ```
+//!
+//! We can easily see that if our candidate value `v` starts with `Variant1` it will not match `q`.
+//! If `v = Variant2(v0, v1)` however, whether or not it matches `p2` and `q` will depend on `v0`
+//! and `v1`. In fact, such a `v` will be a witness of usefulness of `q` exactly when the tuple
+//! `(v0, v1)` is a witness of usefulness of `q'` in the following reduced match:
+//!
+//! ```compile_fail,E0004
+//! # fn foo(x: (Option<bool>, u32)) {
+//! match x {
+//!     (None, 0) => {} // `p2'`
+//!     (Some(_), 0) => {} // `q'`
+//! }
+//! # }
+//! ```
+//!
+//! This motivates a new step in computing usefulness, that we call _specialization_.
+//! Specialization consist of filtering a list of patterns for those that match a constructor, and
+//! then looking into the constructor's fields. This enables usefulness to be computed recursively.
+//!
+//! Instead of acting on a single pattern in each row, we will consider a list of patterns for each
+//! row, and we call such a list a _pattern-stack_. The idea is that we will specialize the
+//! leftmost pattern, which amounts to popping the constructor and pushing its fields, which feels
+//! like a stack. We note a pattern-stack simply with `[p_1 ... p_n]`.
+//! Here's a sequence of specializations of a list of pattern-stacks, to illustrate what's
+//! happening:
+//! ```ignore (illustrative)
+//! [Enum::Variant1(_)]
+//! [Enum::Variant2(None, 0)]
+//! [Enum::Variant2(Some(_), 0)]
+//! //==>> specialize with `Variant2`
+//! [None, 0]
+//! [Some(_), 0]
+//! //==>> specialize with `Some`
+//! [_, 0]
+//! //==>> specialize with `true` (say the type was `bool`)
+//! [0]
+//! //==>> specialize with `0`
+//! []
+//! ```
+//!
+//! The function `specialize(c, p)` takes a value constructor `c` and a pattern `p`, and returns 0
+//! or more pattern-stacks. If `c` does not match the head constructor of `p`, it returns nothing;
+//! otherwise if returns the fields of the constructor. This only returns more than one
+//! pattern-stack if `p` has a pattern-only constructor.
+//!
+//! - Specializing for the wrong constructor returns nothing
+//!
+//!   `specialize(None, Some(p0)) := []`
+//!
+//! - Specializing for the correct constructor returns a single row with the fields
+//!
+//!   `specialize(Variant1, Variant1(p0, p1, p2)) := [[p0, p1, p2]]`
+//!
+//!   `specialize(Foo{..}, Foo { bar: p0, baz: p1 }) := [[p0, p1]]`
+//!
+//! - For or-patterns, we specialize each branch and concatenate the results
+//!
+//!   `specialize(c, p0 | p1) := specialize(c, p0) ++ specialize(c, p1)`
+//!
+//! - We treat the other pattern constructors as if they were a large or-pattern of all the
+//!   possibilities:
+//!
+//!   `specialize(c, _) := specialize(c, Variant1(_) | Variant2(_, _) | ...)`
+//!
+//!   `specialize(c, 1..=100) := specialize(c, 1 | ... | 100)`
+//!
+//!   `specialize(c, [p0, .., p1]) := specialize(c, [p0, p1] | [p0, _, p1] | [p0, _, _, p1] | ...)`
+//!
+//! - If `c` is a pattern-only constructor, `specialize` is defined on a case-by-case basis. See
+//!   the discussion about constructor splitting in [`super::deconstruct_pat`].
+//!
+//!
+//! We then extend this function to work with pattern-stacks as input, by acting on the first
+//! column and keeping the other columns untouched.
+//!
+//! Specialization for the whole matrix is done in [`Matrix::specialize_constructor`]. Note that
+//! or-patterns in the first column are expanded before being stored in the matrix. Specialization
+//! for a single patstack is done from a combination of [`Constructor::is_covered_by`] and
+//! [`PatStack::pop_head_constructor`]. The internals of how it's done mostly live in the
+//! [`Fields`] struct.
+//!
+//!
+//! # Computing usefulness
+//!
+//! We now have all we need to compute usefulness. The inputs to usefulness are a list of
+//! pattern-stacks `p_1 ... p_n` (one per row), and a new pattern_stack `q`. The paper and this
+//! file calls the list of patstacks a _matrix_. They must all have the same number of columns and
+//! the patterns in a given column must all have the same type. `usefulness` returns a (possibly
+//! empty) list of witnesses of usefulness. These witnesses will also be pattern-stacks.
+//!
+//! - base case: `n_columns == 0`.
+//!     Since a pattern-stack functions like a tuple of patterns, an empty one functions like the
+//!     unit type. Thus `q` is useful iff there are no rows above it, i.e. if `n == 0`.
+//!
+//! - inductive case: `n_columns > 0`.
+//!     We need a way to list the constructors we want to try. We will be more clever in the next
+//!     section but for now assume we list all value constructors for the type of the first column.
+//!
+//!     - for each such ctor `c`:
+//!
+//!         - for each `q'` returned by `specialize(c, q)`:
+//!
+//!             - we compute `usefulness(specialize(c, p_1) ... specialize(c, p_n), q')`
+//!
+//!         - for each witness found, we revert specialization by pushing the constructor `c` on top.
+//!
+//!     - We return the concatenation of all the witnesses found, if any.
+//!
+//! Example:
+//! ```ignore (illustrative)
+//! [Some(true)] // p_1
+//! [None] // p_2
+//! [Some(_)] // q
+//! //==>> try `None`: `specialize(None, q)` returns nothing
+//! //==>> try `Some`: `specialize(Some, q)` returns a single row
+//! [true] // p_1'
+//! [_] // q'
+//! //==>> try `true`: `specialize(true, q')` returns a single row
+//! [] // p_1''
+//! [] // q''
+//! //==>> base case; `n != 0` so `q''` is not useful.
+//! //==>> go back up a step
+//! [true] // p_1'
+//! [_] // q'
+//! //==>> try `false`: `specialize(false, q')` returns a single row
+//! [] // q''
+//! //==>> base case; `n == 0` so `q''` is useful. We return the single witness `[]`
+//! witnesses:
+//! []
+//! //==>> undo the specialization with `false`
+//! witnesses:
+//! [false]
+//! //==>> undo the specialization with `Some`
+//! witnesses:
+//! [Some(false)]
+//! //==>> we have tried all the constructors. The output is the single witness `[Some(false)]`.
+//! ```
+//!
+//! This computation is done in [`is_useful`]. In practice we don't care about the list of
+//! witnesses when computing reachability; we only need to know whether any exist. We do keep the
+//! witnesses when computing exhaustiveness to report them to the user.
+//!
+//!
+//! # Making usefulness tractable: constructor splitting
+//!
+//! We're missing one last detail: which constructors do we list? Naively listing all value
+//! constructors cannot work for types like `u64` or `&str`, so we need to be more clever. The
+//! first obvious insight is that we only want to list constructors that are covered by the head
+//! constructor of `q`. If it's a value constructor, we only try that one. If it's a pattern-only
+//! constructor, we use the final clever idea for this algorithm: _constructor splitting_, where we
+//! group together constructors that behave the same.
+//!
+//! The details are not necessary to understand this file, so we explain them in
+//! [`super::deconstruct_pat`]. Splitting is done by the [`Constructor::split`] function.
+
+use self::ArmType::*;
+use self::Usefulness::*;
+
+use super::check_match::{joined_uncovered_patterns, pattern_not_covered_label};
+use super::deconstruct_pat::{Constructor, DeconstructedPat, Fields, SplitWildcard};
+
+use rustc_data_structures::captures::Captures;
+
+use rustc_arena::TypedArena;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir::def_id::DefId;
+use rustc_hir::HirId;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::{Span, DUMMY_SP};
+
+use smallvec::{smallvec, SmallVec};
+use std::fmt;
+use std::iter::once;
+
+pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
+    pub(crate) tcx: TyCtxt<'tcx>,
+    /// The module in which the match occurs. This is necessary for
+    /// checking inhabited-ness of types because whether a type is (visibly)
+    /// inhabited can depend on whether it was defined in the current module or
+    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
+    /// outside its module and should not be matchable with an empty match statement.
+    pub(crate) module: DefId,
+    pub(crate) param_env: ty::ParamEnv<'tcx>,
+    pub(crate) pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+}
+
+impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
+    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
+        if self.tcx.features().exhaustive_patterns {
+            self.tcx.is_ty_uninhabited_from(self.module, ty, self.param_env)
+        } else {
+            false
+        }
+    }
+
+    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
+    pub(super) fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
+        match ty.kind() {
+            ty::Adt(def, ..) => {
+                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
+            }
+            _ => false,
+        }
+    }
+}
+
+#[derive(Copy, Clone)]
+pub(super) struct PatCtxt<'a, 'p, 'tcx> {
+    pub(super) cx: &'a MatchCheckCtxt<'p, 'tcx>,
+    /// Type of the current column under investigation.
+    pub(super) ty: Ty<'tcx>,
+    /// Span of the current pattern under investigation.
+    pub(super) span: Span,
+    /// Whether the current pattern is the whole pattern as found in a match arm, or if it's a
+    /// subpattern.
+    pub(super) is_top_level: bool,
+    /// Whether the current pattern is from a `non_exhaustive` enum.
+    pub(super) is_non_exhaustive: bool,
+}
+
+impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PatCtxt").field("ty", &self.ty).finish()
+    }
+}
+
+/// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]`
+/// works well.
+#[derive(Clone)]
+struct PatStack<'p, 'tcx> {
+    pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
+}
+
+impl<'p, 'tcx> PatStack<'p, 'tcx> {
+    fn from_pattern(pat: &'p DeconstructedPat<'p, 'tcx>) -> Self {
+        Self::from_vec(smallvec![pat])
+    }
+
+    fn from_vec(vec: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>) -> Self {
+        PatStack { pats: vec }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.pats.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.pats.len()
+    }
+
+    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
+        self.pats[0]
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
+        self.pats.iter().copied()
+    }
+
+    // Recursively expand the first pattern into its subpatterns. Only useful if the pattern is an
+    // or-pattern. Panics if `self` is empty.
+    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = PatStack<'p, 'tcx>> + Captures<'a> {
+        self.head().iter_fields().map(move |pat| {
+            let mut new_patstack = PatStack::from_pattern(pat);
+            new_patstack.pats.extend_from_slice(&self.pats[1..]);
+            new_patstack
+        })
+    }
+
+    /// This computes `S(self.head().ctor(), self)`. See top of the file for explanations.
+    ///
+    /// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing
+    /// fields filled with wild patterns.
+    ///
+    /// This is roughly the inverse of `Constructor::apply`.
+    fn pop_head_constructor(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> PatStack<'p, 'tcx> {
+        // We pop the head pattern and push the new fields extracted from the arguments of
+        // `self.head()`.
+        let mut new_fields: SmallVec<[_; 2]> = self.head().specialize(pcx, ctor);
+        new_fields.extend_from_slice(&self.pats[1..]);
+        PatStack::from_vec(new_fields)
+    }
+}
+
+/// Pretty-printing for matrix row.
+impl<'p, 'tcx> fmt::Debug for PatStack<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "+")?;
+        for pat in self.iter() {
+            write!(f, " {:?} +", pat)?;
+        }
+        Ok(())
+    }
+}
+
+/// A 2D matrix.
+#[derive(Clone)]
+pub(super) struct Matrix<'p, 'tcx> {
+    patterns: Vec<PatStack<'p, 'tcx>>,
+}
+
+impl<'p, 'tcx> Matrix<'p, 'tcx> {
+    fn empty() -> Self {
+        Matrix { patterns: vec![] }
+    }
+
+    /// Number of columns of this matrix. `None` is the matrix is empty.
+    pub(super) fn column_count(&self) -> Option<usize> {
+        self.patterns.get(0).map(|r| r.len())
+    }
+
+    /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
+    /// expands it.
+    fn push(&mut self, row: PatStack<'p, 'tcx>) {
+        if !row.is_empty() && row.head().is_or_pat() {
+            self.patterns.extend(row.expand_or_pat());
+        } else {
+            self.patterns.push(row);
+        }
+    }
+
+    /// Iterate over the first component of each row
+    fn heads<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Clone + Captures<'a> {
+        self.patterns.iter().map(|r| r.head())
+    }
+
+    /// This computes `S(constructor, self)`. See top of the file for explanations.
+    fn specialize_constructor(
+        &self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        ctor: &Constructor<'tcx>,
+    ) -> Matrix<'p, 'tcx> {
+        let mut matrix = Matrix::empty();
+        for row in &self.patterns {
+            if ctor.is_covered_by(pcx, row.head().ctor()) {
+                let new_row = row.pop_head_constructor(pcx, ctor);
+                matrix.push(new_row);
+            }
+        }
+        matrix
+    }
+}
+
+/// Pretty-printer for matrices of patterns, example:
+///
+/// ```text
+/// + _     + []                +
+/// + true  + [First]           +
+/// + true  + [Second(true)]    +
+/// + false + [_]               +
+/// + _     + [_, _, tail @ ..] +
+/// ```
+impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "\n")?;
+
+        let Matrix { patterns: m, .. } = self;
+        let pretty_printed_matrix: Vec<Vec<String>> =
+            m.iter().map(|row| row.iter().map(|pat| format!("{:?}", pat)).collect()).collect();
+
+        let column_count = m.iter().map(|row| row.len()).next().unwrap_or(0);
+        assert!(m.iter().all(|row| row.len() == column_count));
+        let column_widths: Vec<usize> = (0..column_count)
+            .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
+            .collect();
+
+        for row in pretty_printed_matrix {
+            write!(f, "+")?;
+            for (column, pat_str) in row.into_iter().enumerate() {
+                write!(f, " ")?;
+                write!(f, "{:1$}", pat_str, column_widths[column])?;
+                write!(f, " +")?;
+            }
+            write!(f, "\n")?;
+        }
+        Ok(())
+    }
+}
+
+/// This carries the results of computing usefulness, as described at the top of the file. When
+/// checking usefulness of a match branch, we use the `NoWitnesses` variant, which also keeps track
+/// of potential unreachable sub-patterns (in the presence of or-patterns). When checking
+/// exhaustiveness of a whole match, we use the `WithWitnesses` variant, which carries a list of
+/// witnesses of non-exhaustiveness when there are any.
+/// Which variant to use is dictated by `ArmType`.
+#[derive(Debug)]
+enum Usefulness<'p, 'tcx> {
+    /// If we don't care about witnesses, simply remember if the pattern was useful.
+    NoWitnesses { useful: bool },
+    /// Carries a list of witnesses of non-exhaustiveness. If empty, indicates that the whole
+    /// pattern is unreachable.
+    WithWitnesses(Vec<Witness<'p, 'tcx>>),
+}
+
+impl<'p, 'tcx> Usefulness<'p, 'tcx> {
+    fn new_useful(preference: ArmType) -> Self {
+        match preference {
+            // A single (empty) witness of reachability.
+            FakeExtraWildcard => WithWitnesses(vec![Witness(vec![])]),
+            RealArm => NoWitnesses { useful: true },
+        }
+    }
+
+    fn new_not_useful(preference: ArmType) -> Self {
+        match preference {
+            FakeExtraWildcard => WithWitnesses(vec![]),
+            RealArm => NoWitnesses { useful: false },
+        }
+    }
+
+    fn is_useful(&self) -> bool {
+        match self {
+            Usefulness::NoWitnesses { useful } => *useful,
+            Usefulness::WithWitnesses(witnesses) => !witnesses.is_empty(),
+        }
+    }
+
+    /// Combine usefulnesses from two branches. This is an associative operation.
+    fn extend(&mut self, other: Self) {
+        match (&mut *self, other) {
+            (WithWitnesses(_), WithWitnesses(o)) if o.is_empty() => {}
+            (WithWitnesses(s), WithWitnesses(o)) if s.is_empty() => *self = WithWitnesses(o),
+            (WithWitnesses(s), WithWitnesses(o)) => s.extend(o),
+            (NoWitnesses { useful: s_useful }, NoWitnesses { useful: o_useful }) => {
+                *s_useful = *s_useful || o_useful
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    /// After calculating usefulness after a specialization, call this to reconstruct a usefulness
+    /// that makes sense for the matrix pre-specialization. This new usefulness can then be merged
+    /// with the results of specializing with the other constructors.
+    fn apply_constructor(
+        self,
+        pcx: &PatCtxt<'_, 'p, 'tcx>,
+        matrix: &Matrix<'p, 'tcx>, // used to compute missing ctors
+        ctor: &Constructor<'tcx>,
+    ) -> Self {
+        match self {
+            NoWitnesses { .. } => self,
+            WithWitnesses(ref witnesses) if witnesses.is_empty() => self,
+            WithWitnesses(witnesses) => {
+                let new_witnesses = if let Constructor::Missing { .. } = ctor {
+                    // We got the special `Missing` constructor, so each of the missing constructors
+                    // gives a new pattern that is not caught by the match. We list those patterns.
+                    let new_patterns = if pcx.is_non_exhaustive {
+                        // Here we don't want the user to try to list all variants, we want them to add
+                        // a wildcard, so we only suggest that.
+                        vec![DeconstructedPat::wildcard(pcx.ty)]
+                    } else {
+                        let mut split_wildcard = SplitWildcard::new(pcx);
+                        split_wildcard.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
+
+                        // This lets us know if we skipped any variants because they are marked
+                        // `doc(hidden)` or they are unstable feature gate (only stdlib types).
+                        let mut hide_variant_show_wild = false;
+                        // Construct for each missing constructor a "wild" version of this
+                        // constructor, that matches everything that can be built with
+                        // it. For example, if `ctor` is a `Constructor::Variant` for
+                        // `Option::Some`, we get the pattern `Some(_)`.
+                        let mut new: Vec<DeconstructedPat<'_, '_>> = split_wildcard
+                            .iter_missing(pcx)
+                            .filter_map(|missing_ctor| {
+                                // Check if this variant is marked `doc(hidden)`
+                                if missing_ctor.is_doc_hidden_variant(pcx)
+                                    || missing_ctor.is_unstable_variant(pcx)
+                                {
+                                    hide_variant_show_wild = true;
+                                    return None;
+                                }
+                                Some(DeconstructedPat::wild_from_ctor(pcx, missing_ctor.clone()))
+                            })
+                            .collect();
+
+                        if hide_variant_show_wild {
+                            new.push(DeconstructedPat::wildcard(pcx.ty));
+                        }
+
+                        new
+                    };
+
+                    witnesses
+                        .into_iter()
+                        .flat_map(|witness| {
+                            new_patterns.iter().map(move |pat| {
+                                Witness(
+                                    witness
+                                        .0
+                                        .iter()
+                                        .chain(once(pat))
+                                        .map(DeconstructedPat::clone_and_forget_reachability)
+                                        .collect(),
+                                )
+                            })
+                        })
+                        .collect()
+                } else {
+                    witnesses
+                        .into_iter()
+                        .map(|witness| witness.apply_constructor(pcx, &ctor))
+                        .collect()
+                };
+                WithWitnesses(new_witnesses)
+            }
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+enum ArmType {
+    FakeExtraWildcard,
+    RealArm,
+}
+
+/// A witness of non-exhaustiveness for error reporting, represented
+/// as a list of patterns (in reverse order of construction) with
+/// wildcards inside to represent elements that can take any inhabitant
+/// of the type as a value.
+///
+/// A witness against a list of patterns should have the same types
+/// and length as the pattern matched against. Because Rust `match`
+/// is always against a single pattern, at the end the witness will
+/// have length 1, but in the middle of the algorithm, it can contain
+/// multiple patterns.
+///
+/// For example, if we are constructing a witness for the match against
+///
+/// ```compile_fail,E0004
+/// # #![feature(type_ascription)]
+/// struct Pair(Option<(u32, u32)>, bool);
+/// # fn foo(p: Pair) {
+/// match (p: Pair) {
+///    Pair(None, _) => {}
+///    Pair(_, false) => {}
+/// }
+/// # }
+/// ```
+///
+/// We'll perform the following steps:
+/// 1. Start with an empty witness
+///     `Witness(vec![])`
+/// 2. Push a witness `true` against the `false`
+///     `Witness(vec![true])`
+/// 3. Push a witness `Some(_)` against the `None`
+///     `Witness(vec![true, Some(_)])`
+/// 4. Apply the `Pair` constructor to the witnesses
+///     `Witness(vec![Pair(Some(_), true)])`
+///
+/// The final `Pair(Some(_), true)` is then the resulting witness.
+#[derive(Debug)]
+pub(crate) struct Witness<'p, 'tcx>(Vec<DeconstructedPat<'p, 'tcx>>);
+
+impl<'p, 'tcx> Witness<'p, 'tcx> {
+    /// Asserts that the witness contains a single pattern, and returns it.
+    fn single_pattern(self) -> DeconstructedPat<'p, 'tcx> {
+        assert_eq!(self.0.len(), 1);
+        self.0.into_iter().next().unwrap()
+    }
+
+    /// Constructs a partial witness for a pattern given a list of
+    /// patterns expanded by the specialization step.
+    ///
+    /// When a pattern P is discovered to be useful, this function is used bottom-up
+    /// to reconstruct a complete witness, e.g., a pattern P' that covers a subset
+    /// of values, V, where each value in that set is not covered by any previously
+    /// used patterns and is covered by the pattern P'. Examples:
+    ///
+    /// left_ty: tuple of 3 elements
+    /// pats: [10, 20, _]           => (10, 20, _)
+    ///
+    /// left_ty: struct X { a: (bool, &'static str), b: usize}
+    /// pats: [(false, "foo"), 42]  => X { a: (false, "foo"), b: 42 }
+    fn apply_constructor(mut self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Self {
+        let pat = {
+            let len = self.0.len();
+            let arity = ctor.arity(pcx);
+            let pats = self.0.drain((len - arity)..).rev();
+            let fields = Fields::from_iter(pcx.cx, pats);
+            DeconstructedPat::new(ctor.clone(), fields, pcx.ty, DUMMY_SP)
+        };
+
+        self.0.push(pat);
+
+        self
+    }
+}
+
+/// Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
+/// is not exhaustive enough.
+///
+/// NB: The partner lint for structs lives in `compiler/rustc_typeck/src/check/pat.rs`.
+fn lint_non_exhaustive_omitted_patterns<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    scrut_ty: Ty<'tcx>,
+    sp: Span,
+    hir_id: HirId,
+    witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
+) {
+    let joined_patterns = joined_uncovered_patterns(cx, &witnesses);
+    cx.tcx.struct_span_lint_hir(NON_EXHAUSTIVE_OMITTED_PATTERNS, hir_id, sp, |build| {
+        let mut lint = build.build("some variants are not matched explicitly");
+        lint.span_label(sp, pattern_not_covered_label(&witnesses, &joined_patterns));
+        lint.help(
+            "ensure that all variants are matched explicitly by adding the suggested match arms",
+        );
+        lint.note(&format!(
+            "the matched value is of type `{}` and the `non_exhaustive_omitted_patterns` attribute was found",
+            scrut_ty,
+        ));
+        lint.emit();
+    });
+}
+
+/// Algorithm from <http://moscova.inria.fr/~maranget/papers/warn/index.html>.
+/// The algorithm from the paper has been modified to correctly handle empty
+/// types. The changes are:
+///   (0) We don't exit early if the pattern matrix has zero rows. We just
+///       continue to recurse over columns.
+///   (1) all_constructors will only return constructors that are statically
+///       possible. E.g., it will only return `Ok` for `Result<T, !>`.
+///
+/// This finds whether a (row) vector `v` of patterns is 'useful' in relation
+/// to a set of such vectors `m` - this is defined as there being a set of
+/// inputs that will match `v` but not any of the sets in `m`.
+///
+/// All the patterns at each column of the `matrix ++ v` matrix must have the same type.
+///
+/// This is used both for reachability checking (if a pattern isn't useful in
+/// relation to preceding patterns, it is not reachable) and exhaustiveness
+/// checking (if a wildcard pattern is useful in relation to a matrix, the
+/// matrix isn't exhaustive).
+///
+/// `is_under_guard` is used to inform if the pattern has a guard. If it
+/// has one it must not be inserted into the matrix. This shouldn't be
+/// relied on for soundness.
+#[instrument(level = "debug", skip(cx, matrix, hir_id))]
+fn is_useful<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    matrix: &Matrix<'p, 'tcx>,
+    v: &PatStack<'p, 'tcx>,
+    witness_preference: ArmType,
+    hir_id: HirId,
+    is_under_guard: bool,
+    is_top_level: bool,
+) -> Usefulness<'p, 'tcx> {
+    debug!(?matrix, ?v);
+    let Matrix { patterns: rows, .. } = matrix;
+
+    // The base case. We are pattern-matching on () and the return value is
+    // based on whether our matrix has a row or not.
+    // NOTE: This could potentially be optimized by checking rows.is_empty()
+    // first and then, if v is non-empty, the return value is based on whether
+    // the type of the tuple we're checking is inhabited or not.
+    if v.is_empty() {
+        let ret = if rows.is_empty() {
+            Usefulness::new_useful(witness_preference)
+        } else {
+            Usefulness::new_not_useful(witness_preference)
+        };
+        debug!(?ret);
+        return ret;
+    }
+
+    debug_assert!(rows.iter().all(|r| r.len() == v.len()));
+
+    // If the first pattern is an or-pattern, expand it.
+    let mut ret = Usefulness::new_not_useful(witness_preference);
+    if v.head().is_or_pat() {
+        debug!("expanding or-pattern");
+        // We try each or-pattern branch in turn.
+        let mut matrix = matrix.clone();
+        for v in v.expand_or_pat() {
+            debug!(?v);
+            let usefulness = ensure_sufficient_stack(|| {
+                is_useful(cx, &matrix, &v, witness_preference, hir_id, is_under_guard, false)
+            });
+            debug!(?usefulness);
+            ret.extend(usefulness);
+            // If pattern has a guard don't add it to the matrix.
+            if !is_under_guard {
+                // We push the already-seen patterns into the matrix in order to detect redundant
+                // branches like `Some(_) | Some(0)`.
+                matrix.push(v);
+            }
+        }
+    } else {
+        let ty = v.head().ty();
+        let is_non_exhaustive = cx.is_foreign_non_exhaustive_enum(ty);
+        debug!("v.head: {:?}, v.span: {:?}", v.head(), v.head().span());
+        let pcx = &PatCtxt { cx, ty, span: v.head().span(), is_top_level, is_non_exhaustive };
+
+        let v_ctor = v.head().ctor();
+        debug!(?v_ctor);
+        if let Constructor::IntRange(ctor_range) = &v_ctor {
+            // Lint on likely incorrect range patterns (#63987)
+            ctor_range.lint_overlapping_range_endpoints(
+                pcx,
+                matrix.heads(),
+                matrix.column_count().unwrap_or(0),
+                hir_id,
+            )
+        }
+        // We split the head constructor of `v`.
+        let split_ctors = v_ctor.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
+        let is_non_exhaustive_and_wild = is_non_exhaustive && v_ctor.is_wildcard();
+        // For each constructor, we compute whether there's a value that starts with it that would
+        // witness the usefulness of `v`.
+        let start_matrix = &matrix;
+        for ctor in split_ctors {
+            debug!("specialize({:?})", ctor);
+            // We cache the result of `Fields::wildcards` because it is used a lot.
+            let spec_matrix = start_matrix.specialize_constructor(pcx, &ctor);
+            let v = v.pop_head_constructor(pcx, &ctor);
+            let usefulness = ensure_sufficient_stack(|| {
+                is_useful(cx, &spec_matrix, &v, witness_preference, hir_id, is_under_guard, false)
+            });
+            let usefulness = usefulness.apply_constructor(pcx, start_matrix, &ctor);
+
+            // When all the conditions are met we have a match with a `non_exhaustive` enum
+            // that has the potential to trigger the `non_exhaustive_omitted_patterns` lint.
+            // To understand the workings checkout `Constructor::split` and `SplitWildcard::new/into_ctors`
+            if is_non_exhaustive_and_wild
+                // We check that the match has a wildcard pattern and that that wildcard is useful,
+                // meaning there are variants that are covered by the wildcard. Without the check
+                // for `witness_preference` the lint would trigger on `if let NonExhaustiveEnum::A = foo {}`
+                && usefulness.is_useful() && matches!(witness_preference, RealArm)
+                && matches!(
+                    &ctor,
+                    Constructor::Missing { nonexhaustive_enum_missing_real_variants: true }
+                )
+            {
+                let patterns = {
+                    let mut split_wildcard = SplitWildcard::new(pcx);
+                    split_wildcard.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
+                    // Construct for each missing constructor a "wild" version of this
+                    // constructor, that matches everything that can be built with
+                    // it. For example, if `ctor` is a `Constructor::Variant` for
+                    // `Option::Some`, we get the pattern `Some(_)`.
+                    split_wildcard
+                        .iter_missing(pcx)
+                        // Filter out the `NonExhaustive` because we want to list only real
+                        // variants. Also remove any unstable feature gated variants.
+                        // Because of how we computed `nonexhaustive_enum_missing_real_variants`,
+                        // this will not return an empty `Vec`.
+                        .filter(|c| !(c.is_non_exhaustive() || c.is_unstable_variant(pcx)))
+                        .cloned()
+                        .map(|missing_ctor| DeconstructedPat::wild_from_ctor(pcx, missing_ctor))
+                        .collect::<Vec<_>>()
+                };
+
+                lint_non_exhaustive_omitted_patterns(pcx.cx, pcx.ty, pcx.span, hir_id, patterns);
+            }
+
+            ret.extend(usefulness);
+        }
+    }
+
+    if ret.is_useful() {
+        v.head().set_reachable();
+    }
+
+    debug!(?ret);
+    ret
+}
+
+/// The arm of a match expression.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct MatchArm<'p, 'tcx> {
+    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
+    pub(crate) pat: &'p DeconstructedPat<'p, 'tcx>,
+    pub(crate) hir_id: HirId,
+    pub(crate) has_guard: bool,
+}
+
+/// Indicates whether or not a given arm is reachable.
+#[derive(Clone, Debug)]
+pub(crate) enum Reachability {
+    /// The arm is reachable. This additionally carries a set of or-pattern branches that have been
+    /// found to be unreachable despite the overall arm being reachable. Used only in the presence
+    /// of or-patterns, otherwise it stays empty.
+    Reachable(Vec<Span>),
+    /// The arm is unreachable.
+    Unreachable,
+}
+
+/// The output of checking a match for exhaustiveness and arm reachability.
+pub(crate) struct UsefulnessReport<'p, 'tcx> {
+    /// For each arm of the input, whether that arm is reachable after the arms above it.
+    pub(crate) arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Reachability)>,
+    /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
+    /// exhaustiveness.
+    pub(crate) non_exhaustiveness_witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
+}
+
+/// The entrypoint for the usefulness algorithm. Computes whether a match is exhaustive and which
+/// of its arms are reachable.
+///
+/// Note: the input patterns must have been lowered through
+/// `check_match::MatchVisitor::lower_pattern`.
+#[instrument(skip(cx, arms), level = "debug")]
+pub(crate) fn compute_match_usefulness<'p, 'tcx>(
+    cx: &MatchCheckCtxt<'p, 'tcx>,
+    arms: &[MatchArm<'p, 'tcx>],
+    scrut_hir_id: HirId,
+    scrut_ty: Ty<'tcx>,
+) -> UsefulnessReport<'p, 'tcx> {
+    let mut matrix = Matrix::empty();
+    let arm_usefulness: Vec<_> = arms
+        .iter()
+        .copied()
+        .map(|arm| {
+            debug!(?arm);
+            let v = PatStack::from_pattern(arm.pat);
+            is_useful(cx, &matrix, &v, RealArm, arm.hir_id, arm.has_guard, true);
+            if !arm.has_guard {
+                matrix.push(v);
+            }
+            let reachability = if arm.pat.is_reachable() {
+                Reachability::Reachable(arm.pat.unreachable_spans())
+            } else {
+                Reachability::Unreachable
+            };
+            (arm, reachability)
+        })
+        .collect();
+
+    let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty));
+    let v = PatStack::from_pattern(wild_pattern);
+    let usefulness = is_useful(cx, &matrix, &v, FakeExtraWildcard, scrut_hir_id, false, true);
+    let non_exhaustiveness_witnesses = match usefulness {
+        WithWitnesses(pats) => pats.into_iter().map(|w| w.single_pattern()).collect(),
+        NoWitnesses { .. } => bug!(),
+    };
+    UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses }
+}
diff --git a/compiler/rustc_mir_build/src/thir/util.rs b/compiler/rustc_mir_build/src/thir/util.rs
new file mode 100644
index 000000000..c58ed1ac0
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/util.rs
@@ -0,0 +1,31 @@
+use rustc_hir as hir;
+use rustc_middle::ty::{self, CanonicalUserType, TyCtxt, UserType};
+
+pub(crate) trait UserAnnotatedTyHelpers<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx>;
+
+    fn typeck_results(&self) -> &ty::TypeckResults<'tcx>;
+
+    /// Looks up the type associated with this hir-id and applies the
+    /// user-given substitutions; the hir-id must map to a suitable
+    /// type.
+    fn user_substs_applied_to_ty_of_hir_id(
+        &self,
+        hir_id: hir::HirId,
+    ) -> Option<CanonicalUserType<'tcx>> {
+        let user_provided_types = self.typeck_results().user_provided_types();
+        let mut user_ty = *user_provided_types.get(hir_id)?;
+        debug!("user_subts_applied_to_ty_of_hir_id: user_ty={:?}", user_ty);
+        let ty = self.typeck_results().node_type(hir_id);
+        match ty.kind() {
+            ty::Adt(adt_def, ..) => {
+                if let UserType::TypeOf(ref mut did, _) = &mut user_ty.value {
+                    *did = adt_def.did();
+                }
+                Some(user_ty)
+            }
+            ty::FnDef(..) => Some(user_ty),
+            _ => bug!("ty: {:?} should not have user provided type {:?} recorded ", ty, user_ty),
+        }
+    }
+}