Merging debian version 1.65.0+dfsg1-2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

24 files changed, 1018 insertions, 869 deletions

diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index 687560012..784583d94 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -1,6 +1,7 @@
 use crate::build::matches::ArmHasGuard;
 use crate::build::ForGuard::OutsideGuard;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+use rustc_middle::middle::region::Scope;
 use rustc_middle::thir::*;
 use rustc_middle::{mir::*, ty};
 use rustc_span::Span;
@@ -10,7 +11,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         destination: Place<'tcx>,
         block: BasicBlock,
-        ast_block: &Block,
+        ast_block: BlockId,
         source_info: SourceInfo,
     ) -> BlockAnd<()> {
         let Block {
@@ -21,7 +22,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             expr,
             targeted_by_break,
             safety_mode,
-        } = *ast_block;
+        } = self.thir[ast_block];
         let expr = expr.map(|expr| &self.thir[expr]);
         self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
             this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
@@ -34,10 +35,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             &stmts,
                             expr,
                             safety_mode,
+                            region_scope,
                         ))
                     })
                 } else {
-                    this.ast_block_stmts(destination, block, span, &stmts, expr, safety_mode)
+                    this.ast_block_stmts(
+                        destination,
+                        block,
+                        span,
+                        &stmts,
+                        expr,
+                        safety_mode,
+                        region_scope,
+                    )
                 }
             })
         })
@@ -51,6 +61,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         stmts: &[StmtId],
         expr: Option<&Expr<'tcx>>,
         safety_mode: BlockSafety,
+        region_scope: Scope,
     ) -> BlockAnd<()> {
         let this = self;
 
@@ -73,6 +84,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let mut let_scope_stack = Vec::with_capacity(8);
         let outer_source_scope = this.source_scope;
         let outer_in_scope_unsafe = this.in_scope_unsafe;
+        // This scope information is kept for breaking out of the parent remainder scope in case
+        // one let-else pattern matching fails.
+        // By doing so, we can be sure that even temporaries that receive extended lifetime
+        // assignments are dropped, too.
+        let mut last_remainder_scope = region_scope;
         this.update_source_scope_for_safety_mode(span, safety_mode);
 
         let source_info = this.source_info(span);
@@ -96,12 +112,169 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 StmtKind::Let {
                     remainder_scope,
                     init_scope,
+                    pattern,
+                    initializer: Some(initializer),
+                    lint_level,
+                    else_block: Some(else_block),
+                } => {
+                    // When lowering the statement `let <pat> = <expr> else { <else> };`,
+                    // the `<else>` block is nested in the parent scope enclosing this statment.
+                    // That scope is usually either the enclosing block scope,
+                    // or the remainder scope of the last statement.
+                    // This is to make sure that temporaries instantiated in `<expr>` are dropped
+                    // as well.
+                    // In addition, even though bindings in `<pat>` only come into scope if
+                    // the pattern matching passes, in the MIR building the storages for them
+                    // are declared as live any way.
+                    // This is similar to `let x;` statements without an initializer expression,
+                    // where the value of `x` in this example may or may be assigned,
+                    // because the storage for their values may not be live after all due to
+                    // failure in pattern matching.
+                    // For this reason, we declare those storages as live but we do not schedule
+                    // any drop yet- they are scheduled later after the pattern matching.
+                    // The generated MIR will have `StorageDead` whenever the control flow breaks out
+                    // of the parent scope, regardless of the result of the pattern matching.
+                    // However, the drops are inserted in MIR only when the control flow breaks out of
+                    // the scope of the remainder scope associated with this `let .. else` statement.
+                    // Pictorial explanation of the scope structure:
+                    // ┌─────────────────────────────────┐
+                    // │  Scope of the enclosing block,  │
+                    // │  or the last remainder scope    │
+                    // │  ┌───────────────────────────┐  │
+                    // │  │  Scope for <else> block   │  │
+                    // │  └───────────────────────────┘  │
+                    // │  ┌───────────────────────────┐  │
+                    // │  │  Remainder scope of       │  │
+                    // │  │  this let-else statement  │  │
+                    // │  │  ┌─────────────────────┐  │  │
+                    // │  │  │ <expr> scope        │  │  │
+                    // │  │  └─────────────────────┘  │  │
+                    // │  │  extended temporaries in  │  │
+                    // │  │  <expr> lives in this     │  │
+                    // │  │  scope                    │  │
+                    // │  │  ┌─────────────────────┐  │  │
+                    // │  │  │ Scopes for the rest │  │  │
+                    // │  │  └─────────────────────┘  │  │
+                    // │  └───────────────────────────┘  │
+                    // └─────────────────────────────────┘
+                    // Generated control flow:
+                    //          │ let Some(x) = y() else { return; }
+                    //          │
+                    // ┌────────▼───────┐
+                    // │ evaluate y()   │
+                    // └────────┬───────┘
+                    //          │              ┌────────────────┐
+                    // ┌────────▼───────┐      │Drop temporaries│
+                    // │Test the pattern├──────►in y()          │
+                    // └────────┬───────┘      │because breaking│
+                    //          │              │out of <expr>   │
+                    // ┌────────▼───────┐      │scope           │
+                    // │Move value into │      └───────┬────────┘
+                    // │binding x       │              │
+                    // └────────┬───────┘      ┌───────▼────────┐
+                    //          │              │Drop extended   │
+                    // ┌────────▼───────┐      │temporaries in  │
+                    // │Drop temporaries│      │<expr> because  │
+                    // │in y()          │      │breaking out of │
+                    // │because breaking│      │remainder scope │
+                    // │out of <expr>   │      └───────┬────────┘
+                    // │scope           │              │
+                    // └────────┬───────┘      ┌───────▼────────┐
+                    //          │              │Enter <else>    ├────────►
+                    // ┌────────▼───────┐      │block           │ return;
+                    // │Continue...     │      └────────────────┘
+                    // └────────────────┘
+
+                    let ignores_expr_result = matches!(pattern.kind, PatKind::Wild);
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result });
+
+                    // Lower the `else` block first because its parent scope is actually
+                    // enclosing the rest of the `let .. else ..` parts.
+                    let else_block_span = this.thir[*else_block].span;
+                    // This place is not really used because this destination place
+                    // should never be used to take values at the end of the failure
+                    // block.
+                    let dummy_place = this.temp(this.tcx.types.never, else_block_span);
+                    let failure_entry = this.cfg.start_new_block();
+                    let failure_block;
+                    unpack!(
+                        failure_block = this.ast_block(
+                            dummy_place,
+                            failure_entry,
+                            *else_block,
+                            this.source_info(else_block_span),
+                        )
+                    );
+                    this.cfg.terminate(
+                        failure_block,
+                        this.source_info(else_block_span),
+                        TerminatorKind::Unreachable,
+                    );
+
+                    // Declare the bindings, which may create a source scope.
+                    let remainder_span = remainder_scope.span(this.tcx, this.region_scope_tree);
+                    this.push_scope((*remainder_scope, source_info));
+                    let_scope_stack.push(remainder_scope);
+
+                    let visibility_scope =
+                        Some(this.new_source_scope(remainder_span, LintLevel::Inherited, None));
+
+                    let init = &this.thir[*initializer];
+                    let initializer_span = init.span;
+                    this.declare_bindings(
+                        visibility_scope,
+                        remainder_span,
+                        pattern,
+                        ArmHasGuard(false),
+                        Some((None, initializer_span)),
+                    );
+                    this.visit_primary_bindings(
+                        pattern,
+                        UserTypeProjections::none(),
+                        &mut |this, _, _, _, node, span, _, _| {
+                            this.storage_live_binding(block, node, span, OutsideGuard, true);
+                        },
+                    );
+                    let failure = unpack!(
+                        block = this.in_opt_scope(
+                            opt_destruction_scope.map(|de| (de, source_info)),
+                            |this| {
+                                let scope = (*init_scope, source_info);
+                                this.in_scope(scope, *lint_level, |this| {
+                                    this.ast_let_else(
+                                        block,
+                                        init,
+                                        initializer_span,
+                                        *else_block,
+                                        &last_remainder_scope,
+                                        pattern,
+                                    )
+                                })
+                            }
+                        )
+                    );
+                    this.cfg.goto(failure, source_info, failure_entry);
+
+                    if let Some(source_scope) = visibility_scope {
+                        this.source_scope = source_scope;
+                    }
+                    last_remainder_scope = *remainder_scope;
+                }
+                StmtKind::Let { init_scope, initializer: None, else_block: Some(_), .. } => {
+                    span_bug!(
+                        init_scope.span(this.tcx, this.region_scope_tree),
+                        "initializer is missing, but else block is present in this let binding",
+                    )
+                }
+                StmtKind::Let {
+                    remainder_scope,
+                    init_scope,
                     ref pattern,
                     initializer,
                     lint_level,
-                    else_block,
+                    else_block: None,
                 } => {
-                    let ignores_expr_result = matches!(*pattern.kind, PatKind::Wild);
+                    let ignores_expr_result = matches!(pattern.kind, PatKind::Wild);
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result });
 
                     // Enter the remainder scope, i.e., the bindings' destruction scope.
@@ -125,27 +298,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 |this| {
                                     let scope = (*init_scope, source_info);
                                     this.in_scope(scope, *lint_level, |this| {
-                                        if let Some(else_block) = else_block {
-                                            this.ast_let_else(
-                                                block,
-                                                init,
-                                                initializer_span,
-                                                else_block,
-                                                visibility_scope,
-                                                *remainder_scope,
-                                                remainder_span,
-                                                pattern,
-                                            )
-                                        } else {
-                                            this.declare_bindings(
-                                                visibility_scope,
-                                                remainder_span,
-                                                pattern,
-                                                ArmHasGuard(false),
-                                                Some((None, initializer_span)),
-                                            );
-                                            this.expr_into_pattern(block, pattern.clone(), init) // irrefutable pattern
-                                        }
+                                        this.declare_bindings(
+                                            visibility_scope,
+                                            remainder_span,
+                                            pattern,
+                                            ArmHasGuard(false),
+                                            Some((None, initializer_span)),
+                                        );
+                                        this.expr_into_pattern(block, &pattern, init) // irrefutable pattern
                                     })
                                 },
                             )
@@ -178,6 +338,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     if let Some(source_scope) = visibility_scope {
                         this.source_scope = source_scope;
                     }
+                    last_remainder_scope = *remainder_scope;
                 }
             }
 
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 648d10b9e..c9bec130c 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -2,26 +2,18 @@
 
 use crate::build::{parse_float_into_constval, Builder};
 use rustc_ast as ast;
-use rustc_hir::def_id::DefId;
 use rustc_middle::mir::interpret::{
     Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
 };
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
-use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, TyCtxt};
 use rustc_target::abi::Size;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, yielding a compile-time constant. Assumes that
     /// `expr` is a valid compile-time constant!
     pub(crate) fn as_constant(&mut self, expr: &Expr<'tcx>) -> Constant<'tcx> {
-        let create_uneval_from_def_id =
-            |tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, substs: SubstsRef<'tcx>| {
-                let uneval = ty::Unevaluated::new(ty::WithOptConstParam::unknown(def_id), substs);
-                tcx.mk_const(ty::ConstS { kind: ty::ConstKind::Unevaluated(uneval), ty })
-            };
-
         let this = self;
         let tcx = this.tcx;
         let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
@@ -41,11 +33,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 Constant { span, user_ty: None, literal }
             }
-            ExprKind::NonHirLiteral { lit, user_ty } => {
-                let user_ty = user_ty.map(|user_ty| {
+            ExprKind::NonHirLiteral { lit, ref user_ty } => {
+                let user_ty = user_ty.as_ref().map(|user_ty| {
                     this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                         span,
-                        user_ty,
+                        user_ty: user_ty.clone(),
                         inferred_ty: ty,
                     })
                 });
@@ -53,11 +45,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 Constant { span, user_ty: user_ty, literal }
             }
-            ExprKind::ZstLiteral { user_ty } => {
-                let user_ty = user_ty.map(|user_ty| {
+            ExprKind::ZstLiteral { ref user_ty } => {
+                let user_ty = user_ty.as_ref().map(|user_ty| {
                     this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                         span,
-                        user_ty,
+                        user_ty: user_ty.clone(),
                         inferred_ty: ty,
                     })
                 });
@@ -65,15 +57,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 Constant { span, user_ty: user_ty, literal }
             }
-            ExprKind::NamedConst { def_id, substs, user_ty } => {
-                let user_ty = user_ty.map(|user_ty| {
+            ExprKind::NamedConst { def_id, substs, ref user_ty } => {
+                let user_ty = user_ty.as_ref().map(|user_ty| {
                     this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                         span,
-                        user_ty,
+                        user_ty: user_ty.clone(),
                         inferred_ty: ty,
                     })
                 });
-                let literal = ConstantKind::Ty(create_uneval_from_def_id(tcx, def_id, ty, substs));
+
+                let uneval = ty::Unevaluated::new(ty::WithOptConstParam::unknown(def_id), substs);
+                let literal = ConstantKind::Unevaluated(uneval, ty);
 
                 Constant { user_ty, span, literal }
             }
@@ -85,7 +79,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Constant { user_ty: None, span, literal }
             }
             ExprKind::ConstBlock { did: def_id, substs } => {
-                let literal = ConstantKind::Ty(create_uneval_from_def_id(tcx, def_id, ty, substs));
+                let uneval = ty::Unevaluated::new(ty::WithOptConstParam::unknown(def_id), substs);
+                let literal = ConstantKind::Unevaluated(uneval, ty);
 
                 Constant { user_ty: None, span, literal }
             }
@@ -144,7 +139,7 @@ pub(crate) fn lit_to_mir_constant<'tcx>(
         }
         (ast::LitKind::Bool(b), ty::Bool) => ConstValue::Scalar(Scalar::from_bool(*b)),
         (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
-        (ast::LitKind::Err(_), _) => return Err(LitToConstError::Reported),
+        (ast::LitKind::Err, _) => return Err(LitToConstError::Reported),
         _ => return Err(LitToConstError::TypeError),
     };
 
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 0c06aad4e..ebb56e5a2 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -2,7 +2,7 @@
 
 use crate::build::expr::category::Category;
 use crate::build::ForGuard::{OutsideGuard, RefWithinGuard};
-use crate::build::{BlockAnd, BlockAndExtension, Builder};
+use crate::build::{BlockAnd, BlockAndExtension, Builder, Capture, CaptureMap};
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::hir::place::Projection as HirProjection;
 use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
@@ -59,8 +59,6 @@ pub(crate) enum PlaceBase {
         var_hir_id: LocalVarId,
         /// DefId of the closure
         closure_def_id: LocalDefId,
-        /// The trait closure implements, `Fn`, `FnMut`, `FnOnce`
-        closure_kind: ty::ClosureKind,
     },
 }
 
@@ -145,27 +143,6 @@ fn is_ancestor_or_same_capture(
     iter::zip(proj_possible_ancestor, proj_capture).all(|(a, b)| a == b)
 }
 
-/// Computes the index of a capture within the desugared closure provided the closure's
-/// `closure_min_captures` and the capture's index of the capture in the
-/// `ty::MinCaptureList` of the root variable `var_hir_id`.
-fn compute_capture_idx<'tcx>(
-    closure_min_captures: &ty::RootVariableMinCaptureList<'tcx>,
-    var_hir_id: LocalVarId,
-    root_var_idx: usize,
-) -> usize {
-    let mut res = 0;
-    for (var_id, capture_list) in closure_min_captures {
-        if *var_id == var_hir_id.0 {
-            res += root_var_idx;
-            break;
-        } else {
-            res += capture_list.len();
-        }
-    }
-
-    res
-}
-
 /// Given a closure, returns the index of a capture within the desugared closure struct and the
 /// `ty::CapturedPlace` which is the ancestor of the Place represented using the `var_hir_id`
 /// and `projection`.
@@ -174,27 +151,17 @@ fn compute_capture_idx<'tcx>(
 ///
 /// Returns None, when the ancestor is not found.
 fn find_capture_matching_projections<'a, 'tcx>(
-    typeck_results: &'a ty::TypeckResults<'tcx>,
+    upvars: &'a CaptureMap<'tcx>,
     var_hir_id: LocalVarId,
-    closure_def_id: LocalDefId,
     projections: &[PlaceElem<'tcx>],
-) -> Option<(usize, &'a ty::CapturedPlace<'tcx>)> {
-    let closure_min_captures = typeck_results.closure_min_captures.get(&closure_def_id)?;
-    let root_variable_min_captures = closure_min_captures.get(&var_hir_id.0)?;
-
+) -> Option<(usize, &'a Capture<'tcx>)> {
     let hir_projections = convert_to_hir_projections_and_truncate_for_capture(projections);
 
-    // If an ancestor is found, `idx` is the index within the list of captured places
-    // for root variable `var_hir_id` and `capture` is the `ty::CapturedPlace` itself.
-    let (idx, capture) = root_variable_min_captures.iter().enumerate().find(|(_, capture)| {
+    upvars.get_by_key_enumerated(var_hir_id.0).find(|(_, capture)| {
         let possible_ancestor_proj_kinds: Vec<_> =
-            capture.place.projections.iter().map(|proj| proj.kind).collect();
+            capture.captured_place.place.projections.iter().map(|proj| proj.kind).collect();
         is_ancestor_or_same_capture(&possible_ancestor_proj_kinds, &hir_projections)
-    })?;
-
-    // Convert index to be from the perspective of the entire closure_min_captures map
-    // instead of just the root variable capture list
-    Some((compute_capture_idx(closure_min_captures, var_hir_id, idx), capture))
+    })
 }
 
 /// Takes a PlaceBuilder and resolves the upvar (if any) within it, so that the
@@ -204,24 +171,15 @@ fn find_capture_matching_projections<'a, 'tcx>(
 fn to_upvars_resolved_place_builder<'a, 'tcx>(
     from_builder: PlaceBuilder<'tcx>,
     tcx: TyCtxt<'tcx>,
-    typeck_results: &'a ty::TypeckResults<'tcx>,
+    upvars: &'a CaptureMap<'tcx>,
 ) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
     match from_builder.base {
         PlaceBase::Local(_) => Ok(from_builder),
-        PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind } => {
-            let mut upvar_resolved_place_builder = PlaceBuilder::from(ty::CAPTURE_STRUCT_LOCAL);
-            match closure_kind {
-                ty::ClosureKind::Fn | ty::ClosureKind::FnMut => {
-                    upvar_resolved_place_builder = upvar_resolved_place_builder.deref();
-                }
-                ty::ClosureKind::FnOnce => {}
-            }
-
+        PlaceBase::Upvar { var_hir_id, closure_def_id } => {
             let Some((capture_index, capture)) =
                 find_capture_matching_projections(
-                    typeck_results,
+                    upvars,
                     var_hir_id,
-                    closure_def_id,
                     &from_builder.projection,
                 ) else {
                 let closure_span = tcx.def_span(closure_def_id);
@@ -241,39 +199,17 @@ fn to_upvars_resolved_place_builder<'a, 'tcx>(
                 return Err(from_builder);
             };
 
-            // We won't be building MIR if the closure wasn't local
-            let closure_hir_id = tcx.hir().local_def_id_to_hir_id(closure_def_id);
-            let closure_ty = typeck_results.node_type(closure_hir_id);
-
-            let substs = match closure_ty.kind() {
-                ty::Closure(_, substs) => ty::UpvarSubsts::Closure(substs),
-                ty::Generator(_, substs, _) => ty::UpvarSubsts::Generator(substs),
-                _ => bug!("Lowering capture for non-closure type {:?}", closure_ty),
-            };
-
             // Access the capture by accessing the field within the Closure struct.
-            //
-            // We must have inferred the capture types since we are building MIR, therefore
-            // it's safe to call `tuple_element_ty` and we can unwrap here because
-            // we know that the capture exists and is the `capture_index`-th capture.
-            let var_ty = substs.tupled_upvars_ty().tuple_fields()[capture_index];
-
-            upvar_resolved_place_builder =
-                upvar_resolved_place_builder.field(Field::new(capture_index), var_ty);
-
-            // If the variable is captured via ByRef(Immutable/Mutable) Borrow,
-            // we need to deref it
-            upvar_resolved_place_builder = match capture.info.capture_kind {
-                ty::UpvarCapture::ByRef(_) => upvar_resolved_place_builder.deref(),
-                ty::UpvarCapture::ByValue => upvar_resolved_place_builder,
-            };
+            let capture_info = &upvars[capture_index];
+
+            let mut upvar_resolved_place_builder = PlaceBuilder::from(capture_info.use_place);
 
             // We used some of the projections to build the capture itself,
             // now we apply the remaining to the upvar resolved place.
             let remaining_projections = strip_prefix(
-                capture.place.base_ty,
+                capture.captured_place.place.base_ty,
                 from_builder.projection,
-                &capture.place.projections,
+                &capture.captured_place.place.projections,
             );
             upvar_resolved_place_builder.projection.extend(remaining_projections);
 
@@ -315,24 +251,24 @@ fn strip_prefix<'tcx>(
 }
 
 impl<'tcx> PlaceBuilder<'tcx> {
-    pub(crate) fn into_place<'a>(
+    pub(in crate::build) fn into_place<'a>(
         self,
         tcx: TyCtxt<'tcx>,
-        typeck_results: &'a ty::TypeckResults<'tcx>,
+        upvars: &'a CaptureMap<'tcx>,
     ) -> Place<'tcx> {
         if let PlaceBase::Local(local) = self.base {
             Place { local, projection: tcx.intern_place_elems(&self.projection) }
         } else {
-            self.expect_upvars_resolved(tcx, typeck_results).into_place(tcx, typeck_results)
+            self.expect_upvars_resolved(tcx, upvars).into_place(tcx, upvars)
         }
     }
 
     fn expect_upvars_resolved<'a>(
         self,
         tcx: TyCtxt<'tcx>,
-        typeck_results: &'a ty::TypeckResults<'tcx>,
+        upvars: &'a CaptureMap<'tcx>,
     ) -> PlaceBuilder<'tcx> {
-        to_upvars_resolved_place_builder(self, tcx, typeck_results).unwrap()
+        to_upvars_resolved_place_builder(self, tcx, upvars).unwrap()
     }
 
     /// Attempts to resolve the `PlaceBuilder`.
@@ -346,12 +282,12 @@ impl<'tcx> PlaceBuilder<'tcx> {
     /// not captured. This can happen because the final mir that will be
     /// generated doesn't require a read for this place. Failures will only
     /// happen inside closures.
-    pub(crate) fn try_upvars_resolved<'a>(
+    pub(in crate::build) fn try_upvars_resolved<'a>(
         self,
         tcx: TyCtxt<'tcx>,
-        typeck_results: &'a ty::TypeckResults<'tcx>,
+        upvars: &'a CaptureMap<'tcx>,
     ) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
-        to_upvars_resolved_place_builder(self, tcx, typeck_results)
+        to_upvars_resolved_place_builder(self, tcx, upvars)
     }
 
     pub(crate) fn base(&self) -> PlaceBase {
@@ -392,6 +328,12 @@ impl<'tcx> From<PlaceBase> for PlaceBuilder<'tcx> {
     }
 }
 
+impl<'tcx> From<Place<'tcx>> for PlaceBuilder<'tcx> {
+    fn from(p: Place<'tcx>) -> Self {
+        Self { base: PlaceBase::Local(p.local), projection: p.projection.to_vec() }
+    }
+}
+
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, yielding a place that we can move from etc.
     ///
@@ -411,7 +353,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr: &Expr<'tcx>,
     ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_place_builder(block, expr));
-        block.and(place_builder.into_place(self.tcx, self.typeck_results))
+        block.and(place_builder.into_place(self.tcx, &self.upvars))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
@@ -435,7 +377,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr: &Expr<'tcx>,
     ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr));
-        block.and(place_builder.into_place(self.tcx, self.typeck_results))
+        block.and(place_builder.into_place(self.tcx, &self.upvars))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
@@ -513,7 +455,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 block.and(place_builder)
             }
 
-            ExprKind::PlaceTypeAscription { source, user_ty } => {
+            ExprKind::PlaceTypeAscription { source, ref user_ty } => {
                 let place_builder = unpack!(
                     block = this.expr_as_place(
                         block,
@@ -526,11 +468,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     let annotation_index =
                         this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                             span: source_info.span,
-                            user_ty,
+                            user_ty: user_ty.clone(),
                             inferred_ty: expr.ty,
                         });
 
-                    let place = place_builder.clone().into_place(this.tcx, this.typeck_results);
+                    let place = place_builder.clone().into_place(this.tcx, &this.upvars);
                     this.cfg.push(
                         block,
                         Statement {
@@ -547,7 +489,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
                 block.and(place_builder)
             }
-            ExprKind::ValueTypeAscription { source, user_ty } => {
+            ExprKind::ValueTypeAscription { source, ref user_ty } => {
                 let source = &this.thir[source];
                 let temp =
                     unpack!(block = this.as_temp(block, source.temp_lifetime, source, mutability));
@@ -555,7 +497,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     let annotation_index =
                         this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                             span: source_info.span,
-                            user_ty,
+                            user_ty: user_ty.clone(),
                             inferred_ty: expr.ty,
                         });
                     this.cfg.push(
@@ -629,17 +571,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         closure_def_id: LocalDefId,
         var_hir_id: LocalVarId,
     ) -> BlockAnd<PlaceBuilder<'tcx>> {
-        let closure_ty =
-            self.typeck_results.node_type(self.tcx.hir().local_def_id_to_hir_id(closure_def_id));
-
-        let closure_kind = if let ty::Closure(_, closure_substs) = closure_ty.kind() {
-            self.infcx.closure_kind(closure_substs).unwrap()
-        } else {
-            // Generators are considered FnOnce.
-            ty::ClosureKind::FnOnce
-        };
-
-        block.and(PlaceBuilder::from(PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind }))
+        block.and(PlaceBuilder::from(PlaceBase::Upvar { var_hir_id, closure_def_id }))
     }
 
     /// Lower an index expression
@@ -678,7 +610,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         if is_outermost_index {
             self.read_fake_borrows(block, fake_borrow_temps, source_info)
         } else {
-            base_place = base_place.expect_upvars_resolved(self.tcx, self.typeck_results);
+            base_place = base_place.expect_upvars_resolved(self.tcx, &self.upvars);
             self.add_fake_borrows_of_base(
                 &base_place,
                 block,
@@ -710,7 +642,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             block,
             source_info,
             len,
-            Rvalue::Len(slice.into_place(self.tcx, self.typeck_results)),
+            Rvalue::Len(slice.into_place(self.tcx, &self.upvars)),
         );
         // lt = idx < len
         self.cfg.push_assign(
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index 15f2d17c4..f88ab63f0 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -197,13 +197,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // create all the steps directly in MIR with operations all backends need to support anyway.
                 let (source, ty) = if let ty::Adt(adt_def, ..) = source.ty.kind() && adt_def.is_enum() {
                     let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
-                    let place = unpack!(block = this.as_place(block, source));
+                    let temp = unpack!(block = this.as_temp(block, scope, source, Mutability::Not));
                     let discr = this.temp(discr_ty, source.span);
                     this.cfg.push_assign(
                         block,
                         source_info,
                         discr,
-                        Rvalue::Discriminant(place),
+                        Rvalue::Discriminant(temp.into()),
                     );
 
                     (Operand::Move(discr), discr_ty)
@@ -216,6 +216,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 };
                 let from_ty = CastTy::from_ty(ty);
                 let cast_ty = CastTy::from_ty(expr.ty);
+                debug!("ExprKind::Cast from_ty={from_ty:?}, cast_ty={:?}/{cast_ty:?}", expr.ty,);
                 let cast_kind = match (from_ty, cast_ty) {
                     (Some(CastTy::Ptr(_) | CastTy::FnPtr), Some(CastTy::Int(_))) => {
                         CastKind::PointerExposeAddress
@@ -223,6 +224,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     (Some(CastTy::Int(_)), Some(CastTy::Ptr(_))) => {
                         CastKind::PointerFromExposedAddress
                     }
+                    (_, Some(CastTy::DynStar)) => CastKind::DynStar,
                     (_, _) => CastKind::Misc,
                 };
                 block.and(Rvalue::Cast(cast_kind, source, expr.ty))
@@ -302,7 +304,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 block.and(Rvalue::Aggregate(Box::new(AggregateKind::Tuple), fields))
             }
-            ExprKind::Closure { closure_id, substs, ref upvars, movability, ref fake_reads } => {
+            ExprKind::Closure(box ClosureExpr {
+                closure_id,
+                substs,
+                ref upvars,
+                movability,
+                ref fake_reads,
+            }) => {
                 // Convert the closure fake reads, if any, from `ExprRef` to mir `Place`
                 // and push the fake reads.
                 // This must come before creating the operands. This is required in case
@@ -322,10 +330,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         unpack!(block = this.as_place_builder(block, &this.thir[*thir_place]));
 
                     if let Ok(place_builder_resolved) =
-                        place_builder.try_upvars_resolved(this.tcx, this.typeck_results)
+                        place_builder.try_upvars_resolved(this.tcx, &this.upvars)
                     {
-                        let mir_place =
-                            place_builder_resolved.into_place(this.tcx, this.typeck_results);
+                        let mir_place = place_builder_resolved.into_place(this.tcx, &this.upvars);
                         this.cfg.push_fake_read(
                             block,
                             this.source_info(this.tcx.hir().span(*hir_id)),
@@ -617,7 +624,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // is same as that of the capture in the parent closure.
             PlaceBase::Upvar { .. } => {
                 let enclosing_upvars_resolved =
-                    arg_place_builder.clone().into_place(this.tcx, this.typeck_results);
+                    arg_place_builder.clone().into_place(this.tcx, &this.upvars);
 
                 match enclosing_upvars_resolved.as_ref() {
                     PlaceRef {
@@ -637,12 +644,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         );
                         // Not in a closure
                         debug_assert!(
-                            this.upvar_mutbls.len() > upvar_index.index(),
-                            "Unexpected capture place, upvar_mutbls={:#?}, upvar_index={:?}",
-                            this.upvar_mutbls,
+                            this.upvars.len() > upvar_index.index(),
+                            "Unexpected capture place, upvars={:#?}, upvar_index={:?}",
+                            this.upvars,
                             upvar_index
                         );
-                        this.upvar_mutbls[upvar_index.index()]
+                        this.upvars[upvar_index.index()].mutability
                     }
                     _ => bug!("Unexpected capture place"),
                 }
@@ -654,7 +661,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
         };
 
-        let arg_place = arg_place_builder.into_place(this.tcx, this.typeck_results);
+        let arg_place = arg_place_builder.into_place(this.tcx, &this.upvars);
 
         this.cfg.push_assign(
             block,
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
index 724b72f87..e5dafb820 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -83,8 +83,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // Don't bother with StorageLive and Dead for these temporaries,
             // they are never assigned.
             ExprKind::Break { .. } | ExprKind::Continue { .. } | ExprKind::Return { .. } => (),
-            ExprKind::Block { body: Block { expr: None, targeted_by_break: false, .. } }
-                if expr_ty.is_never() => {}
+            ExprKind::Block { block }
+                if let Block { expr: None, targeted_by_break: false, .. } = this.thir[block]
+                    && expr_ty.is_never() => {}
             _ => {
                 this.cfg
                     .push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index 017d43d10..224a1b80f 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -46,7 +46,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     })
                 })
             }
-            ExprKind::Block { body: ref ast_block } => {
+            ExprKind::Block { block: ast_block } => {
                 this.ast_block(destination, block, ast_block, source_info)
             }
             ExprKind::Match { scrutinee, ref arms } => {
@@ -75,7 +75,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 this.source_info(then_expr.span)
                             };
                             let (then_block, else_block) =
-                                this.in_if_then_scope(condition_scope, |this| {
+                                this.in_if_then_scope(condition_scope, then_expr.span, |this| {
                                     let then_blk = unpack!(this.then_else_break(
                                         block,
                                         &this.thir[cond],
@@ -108,7 +108,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             ExprKind::Let { expr, ref pat } => {
                 let scope = this.local_scope();
-                let (true_block, false_block) = this.in_if_then_scope(scope, |this| {
+                let (true_block, false_block) = this.in_if_then_scope(scope, expr_span, |this| {
                     this.lower_let_expr(block, &this.thir[expr], pat, scope, None, expr_span)
                 });
 
@@ -314,11 +314,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 this.cfg.push_assign(block, source_info, destination, address_of);
                 block.unit()
             }
-            ExprKind::Adt(box Adt {
+            ExprKind::Adt(box AdtExpr {
                 adt_def,
                 variant_index,
                 substs,
-                user_ty,
+                ref user_ty,
                 ref fields,
                 ref base,
             }) => {
@@ -366,9 +366,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             None => {
                                 let place_builder = place_builder.clone();
                                 this.consume_by_copy_or_move(
-                                    place_builder
-                                        .field(n, *ty)
-                                        .into_place(this.tcx, this.typeck_results),
+                                    place_builder.field(n, *ty).into_place(this.tcx, &this.upvars),
                                 )
                             }
                         })
@@ -378,10 +376,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 };
 
                 let inferred_ty = expr.ty;
-                let user_ty = user_ty.map(|ty| {
+                let user_ty = user_ty.as_ref().map(|user_ty| {
                     this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                         span: source_info.span,
-                        user_ty: ty,
+                        user_ty: user_ty.clone(),
                         inferred_ty,
                     })
                 });
@@ -400,7 +398,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 );
                 block.unit()
             }
-            ExprKind::InlineAsm { template, ref operands, options, line_spans } => {
+            ExprKind::InlineAsm(box InlineAsmExpr {
+                template,
+                ref operands,
+                options,
+                line_spans,
+            }) => {
                 use rustc_middle::{mir, thir};
                 let operands = operands
                     .into_iter()
diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs
index a7e1331aa..00dbcaeb0 100644
--- a/compiler/rustc_mir_build/src/build/expr/stmt.rs
+++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs
@@ -116,14 +116,22 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // it is usually better to focus on `the_value` rather
                 // than the entirety of block(s) surrounding it.
                 let adjusted_span = (|| {
-                    if let ExprKind::Block { body } = &expr.kind && let Some(tail_ex) = body.expr {
+                    if let ExprKind::Block { block } = expr.kind
+                        && let Some(tail_ex) = this.thir[block].expr
+                    {
                         let mut expr = &this.thir[tail_ex];
-                        while let ExprKind::Block {
-                            body: Block { expr: Some(nested_expr), .. },
-                        }
-                        | ExprKind::Scope { value: nested_expr, .. } = expr.kind
-                        {
-                            expr = &this.thir[nested_expr];
+                        loop {
+                            match expr.kind {
+                                ExprKind::Block { block }
+                                    if let Some(nested_expr) = this.thir[block].expr =>
+                                {
+                                    expr = &this.thir[nested_expr];
+                                }
+                                ExprKind::Scope { value: nested_expr, .. } => {
+                                    expr = &this.thir[nested_expr];
+                                }
+                                _ => break,
+                            }
                         }
                         this.block_context.push(BlockFrame::TailExpr {
                             tail_result_is_ignored: true,
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 58b1564cc..93f382cc6 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -155,7 +155,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// * From each pre-binding block to the next pre-binding block.
     /// * From each otherwise block to the next pre-binding block.
-    #[tracing::instrument(level = "debug", skip(self, arms))]
+    #[instrument(level = "debug", skip(self, arms))]
     pub(crate) fn match_expr(
         &mut self,
         destination: Place<'tcx>,
@@ -170,7 +170,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         let mut arm_candidates = self.create_match_candidates(scrutinee_place.clone(), &arms);
 
-        let match_has_guard = arms.iter().copied().any(|arm| self.thir[arm].guard.is_some());
+        let match_has_guard = arm_candidates.iter().any(|(_, candidate)| candidate.has_guard);
         let mut candidates =
             arm_candidates.iter_mut().map(|(_, candidate)| candidate).collect::<Vec<_>>();
 
@@ -221,9 +221,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = self.source_info(scrutinee_span);
 
         if let Ok(scrutinee_builder) =
-            scrutinee_place_builder.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+            scrutinee_place_builder.clone().try_upvars_resolved(self.tcx, &self.upvars)
         {
-            let scrutinee_place = scrutinee_builder.into_place(self.tcx, self.typeck_results);
+            let scrutinee_place = scrutinee_builder.into_place(self.tcx, &self.upvars);
             self.cfg.push_fake_read(block, source_info, cause_matched_place, scrutinee_place);
         }
 
@@ -348,12 +348,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // ```
                     let mut opt_scrutinee_place: Option<(Option<&Place<'tcx>>, Span)> = None;
                     let scrutinee_place: Place<'tcx>;
-                    if let Ok(scrutinee_builder) = scrutinee_place_builder
-                        .clone()
-                        .try_upvars_resolved(this.tcx, this.typeck_results)
+                    if let Ok(scrutinee_builder) =
+                        scrutinee_place_builder.clone().try_upvars_resolved(this.tcx, &this.upvars)
                     {
-                        scrutinee_place =
-                            scrutinee_builder.into_place(this.tcx, this.typeck_results);
+                        scrutinee_place = scrutinee_builder.into_place(this.tcx, &this.upvars);
                         opt_scrutinee_place = Some((Some(&scrutinee_place), scrutinee_span));
                     }
                     let scope = this.declare_bindings(
@@ -373,6 +371,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         Some(arm.span),
                         Some(arm.scope),
                         Some(match_scope),
+                        false,
                     );
 
                     if let Some(source_scope) = scope {
@@ -418,6 +417,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         arm_span: Option<Span>,
         arm_scope: Option<region::Scope>,
         match_scope: Option<region::Scope>,
+        storages_alive: bool,
     ) -> BasicBlock {
         if candidate.subcandidates.is_empty() {
             // Avoid generating another `BasicBlock` when we only have one
@@ -431,6 +431,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 arm_span,
                 match_scope,
                 true,
+                storages_alive,
             )
         } else {
             // It's helpful to avoid scheduling drops multiple times to save
@@ -468,6 +469,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         arm_span,
                         match_scope,
                         schedule_drops,
+                        storages_alive,
                     );
                     if arm_scope.is_none() {
                         schedule_drops = false;
@@ -490,10 +492,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub(super) fn expr_into_pattern(
         &mut self,
         mut block: BasicBlock,
-        irrefutable_pat: Pat<'tcx>,
+        irrefutable_pat: &Pat<'tcx>,
         initializer: &Expr<'tcx>,
     ) -> BlockAnd<()> {
-        match *irrefutable_pat.kind {
+        match irrefutable_pat.kind {
             // Optimize the case of `let x = ...` to write directly into `x`
             PatKind::Binding { mode: BindingMode::ByValue, var, subpattern: None, .. } => {
                 let place =
@@ -518,17 +520,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // broken.
             PatKind::AscribeUserType {
                 subpattern:
-                    Pat {
+                    box Pat {
                         kind:
-                            box PatKind::Binding {
-                                mode: BindingMode::ByValue,
-                                var,
-                                subpattern: None,
-                                ..
+                            PatKind::Binding {
+                                mode: BindingMode::ByValue, var, subpattern: None, ..
                             },
                         ..
                     },
-                ascription: thir::Ascription { annotation, variance: _ },
+                ascription: thir::Ascription { ref annotation, variance: _ },
             } => {
                 let place =
                     self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true);
@@ -541,7 +540,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 let ty_source_info = self.source_info(annotation.span);
 
-                let base = self.canonical_user_type_annotations.push(annotation);
+                let base = self.canonical_user_type_annotations.push(annotation.clone());
                 self.cfg.push(
                     block,
                     Statement {
@@ -573,7 +572,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             _ => {
                 let place_builder = unpack!(block = self.as_place_builder(block, initializer));
-                self.place_into_pattern(block, irrefutable_pat, place_builder, true)
+                self.place_into_pattern(block, &irrefutable_pat, place_builder, true)
             }
         }
     }
@@ -581,7 +580,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub(crate) fn place_into_pattern(
         &mut self,
         block: BasicBlock,
-        irrefutable_pat: Pat<'tcx>,
+        irrefutable_pat: &Pat<'tcx>,
         initializer: PlaceBuilder<'tcx>,
         set_match_place: bool,
     ) -> BlockAnd<()> {
@@ -623,9 +622,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // };
                     // ```
                     if let Ok(match_pair_resolved) =
-                        initializer.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                        initializer.clone().try_upvars_resolved(self.tcx, &self.upvars)
                     {
-                        let place = match_pair_resolved.into_place(self.tcx, self.typeck_results);
+                        let place = match_pair_resolved.into_place(self.tcx, &self.upvars);
                         *match_place = Some(place);
                     }
                 }
@@ -646,6 +645,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             None,
             None,
             None,
+            false,
         )
         .unit()
     }
@@ -702,9 +702,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let local_id = self.var_local_id(var, for_guard);
         let source_info = self.source_info(span);
         self.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(local_id) });
-        // Altough there is almost always scope for given variable in corner cases
+        // Although there is almost always scope for given variable in corner cases
         // like #92893 we might get variable with no scope.
-        if let Some(region_scope) = self.region_scope_tree.var_scope(var.0.local_id) && schedule_drop{
+        if let Some(region_scope) = self.region_scope_tree.var_scope(var.0.local_id) && schedule_drop {
             self.schedule_drop(span, region_scope, local_id, DropKind::Storage);
         }
         Place::from(local_id)
@@ -744,7 +744,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             "visit_primary_bindings: pattern={:?} pattern_user_ty={:?}",
             pattern, pattern_user_ty
         );
-        match *pattern.kind {
+        match pattern.kind {
             PatKind::Binding {
                 mutability,
                 name,
@@ -767,7 +767,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | PatKind::Slice { ref prefix, ref slice, ref suffix } => {
                 let from = u64::try_from(prefix.len()).unwrap();
                 let to = u64::try_from(suffix.len()).unwrap();
-                for subpattern in prefix {
+                for subpattern in prefix.iter() {
                     self.visit_primary_bindings(subpattern, pattern_user_ty.clone().index(), f);
                 }
                 for subpattern in slice {
@@ -777,7 +777,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         f,
                     );
                 }
-                for subpattern in suffix {
+                for subpattern in suffix.iter() {
                     self.visit_primary_bindings(subpattern, pattern_user_ty.clone().index(), f);
                 }
             }
@@ -830,7 +830,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // may not all be in the leftmost subpattern. For example in
                 // `let (x | y) = ...`, the primary binding of `y` occurs in
                 // the right subpattern
-                for subpattern in pats {
+                for subpattern in pats.iter() {
                     self.visit_primary_bindings(subpattern, pattern_user_ty.clone(), f);
                 }
             }
@@ -982,7 +982,7 @@ enum TestKind<'tcx> {
     },
 
     /// Test whether the value falls within an inclusive or exclusive range
-    Range(PatRange<'tcx>),
+    Range(Box<PatRange<'tcx>>),
 
     /// Test that the length of the slice is equal to `len`.
     Len { len: u64, op: BinOp },
@@ -1330,7 +1330,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         // All of the or-patterns have been sorted to the end, so if the first
         // pattern is an or-pattern we only have or-patterns.
-        match *first_candidate.match_pairs[0].pattern.kind {
+        match first_candidate.match_pairs[0].pattern.kind {
             PatKind::Or { .. } => (),
             _ => {
                 self.test_candidates(
@@ -1350,7 +1350,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         let mut otherwise = None;
         for match_pair in match_pairs {
-            let PatKind::Or { ref pats } = &*match_pair.pattern.kind else {
+            let PatKind::Or { ref pats } = &match_pair.pattern.kind else {
                 bug!("Or-patterns should have been sorted to the end");
             };
             let or_span = match_pair.pattern.span;
@@ -1384,7 +1384,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         candidate: &mut Candidate<'pat, 'tcx>,
         otherwise: &mut Option<BasicBlock>,
-        pats: &'pat [Pat<'tcx>],
+        pats: &'pat [Box<Pat<'tcx>>],
         or_span: Span,
         place: PlaceBuilder<'tcx>,
         fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
@@ -1605,9 +1605,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         // Insert a Shallow borrow of any places that is switched on.
         if let Some(fb) = fake_borrows && let Ok(match_place_resolved) =
-            match_place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+            match_place.clone().try_upvars_resolved(self.tcx, &self.upvars)
         {
-            let resolved_place = match_place_resolved.into_place(self.tcx, self.typeck_results);
+            let resolved_place = match_place_resolved.into_place(self.tcx, &self.upvars);
             fb.insert(resolved_place);
         }
 
@@ -1794,10 +1794,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         );
         let mut opt_expr_place: Option<(Option<&Place<'tcx>>, Span)> = None;
         let expr_place: Place<'tcx>;
-        if let Ok(expr_builder) =
-            expr_place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
-        {
-            expr_place = expr_builder.into_place(self.tcx, self.typeck_results);
+        if let Ok(expr_builder) = expr_place_builder.try_upvars_resolved(self.tcx, &self.upvars) {
+            expr_place = expr_builder.into_place(self.tcx, &self.upvars);
             opt_expr_place = Some((Some(&expr_place), expr_span));
         }
         let otherwise_post_guard_block = otherwise_candidate.pre_binding_block.unwrap();
@@ -1820,6 +1818,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             None,
             None,
             None,
+            false,
         );
 
         post_guard_block.unit()
@@ -1843,6 +1842,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         arm_span: Option<Span>,
         match_scope: Option<region::Scope>,
         schedule_drops: bool,
+        storages_alive: bool,
     ) -> BasicBlock {
         debug!("bind_and_guard_matched_candidate(candidate={:?})", candidate);
 
@@ -1978,7 +1978,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             let mut guard_span = rustc_span::DUMMY_SP;
 
             let (post_guard_block, otherwise_post_guard_block) =
-                self.in_if_then_scope(match_scope, |this| match *guard {
+                self.in_if_then_scope(match_scope, guard_span, |this| match *guard {
                     Guard::If(e) => {
                         let e = &this.thir[e];
                         guard_span = e.span;
@@ -2058,7 +2058,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 self.cfg.push_fake_read(post_guard_block, guard_end, cause, Place::from(local_id));
             }
             assert!(schedule_drops, "patterns with guards must schedule drops");
-            self.bind_matched_candidate_for_arm_body(post_guard_block, true, by_value_bindings);
+            self.bind_matched_candidate_for_arm_body(
+                post_guard_block,
+                true,
+                by_value_bindings,
+                storages_alive,
+            );
 
             post_guard_block
         } else {
@@ -2072,6 +2077,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     .iter()
                     .flat_map(|(bindings, _)| bindings)
                     .chain(&candidate.bindings),
+                storages_alive,
             );
             block
         }
@@ -2161,6 +2167,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         block: BasicBlock,
         schedule_drops: bool,
         bindings: impl IntoIterator<Item = &'b Binding<'tcx>>,
+        storages_alive: bool,
     ) where
         'tcx: 'b,
     {
@@ -2170,13 +2177,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // Assign each of the bindings. This may trigger moves out of the candidate.
         for binding in bindings {
             let source_info = self.source_info(binding.span);
-            let local = self.storage_live_binding(
-                block,
-                binding.var_id,
-                binding.span,
-                OutsideGuard,
-                schedule_drops,
-            );
+            let local = if storages_alive {
+                // Here storages are already alive, probably because this is a binding
+                // from let-else.
+                // We just need to schedule drop for the value.
+                self.var_local_id(binding.var_id, OutsideGuard).into()
+            } else {
+                self.storage_live_binding(
+                    block,
+                    binding.var_id,
+                    binding.span,
+                    OutsideGuard,
+                    schedule_drops,
+                )
+            };
             if schedule_drops {
                 self.schedule_drop_for_binding(binding.var_id, binding.span, OutsideGuard);
             }
@@ -2280,23 +2294,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         mut block: BasicBlock,
         init: &Expr<'tcx>,
         initializer_span: Span,
-        else_block: &Block,
-        visibility_scope: Option<SourceScope>,
-        remainder_scope: region::Scope,
-        remainder_span: Span,
+        else_block: BlockId,
+        let_else_scope: &region::Scope,
         pattern: &Pat<'tcx>,
-    ) -> BlockAnd<()> {
-        let (matching, failure) = self.in_if_then_scope(remainder_scope, |this| {
+    ) -> BlockAnd<BasicBlock> {
+        let else_block_span = self.thir[else_block].span;
+        let (matching, failure) = self.in_if_then_scope(*let_else_scope, else_block_span, |this| {
             let scrutinee = unpack!(block = this.lower_scrutinee(block, init, initializer_span));
-            let pat = Pat { ty: init.ty, span: else_block.span, kind: Box::new(PatKind::Wild) };
+            let pat = Pat { ty: init.ty, span: else_block_span, kind: PatKind::Wild };
             let mut wildcard = Candidate::new(scrutinee.clone(), &pat, false);
-            this.declare_bindings(
-                visibility_scope,
-                remainder_span,
-                pattern,
-                ArmHasGuard(false),
-                Some((None, initializer_span)),
-            );
             let mut candidate = Candidate::new(scrutinee.clone(), pattern, false);
             let fake_borrow_temps = this.lower_match_tree(
                 block,
@@ -2315,10 +2321,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 None,
                 None,
                 None,
+                true,
             );
             // This block is for the failure case
             let failure = this.bind_pattern(
-                this.source_info(else_block.span),
+                this.source_info(else_block_span),
                 wildcard,
                 None,
                 &fake_borrow_temps,
@@ -2326,29 +2333,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 None,
                 None,
                 None,
+                true,
             );
-            this.break_for_else(failure, remainder_scope, this.source_info(initializer_span));
+            this.break_for_else(failure, *let_else_scope, this.source_info(initializer_span));
             matching.unit()
         });
-
-        // This place is not really used because this destination place
-        // should never be used to take values at the end of the failure
-        // block.
-        let dummy_place = Place { local: RETURN_PLACE, projection: ty::List::empty() };
-        let failure_block;
-        unpack!(
-            failure_block = self.ast_block(
-                dummy_place,
-                failure,
-                else_block,
-                self.source_info(else_block.span),
-            )
-        );
-        self.cfg.terminate(
-            failure_block,
-            self.source_info(else_block.span),
-            TerminatorKind::Unreachable,
-        );
-        matching.unit()
+        matching.and(failure)
     }
 }
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index c62989041..df221d356 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -67,7 +67,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         loop {
             let match_pairs = mem::take(&mut candidate.match_pairs);
 
-            if let [MatchPair { pattern: Pat { kind: box PatKind::Or { pats }, .. }, place }] =
+            if let [MatchPair { pattern: Pat { kind: PatKind::Or { pats }, .. }, place }] =
                 &*match_pairs
             {
                 existing_bindings.extend_from_slice(&new_bindings);
@@ -113,7 +113,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // late as possible.
                 candidate
                     .match_pairs
-                    .sort_by_key(|pair| matches!(*pair.pattern.kind, PatKind::Or { .. }));
+                    .sort_by_key(|pair| matches!(pair.pattern.kind, PatKind::Or { .. }));
                 debug!(simplified = ?candidate, "simplify_candidate");
                 return false; // if we were not able to simplify any, done.
             }
@@ -127,10 +127,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         candidate: &Candidate<'pat, 'tcx>,
         place: PlaceBuilder<'tcx>,
-        pats: &'pat [Pat<'tcx>],
+        pats: &'pat [Box<Pat<'tcx>>],
     ) -> Vec<Candidate<'pat, 'tcx>> {
         pats.iter()
-            .map(|pat| {
+            .map(|box pat| {
                 let mut candidate = Candidate::new(place.clone(), pat, candidate.has_guard);
                 self.simplify_candidate(&mut candidate);
                 candidate
@@ -149,18 +149,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         candidate: &mut Candidate<'pat, 'tcx>,
     ) -> Result<(), MatchPair<'pat, 'tcx>> {
         let tcx = self.tcx;
-        match *match_pair.pattern.kind {
+        match match_pair.pattern.kind {
             PatKind::AscribeUserType {
                 ref subpattern,
                 ascription: thir::Ascription { ref annotation, variance },
             } => {
                 // Apply the type ascription to the value at `match_pair.place`, which is the
                 if let Ok(place_resolved) =
-                    match_pair.place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                    match_pair.place.clone().try_upvars_resolved(self.tcx, &self.upvars)
                 {
                     candidate.ascriptions.push(Ascription {
                         annotation: annotation.clone(),
-                        source: place_resolved.into_place(self.tcx, self.typeck_results),
+                        source: place_resolved.into_place(self.tcx, &self.upvars),
                         variance,
                     });
                 }
@@ -185,11 +185,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 is_primary: _,
             } => {
                 if let Ok(place_resolved) =
-                    match_pair.place.clone().try_upvars_resolved(self.tcx, self.typeck_results)
+                    match_pair.place.clone().try_upvars_resolved(self.tcx, &self.upvars)
                 {
                     candidate.bindings.push(Binding {
                         span: match_pair.pattern.span,
-                        source: place_resolved.into_place(self.tcx, self.typeck_results),
+                        source: place_resolved.into_place(self.tcx, &self.upvars),
                         var_id: var,
                         binding_mode: mode,
                     });
@@ -208,7 +208,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Err(match_pair)
             }
 
-            PatKind::Range(PatRange { lo, hi, end }) => {
+            PatKind::Range(box PatRange { lo, hi, end }) => {
                 let (range, bias) = match *lo.ty().kind() {
                     ty::Char => {
                         (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
@@ -254,7 +254,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         &mut candidate.match_pairs,
                         &match_pair.place,
                         prefix,
-                        slice.as_ref(),
+                        slice,
                         suffix,
                     );
                     Ok(())
@@ -294,7 +294,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     &mut candidate.match_pairs,
                     &match_pair.place,
                     prefix,
-                    slice.as_ref(),
+                    slice,
                     suffix,
                 );
                 Ok(())
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 598da80c5..47d05a6e3 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -29,7 +29,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// It is a bug to call this with a not-fully-simplified pattern.
     pub(super) fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
-        match *match_pair.pattern.kind {
+        match match_pair.pattern.kind {
             PatKind::Variant { adt_def, substs: _, variant_index: _, subpatterns: _ } => Test {
                 span: match_pair.pattern.span,
                 kind: TestKind::Switch {
@@ -58,10 +58,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 kind: TestKind::Eq { value, ty: match_pair.pattern.ty },
             },
 
-            PatKind::Range(range) => {
+            PatKind::Range(ref range) => {
                 assert_eq!(range.lo.ty(), match_pair.pattern.ty);
                 assert_eq!(range.hi.ty(), match_pair.pattern.ty);
-                Test { span: match_pair.pattern.span, kind: TestKind::Range(range) }
+                Test { span: match_pair.pattern.span, kind: TestKind::Range(range.clone()) }
             }
 
             PatKind::Slice { ref prefix, ref slice, ref suffix } => {
@@ -92,7 +92,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             return false;
         };
 
-        match *match_pair.pattern.kind {
+        match match_pair.pattern.kind {
             PatKind::Constant { value } => {
                 options
                     .entry(value)
@@ -102,9 +102,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             PatKind::Variant { .. } => {
                 panic!("you should have called add_variants_to_switch instead!");
             }
-            PatKind::Range(range) => {
+            PatKind::Range(ref range) => {
                 // Check that none of the switch values are in the range.
-                self.values_not_contained_in_range(range, options).unwrap_or(false)
+                self.values_not_contained_in_range(&*range, options).unwrap_or(false)
             }
             PatKind::Slice { .. }
             | PatKind::Array { .. }
@@ -130,7 +130,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             return false;
         };
 
-        match *match_pair.pattern.kind {
+        match match_pair.pattern.kind {
             PatKind::Variant { adt_def: _, variant_index, .. } => {
                 // We have a pattern testing for variant `variant_index`
                 // set the corresponding index to true
@@ -154,10 +154,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
     ) {
         let place: Place<'tcx>;
-        if let Ok(test_place_builder) =
-            place_builder.try_upvars_resolved(self.tcx, self.typeck_results)
-        {
-            place = test_place_builder.into_place(self.tcx, self.typeck_results);
+        if let Ok(test_place_builder) = place_builder.try_upvars_resolved(self.tcx, &self.upvars) {
+            place = test_place_builder.into_place(self.tcx, &self.upvars);
         } else {
             return;
         }
@@ -272,7 +270,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
             }
 
-            TestKind::Range(PatRange { lo, hi, ref end }) => {
+            TestKind::Range(box PatRange { lo, hi, ref end }) => {
                 let lower_bound_success = self.cfg.start_new_block();
                 let target_blocks = make_target_blocks(self);
 
@@ -506,7 +504,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let (match_pair_index, match_pair) =
             candidate.match_pairs.iter().enumerate().find(|&(_, mp)| mp.place == *test_place)?;
 
-        match (&test.kind, &*match_pair.pattern.kind) {
+        match (&test.kind, &match_pair.pattern.kind) {
             // If we are performing a variant switch, then this
             // informs variant patterns, but nothing else.
             (
@@ -540,9 +538,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Some(index)
             }
 
-            (&TestKind::SwitchInt { switch_ty: _, ref options }, &PatKind::Range(range)) => {
+            (&TestKind::SwitchInt { switch_ty: _, ref options }, &PatKind::Range(ref range)) => {
                 let not_contained =
-                    self.values_not_contained_in_range(range, options).unwrap_or(false);
+                    self.values_not_contained_in_range(&*range, options).unwrap_or(false);
 
                 if not_contained {
                     // No switch values are contained in the pattern range,
@@ -569,7 +567,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             match_pair_index,
                             candidate,
                             prefix,
-                            slice.as_ref(),
+                            slice,
                             suffix,
                         );
                         Some(0)
@@ -607,7 +605,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             match_pair_index,
                             candidate,
                             prefix,
-                            slice.as_ref(),
+                            slice,
                             suffix,
                         );
                         Some(0)
@@ -631,7 +629,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
             }
 
-            (&TestKind::Range(test), &PatKind::Range(pat)) => {
+            (&TestKind::Range(ref test), &PatKind::Range(ref pat)) => {
                 use std::cmp::Ordering::*;
 
                 if test == pat {
@@ -658,8 +656,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 no_overlap
             }
 
-            (&TestKind::Range(range), &PatKind::Constant { value }) => {
-                if let Some(false) = self.const_range_contains(range, value) {
+            (&TestKind::Range(ref range), &PatKind::Constant { value }) => {
+                if let Some(false) = self.const_range_contains(&*range, value) {
                     // `value` is not contained in the testing range,
                     // so `value` can be matched only if this test fails.
                     Some(1)
@@ -678,7 +676,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // However, at this point we can still encounter or-patterns that were extracted
                 // from previous calls to `sort_candidate`, so we need to manually address that
                 // case to avoid panicking in `self.test()`.
-                if let PatKind::Or { .. } = &*match_pair.pattern.kind {
+                if let PatKind::Or { .. } = &match_pair.pattern.kind {
                     return None;
                 }
 
@@ -708,9 +706,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         match_pair_index: usize,
         candidate: &mut Candidate<'pat, 'tcx>,
-        prefix: &'pat [Pat<'tcx>],
-        opt_slice: Option<&'pat Pat<'tcx>>,
-        suffix: &'pat [Pat<'tcx>],
+        prefix: &'pat [Box<Pat<'tcx>>],
+        opt_slice: &'pat Option<Box<Pat<'tcx>>>,
+        suffix: &'pat [Box<Pat<'tcx>>],
     ) {
         let removed_place = candidate.match_pairs.remove(match_pair_index).place;
         self.prefix_slice_suffix(
@@ -754,7 +752,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     fn const_range_contains(
         &self,
-        range: PatRange<'tcx>,
+        range: &PatRange<'tcx>,
         value: ConstantKind<'tcx>,
     ) -> Option<bool> {
         use std::cmp::Ordering::*;
@@ -772,7 +770,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     fn values_not_contained_in_range(
         &self,
-        range: PatRange<'tcx>,
+        range: &PatRange<'tcx>,
         options: &FxIndexMap<ConstantKind<'tcx>, u128>,
     ) -> Option<bool> {
         for &val in options.keys() {
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
index 9a1e98d3b..b61c4fe50 100644
--- a/compiler/rustc_mir_build/src/build/matches/util.rs
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -26,19 +26,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         match_pairs: &mut SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
         place: &PlaceBuilder<'tcx>,
-        prefix: &'pat [Pat<'tcx>],
-        opt_slice: Option<&'pat Pat<'tcx>>,
-        suffix: &'pat [Pat<'tcx>],
+        prefix: &'pat [Box<Pat<'tcx>>],
+        opt_slice: &'pat Option<Box<Pat<'tcx>>>,
+        suffix: &'pat [Box<Pat<'tcx>>],
     ) {
         let tcx = self.tcx;
         let (min_length, exact_size) = if let Ok(place_resolved) =
-            place.clone().try_upvars_resolved(tcx, self.typeck_results)
+            place.clone().try_upvars_resolved(tcx, &self.upvars)
         {
-            match place_resolved
-                .into_place(tcx, self.typeck_results)
-                .ty(&self.local_decls, tcx)
-                .ty
-                .kind()
+            match place_resolved.into_place(tcx, &self.upvars).ty(&self.local_decls, tcx).ty.kind()
             {
                 ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
                 _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 12b8ceede..25c4e51cb 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -1,15 +1,14 @@
-use crate::build;
 pub(crate) use crate::build::expr::as_constant::lit_to_mir_constant;
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::scope::DropKind;
-use crate::thir::pattern::pat_from_hir;
 use rustc_apfloat::ieee::{Double, Single};
 use rustc_apfloat::Float;
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sorted_map::SortedIndexMultiMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::lang_items::LangItem;
 use rustc_hir::{GeneratorKind, Node};
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
@@ -18,8 +17,9 @@ use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::ConstValue;
 use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::*;
-use rustc_middle::thir::{BindingMode, Expr, ExprId, LintLevel, LocalVarId, PatKind, Thir};
-use rustc_middle::ty::subst::Subst;
+use rustc_middle::thir::{
+    self, BindingMode, Expr, ExprId, LintLevel, LocalVarId, Param, ParamId, PatKind, Thir,
+};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable, TypeckResults};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
@@ -47,9 +47,7 @@ pub(crate) fn mir_built<'tcx>(
 
 /// Construct the MIR for a given `DefId`.
 fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> {
-    let id = tcx.hir().local_def_id_to_hir_id(def.did);
     let body_owner_kind = tcx.hir().body_owner_kind(def.did);
-    let typeck_results = tcx.typeck_opt_const_arg(def);
 
     // Ensure unsafeck and abstract const building is ran before we steal the THIR.
     // We can't use `ensure()` for `thir_abstract_const` as it doesn't compute the query
@@ -66,235 +64,42 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
         }
     }
 
-    // Figure out what primary body this item has.
-    let (body_id, return_ty_span, span_with_body) = match tcx.hir().get(id) {
-        Node::Expr(hir::Expr {
-            kind: hir::ExprKind::Closure(hir::Closure { fn_decl, body, .. }),
-            ..
-        }) => (*body, fn_decl.output.span(), None),
-        Node::Item(hir::Item {
-            kind: hir::ItemKind::Fn(hir::FnSig { decl, .. }, _, body_id),
-            span,
-            ..
-        })
-        | Node::ImplItem(hir::ImplItem {
-            kind: hir::ImplItemKind::Fn(hir::FnSig { decl, .. }, body_id),
-            span,
-            ..
-        })
-        | Node::TraitItem(hir::TraitItem {
-            kind: hir::TraitItemKind::Fn(hir::FnSig { decl, .. }, hir::TraitFn::Provided(body_id)),
-            span,
-            ..
-        }) => {
-            // Use the `Span` of the `Item/ImplItem/TraitItem` as the body span,
-            // since the def span of a function does not include the body
-            (*body_id, decl.output.span(), Some(*span))
-        }
-        Node::Item(hir::Item {
-            kind: hir::ItemKind::Static(ty, _, body_id) | hir::ItemKind::Const(ty, body_id),
-            ..
-        })
-        | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(ty, body_id), .. })
-        | Node::TraitItem(hir::TraitItem {
-            kind: hir::TraitItemKind::Const(ty, Some(body_id)),
-            ..
-        }) => (*body_id, ty.span, None),
-        Node::AnonConst(hir::AnonConst { body, hir_id, .. }) => {
-            (*body, tcx.hir().span(*hir_id), None)
-        }
-
-        _ => span_bug!(tcx.hir().span(id), "can't build MIR for {:?}", def.did),
-    };
-
-    // If we don't have a specialized span for the body, just use the
-    // normal def span.
-    let span_with_body = span_with_body.unwrap_or_else(|| tcx.hir().span(id));
-
-    tcx.infer_ctxt().enter(|infcx| {
-        let body = if let Some(error_reported) = typeck_results.tainted_by_errors {
-            build::construct_error(&infcx, def, id, body_id, body_owner_kind, error_reported)
-        } else if body_owner_kind.is_fn_or_closure() {
-            // fetch the fully liberated fn signature (that is, all bound
-            // types/lifetimes replaced)
-            let fn_sig = typeck_results.liberated_fn_sigs()[id];
-            let fn_def_id = tcx.hir().local_def_id(id);
-
-            let safety = match fn_sig.unsafety {
-                hir::Unsafety::Normal => Safety::Safe,
-                hir::Unsafety::Unsafe => Safety::FnUnsafe,
-            };
-
-            let body = tcx.hir().body(body_id);
-            let (thir, expr) = tcx
-                .thir_body(def)
-                .unwrap_or_else(|_| (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0)));
+    let body = match tcx.thir_body(def) {
+        Err(error_reported) => construct_error(tcx, def.did, body_owner_kind, error_reported),
+        Ok((thir, expr)) => {
             // We ran all queries that depended on THIR at the beginning
             // of `mir_build`, so now we can steal it
             let thir = thir.steal();
-            let ty = tcx.type_of(fn_def_id);
-            let mut abi = fn_sig.abi;
-            let implicit_argument = match ty.kind() {
-                ty::Closure(..) => {
-                    // HACK(eddyb) Avoid having RustCall on closures,
-                    // as it adds unnecessary (and wrong) auto-tupling.
-                    abi = Abi::Rust;
-                    vec![ArgInfo(liberated_closure_env_ty(tcx, id, body_id), None, None, None)]
-                }
-                ty::Generator(..) => {
-                    let gen_ty = tcx.typeck_body(body_id).node_type(id);
-
-                    // The resume argument may be missing, in that case we need to provide it here.
-                    // It will always be `()` in this case.
-                    if body.params.is_empty() {
-                        vec![
-                            ArgInfo(gen_ty, None, None, None),
-                            ArgInfo(tcx.mk_unit(), None, None, None),
-                        ]
-                    } else {
-                        vec![ArgInfo(gen_ty, None, None, None)]
-                    }
-                }
-                _ => vec![],
-            };
-
-            let explicit_arguments = body.params.iter().enumerate().map(|(index, arg)| {
-                let owner_id = tcx.hir().body_owner(body_id);
-                let opt_ty_info;
-                let self_arg;
-                if let Some(ref fn_decl) = tcx.hir().fn_decl_by_hir_id(owner_id) {
-                    opt_ty_info = fn_decl
-                        .inputs
-                        .get(index)
-                        // Make sure that inferred closure args have no type span
-                        .and_then(|ty| if arg.pat.span != ty.span { Some(ty.span) } else { None });
-                    self_arg = if index == 0 && fn_decl.implicit_self.has_implicit_self() {
-                        match fn_decl.implicit_self {
-                            hir::ImplicitSelfKind::Imm => Some(ImplicitSelfKind::Imm),
-                            hir::ImplicitSelfKind::Mut => Some(ImplicitSelfKind::Mut),
-                            hir::ImplicitSelfKind::ImmRef => Some(ImplicitSelfKind::ImmRef),
-                            hir::ImplicitSelfKind::MutRef => Some(ImplicitSelfKind::MutRef),
-                            _ => None,
-                        }
-                    } else {
-                        None
-                    };
-                } else {
-                    opt_ty_info = None;
-                    self_arg = None;
-                }
-
-                // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
-                // (as it's created inside the body itself, not passed in from outside).
-                let ty = if fn_sig.c_variadic && index == fn_sig.inputs().len() {
-                    let va_list_did = tcx.require_lang_item(LangItem::VaList, Some(arg.span));
-
-                    tcx.bound_type_of(va_list_did).subst(tcx, &[tcx.lifetimes.re_erased.into()])
-                } else {
-                    fn_sig.inputs()[index]
-                };
-
-                ArgInfo(ty, opt_ty_info, Some(&arg), self_arg)
-            });
 
-            let arguments = implicit_argument.into_iter().chain(explicit_arguments);
-
-            let (yield_ty, return_ty) = if body.generator_kind.is_some() {
-                let gen_ty = tcx.typeck_body(body_id).node_type(id);
-                let gen_sig = match gen_ty.kind() {
-                    ty::Generator(_, gen_substs, ..) => gen_substs.as_generator().sig(),
-                    _ => span_bug!(tcx.hir().span(id), "generator w/o generator type: {:?}", ty),
-                };
-                (Some(gen_sig.yield_ty), gen_sig.return_ty)
+            if body_owner_kind.is_fn_or_closure() {
+                construct_fn(tcx, def, &thir, expr)
             } else {
-                (None, fn_sig.output())
-            };
-
-            let mut mir = build::construct_fn(
-                &thir,
-                &infcx,
-                def,
-                id,
-                arguments,
-                safety,
-                abi,
-                return_ty,
-                return_ty_span,
-                body,
-                expr,
-                span_with_body,
-            );
-            if yield_ty.is_some() {
-                mir.generator.as_mut().unwrap().yield_ty = yield_ty;
+                construct_const(tcx, def, &thir, expr)
             }
-            mir
-        } else {
-            // Get the revealed type of this const. This is *not* the adjusted
-            // type of its body, which may be a subtype of this type. For
-            // example:
-            //
-            // fn foo(_: &()) {}
-            // static X: fn(&'static ()) = foo;
-            //
-            // The adjusted type of the body of X is `for<'a> fn(&'a ())` which
-            // is not the same as the type of X. We need the type of the return
-            // place to be the type of the constant because NLL typeck will
-            // equate them.
-
-            let return_ty = typeck_results.node_type(id);
-
-            let (thir, expr) = tcx
-                .thir_body(def)
-                .unwrap_or_else(|_| (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0)));
-            // We ran all queries that depended on THIR at the beginning
-            // of `mir_build`, so now we can steal it
-            let thir = thir.steal();
-
-            build::construct_const(&thir, &infcx, expr, def, id, return_ty, return_ty_span)
-        };
+        }
+    };
 
-        lints::check(tcx, &body);
-
-        // The borrow checker will replace all the regions here with its own
-        // inference variables. There's no point having non-erased regions here.
-        // The exception is `body.user_type_annotations`, which is used unmodified
-        // by borrow checking.
-        debug_assert!(
-            !(body.local_decls.has_free_regions()
-                || body.basic_blocks().has_free_regions()
-                || body.var_debug_info.has_free_regions()
-                || body.yield_ty().has_free_regions()),
-            "Unexpected free regions in MIR: {:?}",
-            body,
-        );
+    lints::check(tcx, &body);
+
+    // The borrow checker will replace all the regions here with its own
+    // inference variables. There's no point having non-erased regions here.
+    // The exception is `body.user_type_annotations`, which is used unmodified
+    // by borrow checking.
+    debug_assert!(
+        !(body.local_decls.has_free_regions()
+            || body.basic_blocks.has_free_regions()
+            || body.var_debug_info.has_free_regions()
+            || body.yield_ty().has_free_regions()),
+        "Unexpected free regions in MIR: {:?}",
+        body,
+    );
 
-        body
-    })
+    body
 }
 
 ///////////////////////////////////////////////////////////////////////////
 // BuildMir -- walks a crate, looking for fn items and methods to build MIR from
 
-fn liberated_closure_env_ty(
-    tcx: TyCtxt<'_>,
-    closure_expr_id: hir::HirId,
-    body_id: hir::BodyId,
-) -> Ty<'_> {
-    let closure_ty = tcx.typeck_body(body_id).node_type(closure_expr_id);
-
-    let ty::Closure(closure_def_id, closure_substs) = *closure_ty.kind() else {
-        bug!("closure expr does not have closure type: {:?}", closure_ty);
-    };
-
-    let bound_vars =
-        tcx.mk_bound_variable_kinds(std::iter::once(ty::BoundVariableKind::Region(ty::BrEnv)));
-    let br =
-        ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BrEnv };
-    let env_region = ty::ReLateBound(ty::INNERMOST, br);
-    let closure_env_ty = tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap();
-    tcx.erase_late_bound_regions(ty::Binder::bind_with_vars(closure_env_ty, bound_vars))
-}
-
 #[derive(Debug, PartialEq, Eq)]
 enum BlockFrame {
     /// Evaluation is currently within a statement.
@@ -352,7 +157,7 @@ struct BlockContext(Vec<BlockFrame>);
 
 struct Builder<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    infcx: &'a InferCtxt<'a, 'tcx>,
+    infcx: InferCtxt<'a, 'tcx>,
     typeck_results: &'tcx TypeckResults<'tcx>,
     region_scope_tree: &'tcx region::ScopeTree,
     param_env: ty::ParamEnv<'tcx>,
@@ -404,12 +209,21 @@ struct Builder<'a, 'tcx> {
     var_indices: FxHashMap<LocalVarId, LocalsForNode>,
     local_decls: IndexVec<Local, LocalDecl<'tcx>>,
     canonical_user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>,
-    upvar_mutbls: Vec<Mutability>,
+    upvars: CaptureMap<'tcx>,
     unit_temp: Option<Place<'tcx>>,
 
     var_debug_info: Vec<VarDebugInfo<'tcx>>,
 }
 
+type CaptureMap<'tcx> = SortedIndexMultiMap<usize, hir::HirId, Capture<'tcx>>;
+
+#[derive(Debug)]
+struct Capture<'tcx> {
+    captured_place: &'tcx ty::CapturedPlace<'tcx>,
+    use_place: Place<'tcx>,
+    mutability: Mutability,
+}
+
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn is_bound_var_in_guard(&self, id: LocalVarId) -> bool {
         self.guard_context.iter().any(|frame| frame.locals.iter().any(|local| local.id == id))
@@ -615,149 +429,212 @@ macro_rules! unpack {
 ///////////////////////////////////////////////////////////////////////////
 /// the main entry point for building MIR for a function
 
-struct ArgInfo<'tcx>(
-    Ty<'tcx>,
-    Option<Span>,
-    Option<&'tcx hir::Param<'tcx>>,
-    Option<ImplicitSelfKind>,
-);
-
-fn construct_fn<'tcx, A>(
-    thir: &Thir<'tcx>,
-    infcx: &InferCtxt<'_, 'tcx>,
+fn construct_fn<'tcx>(
+    tcx: TyCtxt<'tcx>,
     fn_def: ty::WithOptConstParam<LocalDefId>,
-    fn_id: hir::HirId,
-    arguments: A,
-    safety: Safety,
-    abi: Abi,
-    return_ty: Ty<'tcx>,
-    return_ty_span: Span,
-    body: &'tcx hir::Body<'tcx>,
+    thir: &Thir<'tcx>,
     expr: ExprId,
-    span_with_body: Span,
-) -> Body<'tcx>
-where
-    A: Iterator<Item = ArgInfo<'tcx>>,
-{
-    let arguments: Vec<_> = arguments.collect();
-
-    let tcx = infcx.tcx;
-    let span = tcx.hir().span(fn_id);
-
-    let mut builder = Builder::new(
-        thir,
-        infcx,
-        fn_def,
-        fn_id,
-        span_with_body,
-        arguments.len(),
-        safety,
-        return_ty,
-        return_ty_span,
-        body.generator_kind,
-    );
+) -> Body<'tcx> {
+    let span = tcx.def_span(fn_def.did);
+    let fn_id = tcx.hir().local_def_id_to_hir_id(fn_def.did);
+    let generator_kind = tcx.generator_kind(fn_def.did);
 
-    let call_site_scope =
-        region::Scope { id: body.value.hir_id.local_id, data: region::ScopeData::CallSite };
-    let arg_scope =
-        region::Scope { id: body.value.hir_id.local_id, data: region::ScopeData::Arguments };
-    let source_info = builder.source_info(span);
-    let call_site_s = (call_site_scope, source_info);
-    unpack!(builder.in_scope(call_site_s, LintLevel::Inherited, |builder| {
-        let arg_scope_s = (arg_scope, source_info);
-        // Attribute epilogue to function's closing brace
-        let fn_end = span_with_body.shrink_to_hi();
-        let return_block =
-            unpack!(builder.in_breakable_scope(None, Place::return_place(), fn_end, |builder| {
-                Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
-                    builder.args_and_body(
-                        START_BLOCK,
-                        fn_def.did,
-                        &arguments,
-                        arg_scope,
-                        &thir[expr],
-                    )
-                }))
-            }));
-        let source_info = builder.source_info(fn_end);
-        builder.cfg.terminate(return_block, source_info, TerminatorKind::Return);
-        builder.build_drop_trees();
-        return_block.unit()
-    }));
+    // Figure out what primary body this item has.
+    let body_id = tcx.hir().body_owned_by(fn_def.did);
+    let span_with_body = tcx.hir().span_with_body(fn_id);
+    let return_ty_span = tcx
+        .hir()
+        .fn_decl_by_hir_id(fn_id)
+        .unwrap_or_else(|| span_bug!(span, "can't build MIR for {:?}", fn_def.did))
+        .output
+        .span();
+
+    // fetch the fully liberated fn signature (that is, all bound
+    // types/lifetimes replaced)
+    let typeck_results = tcx.typeck_opt_const_arg(fn_def);
+    let fn_sig = typeck_results.liberated_fn_sigs()[fn_id];
+
+    let safety = match fn_sig.unsafety {
+        hir::Unsafety::Normal => Safety::Safe,
+        hir::Unsafety::Unsafe => Safety::FnUnsafe,
+    };
+
+    let mut abi = fn_sig.abi;
+    if let DefKind::Closure = tcx.def_kind(fn_def.did) {
+        // HACK(eddyb) Avoid having RustCall on closures,
+        // as it adds unnecessary (and wrong) auto-tupling.
+        abi = Abi::Rust;
+    }
+
+    let arguments = &thir.params;
+
+    let (yield_ty, return_ty) = if generator_kind.is_some() {
+        let gen_ty = arguments[thir::UPVAR_ENV_PARAM].ty;
+        let gen_sig = match gen_ty.kind() {
+            ty::Generator(_, gen_substs, ..) => gen_substs.as_generator().sig(),
+            _ => {
+                span_bug!(span, "generator w/o generator type: {:?}", gen_ty)
+            }
+        };
+        (Some(gen_sig.yield_ty), gen_sig.return_ty)
+    } else {
+        (None, fn_sig.output())
+    };
+
+    let mut body = tcx.infer_ctxt().enter(|infcx| {
+        let mut builder = Builder::new(
+            thir,
+            infcx,
+            fn_def,
+            fn_id,
+            span_with_body,
+            arguments.len(),
+            safety,
+            return_ty,
+            return_ty_span,
+            generator_kind,
+        );
+
+        let call_site_scope =
+            region::Scope { id: body_id.hir_id.local_id, data: region::ScopeData::CallSite };
+        let arg_scope =
+            region::Scope { id: body_id.hir_id.local_id, data: region::ScopeData::Arguments };
+        let source_info = builder.source_info(span);
+        let call_site_s = (call_site_scope, source_info);
+        unpack!(builder.in_scope(call_site_s, LintLevel::Inherited, |builder| {
+            let arg_scope_s = (arg_scope, source_info);
+            // Attribute epilogue to function's closing brace
+            let fn_end = span_with_body.shrink_to_hi();
+            let return_block = unpack!(builder.in_breakable_scope(
+                None,
+                Place::return_place(),
+                fn_end,
+                |builder| {
+                    Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
+                        builder.args_and_body(
+                            START_BLOCK,
+                            fn_def.did,
+                            arguments,
+                            arg_scope,
+                            &thir[expr],
+                        )
+                    }))
+                }
+            ));
+            let source_info = builder.source_info(fn_end);
+            builder.cfg.terminate(return_block, source_info, TerminatorKind::Return);
+            builder.build_drop_trees();
+            return_block.unit()
+        }));
+
+        builder.finish()
+    });
 
-    let spread_arg = if abi == Abi::RustCall {
+    body.spread_arg = if abi == Abi::RustCall {
         // RustCall pseudo-ABI untuples the last argument.
         Some(Local::new(arguments.len()))
     } else {
         None
     };
-
-    let mut body = builder.finish();
-    body.spread_arg = spread_arg;
+    if yield_ty.is_some() {
+        body.generator.as_mut().unwrap().yield_ty = yield_ty;
+    }
     body
 }
 
 fn construct_const<'a, 'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def: ty::WithOptConstParam<LocalDefId>,
     thir: &'a Thir<'tcx>,
-    infcx: &'a InferCtxt<'a, 'tcx>,
     expr: ExprId,
-    def: ty::WithOptConstParam<LocalDefId>,
-    hir_id: hir::HirId,
-    const_ty: Ty<'tcx>,
-    const_ty_span: Span,
 ) -> Body<'tcx> {
-    let tcx = infcx.tcx;
-    let span = tcx.hir().span(hir_id);
-    let mut builder = Builder::new(
-        thir,
-        infcx,
-        def,
-        hir_id,
-        span,
-        0,
-        Safety::Safe,
-        const_ty,
-        const_ty_span,
-        None,
-    );
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
+
+    // Figure out what primary body this item has.
+    let (span, const_ty_span) = match tcx.hir().get(hir_id) {
+        Node::Item(hir::Item {
+            kind: hir::ItemKind::Static(ty, _, _) | hir::ItemKind::Const(ty, _),
+            span,
+            ..
+        })
+        | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(ty, _), span, .. })
+        | Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Const(ty, Some(_)),
+            span,
+            ..
+        }) => (*span, ty.span),
+        Node::AnonConst(_) => {
+            let span = tcx.def_span(def.did);
+            (span, span)
+        }
+        _ => span_bug!(tcx.def_span(def.did), "can't build MIR for {:?}", def.did),
+    };
+
+    // Get the revealed type of this const. This is *not* the adjusted
+    // type of its body, which may be a subtype of this type. For
+    // example:
+    //
+    // fn foo(_: &()) {}
+    // static X: fn(&'static ()) = foo;
+    //
+    // The adjusted type of the body of X is `for<'a> fn(&'a ())` which
+    // is not the same as the type of X. We need the type of the return
+    // place to be the type of the constant because NLL typeck will
+    // equate them.
+    let typeck_results = tcx.typeck_opt_const_arg(def);
+    let const_ty = typeck_results.node_type(hir_id);
+
+    tcx.infer_ctxt().enter(|infcx| {
+        let mut builder = Builder::new(
+            thir,
+            infcx,
+            def,
+            hir_id,
+            span,
+            0,
+            Safety::Safe,
+            const_ty,
+            const_ty_span,
+            None,
+        );
 
-    let mut block = START_BLOCK;
-    unpack!(block = builder.expr_into_dest(Place::return_place(), block, &thir[expr]));
+        let mut block = START_BLOCK;
+        unpack!(block = builder.expr_into_dest(Place::return_place(), block, &thir[expr]));
 
-    let source_info = builder.source_info(span);
-    builder.cfg.terminate(block, source_info, TerminatorKind::Return);
+        let source_info = builder.source_info(span);
+        builder.cfg.terminate(block, source_info, TerminatorKind::Return);
 
-    builder.build_drop_trees();
+        builder.build_drop_trees();
 
-    builder.finish()
+        builder.finish()
+    })
 }
 
 /// Construct MIR for an item that has had errors in type checking.
 ///
 /// This is required because we may still want to run MIR passes on an item
 /// with type errors, but normal MIR construction can't handle that in general.
-fn construct_error<'a, 'tcx>(
-    infcx: &'a InferCtxt<'a, 'tcx>,
-    def: ty::WithOptConstParam<LocalDefId>,
-    hir_id: hir::HirId,
-    body_id: hir::BodyId,
+fn construct_error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def: LocalDefId,
     body_owner_kind: hir::BodyOwnerKind,
     err: ErrorGuaranteed,
 ) -> Body<'tcx> {
-    let tcx = infcx.tcx;
-    let span = tcx.hir().span(hir_id);
+    let span = tcx.def_span(def);
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def);
+    let generator_kind = tcx.generator_kind(def);
+
     let ty = tcx.ty_error();
-    let generator_kind = tcx.hir().body(body_id).generator_kind;
     let num_params = match body_owner_kind {
-        hir::BodyOwnerKind::Fn => tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len(),
+        hir::BodyOwnerKind::Fn => tcx.fn_sig(def).inputs().skip_binder().len(),
         hir::BodyOwnerKind::Closure => {
-            if generator_kind.is_some() {
-                // Generators have an implicit `self` parameter *and* a possibly
-                // implicit resume parameter.
-                2
-            } else {
-                // The implicit self parameter adds another local in MIR.
-                1 + tcx.hir().fn_decl_by_hir_id(hir_id).unwrap().inputs.len()
+            let ty = tcx.type_of(def);
+            match ty.kind() {
+                ty::Closure(_, substs) => {
+                    1 + substs.as_closure().sig().inputs().skip_binder().len()
+                }
+                ty::Generator(..) => 2,
+                _ => bug!("expected closure or generator, found {ty:?}"),
             }
         }
         hir::BodyOwnerKind::Const => 0,
@@ -788,7 +665,7 @@ fn construct_error<'a, 'tcx>(
     cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable);
 
     let mut body = Body::new(
-        MirSource::item(def.did.to_def_id()),
+        MirSource::item(def.to_def_id()),
         cfg.basic_blocks,
         source_scopes,
         local_decls,
@@ -806,7 +683,7 @@ fn construct_error<'a, 'tcx>(
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn new(
         thir: &'a Thir<'tcx>,
-        infcx: &'a InferCtxt<'a, 'tcx>,
+        infcx: InferCtxt<'a, 'tcx>,
         def: ty::WithOptConstParam<LocalDefId>,
         hir_id: hir::HirId,
         span: Span,
@@ -855,7 +732,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             in_scope_unsafe: safety,
             local_decls: IndexVec::from_elem_n(LocalDecl::new(return_ty, return_span), 1),
             canonical_user_type_annotations: IndexVec::new(),
-            upvar_mutbls: vec![],
+            upvars: CaptureMap::new(),
             var_indices: Default::default(),
             unit_temp: None,
             var_debug_info: vec![],
@@ -896,20 +773,21 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         mut block: BasicBlock,
         fn_def_id: LocalDefId,
-        arguments: &[ArgInfo<'tcx>],
+        arguments: &IndexVec<ParamId, Param<'tcx>>,
         argument_scope: region::Scope,
         expr: &Expr<'tcx>,
     ) -> BlockAnd<()> {
         // Allocate locals for the function arguments
-        for &ArgInfo(ty, _, arg_opt, _) in arguments.iter() {
+        for param in arguments.iter() {
             let source_info =
-                SourceInfo::outermost(arg_opt.map_or(self.fn_span, |arg| arg.pat.span));
-            let arg_local = self.local_decls.push(LocalDecl::with_source_info(ty, source_info));
+                SourceInfo::outermost(param.pat.as_ref().map_or(self.fn_span, |pat| pat.span));
+            let arg_local =
+                self.local_decls.push(LocalDecl::with_source_info(param.ty, source_info));
 
             // If this is a simple binding pattern, give debuginfo a nice name.
-            if let Some(arg) = arg_opt && let Some(ident) = arg.pat.simple_ident() {
+            if let Some(ref pat) = param.pat && let Some(name) = pat.simple_ident() {
                 self.var_debug_info.push(VarDebugInfo {
-                    name: ident.name,
+                    name,
                     source_info,
                     value: VarDebugInfoContents::Place(arg_local.into()),
                 });
@@ -924,7 +802,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // indexed closure and we stored in a map called closure_min_captures in TypeckResults
         // with the closure's DefId. Here, we run through that vec of UpvarIds for
         // the given closure and use the necessary information to create upvar
-        // debuginfo and to fill `self.upvar_mutbls`.
+        // debuginfo and to fill `self.upvars`.
         if hir_typeck_results.closure_min_captures.get(&fn_def_id).is_some() {
             let mut closure_env_projs = vec![];
             let mut closure_ty = self.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
@@ -944,7 +822,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 .closure_min_captures_flattened(fn_def_id)
                 .zip(capture_tys.zip(capture_syms));
 
-            self.upvar_mutbls = captures_with_tys
+            self.upvars = captures_with_tys
                 .enumerate()
                 .map(|(i, (captured_place, (ty, sym)))| {
                     let capture = captured_place.info.capture_kind;
@@ -964,48 +842,46 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         }
                     };
 
+                    let use_place = Place {
+                        local: ty::CAPTURE_STRUCT_LOCAL,
+                        projection: tcx.intern_place_elems(&projs),
+                    };
                     self.var_debug_info.push(VarDebugInfo {
                         name: *sym,
                         source_info: SourceInfo::outermost(tcx_hir.span(var_id)),
-                        value: VarDebugInfoContents::Place(Place {
-                            local: ty::CAPTURE_STRUCT_LOCAL,
-                            projection: tcx.intern_place_elems(&projs),
-                        }),
+                        value: VarDebugInfoContents::Place(use_place),
                     });
 
-                    mutability
+                    let capture = Capture { captured_place, use_place, mutability };
+                    (var_id, capture)
                 })
                 .collect();
         }
 
         let mut scope = None;
         // Bind the argument patterns
-        for (index, arg_info) in arguments.iter().enumerate() {
+        for (index, param) in arguments.iter().enumerate() {
             // Function arguments always get the first Local indices after the return place
             let local = Local::new(index + 1);
             let place = Place::from(local);
-            let &ArgInfo(_, opt_ty_info, arg_opt, ref self_binding) = arg_info;
 
             // Make sure we drop (parts of) the argument even when not matched on.
             self.schedule_drop(
-                arg_opt.as_ref().map_or(expr.span, |arg| arg.pat.span),
+                param.pat.as_ref().map_or(expr.span, |pat| pat.span),
                 argument_scope,
                 local,
                 DropKind::Value,
             );
 
-            let Some(arg) = arg_opt else {
+            let Some(ref pat) = param.pat else {
                 continue;
             };
-            let pat = match tcx.hir().get(arg.pat.hir_id) {
-                Node::Pat(pat) => pat,
-                node => bug!("pattern became {:?}", node),
-            };
-            let pattern = pat_from_hir(tcx, self.param_env, self.typeck_results, pat);
             let original_source_scope = self.source_scope;
-            let span = pattern.span;
-            self.set_correct_source_scope_for_arg(arg.hir_id, original_source_scope, span);
-            match *pattern.kind {
+            let span = pat.span;
+            if let Some(arg_hir_id) = param.hir_id {
+                self.set_correct_source_scope_for_arg(arg_hir_id, original_source_scope, span);
+            }
+            match pat.kind {
                 // Don't introduce extra copies for simple bindings
                 PatKind::Binding {
                     mutability,
@@ -1016,17 +892,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 } => {
                     self.local_decls[local].mutability = mutability;
                     self.local_decls[local].source_info.scope = self.source_scope;
-                    self.local_decls[local].local_info = if let Some(kind) = self_binding {
+                    self.local_decls[local].local_info = if let Some(kind) = param.self_kind {
                         Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(
-                            BindingForm::ImplicitSelf(*kind),
+                            BindingForm::ImplicitSelf(kind),
                         ))))
                     } else {
                         let binding_mode = ty::BindingMode::BindByValue(mutability);
                         Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
                             VarBindingForm {
                                 binding_mode,
-                                opt_ty_info,
-                                opt_match_place: Some((Some(place), span)),
+                                opt_ty_info: param.ty_span,
+                                opt_match_place: Some((None, span)),
                                 pat_span: span,
                             },
                         )))))
@@ -1037,12 +913,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     scope = self.declare_bindings(
                         scope,
                         expr.span,
-                        &pattern,
+                        &pat,
                         matches::ArmHasGuard(false),
                         Some((Some(&place), span)),
                     );
                     let place_builder = PlaceBuilder::from(local);
-                    unpack!(block = self.place_into_pattern(block, pattern, place_builder, false));
+                    unpack!(block = self.place_into_pattern(block, &pat, place_builder, false));
                 }
             }
             self.source_scope = original_source_scope;
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index b2fd9f25b..ff66ca595 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -466,9 +466,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let normal_exit_block = f(self);
         let breakable_scope = self.scopes.breakable_scopes.pop().unwrap();
         assert!(breakable_scope.region_scope == region_scope);
-        let break_block = self.build_exit_tree(breakable_scope.break_drops, None);
+        let break_block =
+            self.build_exit_tree(breakable_scope.break_drops, region_scope, span, None);
         if let Some(drops) = breakable_scope.continue_drops {
-            self.build_exit_tree(drops, loop_block);
+            self.build_exit_tree(drops, region_scope, span, loop_block);
         }
         match (normal_exit_block, break_block) {
             (Some(block), None) | (None, Some(block)) => block,
@@ -510,6 +511,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub(crate) fn in_if_then_scope<F>(
         &mut self,
         region_scope: region::Scope,
+        span: Span,
         f: F,
     ) -> (BasicBlock, BasicBlock)
     where
@@ -524,7 +526,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         assert!(if_then_scope.region_scope == region_scope);
 
         let else_block = self
-            .build_exit_tree(if_then_scope.else_drops, None)
+            .build_exit_tree(if_then_scope.else_drops, region_scope, span, None)
             .map_or_else(|| self.cfg.start_new_block(), |else_block_and| unpack!(else_block_and));
 
         (then_block, else_block)
@@ -997,10 +999,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Returns the [DropIdx] for the innermost drop if the function unwound at
     /// this point. The `DropIdx` will be created if it doesn't already exist.
     fn diverge_cleanup(&mut self) -> DropIdx {
-        let is_generator = self.generator_kind.is_some();
-        let (uncached_scope, mut cached_drop) = self
-            .scopes
-            .scopes
+        // It is okay to use dummy span because the getting scope index on the topmost scope
+        // must always succeed.
+        self.diverge_cleanup_target(self.scopes.topmost(), DUMMY_SP)
+    }
+
+    /// This is similar to [diverge_cleanup](Self::diverge_cleanup) except its target is set to
+    /// some ancestor scope instead of the current scope.
+    /// It is possible to unwind to some ancestor scope if some drop panics as
+    /// the program breaks out of a if-then scope.
+    fn diverge_cleanup_target(&mut self, target_scope: region::Scope, span: Span) -> DropIdx {
+        let target = self.scopes.scope_index(target_scope, span);
+        let (uncached_scope, mut cached_drop) = self.scopes.scopes[..=target]
             .iter()
             .enumerate()
             .rev()
@@ -1009,7 +1019,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             })
             .unwrap_or((0, ROOT_NODE));
 
-        for scope in &mut self.scopes.scopes[uncached_scope..] {
+        if uncached_scope > target {
+            return cached_drop;
+        }
+
+        let is_generator = self.generator_kind.is_some();
+        for scope in &mut self.scopes.scopes[uncached_scope..=target] {
             for drop in &scope.drops {
                 if is_generator || drop.kind == DropKind::Value {
                     cached_drop = self.scopes.unwind_drops.add_drop(*drop, cached_drop);
@@ -1222,21 +1237,24 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
     fn build_exit_tree(
         &mut self,
         mut drops: DropTree,
+        else_scope: region::Scope,
+        span: Span,
         continue_block: Option<BasicBlock>,
     ) -> Option<BlockAnd<()>> {
         let mut blocks = IndexVec::from_elem(None, &drops.drops);
         blocks[ROOT_NODE] = continue_block;
 
         drops.build_mir::<ExitScopes>(&mut self.cfg, &mut blocks);
+        let is_generator = self.generator_kind.is_some();
 
         // Link the exit drop tree to unwind drop tree.
         if drops.drops.iter().any(|(drop, _)| drop.kind == DropKind::Value) {
-            let unwind_target = self.diverge_cleanup();
+            let unwind_target = self.diverge_cleanup_target(else_scope, span);
             let mut unwind_indices = IndexVec::from_elem_n(unwind_target, 1);
             for (drop_idx, drop_data) in drops.drops.iter_enumerated().skip(1) {
                 match drop_data.0.kind {
                     DropKind::Storage => {
-                        if self.generator_kind.is_some() {
+                        if is_generator {
                             let unwind_drop = self
                                 .scopes
                                 .unwind_drops
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index 864caf0ba..495738ebe 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -75,10 +75,11 @@ impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
         match self.safety_context {
             SafetyContext::BuiltinUnsafeBlock => {}
             SafetyContext::UnsafeBlock { ref mut used, .. } => {
-                if !self.body_unsafety.is_unsafe() || !unsafe_op_in_unsafe_fn_allowed {
-                    // Mark this block as useful
-                    *used = true;
-                }
+                // Mark this block as useful (even inside `unsafe fn`, where it is technically
+                // redundant -- but we want to eventually enable `unsafe_op_in_unsafe_fn` by
+                // default which will require those blocks:
+                // https://github.com/rust-lang/rust/issues/71668#issuecomment-1203075594).
+                *used = true;
             }
             SafetyContext::UnsafeFn if unsafe_op_in_unsafe_fn_allowed => {}
             SafetyContext::UnsafeFn => {
@@ -213,7 +214,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
 
     fn visit_pat(&mut self, pat: &Pat<'tcx>) {
         if self.in_union_destructure {
-            match *pat.kind {
+            match pat.kind {
                 // binding to a variable allows getting stuff out of variable
                 PatKind::Binding { .. }
                 // match is conditional on having this value
@@ -235,7 +236,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
             }
         };
 
-        match &*pat.kind {
+        match &pat.kind {
             PatKind::Leaf { .. } => {
                 if let ty::Adt(adt_def, ..) = pat.ty.kind() {
                     if adt_def.is_union() {
@@ -390,7 +391,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
             ExprKind::InlineAsm { .. } => {
                 self.requires_unsafe(expr.span, UseOfInlineAssembly);
             }
-            ExprKind::Adt(box Adt {
+            ExprKind::Adt(box AdtExpr {
                 adt_def,
                 variant_index: _,
                 substs: _,
@@ -401,13 +402,13 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                 (Bound::Unbounded, Bound::Unbounded) => {}
                 _ => self.requires_unsafe(expr.span, InitializingTypeWith),
             },
-            ExprKind::Closure {
+            ExprKind::Closure(box ClosureExpr {
                 closure_id,
                 substs: _,
                 upvars: _,
                 movability: _,
                 fake_reads: _,
-            } => {
+            }) => {
                 let closure_def = if let Some((did, const_param_id)) =
                     ty::WithOptConstParam::try_lookup(closure_id, self.tcx)
                 {
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index 11cd2a9aa..0c0a2fe9c 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -6,7 +6,7 @@
 #![feature(control_flow_enum)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
 #![feature(min_specialization)]
 #![feature(once_cell)]
 #![recursion_limit = "256"]
diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs
index a7e4403a2..f626571b5 100644
--- a/compiler/rustc_mir_build/src/thir/constant.rs
+++ b/compiler/rustc_mir_build/src/thir/constant.rs
@@ -44,7 +44,7 @@ pub(crate) fn lit_to_const<'tcx>(
         }
         (ast::LitKind::Bool(b), ty::Bool) => ty::ValTree::from_scalar_int((*b).into()),
         (ast::LitKind::Char(c), ty::Char) => ty::ValTree::from_scalar_int((*c).into()),
-        (ast::LitKind::Err(_), _) => return Err(LitToConstError::Reported),
+        (ast::LitKind::Err, _) => return Err(LitToConstError::Reported),
         _ => return Err(LitToConstError::TypeError),
     };
 
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index dccaa61ed..321353ca2 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -9,13 +9,13 @@ use rustc_index::vec::Idx;
 use rustc_middle::ty::CanonicalUserTypeAnnotation;
 
 impl<'tcx> Cx<'tcx> {
-    pub(crate) fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> Block {
+    pub(crate) fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> BlockId {
         // We have to eagerly lower the "spine" of the statements
         // in order to get the lexical scoping correctly.
         let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts);
         let opt_destruction_scope =
             self.region_scope_tree.opt_destruction_scope(block.hir_id.local_id);
-        Block {
+        let block = Block {
             targeted_by_break: block.targeted_by_break,
             region_scope: region::Scope {
                 id: block.hir_id.local_id,
@@ -34,7 +34,9 @@ impl<'tcx> Cx<'tcx> {
                     BlockSafety::ExplicitUnsafe(block.hir_id)
                 }
             },
-        }
+        };
+
+        self.thir.blocks.push(block)
     }
 
     fn mirror_stmts(
@@ -85,21 +87,21 @@ impl<'tcx> Cx<'tcx> {
                             {
                                 debug!("mirror_stmts: user_ty={:?}", user_ty);
                                 let annotation = CanonicalUserTypeAnnotation {
-                                    user_ty,
+                                    user_ty: Box::new(user_ty),
                                     span: ty.span,
                                     inferred_ty: self.typeck_results.node_type(ty.hir_id),
                                 };
-                                pattern = Pat {
+                                pattern = Box::new(Pat {
                                     ty: pattern.ty,
                                     span: pattern.span,
-                                    kind: Box::new(PatKind::AscribeUserType {
+                                    kind: PatKind::AscribeUserType {
                                         ascription: Ascription {
                                             annotation,
                                             variance: ty::Variance::Covariant,
                                         },
                                         subpattern: pattern,
-                                    }),
-                                };
+                                    },
+                                });
                             }
                         }
 
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 985601712..d059877f8 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -108,8 +108,8 @@ impl<'tcx> Cx<'tcx> {
         //   // ^ error message points at this expression.
         // }
         let mut adjust_span = |expr: &mut Expr<'tcx>| {
-            if let ExprKind::Block { body } = &expr.kind {
-                if let Some(last_expr) = body.expr {
+            if let ExprKind::Block { block } = expr.kind {
+                if let Some(last_expr) = self.thir[block].expr {
                     span = self.thir[last_expr].span;
                     expr.span = span;
                 }
@@ -261,15 +261,19 @@ impl<'tcx> Cx<'tcx> {
 
         let kind = match expr.kind {
             // Here comes the interesting stuff:
-            hir::ExprKind::MethodCall(segment, ref args, fn_span) => {
+            hir::ExprKind::MethodCall(segment, receiver, ref args, fn_span) => {
                 // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
                 let expr = self.method_callee(expr, segment.ident.span, None);
                 // When we apply adjustments to the receiver, use the span of
                 // the overall method call for better diagnostics. args[0]
                 // is guaranteed to exist, since a method call always has a receiver.
-                let old_adjustment_span = self.adjustment_span.replace((args[0].hir_id, expr_span));
-                tracing::info!("Using method span: {:?}", expr.span);
-                let args = self.mirror_exprs(args);
+                let old_adjustment_span =
+                    self.adjustment_span.replace((receiver.hir_id, expr_span));
+                info!("Using method span: {:?}", expr.span);
+                let args = std::iter::once(receiver)
+                    .chain(args.iter())
+                    .map(|expr| self.mirror_expr(expr))
+                    .collect();
                 self.adjustment_span = old_adjustment_span;
                 ExprKind::Call {
                     ty: expr.ty,
@@ -329,7 +333,7 @@ impl<'tcx> Cx<'tcx> {
                                 if let UserType::TypeOf(ref mut did, _) = &mut u_ty.value {
                                     *did = adt_def.did();
                                 }
-                                u_ty
+                                Box::new(u_ty)
                             });
                         debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty);
 
@@ -341,7 +345,7 @@ impl<'tcx> Cx<'tcx> {
                                 expr: self.mirror_expr(e),
                             })
                             .collect();
-                        ExprKind::Adt(Box::new(Adt {
+                        ExprKind::Adt(Box::new(AdtExpr {
                             adt_def,
                             substs,
                             variant_index: index,
@@ -369,7 +373,7 @@ impl<'tcx> Cx<'tcx> {
                 ExprKind::AddressOf { mutability, arg: self.mirror_expr(arg) }
             }
 
-            hir::ExprKind::Block(ref blk, _) => ExprKind::Block { body: self.mirror_block(blk) },
+            hir::ExprKind::Block(ref blk, _) => ExprKind::Block { block: self.mirror_block(blk) },
 
             hir::ExprKind::Assign(ref lhs, ref rhs, _) => {
                 ExprKind::Assign { lhs: self.mirror_expr(lhs), rhs: self.mirror_expr(rhs) }
@@ -464,9 +468,9 @@ impl<'tcx> Cx<'tcx> {
                 ty::Adt(adt, substs) => match adt.adt_kind() {
                     AdtKind::Struct | AdtKind::Union => {
                         let user_provided_types = self.typeck_results().user_provided_types();
-                        let user_ty = user_provided_types.get(expr.hir_id).copied();
+                        let user_ty = user_provided_types.get(expr.hir_id).copied().map(Box::new);
                         debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
-                        ExprKind::Adt(Box::new(Adt {
+                        ExprKind::Adt(Box::new(AdtExpr {
                             adt_def: *adt,
                             variant_index: VariantIdx::new(0),
                             substs,
@@ -490,9 +494,10 @@ impl<'tcx> Cx<'tcx> {
                                 let index = adt.variant_index_with_id(variant_id);
                                 let user_provided_types =
                                     self.typeck_results().user_provided_types();
-                                let user_ty = user_provided_types.get(expr.hir_id).copied();
+                                let user_ty =
+                                    user_provided_types.get(expr.hir_id).copied().map(Box::new);
                                 debug!("make_mirror_unadjusted: (variant) user_ty={:?}", user_ty);
-                                ExprKind::Adt(Box::new(Adt {
+                                ExprKind::Adt(Box::new(AdtExpr {
                                     adt_def: *adt,
                                     variant_index: index,
                                     substs,
@@ -547,7 +552,13 @@ impl<'tcx> Cx<'tcx> {
                     None => Vec::new(),
                 };
 
-                ExprKind::Closure { closure_id: def_id, substs, upvars, movability, fake_reads }
+                ExprKind::Closure(Box::new(ClosureExpr {
+                    closure_id: def_id,
+                    substs,
+                    upvars,
+                    movability,
+                    fake_reads,
+                }))
             }
 
             hir::ExprKind::Path(ref qpath) => {
@@ -555,7 +566,7 @@ impl<'tcx> Cx<'tcx> {
                 self.convert_path_expr(expr, res)
             }
 
-            hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm {
+            hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm(Box::new(InlineAsmExpr {
                 template: asm.template,
                 operands: asm
                     .operands
@@ -614,7 +625,7 @@ impl<'tcx> Cx<'tcx> {
                     .collect(),
                 options: asm.options,
                 line_spans: asm.line_spans,
-            },
+            })),
 
             hir::ExprKind::ConstBlock(ref anon_const) => {
                 let ty = self.typeck_results().node_type(anon_const.hir_id);
@@ -679,8 +690,8 @@ impl<'tcx> Cx<'tcx> {
                 let body = self.thir.exprs.push(Expr {
                     ty: block_ty,
                     temp_lifetime,
-                    span: block.span,
-                    kind: ExprKind::Block { body: block },
+                    span: self.thir[block].span,
+                    kind: ExprKind::Block { block },
                 });
                 ExprKind::Loop { body }
             }
@@ -712,14 +723,17 @@ impl<'tcx> Cx<'tcx> {
                     });
                     debug!("make_mirror_unadjusted: (cast) user_ty={:?}", user_ty);
 
-                    ExprKind::ValueTypeAscription { source: cast_expr, user_ty: Some(*user_ty) }
+                    ExprKind::ValueTypeAscription {
+                        source: cast_expr,
+                        user_ty: Some(Box::new(*user_ty)),
+                    }
                 } else {
                     cast
                 }
             }
             hir::ExprKind::Type(ref source, ref ty) => {
                 let user_provided_types = self.typeck_results.user_provided_types();
-                let user_ty = user_provided_types.get(ty.hir_id).copied();
+                let user_ty = user_provided_types.get(ty.hir_id).copied().map(Box::new);
                 debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
                 let mirrored = self.mirror_expr(source);
                 if source.is_syntactic_place_expr() {
@@ -748,7 +762,7 @@ impl<'tcx> Cx<'tcx> {
         &mut self,
         hir_id: hir::HirId,
         res: Res,
-    ) -> Option<ty::CanonicalUserType<'tcx>> {
+    ) -> Option<Box<ty::CanonicalUserType<'tcx>>> {
         debug!("user_substs_applied_to_res: res={:?}", res);
         let user_provided_type = match res {
             // A reference to something callable -- e.g., a fn, method, or
@@ -759,7 +773,7 @@ impl<'tcx> Cx<'tcx> {
             | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
             | Res::Def(DefKind::Const, _)
             | Res::Def(DefKind::AssocConst, _) => {
-                self.typeck_results().user_provided_types().get(hir_id).copied()
+                self.typeck_results().user_provided_types().get(hir_id).copied().map(Box::new)
             }
 
             // A unit struct/variant which is used as a value (e.g.,
@@ -767,11 +781,11 @@ impl<'tcx> Cx<'tcx> {
             // this variant -- but with the substitutions given by the
             // user.
             Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => {
-                self.user_substs_applied_to_ty_of_hir_id(hir_id)
+                self.user_substs_applied_to_ty_of_hir_id(hir_id).map(Box::new)
             }
 
             // `Self` is used in expression as a tuple struct constructor or a unit struct constructor
-            Res::SelfCtor(_) => self.user_substs_applied_to_ty_of_hir_id(hir_id),
+            Res::SelfCtor(_) => self.user_substs_applied_to_ty_of_hir_id(hir_id).map(Box::new),
 
             _ => bug!("user_substs_applied_to_res: unexpected res {:?} at {:?}", res, hir_id),
         };
@@ -846,22 +860,22 @@ impl<'tcx> Cx<'tcx> {
 
             Res::Def(DefKind::Const, def_id) | Res::Def(DefKind::AssocConst, def_id) => {
                 let user_ty = self.user_substs_applied_to_res(expr.hir_id, res);
-                ExprKind::NamedConst { def_id, substs, user_ty: user_ty }
+                ExprKind::NamedConst { def_id, substs, user_ty }
             }
 
             Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id) => {
                 let user_provided_types = self.typeck_results.user_provided_types();
-                let user_provided_type = user_provided_types.get(expr.hir_id).copied();
-                debug!("convert_path_expr: user_provided_type={:?}", user_provided_type);
+                let user_ty = user_provided_types.get(expr.hir_id).copied().map(Box::new);
+                debug!("convert_path_expr: user_ty={:?}", user_ty);
                 let ty = self.typeck_results().node_type(expr.hir_id);
                 match ty.kind() {
                     // A unit struct/variant which is used as a value.
                     // We return a completely different ExprKind here to account for this special case.
-                    ty::Adt(adt_def, substs) => ExprKind::Adt(Box::new(Adt {
+                    ty::Adt(adt_def, substs) => ExprKind::Adt(Box::new(AdtExpr {
                         adt_def: *adt_def,
                         variant_index: adt_def.variant_index_with_ctor_id(def_id),
                         substs,
-                        user_ty: user_provided_type,
+                        user_ty,
                         fields: Box::new([]),
                         base: None,
                     })),
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index f7351a4ca..3ef1b263f 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -8,12 +8,14 @@ use crate::thir::util::UserAnnotatedTyHelpers;
 use rustc_data_structures::steal::Steal;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::lang_items::LangItem;
 use rustc_hir::HirId;
 use rustc_hir::Node;
 use rustc_middle::middle::region;
 use rustc_middle::thir::*;
-use rustc_middle::ty::{self, RvalueScopes, TyCtxt};
+use rustc_middle::ty::{self, RvalueScopes, Subst, TyCtxt};
 use rustc_span::Span;
 
 pub(crate) fn thir_body<'tcx>(
@@ -27,6 +29,26 @@ pub(crate) fn thir_body<'tcx>(
         return Err(reported);
     }
     let expr = cx.mirror_expr(&body.value);
+
+    let owner_id = hir.local_def_id_to_hir_id(owner_def.did);
+    if let Some(ref fn_decl) = hir.fn_decl_by_hir_id(owner_id) {
+        let closure_env_param = cx.closure_env_param(owner_def.did, owner_id);
+        let explicit_params = cx.explicit_params(owner_id, fn_decl, body);
+        cx.thir.params = closure_env_param.into_iter().chain(explicit_params).collect();
+
+        // The resume argument may be missing, in that case we need to provide it here.
+        // It will always be `()` in this case.
+        if tcx.def_kind(owner_def.did) == DefKind::Generator && body.params.is_empty() {
+            cx.thir.params.push(Param {
+                ty: tcx.mk_unit(),
+                pat: None,
+                ty_span: None,
+                self_kind: None,
+                hir_id: None,
+            });
+        }
+    }
+
     Ok((tcx.alloc_steal_thir(cx.thir), expr))
 }
 
@@ -44,11 +66,11 @@ struct Cx<'tcx> {
     tcx: TyCtxt<'tcx>,
     thir: Thir<'tcx>,
 
-    pub(crate) param_env: ty::ParamEnv<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
 
-    pub(crate) region_scope_tree: &'tcx region::ScopeTree,
-    pub(crate) typeck_results: &'tcx ty::TypeckResults<'tcx>,
-    pub(crate) rvalue_scopes: &'tcx RvalueScopes,
+    region_scope_tree: &'tcx region::ScopeTree,
+    typeck_results: &'tcx ty::TypeckResults<'tcx>,
+    rvalue_scopes: &'tcx RvalueScopes,
 
     /// When applying adjustments to the expression
     /// with the given `HirId`, use the given `Span`,
@@ -77,14 +99,94 @@ impl<'tcx> Cx<'tcx> {
         }
     }
 
-    #[tracing::instrument(level = "debug", skip(self))]
-    pub(crate) fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Pat<'tcx> {
+    #[instrument(level = "debug", skip(self))]
+    fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Box<Pat<'tcx>> {
         let p = match self.tcx.hir().get(p.hir_id) {
             Node::Pat(p) => p,
             node => bug!("pattern became {:?}", node),
         };
         pat_from_hir(self.tcx, self.param_env, self.typeck_results(), p)
     }
+
+    fn closure_env_param(&self, owner_def: LocalDefId, owner_id: HirId) -> Option<Param<'tcx>> {
+        match self.tcx.def_kind(owner_def) {
+            DefKind::Closure => {
+                let closure_ty = self.typeck_results.node_type(owner_id);
+
+                let ty::Closure(closure_def_id, closure_substs) = *closure_ty.kind() else {
+                    bug!("closure expr does not have closure type: {:?}", closure_ty);
+                };
+
+                let bound_vars = self.tcx.mk_bound_variable_kinds(std::iter::once(
+                    ty::BoundVariableKind::Region(ty::BrEnv),
+                ));
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(bound_vars.len() - 1),
+                    kind: ty::BrEnv,
+                };
+                let env_region = ty::ReLateBound(ty::INNERMOST, br);
+                let closure_env_ty =
+                    self.tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap();
+                let liberated_closure_env_ty = self.tcx.erase_late_bound_regions(
+                    ty::Binder::bind_with_vars(closure_env_ty, bound_vars),
+                );
+                let env_param = Param {
+                    ty: liberated_closure_env_ty,
+                    pat: None,
+                    ty_span: None,
+                    self_kind: None,
+                    hir_id: None,
+                };
+
+                Some(env_param)
+            }
+            DefKind::Generator => {
+                let gen_ty = self.typeck_results.node_type(owner_id);
+                let gen_param =
+                    Param { ty: gen_ty, pat: None, ty_span: None, self_kind: None, hir_id: None };
+                Some(gen_param)
+            }
+            _ => None,
+        }
+    }
+
+    fn explicit_params<'a>(
+        &'a mut self,
+        owner_id: HirId,
+        fn_decl: &'tcx hir::FnDecl<'tcx>,
+        body: &'tcx hir::Body<'tcx>,
+    ) -> impl Iterator<Item = Param<'tcx>> + 'a {
+        let fn_sig = self.typeck_results.liberated_fn_sigs()[owner_id];
+
+        body.params.iter().enumerate().map(move |(index, param)| {
+            let ty_span = fn_decl
+                .inputs
+                .get(index)
+                // Make sure that inferred closure args have no type span
+                .and_then(|ty| if param.pat.span != ty.span { Some(ty.span) } else { None });
+
+            let self_kind = if index == 0 && fn_decl.implicit_self.has_implicit_self() {
+                Some(fn_decl.implicit_self)
+            } else {
+                None
+            };
+
+            // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
+            // (as it's created inside the body itself, not passed in from outside).
+            let ty = if fn_decl.c_variadic && index == fn_decl.inputs.len() {
+                let va_list_did = self.tcx.require_lang_item(LangItem::VaList, Some(param.span));
+
+                self.tcx
+                    .bound_type_of(va_list_did)
+                    .subst(self.tcx, &[self.tcx.lifetimes.re_erased.into()])
+            } else {
+                fn_sig.inputs()[index]
+            };
+
+            let pat = self.pattern_from_hir(param.pat);
+            Param { pat: Some(pat), ty, ty_span, self_kind, hir_id: Some(param.hir_id) }
+        })
+    }
 }
 
 impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'tcx> {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 063c07647..d45b88690 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -491,8 +491,8 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
                     err.span_suggestion_verbose(
                         semi_span.shrink_to_lo(),
                         &format!(
-                            "alternatively, on nightly, you might want to use \
-                             `#![feature(let_else)]` to handle the variant{} that {} matched",
+                            "alternatively, you might want to use \
+                             let else to handle the variant{} that {} matched",
                             pluralize!(witnesses.len()),
                             match witnesses.len() {
                                 1 => "isn't",
@@ -849,22 +849,22 @@ fn non_exhaustive_match<'p, 'tcx>(
             ));
         }
         [.., prev, last] if prev.span.eq_ctxt(last.span) => {
-            if let Ok(snippet) = sm.span_to_snippet(prev.span.between(last.span)) {
-                let comma = if matches!(last.body.kind, hir::ExprKind::Block(..))
-                    && last.span.eq_ctxt(last.body.span)
-                {
-                    ""
-                } else {
-                    ","
-                };
+            let comma = if matches!(last.body.kind, hir::ExprKind::Block(..))
+                && last.span.eq_ctxt(last.body.span)
+            {
+                ""
+            } else {
+                ","
+            };
+            let spacing = if sm.is_multiline(prev.span.between(last.span)) {
+                sm.indentation_before(last.span).map(|indent| format!("\n{indent}"))
+            } else {
+                Some(" ".to_string())
+            };
+            if let Some(spacing) = spacing {
                 suggestion = Some((
                     last.span.shrink_to_hi(),
-                    format!(
-                        "{}{}{} => todo!()",
-                        comma,
-                        snippet.strip_prefix(',').unwrap_or(&snippet),
-                        pattern
-                    ),
+                    format!("{}{}{} => todo!()", comma, spacing, pattern),
                 ));
             }
         }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index d6dd0f017..b58685e89 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -19,21 +19,18 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     /// Converts an evaluated constant to a pattern (if possible).
     /// This means aggregate values (like structs and enums) are converted
     /// to a pattern that matches the value (as if you'd compared via structural equality).
-    #[instrument(level = "debug", skip(self))]
+    #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn const_to_pat(
         &self,
         cv: mir::ConstantKind<'tcx>,
         id: hir::HirId,
         span: Span,
         mir_structural_match_violation: bool,
-    ) -> Pat<'tcx> {
-        let pat = self.tcx.infer_ctxt().enter(|infcx| {
+    ) -> Box<Pat<'tcx>> {
+        self.tcx.infer_ctxt().enter(|infcx| {
             let mut convert = ConstToPat::new(self, id, span, infcx);
             convert.to_pat(cv, mir_structural_match_violation)
-        });
-
-        debug!(?pat);
-        pat
+        })
     }
 }
 
@@ -159,7 +156,7 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
         &mut self,
         cv: mir::ConstantKind<'tcx>,
         mir_structural_match_violation: bool,
-    ) -> Pat<'tcx> {
+    ) -> Box<Pat<'tcx>> {
         trace!(self.treat_byte_string_as_slice);
         // This method is just a wrapper handling a validity check; the heavy lifting is
         // performed by the recursive `recur` method, which is not meant to be
@@ -168,7 +165,14 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
         // once indirect_structural_match is a full fledged error, this
         // level of indirection can be eliminated
 
-        let inlined_const_as_pat = self.recur(cv, mir_structural_match_violation).unwrap();
+        let inlined_const_as_pat =
+            self.recur(cv, mir_structural_match_violation).unwrap_or_else(|_| {
+                Box::new(Pat {
+                    span: self.span,
+                    ty: cv.ty(),
+                    kind: PatKind::Constant { value: cv },
+                })
+            });
 
         if self.include_lint_checks && !self.saw_const_match_error.get() {
             // If we were able to successfully convert the const to some pat,
@@ -269,7 +273,7 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
         &self,
         cv: mir::ConstantKind<'tcx>,
         mir_structural_match_violation: bool,
-    ) -> Result<Pat<'tcx>, FallbackToConstRef> {
+    ) -> Result<Box<Pat<'tcx>>, FallbackToConstRef> {
         let id = self.id;
         let span = self.span;
         let tcx = self.tcx();
@@ -396,7 +400,7 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                     .map(|val| self.recur(*val, false))
                     .collect::<Result<_, _>>()?,
                 slice: None,
-                suffix: Vec::new(),
+                suffix: Box::new([]),
             },
             ty::Ref(_, pointee_ty, ..) => match *pointee_ty.kind() {
                 // These are not allowed and will error elsewhere anyway.
@@ -423,8 +427,8 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                     let old = self.behind_reference.replace(true);
                     let array = tcx.deref_mir_constant(self.param_env.and(cv));
                     let val = PatKind::Deref {
-                        subpattern: Pat {
-                            kind: Box::new(PatKind::Array {
+                        subpattern: Box::new(Pat {
+                            kind: PatKind::Array {
                                 prefix: tcx
                                     .destructure_mir_constant(param_env, array)
                                     .fields
@@ -432,11 +436,11 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                                     .map(|val| self.recur(*val, false))
                                     .collect::<Result<_, _>>()?,
                                 slice: None,
-                                suffix: vec![],
-                            }),
+                                suffix: Box::new([]),
+                            },
                             span,
                             ty: *pointee_ty,
-                        },
+                        }),
                     };
                     self.behind_reference.set(old);
                     val
@@ -449,8 +453,8 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                     let old = self.behind_reference.replace(true);
                     let array = tcx.deref_mir_constant(self.param_env.and(cv));
                     let val = PatKind::Deref {
-                        subpattern: Pat {
-                            kind: Box::new(PatKind::Slice {
+                        subpattern: Box::new(Pat {
+                            kind: PatKind::Slice {
                                 prefix: tcx
                                     .destructure_mir_constant(param_env, array)
                                     .fields
@@ -458,11 +462,11 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                                     .map(|val| self.recur(*val, false))
                                     .collect::<Result<_, _>>()?,
                                 slice: None,
-                                suffix: vec![],
-                            }),
+                                suffix: Box::new([]),
+                            },
                             span,
                             ty: tcx.mk_slice(elem_ty),
-                        },
+                        }),
                     };
                     self.behind_reference.set(old);
                     val
@@ -596,6 +600,6 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
             );
         }
 
-        Ok(Pat { span, ty: cv.ty(), kind: Box::new(kind) })
+        Ok(Box::new(Pat { span, ty: cv.ty(), kind }))
     }
 }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 8d6f8efb6..5105f059f 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -71,9 +71,9 @@ use std::ops::RangeInclusive;
 /// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
 fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
     fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
-        if let PatKind::Or { pats } = pat.kind.as_ref() {
-            for pat in pats {
-                expand(pat, vec);
+        if let PatKind::Or { pats } = &pat.kind {
+            for pat in pats.iter() {
+                expand(&pat, vec);
             }
         } else {
             vec.push(pat)
@@ -252,10 +252,14 @@ impl IntRange {
         let kind = if lo == hi {
             PatKind::Constant { value: lo_const }
         } else {
-            PatKind::Range(PatRange { lo: lo_const, hi: hi_const, end: RangeEnd::Included })
+            PatKind::Range(Box::new(PatRange {
+                lo: lo_const,
+                hi: hi_const,
+                end: RangeEnd::Included,
+            }))
         };
 
-        Pat { ty, span: DUMMY_SP, kind: Box::new(kind) }
+        Pat { ty, span: DUMMY_SP, kind }
     }
 
     /// Lint on likely incorrect range patterns (#63987)
@@ -1297,7 +1301,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
         let ctor;
         let fields;
-        match pat.kind.as_ref() {
+        match &pat.kind {
             PatKind::AscribeUserType { subpattern, .. } => return mkpat(subpattern),
             PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
             PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
@@ -1342,9 +1346,9 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                         fields = Fields::singleton(cx, pat);
                     }
                     ty::Adt(adt, _) => {
-                        ctor = match pat.kind.as_ref() {
+                        ctor = match pat.kind {
                             PatKind::Leaf { .. } => Single,
-                            PatKind::Variant { variant_index, .. } => Variant(*variant_index),
+                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
                             _ => bug!(),
                         };
                         let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
@@ -1402,7 +1406,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     }
                 }
             }
-            &PatKind::Range(PatRange { lo, hi, end }) => {
+            &PatKind::Range(box PatRange { lo, hi, end }) => {
                 let ty = lo.ty();
                 ctor = if let Some(int_range) = IntRange::from_range(
                     cx.tcx,
@@ -1429,7 +1433,8 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     FixedLen(prefix.len() + suffix.len())
                 };
                 ctor = Slice(Slice::new(array_len, kind));
-                fields = Fields::from_iter(cx, prefix.iter().chain(suffix).map(mkpat));
+                fields =
+                    Fields::from_iter(cx, prefix.iter().chain(suffix.iter()).map(|p| mkpat(&*p)));
             }
             PatKind::Or { .. } => {
                 ctor = Or;
@@ -1442,15 +1447,15 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
 
     pub(crate) fn to_pat(&self, cx: &MatchCheckCtxt<'p, 'tcx>) -> Pat<'tcx> {
         let is_wildcard = |pat: &Pat<'_>| {
-            matches!(*pat.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild)
+            matches!(pat.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild)
         };
-        let mut subpatterns = self.iter_fields().map(|p| p.to_pat(cx));
-        let pat = match &self.ctor {
+        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_pat(cx)));
+        let kind = match &self.ctor {
             Single | Variant(_) => match self.ty.kind() {
                 ty::Tuple(..) => PatKind::Leaf {
                     subpatterns: subpatterns
                         .enumerate()
-                        .map(|(i, p)| FieldPat { field: Field::new(i), pattern: p })
+                        .map(|(i, pattern)| FieldPat { field: Field::new(i), pattern })
                         .collect(),
                 },
                 ty::Adt(adt_def, _) if adt_def.is_box() => {
@@ -1485,7 +1490,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     FixedLen(_) => PatKind::Slice {
                         prefix: subpatterns.collect(),
                         slice: None,
-                        suffix: vec![],
+                        suffix: Box::new([]),
                     },
                     VarLen(prefix, _) => {
                         let mut subpatterns = subpatterns.peekable();
@@ -1504,14 +1509,18 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                                 subpatterns.next();
                             }
                         }
-                        let suffix: Vec<_> = subpatterns.collect();
+                        let suffix: Box<[_]> = subpatterns.collect();
                         let wild = Pat::wildcard_from_ty(self.ty);
-                        PatKind::Slice { prefix, slice: Some(wild), suffix }
+                        PatKind::Slice {
+                            prefix: prefix.into_boxed_slice(),
+                            slice: Some(Box::new(wild)),
+                            suffix,
+                        }
                     }
                 }
             }
             &Str(value) => PatKind::Constant { value },
-            &FloatRange(lo, hi, end) => PatKind::Range(PatRange { lo, hi, end }),
+            &FloatRange(lo, hi, end) => PatKind::Range(Box::new(PatRange { lo, hi, end })),
             IntRange(range) => return range.to_pat(cx.tcx, self.ty),
             Wildcard | NonExhaustive => PatKind::Wild,
             Missing { .. } => bug!(
@@ -1523,7 +1532,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
             }
         };
 
-        Pat { ty: self.ty, span: DUMMY_SP, kind: Box::new(pat) }
+        Pat { ty: self.ty, span: DUMMY_SP, kind }
     }
 
     pub(super) fn is_or_pat(&self) -> bool {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index a13748a2d..0edf6a983 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -29,27 +29,27 @@ use rustc_span::{Span, Symbol};
 use std::cmp::Ordering;
 
 #[derive(Clone, Debug)]
-pub(crate) enum PatternError {
+enum PatternError {
     AssocConstInPattern(Span),
     ConstParamInPattern(Span),
     StaticInPattern(Span),
     NonConstPath(Span),
 }
 
-pub(crate) struct PatCtxt<'a, 'tcx> {
-    pub(crate) tcx: TyCtxt<'tcx>,
-    pub(crate) param_env: ty::ParamEnv<'tcx>,
-    pub(crate) typeck_results: &'a ty::TypeckResults<'tcx>,
-    pub(crate) errors: Vec<PatternError>,
+struct PatCtxt<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    typeck_results: &'a ty::TypeckResults<'tcx>,
+    errors: Vec<PatternError>,
     include_lint_checks: bool,
 }
 
-pub(crate) fn pat_from_hir<'a, 'tcx>(
+pub(super) fn pat_from_hir<'a, 'tcx>(
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     typeck_results: &'a ty::TypeckResults<'tcx>,
     pat: &'tcx hir::Pat<'tcx>,
-) -> Pat<'tcx> {
+) -> Box<Pat<'tcx>> {
     let mut pcx = PatCtxt::new(tcx, param_env, typeck_results);
     let result = pcx.lower_pattern(pat);
     if !pcx.errors.is_empty() {
@@ -61,7 +61,7 @@ pub(crate) fn pat_from_hir<'a, 'tcx>(
 }
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
-    pub(crate) fn new(
+    fn new(
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
         typeck_results: &'a ty::TypeckResults<'tcx>,
@@ -69,12 +69,12 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         PatCtxt { tcx, param_env, typeck_results, errors: vec![], include_lint_checks: false }
     }
 
-    pub(crate) fn include_lint_checks(&mut self) -> &mut Self {
+    fn include_lint_checks(&mut self) -> &mut Self {
         self.include_lint_checks = true;
         self
     }
 
-    pub(crate) fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
+    fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Box<Pat<'tcx>> {
         // When implicit dereferences have been inserted in this pattern, the unadjusted lowered
         // pattern has the type that results *after* dereferencing. For example, in this code:
         //
@@ -97,13 +97,13 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let unadjusted_pat = self.lower_pattern_unadjusted(pat);
         self.typeck_results.pat_adjustments().get(pat.hir_id).unwrap_or(&vec![]).iter().rev().fold(
             unadjusted_pat,
-            |pat, ref_ty| {
+            |pat: Box<_>, ref_ty| {
                 debug!("{:?}: wrapping pattern with type {:?}", pat, ref_ty);
-                Pat {
+                Box::new(Pat {
                     span: pat.span,
                     ty: *ref_ty,
-                    kind: Box::new(PatKind::Deref { subpattern: pat }),
-                }
+                    kind: PatKind::Deref { subpattern: pat },
+                })
             },
         )
     }
@@ -113,7 +113,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         expr: &'tcx hir::Expr<'tcx>,
     ) -> (PatKind<'tcx>, Option<Ascription<'tcx>>) {
         match self.lower_lit(expr) {
-            PatKind::AscribeUserType { ascription, subpattern: Pat { kind: box kind, .. } } => {
+            PatKind::AscribeUserType { ascription, subpattern: box Pat { kind, .. } } => {
                 (kind, Some(ascription))
             }
             kind => (kind, None),
@@ -134,7 +134,9 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         match (end, cmp) {
             // `x..y` where `x < y`.
             // Non-empty because the range includes at least `x`.
-            (RangeEnd::Excluded, Some(Ordering::Less)) => PatKind::Range(PatRange { lo, hi, end }),
+            (RangeEnd::Excluded, Some(Ordering::Less)) => {
+                PatKind::Range(Box::new(PatRange { lo, hi, end }))
+            }
             // `x..y` where `x >= y`. The range is empty => error.
             (RangeEnd::Excluded, _) => {
                 struct_span_err!(
@@ -149,7 +151,9 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             // `x..=y` where `x == y`.
             (RangeEnd::Included, Some(Ordering::Equal)) => PatKind::Constant { value: lo },
             // `x..=y` where `x < y`.
-            (RangeEnd::Included, Some(Ordering::Less)) => PatKind::Range(PatRange { lo, hi, end }),
+            (RangeEnd::Included, Some(Ordering::Less)) => {
+                PatKind::Range(Box::new(PatRange { lo, hi, end }))
+            }
             // `x..=y` where `x > y` hence the range is empty => error.
             (RangeEnd::Included, _) => {
                 let mut err = struct_span_err!(
@@ -196,8 +200,9 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         }
     }
 
-    fn lower_pattern_unadjusted(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
+    fn lower_pattern_unadjusted(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Box<Pat<'tcx>> {
         let mut ty = self.typeck_results.node_type(pat.hir_id);
+        let mut span = pat.span;
 
         let kind = match pat.kind {
             hir::PatKind::Wild => PatKind::Wild,
@@ -228,7 +233,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 // constants somewhere. Have them on the range pattern.
                 for end in &[lo, hi] {
                     if let Some((_, Some(ascription))) = end {
-                        let subpattern = Pat { span: pat.span, ty, kind: Box::new(kind) };
+                        let subpattern = Box::new(Pat { span: pat.span, ty, kind });
                         kind =
                             PatKind::AscribeUserType { ascription: ascription.clone(), subpattern };
                     }
@@ -258,6 +263,10 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             }
 
             hir::PatKind::Binding(_, id, ident, ref sub) => {
+                if let Some(ident_span) = ident.span.find_ancestor_inside(span) {
+                    span = span.with_hi(ident_span.hi());
+                }
+
                 let bm = *self
                     .typeck_results
                     .pat_binding_modes()
@@ -322,14 +331,14 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             hir::PatKind::Or(ref pats) => PatKind::Or { pats: self.lower_patterns(pats) },
         };
 
-        Pat { span: pat.span, ty, kind: Box::new(kind) }
+        Box::new(Pat { span, ty, kind })
     }
 
     fn lower_tuple_subpats(
         &mut self,
         pats: &'tcx [hir::Pat<'tcx>],
         expected_len: usize,
-        gap_pos: Option<usize>,
+        gap_pos: hir::DotDotPos,
     ) -> Vec<FieldPat<'tcx>> {
         pats.iter()
             .enumerate_and_adjust(expected_len, gap_pos)
@@ -340,11 +349,14 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             .collect()
     }
 
-    fn lower_patterns(&mut self, pats: &'tcx [hir::Pat<'tcx>]) -> Vec<Pat<'tcx>> {
+    fn lower_patterns(&mut self, pats: &'tcx [hir::Pat<'tcx>]) -> Box<[Box<Pat<'tcx>>]> {
         pats.iter().map(|p| self.lower_pattern(p)).collect()
     }
 
-    fn lower_opt_pattern(&mut self, pat: &'tcx Option<&'tcx hir::Pat<'tcx>>) -> Option<Pat<'tcx>> {
+    fn lower_opt_pattern(
+        &mut self,
+        pat: &'tcx Option<&'tcx hir::Pat<'tcx>>,
+    ) -> Option<Box<Pat<'tcx>>> {
         pat.as_ref().map(|p| self.lower_pattern(p))
     }
 
@@ -436,12 +448,12 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         if let Some(user_ty) = self.user_substs_applied_to_ty_of_hir_id(hir_id) {
             debug!("lower_variant_or_leaf: kind={:?} user_ty={:?} span={:?}", kind, user_ty, span);
             let annotation = CanonicalUserTypeAnnotation {
-                user_ty,
+                user_ty: Box::new(user_ty),
                 span,
                 inferred_ty: self.typeck_results.node_type(hir_id),
             };
             kind = PatKind::AscribeUserType {
-                subpattern: Pat { span, ty, kind: Box::new(kind) },
+                subpattern: Box::new(Pat { span, ty, kind }),
                 ascription: Ascription { annotation, variance: ty::Variance::Covariant },
             };
         }
@@ -453,11 +465,11 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     /// it to `const_to_pat`. Any other path (like enum variants without fields)
     /// is converted to the corresponding pattern via `lower_variant_or_leaf`.
     #[instrument(skip(self), level = "debug")]
-    fn lower_path(&mut self, qpath: &hir::QPath<'_>, id: hir::HirId, span: Span) -> Pat<'tcx> {
+    fn lower_path(&mut self, qpath: &hir::QPath<'_>, id: hir::HirId, span: Span) -> Box<Pat<'tcx>> {
         let ty = self.typeck_results.node_type(id);
         let res = self.typeck_results.qpath_res(qpath, id);
 
-        let pat_from_kind = |kind| Pat { span, ty, kind: Box::new(kind) };
+        let pat_from_kind = |kind| Box::new(Pat { span, ty, kind });
 
         let (def_id, is_associated_const) = match res {
             Res::Def(DefKind::Const, def_id) => (def_id, false),
@@ -469,7 +481,12 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         // Use `Reveal::All` here because patterns are always monomorphic even if their function
         // isn't.
         let param_env_reveal_all = self.param_env.with_reveal_all_normalized(self.tcx);
-        let substs = self.typeck_results.node_substs(id);
+        // N.B. There is no guarantee that substs collected in typeck results are fully normalized,
+        // so they need to be normalized in order to pass to `Instance::resolve`, which will ICE
+        // if given unnormalized types.
+        let substs = self
+            .tcx
+            .normalize_erasing_regions(param_env_reveal_all, self.typeck_results.node_substs(id));
         let instance = match ty::Instance::resolve(self.tcx, param_env_reveal_all, def_id, substs) {
             Ok(Some(i)) => i,
             Ok(None) => {
@@ -505,13 +522,13 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 let user_provided_types = self.typeck_results().user_provided_types();
                 if let Some(&user_ty) = user_provided_types.get(id) {
                     let annotation = CanonicalUserTypeAnnotation {
-                        user_ty,
+                        user_ty: Box::new(user_ty),
                         span,
                         inferred_ty: self.typeck_results().node_type(id),
                     };
-                    Pat {
+                    Box::new(Pat {
                         span,
-                        kind: Box::new(PatKind::AscribeUserType {
+                        kind: PatKind::AscribeUserType {
                             subpattern: pattern,
                             ascription: Ascription {
                                 annotation,
@@ -519,9 +536,9 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                                 /// `variance` field documentation for details.
                                 variance: ty::Variance::Contravariant,
                             },
-                        }),
+                        },
                         ty: const_.ty(),
-                    }
+                    })
                 } else {
                     pattern
                 }
@@ -553,23 +570,19 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let value = value.eval(self.tcx, self.param_env);
 
         match value {
-            mir::ConstantKind::Ty(c) => {
-                match c.kind() {
-                    ConstKind::Param(_) => {
-                        self.errors.push(PatternError::ConstParamInPattern(span));
-                        return PatKind::Wild;
-                    }
-                    ConstKind::Unevaluated(_) => {
-                        // If we land here it means the const can't be evaluated because it's `TooGeneric`.
-                        self.tcx
-                            .sess
-                            .span_err(span, "constant pattern depends on a generic parameter");
-                        return PatKind::Wild;
-                    }
-                    _ => bug!("Expected either ConstKind::Param or ConstKind::Unevaluated"),
+            mir::ConstantKind::Ty(c) => match c.kind() {
+                ConstKind::Param(_) => {
+                    self.errors.push(PatternError::ConstParamInPattern(span));
+                    return PatKind::Wild;
                 }
+                _ => bug!("Expected ConstKind::Param"),
+            },
+            mir::ConstantKind::Val(_, _) => self.const_to_pat(value, id, span, false).kind,
+            mir::ConstantKind::Unevaluated(..) => {
+                // If we land here it means the const can't be evaluated because it's `TooGeneric`.
+                self.tcx.sess.span_err(span, "constant pattern depends on a generic parameter");
+                return PatKind::Wild;
             }
-            mir::ConstantKind::Val(_, _) => *self.const_to_pat(value, id, span, false).kind,
         }
     }
 
@@ -580,7 +593,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     fn lower_lit(&mut self, expr: &'tcx hir::Expr<'tcx>) -> PatKind<'tcx> {
         let (lit, neg) = match expr.kind {
             hir::ExprKind::Path(ref qpath) => {
-                return *self.lower_path(qpath, expr.hir_id, expr.span).kind;
+                return self.lower_path(qpath, expr.hir_id, expr.span).kind;
             }
             hir::ExprKind::ConstBlock(ref anon_const) => {
                 return self.lower_inline_const(anon_const, expr.hir_id, expr.span);
@@ -598,7 +611,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let lit_input =
             LitToConstInput { lit: &lit.node, ty: self.typeck_results.expr_ty(expr), neg };
         match self.tcx.at(expr.span).lit_to_mir_constant(lit_input) {
-            Ok(constant) => *self.const_to_pat(constant, expr.hir_id, lit.span, false).kind,
+            Ok(constant) => self.const_to_pat(constant, expr.hir_id, lit.span, false).kind,
             Err(LitToConstError::Reported) => PatKind::Wild,
             Err(LitToConstError::TypeError) => bug!("lower_lit: had type error"),
         }
@@ -615,7 +628,7 @@ impl<'tcx> UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
     }
 }
 
-pub(crate) trait PatternFoldable<'tcx>: Sized {
+trait PatternFoldable<'tcx>: Sized {
     fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
         self.super_fold_with(folder)
     }
@@ -623,7 +636,7 @@ pub(crate) trait PatternFoldable<'tcx>: Sized {
     fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self;
 }
 
-pub(crate) trait PatternFolder<'tcx>: Sized {
+trait PatternFolder<'tcx>: Sized {
     fn fold_pattern(&mut self, pattern: &Pat<'tcx>) -> Pat<'tcx> {
         pattern.super_fold_with(self)
     }
@@ -646,6 +659,12 @@ impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Vec<T> {
     }
 }
 
+impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Box<[T]> {
+    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        self.iter().map(|t| t.fold_with(folder)).collect()
+    }
+}
+
 impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Option<T> {
     fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
         self.as_ref().map(|t| t.fold_with(folder))
@@ -732,7 +751,7 @@ impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
                 PatKind::Deref { subpattern: subpattern.fold_with(folder) }
             }
             PatKind::Constant { value } => PatKind::Constant { value },
-            PatKind::Range(range) => PatKind::Range(range),
+            PatKind::Range(ref range) => PatKind::Range(range.clone()),
             PatKind::Slice { ref prefix, ref slice, ref suffix } => PatKind::Slice {
                 prefix: prefix.fold_with(folder),
                 slice: slice.fold_with(folder),
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 0a660ef30..115d34ff8 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -364,8 +364,8 @@ impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
 /// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]`
 /// works well.
 #[derive(Clone)]
-struct PatStack<'p, 'tcx> {
-    pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
+pub(crate) struct PatStack<'p, 'tcx> {
+    pub(crate) pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
 }
 
 impl<'p, 'tcx> PatStack<'p, 'tcx> {
@@ -403,6 +403,21 @@ impl<'p, 'tcx> PatStack<'p, 'tcx> {
         })
     }
 
+    // Recursively expand all patterns into their subpatterns and push each `PatStack` to matrix.
+    fn expand_and_extend<'a>(&'a self, matrix: &mut Matrix<'p, 'tcx>) {
+        if !self.is_empty() && self.head().is_or_pat() {
+            for pat in self.head().iter_fields() {
+                let mut new_patstack = PatStack::from_pattern(pat);
+                new_patstack.pats.extend_from_slice(&self.pats[1..]);
+                if !new_patstack.is_empty() && new_patstack.head().is_or_pat() {
+                    new_patstack.expand_and_extend(matrix);
+                } else if !new_patstack.is_empty() {
+                    matrix.push(new_patstack);
+                }
+            }
+        }
+    }
+
     /// This computes `S(self.head().ctor(), self)`. See top of the file for explanations.
     ///
     /// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing
@@ -436,7 +451,7 @@ impl<'p, 'tcx> fmt::Debug for PatStack<'p, 'tcx> {
 /// A 2D matrix.
 #[derive(Clone)]
 pub(super) struct Matrix<'p, 'tcx> {
-    patterns: Vec<PatStack<'p, 'tcx>>,
+    pub patterns: Vec<PatStack<'p, 'tcx>>,
 }
 
 impl<'p, 'tcx> Matrix<'p, 'tcx> {
@@ -453,7 +468,7 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> {
     /// expands it.
     fn push(&mut self, row: PatStack<'p, 'tcx>) {
         if !row.is_empty() && row.head().is_or_pat() {
-            self.patterns.extend(row.expand_or_pat());
+            row.expand_and_extend(self);
         } else {
             self.patterns.push(row);
         }
@@ -739,8 +754,8 @@ fn lint_non_exhaustive_omitted_patterns<'p, 'tcx>(
     hir_id: HirId,
     witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
 ) {
-    let joined_patterns = joined_uncovered_patterns(cx, &witnesses);
     cx.tcx.struct_span_lint_hir(NON_EXHAUSTIVE_OMITTED_PATTERNS, hir_id, sp, |build| {
+        let joined_patterns = joined_uncovered_patterns(cx, &witnesses);
         let mut lint = build.build("some variants are not matched explicitly");
         lint.span_label(sp, pattern_not_covered_label(&witnesses, &joined_patterns));
         lint.help(
@@ -776,7 +791,7 @@ fn lint_non_exhaustive_omitted_patterns<'p, 'tcx>(
 /// `is_under_guard` is used to inform if the pattern has a guard. If it
 /// has one it must not be inserted into the matrix. This shouldn't be
 /// relied on for soundness.
-#[instrument(level = "debug", skip(cx, matrix, hir_id))]
+#[instrument(level = "debug", skip(cx, matrix, hir_id), ret)]
 fn is_useful<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
     matrix: &Matrix<'p, 'tcx>,
@@ -902,7 +917,6 @@ fn is_useful<'p, 'tcx>(
         v.head().set_reachable();
     }
 
-    debug!(?ret);
     ret
 }