16 files changed, 955 insertions, 344 deletions

diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index 183db56d7..49d7136a2 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -1,4 +1,3 @@
-use crate::build::matches::ArmHasGuard;
 use crate::build::ForGuard::OutsideGuard;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
 use rustc_middle::middle::region::Scope;
@@ -118,7 +117,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     else_block: Some(else_block),
                 } => {
                     // When lowering the statement `let <pat> = <expr> else { <else> };`,
-                    // the `<else>` block is nested in the parent scope enclosing this statment.
+                    // the `<else>` block is nested in the parent scope enclosing this statement.
                     // That scope is usually either the enclosing block scope,
                     // or the remainder scope of the last statement.
                     // This is to make sure that temporaries instantiated in `<expr>` are dropped
@@ -231,7 +230,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                         visibility_scope,
                                         remainder_span,
                                         pattern,
-                                        ArmHasGuard(false),
+                                        None,
                                         Some((None, initializer_span)),
                                     );
                                     this.visit_primary_bindings(
@@ -308,7 +307,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                             visibility_scope,
                                             remainder_span,
                                             pattern,
-                                            ArmHasGuard(false),
+                                            None,
                                             Some((None, initializer_span)),
                                         );
                                         this.expr_into_pattern(block, &pattern, init)
@@ -324,7 +323,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 visibility_scope,
                                 remainder_span,
                                 pattern,
-                                ArmHasGuard(false),
+                                None,
                                 None,
                             );
                             block.unit()
diff --git a/compiler/rustc_mir_build/src/build/custom/mod.rs b/compiler/rustc_mir_build/src/build/custom/mod.rs
new file mode 100644
index 000000000..eb021f477
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/custom/mod.rs
@@ -0,0 +1,155 @@
+//! Provides the implementation of the `custom_mir` attribute.
+//!
+//! Up until MIR building, this attribute has absolutely no effect. The `mir!` macro is a normal
+//! decl macro that expands like any other, and the code goes through parsing, name resolution and
+//! type checking like all other code. In MIR building we finally detect whether this attribute is
+//! present, and if so we branch off into this module, which implements the attribute by
+//! implementing a custom lowering from THIR to MIR.
+//!
+//! The result of this lowering is returned "normally" from the `mir_built` query, with the only
+//! notable difference being that the `injected` field in the body is set. Various components of the
+//! MIR pipeline, like borrowck and the pass manager will then consult this field (via
+//! `body.should_skip()`) to skip the parts of the MIR pipeline that precede the MIR phase the user
+//! specified.
+//!
+//! This file defines the general framework for the custom parsing. The parsing for all the
+//! "top-level" constructs can be found in the `parse` submodule, while the parsing for statements,
+//! terminators, and everything below can be found in the `parse::instruction` submodule.
+//!
+
+use rustc_ast::Attribute;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def_id::DefId;
+use rustc_index::vec::IndexVec;
+use rustc_middle::{
+    mir::*,
+    thir::*,
+    ty::{Ty, TyCtxt},
+};
+use rustc_span::Span;
+
+mod parse;
+
+pub(super) fn build_custom_mir<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    did: DefId,
+    thir: &Thir<'tcx>,
+    expr: ExprId,
+    params: &IndexVec<ParamId, Param<'tcx>>,
+    return_ty: Ty<'tcx>,
+    return_ty_span: Span,
+    span: Span,
+    attr: &Attribute,
+) -> Body<'tcx> {
+    let mut body = Body {
+        basic_blocks: BasicBlocks::new(IndexVec::new()),
+        source: MirSource::item(did),
+        phase: MirPhase::Built,
+        source_scopes: IndexVec::new(),
+        generator: None,
+        local_decls: LocalDecls::new(),
+        user_type_annotations: IndexVec::new(),
+        arg_count: params.len(),
+        spread_arg: None,
+        var_debug_info: Vec::new(),
+        span,
+        required_consts: Vec::new(),
+        is_polymorphic: false,
+        tainted_by_errors: None,
+        injection_phase: None,
+        pass_count: 0,
+    };
+
+    body.local_decls.push(LocalDecl::new(return_ty, return_ty_span));
+    body.basic_blocks_mut().push(BasicBlockData::new(None));
+    body.source_scopes.push(SourceScopeData {
+        span,
+        parent_scope: None,
+        inlined: None,
+        inlined_parent_scope: None,
+        local_data: ClearCrossCrate::Clear,
+    });
+    body.injection_phase = Some(parse_attribute(attr));
+
+    let mut pctxt = ParseCtxt {
+        tcx,
+        thir,
+        source_scope: OUTERMOST_SOURCE_SCOPE,
+        body: &mut body,
+        local_map: FxHashMap::default(),
+        block_map: FxHashMap::default(),
+    };
+
+    let res = (|| {
+        pctxt.parse_args(&params)?;
+        pctxt.parse_body(expr)
+    })();
+    if let Err(err) = res {
+        tcx.sess.diagnostic().span_fatal(
+            err.span,
+            format!("Could not parse {}, found: {:?}", err.expected, err.item_description),
+        )
+    }
+
+    body
+}
+
+fn parse_attribute(attr: &Attribute) -> MirPhase {
+    let meta_items = attr.meta_item_list().unwrap();
+    let mut dialect: Option<String> = None;
+    let mut phase: Option<String> = None;
+
+    for nested in meta_items {
+        let name = nested.name_or_empty();
+        let value = nested.value_str().unwrap().as_str().to_string();
+        match name.as_str() {
+            "dialect" => {
+                assert!(dialect.is_none());
+                dialect = Some(value);
+            }
+            "phase" => {
+                assert!(phase.is_none());
+                phase = Some(value);
+            }
+            other => {
+                panic!("Unexpected key {}", other);
+            }
+        }
+    }
+
+    let Some(dialect) = dialect else {
+        assert!(phase.is_none());
+        return MirPhase::Built;
+    };
+
+    MirPhase::parse(dialect, phase)
+}
+
+struct ParseCtxt<'tcx, 'body> {
+    tcx: TyCtxt<'tcx>,
+    thir: &'body Thir<'tcx>,
+    source_scope: SourceScope,
+
+    body: &'body mut Body<'tcx>,
+    local_map: FxHashMap<LocalVarId, Local>,
+    block_map: FxHashMap<LocalVarId, BasicBlock>,
+}
+
+struct ParseError {
+    span: Span,
+    item_description: String,
+    expected: String,
+}
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    fn expr_error(&self, expr: ExprId, expected: &'static str) -> ParseError {
+        let expr = &self.thir[expr];
+        ParseError {
+            span: expr.span,
+            item_description: format!("{:?}", expr.kind),
+            expected: expected.to_string(),
+        }
+    }
+}
+
+type PResult<T> = Result<T, ParseError>;
diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs
new file mode 100644
index 000000000..d72770e70
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/custom/parse.rs
@@ -0,0 +1,256 @@
+use rustc_index::vec::IndexVec;
+use rustc_middle::{mir::*, thir::*, ty::Ty};
+use rustc_span::Span;
+
+use super::{PResult, ParseCtxt, ParseError};
+
+mod instruction;
+
+/// Helper macro for parsing custom MIR.
+///
+/// Example usage looks something like:
+/// ```rust,ignore (incomplete example)
+/// parse_by_kind!(
+///     self, // : &ParseCtxt
+///     expr_id, // what you're matching against
+///     "assignment", // the thing you're trying to parse
+///     @call("mir_assign", args) => { args[0] }, // match invocations of the `mir_assign` special function
+///     ExprKind::Assign { lhs, .. } => { lhs }, // match thir assignment expressions
+///     // no need for fallthrough case - reasonable error is automatically generated
+/// )
+/// ```
+macro_rules! parse_by_kind {
+    (
+        $self:ident,
+        $expr_id:expr,
+        $expr_name:pat,
+        $expected:literal,
+        $(
+            @call($name:literal, $args:ident) => $call_expr:expr,
+        )*
+        $(
+            $pat:pat => $expr:expr,
+        )*
+    ) => {{
+        let expr_id = $self.preparse($expr_id);
+        let expr = &$self.thir[expr_id];
+        debug!("Trying to parse {:?} as {}", expr.kind, $expected);
+        let $expr_name = expr;
+        match &expr.kind {
+            $(
+                ExprKind::Call { ty, fun: _, args: $args, .. } if {
+                    match ty.kind() {
+                        ty::FnDef(did, _) => {
+                            $self.tcx.is_diagnostic_item(rustc_span::Symbol::intern($name), *did)
+                        }
+                        _ => false,
+                    }
+                } => $call_expr,
+            )*
+            $(
+                $pat => $expr,
+            )*
+            #[allow(unreachable_patterns)]
+            _ => return Err($self.expr_error(expr_id, $expected))
+        }
+    }};
+}
+pub(crate) use parse_by_kind;
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    /// Expressions should only ever be matched on after preparsing them. This removes extra scopes
+    /// we don't care about.
+    fn preparse(&self, expr_id: ExprId) -> ExprId {
+        let expr = &self.thir[expr_id];
+        match expr.kind {
+            ExprKind::Scope { value, .. } => self.preparse(value),
+            _ => expr_id,
+        }
+    }
+
+    fn statement_as_expr(&self, stmt_id: StmtId) -> PResult<ExprId> {
+        match &self.thir[stmt_id].kind {
+            StmtKind::Expr { expr, .. } => Ok(*expr),
+            kind @ StmtKind::Let { pattern, .. } => {
+                return Err(ParseError {
+                    span: pattern.span,
+                    item_description: format!("{:?}", kind),
+                    expected: "expression".to_string(),
+                });
+            }
+        }
+    }
+
+    pub fn parse_args(&mut self, params: &IndexVec<ParamId, Param<'tcx>>) -> PResult<()> {
+        for param in params.iter() {
+            let (var, span) = {
+                let pat = param.pat.as_ref().unwrap();
+                match &pat.kind {
+                    PatKind::Binding { var, .. } => (*var, pat.span),
+                    _ => {
+                        return Err(ParseError {
+                            span: pat.span,
+                            item_description: format!("{:?}", pat.kind),
+                            expected: "local".to_string(),
+                        });
+                    }
+                }
+            };
+            let decl = LocalDecl::new(param.ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    /// Bodies are of the form:
+    ///
+    /// ```text
+    /// {
+    ///     let bb1: BasicBlock;
+    ///     let bb2: BasicBlock;
+    ///     {
+    ///         let RET: _;
+    ///         let local1;
+    ///         let local2;
+    ///
+    ///         {
+    ///             { // entry block
+    ///                 statement1;
+    ///                 terminator1
+    ///             };
+    ///
+    ///             bb1 = {
+    ///                 statement2;
+    ///                 terminator2
+    ///             };
+    ///
+    ///             bb2 = {
+    ///                 statement3;
+    ///                 terminator3
+    ///             }
+    ///
+    ///             RET
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// This allows us to easily parse the basic blocks declarations, local declarations, and
+    /// basic block definitions in order.
+    pub fn parse_body(&mut self, expr_id: ExprId) -> PResult<()> {
+        let body = parse_by_kind!(self, expr_id, _, "whole body",
+            ExprKind::Block { block } => self.thir[*block].expr.unwrap(),
+        );
+        let (block_decls, rest) = parse_by_kind!(self, body, _, "body with block decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_block_decls(block_decls.iter().copied())?;
+
+        let (local_decls, rest) = parse_by_kind!(self, rest, _, "body with local decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_local_decls(local_decls.iter().copied())?;
+
+        let block_defs = parse_by_kind!(self, rest, _, "body with block defs",
+            ExprKind::Block { block } => &self.thir[*block].stmts,
+        );
+        for (i, block_def) in block_defs.iter().enumerate() {
+            let block = self.parse_block_def(self.statement_as_expr(*block_def)?)?;
+            self.body.basic_blocks_mut()[BasicBlock::from_usize(i)] = block;
+        }
+
+        Ok(())
+    }
+
+    fn parse_block_decls(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            let (var, _, _) = self.parse_let_statement(stmt)?;
+            let data = BasicBlockData::new(None);
+            let block = self.body.basic_blocks_mut().push(data);
+            self.block_map.insert(var, block);
+        }
+
+        Ok(())
+    }
+
+    fn parse_local_decls(&mut self, mut stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        let (ret_var, ..) = self.parse_let_statement(stmts.next().unwrap())?;
+        self.local_map.insert(ret_var, Local::from_u32(0));
+
+        for stmt in stmts {
+            let (var, ty, span) = self.parse_let_statement(stmt)?;
+            let decl = LocalDecl::new(ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        let pattern = match &self.thir[stmt_id].kind {
+            StmtKind::Let { pattern, .. } => pattern,
+            StmtKind::Expr { expr, .. } => {
+                return Err(self.expr_error(*expr, "let statement"));
+            }
+        };
+
+        self.parse_var(pattern)
+    }
+
+    fn parse_var(&mut self, mut pat: &Pat<'tcx>) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        // Make sure we throw out any `AscribeUserType` we find
+        loop {
+            match &pat.kind {
+                PatKind::Binding { var, ty, .. } => break Ok((*var, *ty, pat.span)),
+                PatKind::AscribeUserType { subpattern, .. } => {
+                    pat = subpattern;
+                }
+                _ => {
+                    break Err(ParseError {
+                        span: pat.span,
+                        item_description: format!("{:?}", pat.kind),
+                        expected: "local".to_string(),
+                    });
+                }
+            }
+        }
+    }
+
+    fn parse_block_def(&self, expr_id: ExprId) -> PResult<BasicBlockData<'tcx>> {
+        let block = parse_by_kind!(self, expr_id, _, "basic block",
+            ExprKind::Block { block } => &self.thir[*block],
+        );
+
+        let mut data = BasicBlockData::new(None);
+        for stmt_id in &*block.stmts {
+            let stmt = self.statement_as_expr(*stmt_id)?;
+            let span = self.thir[stmt].span;
+            let statement = self.parse_statement(stmt)?;
+            data.statements.push(Statement {
+                source_info: SourceInfo { span, scope: self.source_scope },
+                kind: statement,
+            });
+        }
+
+        let Some(trailing) = block.expr else {
+            return Err(self.expr_error(expr_id, "terminator"))
+        };
+        let span = self.thir[trailing].span;
+        let terminator = self.parse_terminator(trailing)?;
+        data.terminator = Some(Terminator {
+            source_info: SourceInfo { span, scope: self.source_scope },
+            kind: terminator,
+        });
+
+        Ok(data)
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
new file mode 100644
index 000000000..03206af33
--- /dev/null
+++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
@@ -0,0 +1,105 @@
+use rustc_middle::mir::interpret::{ConstValue, Scalar};
+use rustc_middle::{mir::*, thir::*, ty};
+
+use super::{parse_by_kind, PResult, ParseCtxt};
+
+impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
+    pub fn parse_statement(&self, expr_id: ExprId) -> PResult<StatementKind<'tcx>> {
+        parse_by_kind!(self, expr_id, _, "statement",
+            @call("mir_retag", args) => {
+                Ok(StatementKind::Retag(RetagKind::Default, Box::new(self.parse_place(args[0])?)))
+            },
+            @call("mir_retag_raw", args) => {
+                Ok(StatementKind::Retag(RetagKind::Raw, Box::new(self.parse_place(args[0])?)))
+            },
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs = self.parse_place(*lhs)?;
+                let rhs = self.parse_rvalue(*rhs)?;
+                Ok(StatementKind::Assign(Box::new((lhs, rhs))))
+            },
+        )
+    }
+
+    pub fn parse_terminator(&self, expr_id: ExprId) -> PResult<TerminatorKind<'tcx>> {
+        parse_by_kind!(self, expr_id, _, "terminator",
+            @call("mir_return", _args) => {
+                Ok(TerminatorKind::Return)
+            },
+            @call("mir_goto", args) => {
+                Ok(TerminatorKind::Goto { target: self.parse_block(args[0])? } )
+            },
+        )
+    }
+
+    fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
+        parse_by_kind!(self, expr_id, _, "rvalue",
+            ExprKind::Borrow { borrow_kind, arg } => Ok(
+                Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
+            ),
+            ExprKind::AddressOf { mutability, arg } => Ok(
+                Rvalue::AddressOf(*mutability, self.parse_place(*arg)?)
+            ),
+            _ => self.parse_operand(expr_id).map(Rvalue::Use),
+        )
+    }
+
+    fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+        parse_by_kind!(self, expr_id, expr, "operand",
+            @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move),
+            @call("mir_static", args) => self.parse_static(args[0]),
+            @call("mir_static_mut", args) => self.parse_static(args[0]),
+            ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. } => {
+                Ok(Operand::Constant(Box::new(
+                    crate::build::expr::as_constant::as_constant_inner(expr, |_| None, self.tcx)
+                )))
+            },
+            _ => self.parse_place(expr_id).map(Operand::Copy),
+        )
+    }
+
+    fn parse_place(&self, expr_id: ExprId) -> PResult<Place<'tcx>> {
+        parse_by_kind!(self, expr_id, _, "place",
+            ExprKind::Deref { arg } => Ok(
+                self.parse_place(*arg)?.project_deeper(&[PlaceElem::Deref], self.tcx)
+            ),
+            _ => self.parse_local(expr_id).map(Place::from),
+        )
+    }
+
+    fn parse_local(&self, expr_id: ExprId) -> PResult<Local> {
+        parse_by_kind!(self, expr_id, _, "local",
+            ExprKind::VarRef { id } => Ok(self.local_map[id]),
+        )
+    }
+
+    fn parse_block(&self, expr_id: ExprId) -> PResult<BasicBlock> {
+        parse_by_kind!(self, expr_id, _, "basic block",
+            ExprKind::VarRef { id } => Ok(self.block_map[id]),
+        )
+    }
+
+    fn parse_static(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+        let expr_id = parse_by_kind!(self, expr_id, _, "static",
+            ExprKind::Deref { arg } => *arg,
+        );
+
+        parse_by_kind!(self, expr_id, expr, "static",
+            ExprKind::StaticRef { alloc_id, ty, .. } => {
+                let const_val =
+                    ConstValue::Scalar(Scalar::from_pointer((*alloc_id).into(), &self.tcx));
+                let literal = ConstantKind::Val(const_val, *ty);
+
+                Ok(Operand::Constant(Box::new(Constant {
+                    span: expr.span,
+                    user_ty: None,
+                    literal
+                })))
+            },
+        )
+    }
+}
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 37dc1ad9f..717c62315 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -8,7 +8,10 @@ use rustc_middle::mir::interpret::{
 };
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
-use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, TyCtxt};
+use rustc_middle::ty::{
+    self, CanonicalUserType, CanonicalUserTypeAnnotation, TyCtxt, UserTypeAnnotationIndex,
+};
+use rustc_span::DUMMY_SP;
 use rustc_target::abi::Size;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
@@ -18,83 +21,87 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let this = self;
         let tcx = this.tcx;
         let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
-        match *kind {
+        match kind {
             ExprKind::Scope { region_scope: _, lint_level: _, value } => {
-                this.as_constant(&this.thir[value])
-            }
-            ExprKind::Literal { lit, neg } => {
-                let literal =
-                    match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg }) {
-                        Ok(c) => c,
-                        Err(LitToConstError::Reported) => ConstantKind::Ty(tcx.const_error(ty)),
-                        Err(LitToConstError::TypeError) => {
-                            bug!("encountered type error in `lit_to_mir_constant")
-                        }
-                    };
-
-                Constant { span, user_ty: None, literal }
+                this.as_constant(&this.thir[*value])
             }
-            ExprKind::NonHirLiteral { lit, ref user_ty } => {
-                let user_ty = user_ty.as_ref().map(|user_ty| {
-                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+            _ => as_constant_inner(
+                expr,
+                |user_ty| {
+                    Some(this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
                         span,
                         user_ty: user_ty.clone(),
                         inferred_ty: ty,
-                    })
-                });
-                let literal = ConstantKind::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
+                    }))
+                },
+                tcx,
+            ),
+        }
+    }
+}
 
-                Constant { span, user_ty: user_ty, literal }
-            }
-            ExprKind::ZstLiteral { ref user_ty } => {
-                let user_ty = user_ty.as_ref().map(|user_ty| {
-                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
-                        span,
-                        user_ty: user_ty.clone(),
-                        inferred_ty: ty,
-                    })
-                });
-                let literal = ConstantKind::Val(ConstValue::ZeroSized, ty);
+pub fn as_constant_inner<'tcx>(
+    expr: &Expr<'tcx>,
+    push_cuta: impl FnMut(&Box<CanonicalUserType<'tcx>>) -> Option<UserTypeAnnotationIndex>,
+    tcx: TyCtxt<'tcx>,
+) -> Constant<'tcx> {
+    let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
+    match *kind {
+        ExprKind::Literal { lit, neg } => {
+            let literal =
+                match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg }) {
+                    Ok(c) => c,
+                    Err(LitToConstError::Reported(guar)) => {
+                        ConstantKind::Ty(tcx.const_error_with_guaranteed(ty, guar))
+                    }
+                    Err(LitToConstError::TypeError) => {
+                        bug!("encountered type error in `lit_to_mir_constant")
+                    }
+                };
 
-                Constant { span, user_ty: user_ty, literal }
-            }
-            ExprKind::NamedConst { def_id, substs, ref user_ty } => {
-                let user_ty = user_ty.as_ref().map(|user_ty| {
-                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
-                        span,
-                        user_ty: user_ty.clone(),
-                        inferred_ty: ty,
-                    })
-                });
+            Constant { span, user_ty: None, literal }
+        }
+        ExprKind::NonHirLiteral { lit, ref user_ty } => {
+            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
 
-                let uneval =
-                    mir::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
-                let literal = ConstantKind::Unevaluated(uneval, ty);
+            let literal = ConstantKind::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
 
-                Constant { user_ty, span, literal }
-            }
-            ExprKind::ConstParam { param, def_id: _ } => {
-                let const_param =
-                    tcx.mk_const(ty::ConstS { kind: ty::ConstKind::Param(param), ty: expr.ty });
-                let literal = ConstantKind::Ty(const_param);
+            Constant { span, user_ty: user_ty, literal }
+        }
+        ExprKind::ZstLiteral { ref user_ty } => {
+            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
 
-                Constant { user_ty: None, span, literal }
-            }
-            ExprKind::ConstBlock { did: def_id, substs } => {
-                let uneval =
-                    mir::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
-                let literal = ConstantKind::Unevaluated(uneval, ty);
+            let literal = ConstantKind::Val(ConstValue::ZeroSized, ty);
 
-                Constant { user_ty: None, span, literal }
-            }
-            ExprKind::StaticRef { alloc_id, ty, .. } => {
-                let const_val = ConstValue::Scalar(Scalar::from_pointer(alloc_id.into(), &tcx));
-                let literal = ConstantKind::Val(const_val, ty);
+            Constant { span, user_ty: user_ty, literal }
+        }
+        ExprKind::NamedConst { def_id, substs, ref user_ty } => {
+            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
 
-                Constant { span, user_ty: None, literal }
-            }
-            _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
+            let uneval = mir::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
+            let literal = ConstantKind::Unevaluated(uneval, ty);
+
+            Constant { user_ty, span, literal }
         }
+        ExprKind::ConstParam { param, def_id: _ } => {
+            let const_param = tcx.mk_const(ty::ConstKind::Param(param), expr.ty);
+            let literal = ConstantKind::Ty(const_param);
+
+            Constant { user_ty: None, span, literal }
+        }
+        ExprKind::ConstBlock { did: def_id, substs } => {
+            let uneval = mir::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
+            let literal = ConstantKind::Unevaluated(uneval, ty);
+
+            Constant { user_ty: None, span, literal }
+        }
+        ExprKind::StaticRef { alloc_id, ty, .. } => {
+            let const_val = ConstValue::Scalar(Scalar::from_pointer(alloc_id.into(), &tcx));
+            let literal = ConstantKind::Val(const_val, ty);
+
+            Constant { span, user_ty: None, literal }
+        }
+        _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
     }
 }
 
@@ -106,7 +113,15 @@ pub(crate) fn lit_to_mir_constant<'tcx>(
     let LitToConstInput { lit, ty, neg } = lit_input;
     let trunc = |n| {
         let param_ty = ty::ParamEnv::reveal_all().and(ty);
-        let width = tcx.layout_of(param_ty).map_err(|_| LitToConstError::Reported)?.size;
+        let width = tcx
+            .layout_of(param_ty)
+            .map_err(|_| {
+                LitToConstError::Reported(tcx.sess.delay_span_bug(
+                    DUMMY_SP,
+                    format!("couldn't compute width of literal: {:?}", lit_input.lit),
+                ))
+            })?
+            .size;
         trace!("trunc {} with size {} and shift {}", n, width.bits(), 128 - width.bits());
         let result = width.truncate(n);
         trace!("trunc result: {}", result);
@@ -137,12 +152,20 @@ pub(crate) fn lit_to_mir_constant<'tcx>(
         (ast::LitKind::Int(n, _), ty::Uint(_)) | (ast::LitKind::Int(n, _), ty::Int(_)) => {
             trunc(if neg { (*n as i128).overflowing_neg().0 as u128 } else { *n })?
         }
-        (ast::LitKind::Float(n, _), ty::Float(fty)) => {
-            parse_float_into_constval(*n, *fty, neg).ok_or(LitToConstError::Reported)?
-        }
+        (ast::LitKind::Float(n, _), ty::Float(fty)) => parse_float_into_constval(*n, *fty, neg)
+            .ok_or_else(|| {
+                LitToConstError::Reported(tcx.sess.delay_span_bug(
+                    DUMMY_SP,
+                    format!("couldn't parse float literal: {:?}", lit_input.lit),
+                ))
+            })?,
         (ast::LitKind::Bool(b), ty::Bool) => ConstValue::Scalar(Scalar::from_bool(*b)),
         (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
-        (ast::LitKind::Err, _) => return Err(LitToConstError::Reported),
+        (ast::LitKind::Err, _) => {
+            return Err(LitToConstError::Reported(
+                tcx.sess.delay_span_bug(DUMMY_SP, "encountered LitKind::Err during mir build"),
+            ));
+        }
         _ => return Err(LitToConstError::TypeError),
     };
 
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 396782d45..edd527286 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -65,7 +65,7 @@ pub(crate) enum PlaceBase {
 
 /// `PlaceBuilder` is used to create places during MIR construction. It allows you to "build up" a
 /// place by pushing more and more projections onto the end, and then convert the final set into a
-/// place using the `into_place` method.
+/// place using the `to_place` method.
 ///
 /// This is used internally when building a place for an expression like `a.b.c`. The fields `b`
 /// and `c` can be progressively pushed onto the place builder that is created when converting `a`.
@@ -167,59 +167,54 @@ fn find_capture_matching_projections<'a, 'tcx>(
     })
 }
 
-/// Takes a PlaceBuilder and resolves the upvar (if any) within it, so that the
-/// `PlaceBuilder` now starts from `PlaceBase::Local`.
-///
-/// Returns a Result with the error being the PlaceBuilder (`from_builder`) that was not found.
+/// Takes an upvar place and tries to resolve it into a `PlaceBuilder`
+/// with `PlaceBase::Local`
 #[instrument(level = "trace", skip(cx), ret)]
 fn to_upvars_resolved_place_builder<'tcx>(
-    from_builder: PlaceBuilder<'tcx>,
     cx: &Builder<'_, 'tcx>,
-) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
-    match from_builder.base {
-        PlaceBase::Local(_) => Ok(from_builder),
-        PlaceBase::Upvar { var_hir_id, closure_def_id } => {
-            let Some((capture_index, capture)) =
-                find_capture_matching_projections(
-                    &cx.upvars,
-                    var_hir_id,
-                    &from_builder.projection,
-                ) else {
-                let closure_span = cx.tcx.def_span(closure_def_id);
-                if !enable_precise_capture(cx.tcx, closure_span) {
-                    bug!(
-                        "No associated capture found for {:?}[{:#?}] even though \
-                            capture_disjoint_fields isn't enabled",
-                        var_hir_id,
-                        from_builder.projection
-                    )
-                } else {
-                    debug!(
-                        "No associated capture found for {:?}[{:#?}]",
-                        var_hir_id, from_builder.projection,
-                    );
-                }
-                return Err(from_builder);
-            };
+    var_hir_id: LocalVarId,
+    closure_def_id: LocalDefId,
+    projection: &[PlaceElem<'tcx>],
+) -> Option<PlaceBuilder<'tcx>> {
+    let Some((capture_index, capture)) =
+        find_capture_matching_projections(
+            &cx.upvars,
+            var_hir_id,
+            &projection,
+        ) else {
+        let closure_span = cx.tcx.def_span(closure_def_id);
+        if !enable_precise_capture(cx.tcx, closure_span) {
+            bug!(
+                "No associated capture found for {:?}[{:#?}] even though \
+                    capture_disjoint_fields isn't enabled",
+                var_hir_id,
+                projection
+            )
+        } else {
+            debug!(
+                "No associated capture found for {:?}[{:#?}]",
+                var_hir_id, projection,
+            );
+        }
+        return None;
+    };
 
-            // Access the capture by accessing the field within the Closure struct.
-            let capture_info = &cx.upvars[capture_index];
+    // Access the capture by accessing the field within the Closure struct.
+    let capture_info = &cx.upvars[capture_index];
 
-            let mut upvar_resolved_place_builder = PlaceBuilder::from(capture_info.use_place);
+    let mut upvar_resolved_place_builder = PlaceBuilder::from(capture_info.use_place);
 
-            // We used some of the projections to build the capture itself,
-            // now we apply the remaining to the upvar resolved place.
-            trace!(?capture.captured_place, ?from_builder.projection);
-            let remaining_projections = strip_prefix(
-                capture.captured_place.place.base_ty,
-                from_builder.projection,
-                &capture.captured_place.place.projections,
-            );
-            upvar_resolved_place_builder.projection.extend(remaining_projections);
+    // We used some of the projections to build the capture itself,
+    // now we apply the remaining to the upvar resolved place.
+    trace!(?capture.captured_place, ?projection);
+    let remaining_projections = strip_prefix(
+        capture.captured_place.place.base_ty,
+        projection,
+        &capture.captured_place.place.projections,
+    );
+    upvar_resolved_place_builder.projection.extend(remaining_projections);
 
-            Ok(upvar_resolved_place_builder)
-        }
-    }
+    Some(upvar_resolved_place_builder)
 }
 
 /// Returns projections remaining after stripping an initial prefix of HIR
@@ -228,13 +223,14 @@ fn to_upvars_resolved_place_builder<'tcx>(
 /// Supports only HIR projection kinds that represent a path that might be
 /// captured by a closure or a generator, i.e., an `Index` or a `Subslice`
 /// projection kinds are unsupported.
-fn strip_prefix<'tcx>(
+fn strip_prefix<'a, 'tcx>(
     mut base_ty: Ty<'tcx>,
-    projections: Vec<PlaceElem<'tcx>>,
+    projections: &'a [PlaceElem<'tcx>],
     prefix_projections: &[HirProjection<'tcx>],
-) -> impl Iterator<Item = PlaceElem<'tcx>> {
+) -> impl Iterator<Item = PlaceElem<'tcx>> + 'a {
     let mut iter = projections
-        .into_iter()
+        .iter()
+        .copied()
         // Filter out opaque casts, they are unnecessary in the prefix.
         .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(..)));
     for projection in prefix_projections {
@@ -258,21 +254,21 @@ fn strip_prefix<'tcx>(
 }
 
 impl<'tcx> PlaceBuilder<'tcx> {
-    pub(in crate::build) fn into_place(self, cx: &Builder<'_, 'tcx>) -> Place<'tcx> {
-        if let PlaceBase::Local(local) = self.base {
-            Place { local, projection: cx.tcx.intern_place_elems(&self.projection) }
-        } else {
-            self.expect_upvars_resolved(cx).into_place(cx)
-        }
+    pub(in crate::build) fn to_place(&self, cx: &Builder<'_, 'tcx>) -> Place<'tcx> {
+        self.try_to_place(cx).unwrap()
     }
 
-    fn expect_upvars_resolved(self, cx: &Builder<'_, 'tcx>) -> PlaceBuilder<'tcx> {
-        to_upvars_resolved_place_builder(self, cx).unwrap()
+    /// Creates a `Place` or returns `None` if an upvar cannot be resolved
+    pub(in crate::build) fn try_to_place(&self, cx: &Builder<'_, 'tcx>) -> Option<Place<'tcx>> {
+        let resolved = self.resolve_upvar(cx);
+        let builder = resolved.as_ref().unwrap_or(self);
+        let PlaceBase::Local(local) = builder.base else { return None };
+        let projection = cx.tcx.intern_place_elems(&builder.projection);
+        Some(Place { local, projection })
     }
 
     /// Attempts to resolve the `PlaceBuilder`.
-    /// On success, it will return the resolved `PlaceBuilder`.
-    /// On failure, it will return itself.
+    /// Returns `None` if this is not an upvar.
     ///
     /// Upvars resolve may fail for a `PlaceBuilder` when attempting to
     /// resolve a disjoint field whose root variable is not captured
@@ -281,11 +277,14 @@ impl<'tcx> PlaceBuilder<'tcx> {
     /// not captured. This can happen because the final mir that will be
     /// generated doesn't require a read for this place. Failures will only
     /// happen inside closures.
-    pub(in crate::build) fn try_upvars_resolved(
-        self,
+    pub(in crate::build) fn resolve_upvar(
+        &self,
         cx: &Builder<'_, 'tcx>,
-    ) -> Result<PlaceBuilder<'tcx>, PlaceBuilder<'tcx>> {
-        to_upvars_resolved_place_builder(self, cx)
+    ) -> Option<PlaceBuilder<'tcx>> {
+        let PlaceBase::Upvar { var_hir_id, closure_def_id } = self.base else {
+            return None;
+        };
+        to_upvars_resolved_place_builder(cx, var_hir_id, closure_def_id, &self.projection)
     }
 
     pub(crate) fn base(&self) -> PlaceBase {
@@ -316,6 +315,14 @@ impl<'tcx> PlaceBuilder<'tcx> {
         self.projection.push(elem);
         self
     }
+
+    /// Same as `.clone().project(..)` but more efficient
+    pub(crate) fn clone_project(&self, elem: PlaceElem<'tcx>) -> Self {
+        Self {
+            base: self.base,
+            projection: Vec::from_iter(self.projection.iter().copied().chain([elem])),
+        }
+    }
 }
 
 impl<'tcx> From<Local> for PlaceBuilder<'tcx> {
@@ -355,7 +362,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr: &Expr<'tcx>,
     ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_place_builder(block, expr));
-        block.and(place_builder.into_place(self))
+        block.and(place_builder.to_place(self))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
@@ -379,7 +386,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr: &Expr<'tcx>,
     ) -> BlockAnd<Place<'tcx>> {
         let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr));
-        block.and(place_builder.into_place(self))
+        block.and(place_builder.to_place(self))
     }
 
     /// This is used when constructing a compound `Place`, so that we can avoid creating
@@ -474,7 +481,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             inferred_ty: expr.ty,
                         });
 
-                    let place = place_builder.clone().into_place(this);
+                    let place = place_builder.to_place(this);
                     this.cfg.push(
                         block,
                         Statement {
@@ -599,7 +606,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let is_outermost_index = fake_borrow_temps.is_none();
         let fake_borrow_temps = fake_borrow_temps.unwrap_or(base_fake_borrow_temps);
 
-        let mut base_place =
+        let base_place =
             unpack!(block = self.expr_as_place(block, base, mutability, Some(fake_borrow_temps),));
 
         // Making this a *fresh* temporary means we do not have to worry about
@@ -607,14 +614,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // The "retagging" transformation (for Stacked Borrows) relies on this.
         let idx = unpack!(block = self.as_temp(block, temp_lifetime, index, Mutability::Not,));
 
-        block = self.bounds_check(block, base_place.clone(), idx, expr_span, source_info);
+        block = self.bounds_check(block, &base_place, idx, expr_span, source_info);
 
         if is_outermost_index {
             self.read_fake_borrows(block, fake_borrow_temps, source_info)
         } else {
-            base_place = base_place.expect_upvars_resolved(self);
             self.add_fake_borrows_of_base(
-                &base_place,
+                base_place.to_place(self),
                 block,
                 fake_borrow_temps,
                 expr_span,
@@ -628,7 +634,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn bounds_check(
         &mut self,
         block: BasicBlock,
-        slice: PlaceBuilder<'tcx>,
+        slice: &PlaceBuilder<'tcx>,
         index: Local,
         expr_span: Span,
         source_info: SourceInfo,
@@ -640,7 +646,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let lt = self.temp(bool_ty, expr_span);
 
         // len = len(slice)
-        self.cfg.push_assign(block, source_info, len, Rvalue::Len(slice.into_place(self)));
+        self.cfg.push_assign(block, source_info, len, Rvalue::Len(slice.to_place(self)));
         // lt = idx < len
         self.cfg.push_assign(
             block,
@@ -658,19 +664,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     fn add_fake_borrows_of_base(
         &mut self,
-        base_place: &PlaceBuilder<'tcx>,
+        base_place: Place<'tcx>,
         block: BasicBlock,
         fake_borrow_temps: &mut Vec<Local>,
         expr_span: Span,
         source_info: SourceInfo,
     ) {
         let tcx = self.tcx;
-        let local = match base_place.base {
-            PlaceBase::Local(local) => local,
-            PlaceBase::Upvar { .. } => bug!("Expected PlacseBase::Local found Upvar"),
-        };
 
-        let place_ty = Place::ty_from(local, &base_place.projection, &self.local_decls, tcx);
+        let place_ty = base_place.ty(&self.local_decls, tcx);
         if let ty::Slice(_) = place_ty.ty.kind() {
             // We need to create fake borrows to ensure that the bounds
             // check that we just did stays valid. Since we can't assign to
@@ -680,7 +682,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 match elem {
                     ProjectionElem::Deref => {
                         let fake_borrow_deref_ty = Place::ty_from(
-                            local,
+                            base_place.local,
                             &base_place.projection[..idx],
                             &self.local_decls,
                             tcx,
@@ -698,14 +700,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             Rvalue::Ref(
                                 tcx.lifetimes.re_erased,
                                 BorrowKind::Shallow,
-                                Place { local, projection },
+                                Place { local: base_place.local, projection },
                             ),
                         );
                         fake_borrow_temps.push(fake_borrow_temp);
                     }
                     ProjectionElem::Index(_) => {
                         let index_ty = Place::ty_from(
-                            local,
+                            base_place.local,
                             &base_place.projection[..idx],
                             &self.local_decls,
                             tcx,
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index 3dafdcb78..0814793f2 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -2,6 +2,7 @@
 
 use rustc_index::vec::Idx;
 use rustc_middle::ty::util::IntTypeExt;
+use rustc_target::abi::{Abi, Primitive};
 
 use crate::build::expr::as_place::PlaceBase;
 use crate::build::expr::category::{Category, RvalueFunc};
@@ -198,6 +199,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let (source, ty) = if let ty::Adt(adt_def, ..) = source.ty.kind() && adt_def.is_enum() {
                     let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
                     let temp = unpack!(block = this.as_temp(block, scope, source, Mutability::Not));
+                    let layout = this.tcx.layout_of(this.param_env.and(source.ty));
                     let discr = this.temp(discr_ty, source.span);
                     this.cfg.push_assign(
                         block,
@@ -205,8 +207,55 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         discr,
                         Rvalue::Discriminant(temp.into()),
                     );
+                    let (op,ty) = (Operand::Move(discr), discr_ty);
+
+                    if let Abi::Scalar(scalar) = layout.unwrap().abi{
+                        if let Primitive::Int(_, signed) = scalar.primitive() {
+                            let range = scalar.valid_range(&this.tcx);
+                            // FIXME: Handle wraparound cases too.
+                            if range.end >= range.start {
+                                let mut assumer = |range: u128, bin_op: BinOp| {
+                                    // We will be overwriting this val if our scalar is signed value
+                                    // because sign extension on unsigned types might cause unintended things
+                                    let mut range_val =
+                                        ConstantKind::from_bits(this.tcx, range, ty::ParamEnv::empty().and(discr_ty));
+                                    let bool_ty = this.tcx.types.bool;
+                                    if signed {
+                                        let scalar_size_extend = scalar.size(&this.tcx).sign_extend(range);
+                                        let discr_layout = this.tcx.layout_of(this.param_env.and(discr_ty));
+                                        let truncated_val = discr_layout.unwrap().size.truncate(scalar_size_extend);
+                                        range_val = ConstantKind::from_bits(
+                                            this.tcx,
+                                            truncated_val,
+                                            ty::ParamEnv::empty().and(discr_ty),
+                                        );
+                                    }
+                                    let lit_op = this.literal_operand(expr.span, range_val);
+                                    let is_bin_op = this.temp(bool_ty, expr_span);
+                                    this.cfg.push_assign(
+                                        block,
+                                        source_info,
+                                        is_bin_op,
+                                        Rvalue::BinaryOp(bin_op, Box::new(((lit_op), (Operand::Copy(discr))))),
+                                    );
+                                    this.cfg.push(
+                                        block,
+                                        Statement {
+                                            source_info,
+                                            kind: StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
+                                                Operand::Copy(is_bin_op),
+                                            ))),
+                                        },
+                                    )
+                                };
+                                assumer(range.end, BinOp::Ge);
+                                assumer(range.start, BinOp::Le);
+                            }
+                        }
+                    }
+
+                    (op,ty)
 
-                    (Operand::Move(discr), discr_ty)
                 } else {
                     let ty = source.ty;
                     let source = unpack!(
@@ -320,8 +369,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     let place_builder =
                         unpack!(block = this.as_place_builder(block, &this.thir[*thir_place]));
 
-                    if let Ok(place_builder_resolved) = place_builder.try_upvars_resolved(this) {
-                        let mir_place = place_builder_resolved.into_place(this);
+                    if let Some(mir_place) = place_builder.try_to_place(this) {
                         this.cfg.push_fake_read(
                             block,
                             this.source_info(this.tcx.hir().span(*hir_id)),
@@ -612,7 +660,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // by the parent itself. The mutability of the current capture
             // is same as that of the capture in the parent closure.
             PlaceBase::Upvar { .. } => {
-                let enclosing_upvars_resolved = arg_place_builder.clone().into_place(this);
+                let enclosing_upvars_resolved = arg_place_builder.to_place(this);
 
                 match enclosing_upvars_resolved.as_ref() {
                     PlaceRef {
@@ -649,7 +697,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             Mutability::Mut => BorrowKind::Mut { allow_two_phase_borrow: false },
         };
 
-        let arg_place = arg_place_builder.into_place(this);
+        let arg_place = arg_place_builder.to_place(this);
 
         this.cfg.push_assign(
             block,
diff --git a/compiler/rustc_mir_build/src/build/expr/category.rs b/compiler/rustc_mir_build/src/build/expr/category.rs
index a4386319d..d33401f07 100644
--- a/compiler/rustc_mir_build/src/build/expr/category.rs
+++ b/compiler/rustc_mir_build/src/build/expr/category.rs
@@ -2,35 +2,35 @@ use rustc_middle::thir::*;
 
 #[derive(Debug, PartialEq)]
 pub(crate) enum Category {
-    // An assignable memory location like `x`, `x.f`, `foo()[3]`, that
-    // sort of thing. Something that could appear on the LHS of an `=`
-    // sign.
+    /// An assignable memory location like `x`, `x.f`, `foo()[3]`, that
+    /// sort of thing. Something that could appear on the LHS of an `=`
+    /// sign.
     Place,
 
-    // A literal like `23` or `"foo"`. Does not include constant
-    // expressions like `3 + 5`.
+    /// A literal like `23` or `"foo"`. Does not include constant
+    /// expressions like `3 + 5`.
     Constant,
 
-    // Something that generates a new value at runtime, like `x + y`
-    // or `foo()`.
+    /// Something that generates a new value at runtime, like `x + y`
+    /// or `foo()`.
     Rvalue(RvalueFunc),
 }
 
-// Rvalues fall into different "styles" that will determine which fn
-// is best suited to generate them.
+/// Rvalues fall into different "styles" that will determine which fn
+/// is best suited to generate them.
 #[derive(Debug, PartialEq)]
 pub(crate) enum RvalueFunc {
-    // Best generated by `into`. This is generally exprs that
-    // cause branching, like `match`, but also includes calls.
+    /// Best generated by `into`. This is generally exprs that
+    /// cause branching, like `match`, but also includes calls.
     Into,
 
-    // Best generated by `as_rvalue`. This is usually the case.
+    /// Best generated by `as_rvalue`. This is usually the case.
     AsRvalue,
 }
 
-/// Determines the category for a given expression. Note that scope
-/// and paren expressions have no category.
 impl Category {
+    /// Determines the category for a given expression. Note that scope
+    /// and paren expressions have no category.
     pub(crate) fn of(ek: &ExprKind<'_>) -> Option<Category> {
         match *ek {
             ExprKind::Scope { .. } => None,
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index 24ecd0a53..218a26e62 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -108,7 +108,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Let { expr, ref pat } => {
                 let scope = this.local_scope();
                 let (true_block, false_block) = this.in_if_then_scope(scope, expr_span, |this| {
-                    this.lower_let_expr(block, &this.thir[expr], pat, scope, None, expr_span)
+                    this.lower_let_expr(block, &this.thir[expr], pat, scope, None, expr_span, true)
                 });
 
                 this.cfg.push_assign_constant(
@@ -271,15 +271,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // MIR checks and ultimately whether code is accepted or not. We can only
                         // omit the return edge if a return type is visibly uninhabited to a module
                         // that makes the call.
-                        target: if this.tcx.is_ty_uninhabited_from(
-                            this.parent_module,
-                            expr.ty,
-                            this.param_env,
-                        ) {
-                            None
-                        } else {
-                            Some(success)
-                        },
+                        target: expr
+                            .ty
+                            .is_inhabited_from(this.tcx, this.parent_module, this.param_env)
+                            .then_some(success),
                         from_hir_call,
                         fn_span,
                     },
@@ -363,10 +358,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         .map(|(n, ty)| match fields_map.get(&n) {
                             Some(v) => v.clone(),
                             None => {
-                                let place_builder = place_builder.clone();
-                                this.consume_by_copy_or_move(
-                                    place_builder.field(n, *ty).into_place(this),
-                                )
+                                let place = place_builder.clone_project(PlaceElem::Field(n, *ty));
+                                this.consume_by_copy_or_move(place.to_place(this))
                             }
                         })
                         .collect()
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 3f813e0af..691cbee2c 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -84,6 +84,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 break_scope,
                 Some(variable_source_info.scope),
                 variable_source_info.span,
+                true,
             ),
             _ => {
                 let temp_scope = temp_scope_override.unwrap_or_else(|| this.local_scope());
@@ -168,7 +169,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let scrutinee_place =
             unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span,));
 
-        let mut arm_candidates = self.create_match_candidates(scrutinee_place.clone(), &arms);
+        let mut arm_candidates = self.create_match_candidates(&scrutinee_place, &arms);
 
         let match_has_guard = arm_candidates.iter().any(|(_, candidate)| candidate.has_guard);
         let mut candidates =
@@ -220,8 +221,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let cause_matched_place = FakeReadCause::ForMatchedPlace(None);
         let source_info = self.source_info(scrutinee_span);
 
-        if let Ok(scrutinee_builder) = scrutinee_place_builder.clone().try_upvars_resolved(self) {
-            let scrutinee_place = scrutinee_builder.into_place(self);
+        if let Some(scrutinee_place) = scrutinee_place_builder.try_to_place(self) {
             self.cfg.push_fake_read(block, source_info, cause_matched_place, scrutinee_place);
         }
 
@@ -231,7 +231,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Create the initial `Candidate`s for a `match` expression.
     fn create_match_candidates<'pat>(
         &mut self,
-        scrutinee: PlaceBuilder<'tcx>,
+        scrutinee: &PlaceBuilder<'tcx>,
         arms: &'pat [ArmId],
     ) -> Vec<(&'pat Arm<'tcx>, Candidate<'pat, 'tcx>)>
     where
@@ -334,7 +334,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let arm_scope = (arm.scope, arm_source_info);
                 let match_scope = self.local_scope();
                 self.in_scope(arm_scope, arm.lint_level, |this| {
-                    // `try_upvars_resolved` may fail if it is unable to resolve the given
+                    // `try_to_place` may fail if it is unable to resolve the given
                     // `PlaceBuilder` inside a closure. In this case, we don't want to include
                     // a scrutinee place. `scrutinee_place_builder` will fail to be resolved
                     // if the only match arm is a wildcard (`_`).
@@ -345,31 +345,23 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     //    match foo { _ => () };
                     // };
                     // ```
-                    let mut opt_scrutinee_place: Option<(Option<&Place<'tcx>>, Span)> = None;
-                    let scrutinee_place: Place<'tcx>;
-                    if let Ok(scrutinee_builder) =
-                        scrutinee_place_builder.clone().try_upvars_resolved(this)
-                    {
-                        scrutinee_place = scrutinee_builder.into_place(this);
-                        opt_scrutinee_place = Some((Some(&scrutinee_place), scrutinee_span));
-                    }
+                    let scrutinee_place = scrutinee_place_builder.try_to_place(this);
+                    let opt_scrutinee_place =
+                        scrutinee_place.as_ref().map(|place| (Some(place), scrutinee_span));
                     let scope = this.declare_bindings(
                         None,
                         arm.span,
                         &arm.pattern,
-                        ArmHasGuard(arm.guard.is_some()),
+                        arm.guard.as_ref(),
                         opt_scrutinee_place,
                     );
 
                     let arm_block = this.bind_pattern(
                         outer_source_info,
                         candidate,
-                        arm.guard.as_ref(),
                         &fake_borrow_temps,
                         scrutinee_span,
-                        Some(arm.span),
-                        Some(arm.scope),
-                        Some(match_scope),
+                        Some((arm, match_scope)),
                         false,
                     );
 
@@ -410,12 +402,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         outer_source_info: SourceInfo,
         candidate: Candidate<'_, 'tcx>,
-        guard: Option<&Guard<'tcx>>,
         fake_borrow_temps: &[(Place<'tcx>, Local)],
         scrutinee_span: Span,
-        arm_span: Option<Span>,
-        arm_scope: Option<region::Scope>,
-        match_scope: Option<region::Scope>,
+        arm_match_scope: Option<(&Arm<'tcx>, region::Scope)>,
         storages_alive: bool,
     ) -> BasicBlock {
         if candidate.subcandidates.is_empty() {
@@ -424,11 +413,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             self.bind_and_guard_matched_candidate(
                 candidate,
                 &[],
-                guard,
                 fake_borrow_temps,
                 scrutinee_span,
-                arm_span,
-                match_scope,
+                arm_match_scope,
                 true,
                 storages_alive,
             )
@@ -449,6 +436,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // we lower the guard.
             let target_block = self.cfg.start_new_block();
             let mut schedule_drops = true;
+            let arm = arm_match_scope.unzip().0;
             // We keep a stack of all of the bindings and type ascriptions
             // from the parent candidates that we visit, that also need to
             // be bound for each candidate.
@@ -456,21 +444,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 candidate,
                 &mut Vec::new(),
                 &mut |leaf_candidate, parent_bindings| {
-                    if let Some(arm_scope) = arm_scope {
-                        self.clear_top_scope(arm_scope);
+                    if let Some(arm) = arm {
+                        self.clear_top_scope(arm.scope);
                     }
                     let binding_end = self.bind_and_guard_matched_candidate(
                         leaf_candidate,
                         parent_bindings,
-                        guard,
                         &fake_borrow_temps,
                         scrutinee_span,
-                        arm_span,
-                        match_scope,
+                        arm_match_scope,
                         schedule_drops,
                         storages_alive,
                     );
-                    if arm_scope.is_none() {
+                    if arm.is_none() {
                         schedule_drops = false;
                     }
                     self.cfg.goto(binding_end, outer_source_info, target_block);
@@ -600,7 +586,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             while let Some(next) = {
                 for binding in &candidate_ref.bindings {
                     let local = self.var_local_id(binding.var_id, OutsideGuard);
-                    // `try_upvars_resolved` may fail if it is unable to resolve the given
+                    // `try_to_place` may fail if it is unable to resolve the given
                     // `PlaceBuilder` inside a closure. In this case, we don't want to include
                     // a scrutinee place. `scrutinee_place_builder` will fail for destructured
                     // assignments. This is because a closure only captures the precise places
@@ -614,9 +600,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     //    let (v1, v2) = foo;
                     // };
                     // ```
-                    if let Ok(match_pair_resolved) = initializer.clone().try_upvars_resolved(self) {
-                        let place = match_pair_resolved.into_place(self);
-
+                    if let Some(place) = initializer.try_to_place(self) {
                         let Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
                             VarBindingForm { opt_match_place: Some((ref mut match_place, _)), .. },
                         )))) = self.local_decls[local].local_info else {
@@ -636,12 +620,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.bind_pattern(
             self.source_info(irrefutable_pat.span),
             candidate,
-            None,
             &fake_borrow_temps,
             irrefutable_pat.span,
             None,
-            None,
-            None,
             false,
         )
         .unit()
@@ -657,7 +638,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         mut visibility_scope: Option<SourceScope>,
         scope_span: Span,
         pattern: &Pat<'tcx>,
-        has_guard: ArmHasGuard,
+        guard: Option<&Guard<'tcx>>,
         opt_match_place: Option<(Option<&Place<'tcx>>, Span)>,
     ) -> Option<SourceScope> {
         self.visit_primary_bindings(
@@ -679,12 +660,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     var,
                     ty,
                     user_ty,
-                    has_guard,
+                    ArmHasGuard(guard.is_some()),
                     opt_match_place.map(|(x, y)| (x.cloned(), y)),
                     pattern.span,
                 );
             },
         );
+        if let Some(Guard::IfLet(guard_pat, _)) = guard {
+            // FIXME: pass a proper `opt_match_place`
+            self.declare_bindings(visibility_scope, scope_span, guard_pat, None, None);
+        }
         visibility_scope
     }
 
@@ -1352,7 +1337,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 bug!("Or-patterns should have been sorted to the end");
             };
             let or_span = match_pair.pattern.span;
-            let place = match_pair.place;
 
             first_candidate.visit_leaves(|leaf_candidate| {
                 self.test_or_pattern(
@@ -1360,7 +1344,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     &mut otherwise,
                     pats,
                     or_span,
-                    place.clone(),
+                    &match_pair.place,
                     fake_borrows,
                 );
             });
@@ -1388,7 +1372,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         otherwise: &mut Option<BasicBlock>,
         pats: &'pat [Box<Pat<'tcx>>],
         or_span: Span,
-        place: PlaceBuilder<'tcx>,
+        place: &PlaceBuilder<'tcx>,
         fake_borrows: &mut Option<FxIndexSet<Place<'tcx>>>,
     ) {
         debug!("candidate={:#?}\npats={:#?}", candidate, pats);
@@ -1606,10 +1590,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
 
         // Insert a Shallow borrow of any places that is switched on.
-        if let Some(fb) = fake_borrows && let Ok(match_place_resolved) =
-            match_place.clone().try_upvars_resolved(self)
+        if let Some(fb) = fake_borrows
+            && let Some(resolved_place) = match_place.try_to_place(self)
         {
-            let resolved_place = match_place_resolved.into_place(self);
             fb.insert(resolved_place);
         }
 
@@ -1628,7 +1611,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // encounter a candidate where the test is not relevant; at
         // that point, we stop sorting.
         while let Some(candidate) = candidates.first_mut() {
-            let Some(idx) = self.sort_candidate(&match_place.clone(), &test, candidate) else {
+            let Some(idx) = self.sort_candidate(&match_place, &test, candidate) else {
                 break;
             };
             let (candidate, rest) = candidates.split_first_mut().unwrap();
@@ -1697,7 +1680,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             target_blocks
         };
 
-        self.perform_test(span, scrutinee_span, block, match_place, &test, make_target_blocks);
+        self.perform_test(span, scrutinee_span, block, &match_place, &test, make_target_blocks);
     }
 
     /// Determine the fake borrows that are needed from a set of places that
@@ -1778,6 +1761,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 // Pat binding - used for `let` and function parameters as well.
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// If the bindings have already been declared, set `declare_bindings` to
+    /// `false` to avoid duplicated bindings declaration. Used for if-let guards.
     pub(crate) fn lower_let_expr(
         &mut self,
         mut block: BasicBlock,
@@ -1786,6 +1771,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         else_target: region::Scope,
         source_scope: Option<SourceScope>,
         span: Span,
+        declare_bindings: bool,
     ) -> BlockAnd<()> {
         let expr_span = expr.span;
         let expr_place_builder = unpack!(block = self.lower_scrutinee(block, expr, expr_span));
@@ -1800,32 +1786,21 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             false,
             &mut [&mut guard_candidate, &mut otherwise_candidate],
         );
-        let mut opt_expr_place: Option<(Option<&Place<'tcx>>, Span)> = None;
-        let expr_place: Place<'tcx>;
-        if let Ok(expr_builder) = expr_place_builder.try_upvars_resolved(self) {
-            expr_place = expr_builder.into_place(self);
-            opt_expr_place = Some((Some(&expr_place), expr_span));
-        }
+        let expr_place = expr_place_builder.try_to_place(self);
+        let opt_expr_place = expr_place.as_ref().map(|place| (Some(place), expr_span));
         let otherwise_post_guard_block = otherwise_candidate.pre_binding_block.unwrap();
         self.break_for_else(otherwise_post_guard_block, else_target, self.source_info(expr_span));
 
-        self.declare_bindings(
-            source_scope,
-            pat.span.to(span),
-            pat,
-            ArmHasGuard(false),
-            opt_expr_place,
-        );
+        if declare_bindings {
+            self.declare_bindings(source_scope, pat.span.to(span), pat, None, opt_expr_place);
+        }
 
         let post_guard_block = self.bind_pattern(
             self.source_info(pat.span),
             guard_candidate,
-            None,
             &fake_borrow_temps,
             expr.span,
             None,
-            None,
-            None,
             false,
         );
 
@@ -1844,11 +1819,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         candidate: Candidate<'pat, 'tcx>,
         parent_bindings: &[(Vec<Binding<'tcx>>, Vec<Ascription<'tcx>>)],
-        guard: Option<&Guard<'tcx>>,
         fake_borrows: &[(Place<'tcx>, Local)],
         scrutinee_span: Span,
-        arm_span: Option<Span>,
-        match_scope: Option<region::Scope>,
+        arm_match_scope: Option<(&Arm<'tcx>, region::Scope)>,
         schedule_drops: bool,
         storages_alive: bool,
     ) -> BasicBlock {
@@ -1960,7 +1933,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         //      the reference that we create for the arm.
         //    * So we eagerly create the reference for the arm and then take a
         //      reference to that.
-        if let Some(guard) = guard {
+        if let Some((arm, match_scope)) = arm_match_scope
+            && let Some(guard) = &arm.guard
+        {
             let tcx = self.tcx;
             let bindings = parent_bindings
                 .iter()
@@ -1981,8 +1956,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 self.cfg.push_assign(block, scrutinee_source_info, Place::from(temp), borrow);
             }
 
-            let arm_span = arm_span.unwrap();
-            let match_scope = match_scope.unwrap();
             let mut guard_span = rustc_span::DUMMY_SP;
 
             let (post_guard_block, otherwise_post_guard_block) =
@@ -1995,13 +1968,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             e,
                             None,
                             match_scope,
-                            this.source_info(arm_span),
+                            this.source_info(arm.span),
                         )
                     }
                     Guard::IfLet(ref pat, scrutinee) => {
                         let s = &this.thir[scrutinee];
                         guard_span = s.span;
-                        this.lower_let_expr(block, s, pat, match_scope, None, arm_span)
+                        this.lower_let_expr(block, s, pat, match_scope, None, arm.span, false)
                     }
                 });
 
@@ -2317,24 +2290,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             let matching = this.bind_pattern(
                 this.source_info(pattern.span),
                 candidate,
-                None,
                 &fake_borrow_temps,
                 initializer_span,
                 None,
-                None,
-                None,
                 true,
             );
             // This block is for the failure case
             let failure = this.bind_pattern(
                 this.source_info(else_block_span),
                 wildcard,
-                None,
                 &fake_borrow_temps,
                 initializer_span,
                 None,
-                None,
-                None,
                 true,
             );
             this.break_for_else(failure, *let_else_scope, this.source_info(initializer_span));
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index 924d2f555..f6b1955fd 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -73,8 +73,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             {
                 existing_bindings.extend_from_slice(&new_bindings);
                 mem::swap(&mut candidate.bindings, &mut existing_bindings);
-                candidate.subcandidates =
-                    self.create_or_subcandidates(candidate, place.clone(), pats);
+                candidate.subcandidates = self.create_or_subcandidates(candidate, &place, pats);
                 return true;
             }
 
@@ -127,7 +126,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn create_or_subcandidates<'pat>(
         &mut self,
         candidate: &Candidate<'pat, 'tcx>,
-        place: PlaceBuilder<'tcx>,
+        place: &PlaceBuilder<'tcx>,
         pats: &'pat [Box<Pat<'tcx>>],
     ) -> Vec<Candidate<'pat, 'tcx>> {
         pats.iter()
@@ -156,10 +155,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 ascription: thir::Ascription { ref annotation, variance },
             } => {
                 // Apply the type ascription to the value at `match_pair.place`, which is the
-                if let Ok(place_resolved) = match_pair.place.clone().try_upvars_resolved(self) {
+                if let Some(source) = match_pair.place.try_to_place(self) {
                     candidate.ascriptions.push(Ascription {
                         annotation: annotation.clone(),
-                        source: place_resolved.into_place(self),
+                        source,
                         variance,
                     });
                 }
@@ -183,10 +182,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 ref subpattern,
                 is_primary: _,
             } => {
-                if let Ok(place_resolved) = match_pair.place.clone().try_upvars_resolved(self) {
+                if let Some(source) = match_pair.place.try_to_place(self) {
                     candidate.bindings.push(Binding {
                         span: match_pair.pattern.span,
-                        source: place_resolved.into_place(self),
+                        source,
                         var_id: var,
                         binding_mode: mode,
                     });
@@ -264,10 +263,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let irrefutable = adt_def.variants().iter_enumerated().all(|(i, v)| {
                     i == variant_index || {
                         self.tcx.features().exhaustive_patterns
-                            && v.inhabited_predicate(self.tcx, adt_def)
+                            && !v
+                                .inhabited_predicate(self.tcx, adt_def)
                                 .subst(self.tcx, substs)
-                                .apply_any_module(self.tcx, self.param_env)
-                                != Some(true)
+                                .apply_ignore_module(self.tcx, self.param_env)
                     }
                 }) && (adt_def.did().is_local()
                     || !adt_def.is_variant_list_non_exhaustive());
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index b597ecfaa..58513bde2 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -150,11 +150,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_start_span: Span,
         scrutinee_span: Span,
         block: BasicBlock,
-        place_builder: PlaceBuilder<'tcx>,
+        place_builder: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
     ) {
-        let place = place_builder.into_place(self);
+        let place = place_builder.to_place(self);
         let place_ty = place.ty(&self.local_decls, self.tcx);
         debug!(?place, ?place_ty,);
 
@@ -240,6 +240,39 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::Eq { value, ty } => {
+                let tcx = self.tcx;
+                if let ty::Adt(def, _) = ty.kind() && Some(def.did()) == tcx.lang_items().string() {
+                    if !tcx.features().string_deref_patterns {
+                        bug!("matching on `String` went through without enabling string_deref_patterns");
+                    }
+                    let re_erased = tcx.lifetimes.re_erased;
+                    let ref_string = self.temp(tcx.mk_imm_ref(re_erased, ty), test.span);
+                    let ref_str_ty = tcx.mk_imm_ref(re_erased, tcx.types.str_);
+                    let ref_str = self.temp(ref_str_ty, test.span);
+                    let deref = tcx.require_lang_item(LangItem::Deref, None);
+                    let method = trait_method(tcx, deref, sym::deref, [ty]);
+                    let eq_block = self.cfg.start_new_block();
+                    self.cfg.push_assign(block, source_info, ref_string, Rvalue::Ref(re_erased, BorrowKind::Shared, place));
+                    self.cfg.terminate(
+                        block,
+                        source_info,
+                        TerminatorKind::Call {
+                            func: Operand::Constant(Box::new(Constant {
+                                span: test.span,
+                                user_ty: None,
+                                literal: method,
+                            })),
+                            args: vec![Operand::Move(ref_string)],
+                            destination: ref_str,
+                            target: Some(eq_block),
+                            cleanup: None,
+                            from_hir_call: false,
+                            fn_span: source_info.span
+                        }
+                    );
+                    self.non_scalar_compare(eq_block, make_target_blocks, source_info, value, ref_str, ref_str_ty);
+                    return;
+                }
                 if !ty.is_scalar() {
                     // Use `PartialEq::eq` instead of `BinOp::Eq`
                     // (the binop can only handle primitives)
@@ -411,8 +444,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             bug!("non_scalar_compare called on non-reference type: {}", ty);
         };
 
-        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, None);
-        let method = trait_method(self.tcx, eq_def_id, sym::eq, deref_ty, &[deref_ty.into()]);
+        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, Some(source_info.span));
+        let method = trait_method(self.tcx, eq_def_id, sym::eq, [deref_ty, deref_ty]);
 
         let bool_ty = self.tcx.types.bool;
         let eq_result = self.temp(bool_ty, source_info.span);
@@ -727,7 +760,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let downcast_place = match_pair.place.downcast(adt_def, variant_index); // `(x as Variant)`
         let consequent_match_pairs = subpatterns.iter().map(|subpattern| {
             // e.g., `(x as Variant).0`
-            let place = downcast_place.clone().field(subpattern.field, subpattern.pattern.ty);
+            let place = downcast_place
+                .clone_project(PlaceElem::Field(subpattern.field, subpattern.pattern.ty));
             // e.g., `(x as Variant).0 @ P1`
             MatchPair::new(place, &subpattern.pattern, self)
         });
@@ -804,10 +838,9 @@ fn trait_method<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_def_id: DefId,
     method_name: Symbol,
-    self_ty: Ty<'tcx>,
-    params: &[GenericArg<'tcx>],
+    substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
 ) -> ConstantKind<'tcx> {
-    let substs = tcx.mk_substs_trait(self_ty, params);
+    let substs = tcx.mk_substs(substs.into_iter().map(Into::into));
 
     // The unhygienic comparison here is acceptable because this is only
     // used on known traits.
@@ -817,8 +850,7 @@ fn trait_method<'tcx>(
         .find(|item| item.kind == ty::AssocKind::Fn)
         .expect("trait method not found");
 
-    let method_ty = tcx.bound_type_of(item.def_id);
-    let method_ty = method_ty.subst(tcx, substs);
+    let method_ty = tcx.mk_fn_def(item.def_id, substs);
 
     ConstantKind::zero_sized(method_ty)
 }
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
index b854ba47f..bd435f9ab 100644
--- a/compiler/rustc_mir_build/src/build/matches/util.rs
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -18,7 +18,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         subpatterns
             .iter()
             .map(|fieldpat| {
-                let place = place.clone().field(fieldpat.field, fieldpat.pattern.ty);
+                let place =
+                    place.clone_project(PlaceElem::Field(fieldpat.field, fieldpat.pattern.ty));
                 MatchPair::new(place, &fieldpat.pattern, self)
             })
             .collect()
@@ -33,26 +34,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         suffix: &'pat [Box<Pat<'tcx>>],
     ) {
         let tcx = self.tcx;
-        let (min_length, exact_size) =
-            if let Ok(place_resolved) = place.clone().try_upvars_resolved(self) {
-                match place_resolved.into_place(self).ty(&self.local_decls, tcx).ty.kind() {
-                    ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
-                    _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
-                }
-            } else {
-                ((prefix.len() + suffix.len()).try_into().unwrap(), false)
-            };
+        let (min_length, exact_size) = if let Some(place_resolved) = place.try_to_place(self) {
+            match place_resolved.ty(&self.local_decls, tcx).ty.kind() {
+                ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
+                _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
+            }
+        } else {
+            ((prefix.len() + suffix.len()).try_into().unwrap(), false)
+        };
 
         match_pairs.extend(prefix.iter().enumerate().map(|(idx, subpattern)| {
             let elem =
                 ProjectionElem::ConstantIndex { offset: idx as u64, min_length, from_end: false };
-            let place = place.clone().project(elem);
-            MatchPair::new(place, subpattern, self)
+            MatchPair::new(place.clone_project(elem), subpattern, self)
         }));
 
         if let Some(subslice_pat) = opt_slice {
             let suffix_len = suffix.len() as u64;
-            let subslice = place.clone().project(ProjectionElem::Subslice {
+            let subslice = place.clone_project(PlaceElem::Subslice {
                 from: prefix.len() as u64,
                 to: if exact_size { min_length - suffix_len } else { suffix_len },
                 from_end: !exact_size,
@@ -67,7 +66,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 min_length,
                 from_end: !exact_size,
             };
-            let place = place.clone().project(elem);
+            let place = place.clone_project(elem);
             MatchPair::new(place, subpattern, self)
         }));
     }
@@ -97,15 +96,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
 impl<'pat, 'tcx> MatchPair<'pat, 'tcx> {
     pub(in crate::build) fn new(
-        place: PlaceBuilder<'tcx>,
+        mut place: PlaceBuilder<'tcx>,
         pattern: &'pat Pat<'tcx>,
         cx: &Builder<'_, 'tcx>,
     ) -> MatchPair<'pat, 'tcx> {
         // Force the place type to the pattern's type.
         // FIXME(oli-obk): can we use this to simplify slice/array pattern hacks?
-        let mut place = match place.try_upvars_resolved(cx) {
-            Ok(val) | Err(val) => val,
-        };
+        if let Some(resolved) = place.resolve_upvar(cx) {
+            place = resolved;
+        }
 
         // Only add the OpaqueCast projection if the given place is an opaque type and the
         // expected type from the pattern is not.
diff --git a/compiler/rustc_mir_build/src/build/misc.rs b/compiler/rustc_mir_build/src/build/misc.rs
index 86f466ff7..baeb2718c 100644
--- a/compiler/rustc_mir_build/src/build/misc.rs
+++ b/compiler/rustc_mir_build/src/build/misc.rs
@@ -34,8 +34,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         Operand::Constant(constant)
     }
 
-    // Returns a zero literal operand for the appropriate type, works for
-    // bool, char and integers.
+    /// Returns a zero literal operand for the appropriate type, works for
+    /// bool, char and integers.
     pub(crate) fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
         let literal = ConstantKind::from_bits(self.tcx, 0, ty::ParamEnv::empty().and(ty));
 
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index cbcf9cd12..007f3b55e 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -481,6 +481,22 @@ fn construct_fn<'tcx>(
         (None, fn_sig.output())
     };
 
+    if let Some(custom_mir_attr) =
+        tcx.hir().attrs(fn_id).iter().find(|attr| attr.name_or_empty() == sym::custom_mir)
+    {
+        return custom::build_custom_mir(
+            tcx,
+            fn_def.did.to_def_id(),
+            thir,
+            expr,
+            arguments,
+            return_ty,
+            return_ty_span,
+            span_with_body,
+            custom_mir_attr,
+        );
+    }
+
     let infcx = tcx.infer_ctxt().build();
     let mut builder = Builder::new(
         thir,
@@ -908,7 +924,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         scope,
                         expr.span,
                         &pat,
-                        matches::ArmHasGuard(false),
+                        None,
                         Some((Some(&place), span)),
                     );
                     let place_builder = PlaceBuilder::from(local);
@@ -932,20 +948,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         original_source_scope: SourceScope,
         pattern_span: Span,
     ) {
-        let tcx = self.tcx;
-        let current_root = tcx.maybe_lint_level_root_bounded(arg_hir_id, self.hir_id);
-        let parent_root = tcx.maybe_lint_level_root_bounded(
-            self.source_scopes[original_source_scope]
-                .local_data
-                .as_ref()
-                .assert_crate_local()
-                .lint_root,
-            self.hir_id,
-        );
-        if current_root != parent_root {
-            self.source_scope =
-                self.new_source_scope(pattern_span, LintLevel::Explicit(current_root), None);
-        }
+        let parent_id = self.source_scopes[original_source_scope]
+            .local_data
+            .as_ref()
+            .assert_crate_local()
+            .lint_root;
+        self.maybe_new_source_scope(pattern_span, None, arg_hir_id, parent_id);
     }
 
     fn get_unit_temp(&mut self) -> Place<'tcx> {
@@ -1033,6 +1041,7 @@ pub(crate) fn parse_float_into_scalar(
 
 mod block;
 mod cfg;
+mod custom;
 mod expr;
 mod matches;
 mod misc;
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index 3cebd5ebe..33f49ffda 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -85,6 +85,7 @@ use std::mem;
 
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::HirId;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
@@ -443,8 +444,9 @@ impl<'tcx> Scopes<'tcx> {
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     // Adding and removing scopes
     // ==========================
-    //  Start a breakable scope, which tracks where `continue`, `break` and
-    //  `return` should branch to.
+
+    ///  Start a breakable scope, which tracks where `continue`, `break` and
+    ///  `return` should branch to.
     pub(crate) fn in_breakable_scope<F>(
         &mut self,
         loop_block: Option<BasicBlock>,
@@ -566,25 +568,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<R>,
     {
         let source_scope = self.source_scope;
-        let tcx = self.tcx;
         if let LintLevel::Explicit(current_hir_id) = lint_level {
-            // Use `maybe_lint_level_root_bounded` with `root_lint_level` as a bound
-            // to avoid adding Hir dependencies on our parents.
-            // We estimate the true lint roots here to avoid creating a lot of source scopes.
-
-            let parent_root = tcx.maybe_lint_level_root_bounded(
-                self.source_scopes[source_scope].local_data.as_ref().assert_crate_local().lint_root,
-                self.hir_id,
-            );
-            let current_root = tcx.maybe_lint_level_root_bounded(current_hir_id, self.hir_id);
-
-            if parent_root != current_root {
-                self.source_scope = self.new_source_scope(
-                    region_scope.1.span,
-                    LintLevel::Explicit(current_root),
-                    None,
-                );
-            }
+            let parent_id =
+                self.source_scopes[source_scope].local_data.as_ref().assert_crate_local().lint_root;
+            self.maybe_new_source_scope(region_scope.1.span, None, current_hir_id, parent_id);
         }
         self.push_scope(region_scope);
         let mut block;
@@ -757,6 +744,40 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         ))
     }
 
+    /// Possibly creates a new source scope if `current_root` and `parent_root`
+    /// are different, or if -Zmaximal-hir-to-mir-coverage is enabled.
+    pub(crate) fn maybe_new_source_scope(
+        &mut self,
+        span: Span,
+        safety: Option<Safety>,
+        current_id: HirId,
+        parent_id: HirId,
+    ) {
+        let (current_root, parent_root) =
+            if self.tcx.sess.opts.unstable_opts.maximal_hir_to_mir_coverage {
+                // Some consumers of rustc need to map MIR locations back to HIR nodes. Currently the
+                // the only part of rustc that tracks MIR -> HIR is the `SourceScopeLocalData::lint_root`
+                // field that tracks lint levels for MIR locations.  Normally the number of source scopes
+                // is limited to the set of nodes with lint annotations. The -Zmaximal-hir-to-mir-coverage
+                // flag changes this behavior to maximize the number of source scopes, increasing the
+                // granularity of the MIR->HIR mapping.
+                (current_id, parent_id)
+            } else {
+                // Use `maybe_lint_level_root_bounded` with `self.hir_id` as a bound
+                // to avoid adding Hir dependencies on our parents.
+                // We estimate the true lint roots here to avoid creating a lot of source scopes.
+                (
+                    self.tcx.maybe_lint_level_root_bounded(current_id, self.hir_id),
+                    self.tcx.maybe_lint_level_root_bounded(parent_id, self.hir_id),
+                )
+            };
+
+        if current_root != parent_root {
+            let lint_level = LintLevel::Explicit(current_root);
+            self.source_scope = self.new_source_scope(span, lint_level, safety);
+        }
+    }
+
     /// Creates a new source scope, nested in the current one.
     pub(crate) fn new_source_scope(
         &mut self,
@@ -799,6 +820,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     // Finding scopes
     // ==============
+
     /// Returns the scope that we should use as the lifetime of an
     /// operand. Basically, an operand must live until it is consumed.
     /// This is similar to, but not quite the same as, the temporary
@@ -824,6 +846,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     // Scheduling drops
     // ================
+
     pub(crate) fn schedule_drop_storage_and_value(
         &mut self,
         span: Span,
@@ -996,6 +1019,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     // Other
     // =====
+
     /// Returns the [DropIdx] for the innermost drop if the function unwound at
     /// this point. The `DropIdx` will be created if it doesn't already exist.
     fn diverge_cleanup(&mut self) -> DropIdx {