Merging upstream version 1.76.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 289 insertions, 169 deletions

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 0fff9a6be..704f124db 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,22 +1,22 @@
 use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
-    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd,
-    NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
+    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
+    NeverPatternWithBody, NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure,
+    UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
 use crate::{FnDeclKind, ImplTraitPosition};
-use rustc_ast::attr;
 use rustc_ast::ptr::P as AstP;
 use rustc_ast::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
-use rustc_hir::def::Res;
-use rustc_hir::definitions::DefPathData;
+use rustc_hir::def::{DefKind, Res};
+use rustc_middle::span_bug;
 use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, Spanned};
-use rustc_span::symbol::{sym, Ident, Symbol};
+use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
 use rustc_span::{DesugaringKind, Span};
 use thin_vec::{thin_vec, ThinVec};
@@ -42,8 +42,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     }
                     // Merge attributes into the inner expression.
                     if !e.attrs.is_empty() {
-                        let old_attrs =
-                            self.attrs.get(&ex.hir_id.local_id).map(|la| *la).unwrap_or(&[]);
+                        let old_attrs = self.attrs.get(&ex.hir_id.local_id).copied().unwrap_or(&[]);
                         self.attrs.insert(
                             ex.hir_id.local_id,
                             &*self.arena.alloc_from_iter(
@@ -73,7 +72,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let kind = match &e.kind {
                 ExprKind::Array(exprs) => hir::ExprKind::Array(self.lower_exprs(exprs)),
                 ExprKind::ConstBlock(c) => {
-                    let c = self.with_new_scopes(|this| hir::ConstBlock {
+                    let c = self.with_new_scopes(c.value.span, |this| hir::ConstBlock {
                         def_id: this.local_def_id(c.id),
                         hir_id: this.lower_node_id(c.id),
                         body: this.lower_const_body(c.value.span, Some(&c.value)),
@@ -190,46 +189,43 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
                 ExprKind::Await(expr, await_kw_span) => self.lower_expr_await(*await_kw_span, expr),
                 ExprKind::Closure(box Closure {
                     binder,
                     capture_clause,
                     constness,
-                    asyncness,
+                    coroutine_kind,
                     movability,
                     fn_decl,
                     body,
                     fn_decl_span,
                     fn_arg_span,
-                }) => {
-                    if let Async::Yes { closure_id, .. } = asyncness {
-                        self.lower_expr_async_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            hir_id,
-                            *closure_id,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    } else {
-                        self.lower_expr_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            *constness,
-                            *movability,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    }
-                }
+                }) => match coroutine_kind {
+                    Some(coroutine_kind) => self.lower_expr_coroutine_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        hir_id,
+                        *coroutine_kind,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                    None => self.lower_expr_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        *constness,
+                        *movability,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                },
                 ExprKind::Block(blk, opt_label) => {
                     let opt_label = self.lower_label(*opt_label);
                     hir::ExprKind::Block(self.lower_block(blk, opt_label.is_some()), opt_label)
@@ -324,11 +320,20 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
+                ExprKind::Gen(capture_clause, block, GenBlockKind::AsyncGen) => self
+                    .make_async_gen_expr(
+                        *capture_clause,
+                        e.id,
+                        None,
+                        e.span,
+                        hir::CoroutineSource::Block,
+                        |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
+                    ),
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
                 ExprKind::Err => hir::ExprKind::Err(
-                    self.tcx.sess.delay_span_bug(e.span, "lowered ExprKind::Err"),
+                    self.tcx.sess.span_delayed_bug(e.span, "lowered ExprKind::Err"),
                 ),
                 ExprKind::Try(sub_expr) => self.lower_expr_try(e.span, sub_expr),
 
@@ -351,30 +356,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_binop(&mut self, b: BinOp) -> hir::BinOp {
-        Spanned {
-            node: match b.node {
-                BinOpKind::Add => hir::BinOpKind::Add,
-                BinOpKind::Sub => hir::BinOpKind::Sub,
-                BinOpKind::Mul => hir::BinOpKind::Mul,
-                BinOpKind::Div => hir::BinOpKind::Div,
-                BinOpKind::Rem => hir::BinOpKind::Rem,
-                BinOpKind::And => hir::BinOpKind::And,
-                BinOpKind::Or => hir::BinOpKind::Or,
-                BinOpKind::BitXor => hir::BinOpKind::BitXor,
-                BinOpKind::BitAnd => hir::BinOpKind::BitAnd,
-                BinOpKind::BitOr => hir::BinOpKind::BitOr,
-                BinOpKind::Shl => hir::BinOpKind::Shl,
-                BinOpKind::Shr => hir::BinOpKind::Shr,
-                BinOpKind::Eq => hir::BinOpKind::Eq,
-                BinOpKind::Lt => hir::BinOpKind::Lt,
-                BinOpKind::Le => hir::BinOpKind::Le,
-                BinOpKind::Ne => hir::BinOpKind::Ne,
-                BinOpKind::Ge => hir::BinOpKind::Ge,
-                BinOpKind::Gt => hir::BinOpKind::Gt,
-            },
-            span: self.lower_span(b.span),
-        }
+    fn lower_binop(&mut self, b: BinOp) -> BinOp {
+        Spanned { node: b.node, span: self.lower_span(b.span) }
     }
 
     fn lower_legacy_const_generics(
@@ -396,7 +379,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let node_id = self.next_node_id();
 
                 // Add a definition for the in-band const def.
-                self.create_def(parent_def_id.def_id, node_id, DefPathData::AnonConst, f.span);
+                self.create_def(
+                    parent_def_id.def_id,
+                    node_id,
+                    kw::Empty,
+                    DefKind::AnonConst,
+                    f.span,
+                );
 
                 let anon_const = AnonConst { id: node_id, value: arg };
                 generic_args.push(AngleBracketedArg::Arg(GenericArg::Const(anon_const)));
@@ -525,7 +514,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     this.mark_span_with_reason(
                         DesugaringKind::TryBlock,
                         expr.span,
-                        this.allow_try_trait.clone(),
+                        Some(this.allow_try_trait.clone()),
                     ),
                     expr,
                 )
@@ -533,7 +522,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let try_span = this.mark_span_with_reason(
                     DesugaringKind::TryBlock,
                     this.tcx.sess.source_map().end_point(body.span),
-                    this.allow_try_trait.clone(),
+                    Some(this.allow_try_trait.clone()),
                 );
 
                 (try_span, this.expr_unit(try_span))
@@ -561,13 +550,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         expr: &'hir hir::Expr<'hir>,
         overall_span: Span,
     ) -> &'hir hir::Expr<'hir> {
-        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item, None));
+        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item));
         self.expr_call(overall_span, constructor, std::slice::from_ref(expr))
     }
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm<'hir> {
         let pat = self.lower_pat(&arm.pat);
-        let guard = arm.guard.as_ref().map(|cond| {
+        let mut guard = arm.guard.as_ref().map(|cond| {
             if let ExprKind::Let(pat, scrutinee, span, is_recovered) = &cond.kind {
                 hir::Guard::IfLet(self.arena.alloc(hir::Let {
                     hir_id: self.next_id(),
@@ -582,14 +571,46 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
         });
         let hir_id = self.next_id();
+        let span = self.lower_span(arm.span);
         self.lower_attrs(hir_id, &arm.attrs);
-        hir::Arm {
-            hir_id,
-            pat,
-            guard,
-            body: self.lower_expr(&arm.body),
-            span: self.lower_span(arm.span),
-        }
+        let is_never_pattern = pat.is_never_pattern();
+        let body = if let Some(body) = &arm.body
+            && !is_never_pattern
+        {
+            self.lower_expr(body)
+        } else {
+            // Either `body.is_none()` or `is_never_pattern` here.
+            if !is_never_pattern {
+                if self.tcx.features().never_patterns {
+                    // If the feature is off we already emitted the error after parsing.
+                    let suggestion = span.shrink_to_hi();
+                    self.tcx.sess.emit_err(MatchArmWithNoBody { span, suggestion });
+                }
+            } else if let Some(body) = &arm.body {
+                self.tcx.sess.emit_err(NeverPatternWithBody { span: body.span });
+                guard = None;
+            } else if let Some(g) = &arm.guard {
+                self.tcx.sess.emit_err(NeverPatternWithGuard { span: g.span });
+                guard = None;
+            }
+
+            // We add a fake `loop {}` arm body so that it typecks to `!`.
+            // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
+            let block = self.arena.alloc(hir::Block {
+                stmts: &[],
+                expr: None,
+                hir_id: self.next_id(),
+                rules: hir::BlockCheckMode::DefaultBlock,
+                span,
+                targeted_by_break: false,
+            });
+            self.arena.alloc(hir::Expr {
+                hir_id: self.next_id(),
+                kind: hir::ExprKind::Loop(block, None, hir::LoopSource::Loop, span),
+                span,
+            })
+        };
+        hir::Arm { hir_id, pat, guard, body, span }
     }
 
     /// Lower an `async` construct to a coroutine that implements `Future`.
@@ -613,9 +634,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let output = ret_ty.unwrap_or_else(|| hir::FnRetTy::DefaultReturn(self.lower_span(span)));
 
         // Resume argument type: `ResumeTy`
-        let unstable_span =
-            self.mark_span_with_reason(DesugaringKind::Async, span, self.allow_gen_future.clone());
-        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span, None);
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
         let input_ty = hir::Ty {
             hir_id: self.next_id(),
             kind: hir::TyKind::Path(resume_ty),
@@ -721,6 +745,87 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }))
     }
 
+    /// Lower a `async gen` construct to a generator that implements `AsyncIterator`.
+    ///
+    /// This results in:
+    ///
+    /// ```text
+    /// static move? |_task_context| -> () {
+    ///     <body>
+    /// }
+    /// ```
+    pub(super) fn make_async_gen_expr(
+        &mut self,
+        capture_clause: CaptureBy,
+        closure_node_id: NodeId,
+        _yield_ty: Option<hir::FnRetTy<'hir>>,
+        span: Span,
+        async_coroutine_source: hir::CoroutineSource,
+        body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
+    ) -> hir::ExprKind<'hir> {
+        let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
+
+        // Resume argument type: `ResumeTy`
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
+        let input_ty = hir::Ty {
+            hir_id: self.next_id(),
+            kind: hir::TyKind::Path(resume_ty),
+            span: unstable_span,
+        };
+
+        // The closure/coroutine `FnDecl` takes a single (resume) argument of type `input_ty`.
+        let fn_decl = self.arena.alloc(hir::FnDecl {
+            inputs: arena_vec![self; input_ty],
+            output,
+            c_variadic: false,
+            implicit_self: hir::ImplicitSelfKind::None,
+            lifetime_elision_allowed: false,
+        });
+
+        // Lower the argument pattern/ident. The ident is used again in the `.await` lowering.
+        let (pat, task_context_hid) = self.pat_ident_binding_mode(
+            span,
+            Ident::with_dummy_span(sym::_task_context),
+            hir::BindingAnnotation::MUT,
+        );
+        let param = hir::Param {
+            hir_id: self.next_id(),
+            pat,
+            ty_span: self.lower_span(span),
+            span: self.lower_span(span),
+        };
+        let params = arena_vec![self; param];
+
+        let body = self.lower_body(move |this| {
+            this.coroutine_kind = Some(hir::CoroutineKind::AsyncGen(async_coroutine_source));
+
+            let old_ctx = this.task_context;
+            this.task_context = Some(task_context_hid);
+            let res = body(this);
+            this.task_context = old_ctx;
+            (params, res)
+        });
+
+        // `static |_task_context| -> <ret_ty> { body }`:
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
+
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
     pub(super) fn maybe_forward_track_caller(
@@ -736,7 +841,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let unstable_span = self.mark_span_with_reason(
                 DesugaringKind::Async,
                 span,
-                self.allow_gen_future.clone(),
+                Some(self.allow_gen_future.clone()),
             );
             self.lower_attrs(
                 inner_hir_id,
@@ -770,20 +875,23 @@ impl<'hir> LoweringContext<'_, 'hir> {
     /// ```
     fn lower_expr_await(&mut self, await_kw_span: Span, expr: &Expr) -> hir::ExprKind<'hir> {
         let full_span = expr.span.to(await_kw_span);
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Async(_)) => {}
+
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Async(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) | None => {
-                self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
+                return hir::ExprKind::Err(self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
                     await_kw_span,
                     item_span: self.current_item,
-                });
+                }));
             }
-        }
+        };
+
         let span = self.mark_span_with_reason(DesugaringKind::Await, await_kw_span, None);
         let gen_future_span = self.mark_span_with_reason(
             DesugaringKind::Await,
             full_span,
-            self.allow_gen_future.clone(),
+            Some(self.allow_gen_future.clone()),
         );
         let expr = self.lower_expr_mut(expr);
         let expr_hir_id = expr.hir_id;
@@ -792,8 +900,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // debuggers and debugger extensions expect it to be called `__awaitee`. They use
         // this name to identify what is being awaited by a suspended async functions.
         let awaitee_ident = Ident::with_dummy_span(sym::__awaitee);
-        let (awaitee_pat, awaitee_pat_hid) =
-            self.pat_ident_binding_mode(span, awaitee_ident, hir::BindingAnnotation::MUT);
+        let (awaitee_pat, awaitee_pat_hid) = self.pat_ident_binding_mode(
+            gen_future_span,
+            awaitee_ident,
+            hir::BindingAnnotation::MUT,
+        );
 
         let task_context_ident = Ident::with_dummy_span(sym::_task_context);
 
@@ -806,29 +917,27 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let poll_expr = {
             let awaitee = self.expr_ident(span, awaitee_ident, awaitee_pat_hid);
             let ref_mut_awaitee = self.expr_mut_addr_of(span, awaitee);
-            let task_context = if let Some(task_context_hid) = self.task_context {
-                self.expr_ident_mut(span, task_context_ident, task_context_hid)
-            } else {
-                // Use of `await` outside of an async context, we cannot use `task_context` here.
-                self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no task_context hir id"))
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
             };
+
+            let task_context = self.expr_ident_mut(span, task_context_ident, task_context_hid);
+
             let new_unchecked = self.expr_call_lang_item_fn_mut(
                 span,
                 hir::LangItem::PinNewUnchecked,
                 arena_vec![self; ref_mut_awaitee],
-                Some(expr_hir_id),
             );
             let get_context = self.expr_call_lang_item_fn_mut(
                 gen_future_span,
                 hir::LangItem::GetContext,
                 arena_vec![self; task_context],
-                Some(expr_hir_id),
             );
             let call = self.expr_call_lang_item_fn(
                 span,
                 hir::LangItem::FuturePoll,
                 arena_vec![self; new_unchecked, get_context],
-                Some(expr_hir_id),
             );
             self.arena.alloc(self.expr_unsafe(call))
         };
@@ -841,12 +950,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let (x_pat, x_pat_hid) = self.pat_ident(gen_future_span, x_ident);
             let x_expr = self.expr_ident(gen_future_span, x_ident, x_pat_hid);
             let ready_field = self.single_pat_field(gen_future_span, x_pat);
-            let ready_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollReady,
-                ready_field,
-                Some(expr_hir_id),
-            );
+            let ready_pat = self.pat_lang_item_variant(span, hir::LangItem::PollReady, ready_field);
             let break_x = self.with_loop_scope(loop_node_id, move |this| {
                 let expr_break =
                     hir::ExprKind::Break(this.lower_loop_destination(None), Some(x_expr));
@@ -857,12 +961,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // `::std::task::Poll::Pending => {}`
         let pending_arm = {
-            let pending_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollPending,
-                &[],
-                Some(expr_hir_id),
-            );
+            let pending_pat = self.pat_lang_item_variant(span, hir::LangItem::PollPending, &[]);
             let empty_block = self.expr_block_empty(span);
             self.arm(pending_pat, empty_block)
         };
@@ -877,25 +976,30 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.stmt_expr(span, match_expr)
         };
 
-        // task_context = yield ();
+        // Depending on `async` of `async gen`:
+        // async     - task_context = yield ();
+        // async gen - task_context = yield ASYNC_GEN_PENDING;
         let yield_stmt = {
-            let unit = self.expr_unit(span);
+            let yielded = if is_async_gen {
+                self.arena.alloc(self.expr_lang_item_path(span, hir::LangItem::AsyncGenPending))
+            } else {
+                self.expr_unit(span)
+            };
+
             let yield_expr = self.expr(
                 span,
-                hir::ExprKind::Yield(unit, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
+                hir::ExprKind::Yield(yielded, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
             );
             let yield_expr = self.arena.alloc(yield_expr);
 
-            if let Some(task_context_hid) = self.task_context {
-                let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
-                let assign =
-                    self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
-                self.stmt_expr(span, assign)
-            } else {
-                // Use of `await` outside of an async context. Return `yield_expr` so that we can
-                // proceed with type checking.
-                self.stmt(span, hir::StmtKind::Semi(yield_expr))
-            }
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+            let assign =
+                self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
+            self.stmt_expr(span, assign)
         };
 
         let loop_block = self.block_all(span, arena_vec![self; inner_match_stmt, yield_stmt], None);
@@ -920,7 +1024,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             span,
             hir::LangItem::IntoFutureIntoFuture,
             arena_vec![self; expr],
-            Some(expr_hir_id),
         );
 
         // match <into_future_expr> {
@@ -947,9 +1050,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     ) -> hir::ExprKind<'hir> {
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        let (body_id, coroutine_option) = self.with_new_scopes(move |this| {
-            let prev = this.current_item;
-            this.current_item = Some(fn_decl_span);
+        let (body_id, coroutine_option) = self.with_new_scopes(fn_decl_span, move |this| {
             let mut coroutine_kind = None;
             let body_id = this.lower_fn_body(decl, |this| {
                 let e = this.lower_expr_mut(body);
@@ -957,8 +1058,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 e
             });
             let coroutine_option =
-                this.coroutine_movability_for_fn(&decl, fn_decl_span, coroutine_kind, movability);
-            this.current_item = prev;
+                this.coroutine_movability_for_fn(decl, fn_decl_span, coroutine_kind, movability);
             (body_id, coroutine_option)
         });
 
@@ -996,7 +1096,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 Some(movability)
             }
-            Some(hir::CoroutineKind::Gen(_)) | Some(hir::CoroutineKind::Async(_)) => {
+            Some(
+                hir::CoroutineKind::Gen(_)
+                | hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_),
+            ) => {
                 panic!("non-`async`/`gen` closure body turned `async`/`gen` during lowering");
             }
             None => {
@@ -1023,18 +1127,22 @@ impl<'hir> LoweringContext<'_, 'hir> {
         (binder, params)
     }
 
-    fn lower_expr_async_closure(
+    fn lower_expr_coroutine_closure(
         &mut self,
         binder: &ClosureBinder,
         capture_clause: CaptureBy,
         closure_id: NodeId,
         closure_hir_id: hir::HirId,
-        inner_closure_id: NodeId,
+        coroutine_kind: CoroutineKind,
         decl: &FnDecl,
         body: &Expr,
         fn_decl_span: Span,
         fn_arg_span: Span,
     ) -> hir::ExprKind<'hir> {
+        let CoroutineKind::Async { closure_id: inner_closure_id, .. } = coroutine_kind else {
+            span_bug!(fn_decl_span, "`async gen` and `gen` closures are not supported, yet");
+        };
+
         if let &ClosureBinder::For { span, .. } = binder {
             self.tcx.sess.emit_err(NotSupportedForLifetimeBinderAsyncClosure { span });
         }
@@ -1044,7 +1152,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let outer_decl =
             FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) };
 
-        let body = self.with_new_scopes(|this| {
+        let body = self.with_new_scopes(fn_decl_span, |this| {
             // FIXME(cramertj): allow `async` non-`move` closures with arguments.
             if capture_clause == CaptureBy::Ref && !decl.inputs.is_empty() {
                 this.tcx.sess.emit_err(AsyncNonMoveClosureNotSupported { fn_decl_span });
@@ -1055,7 +1163,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let body_id = this.lower_fn_body(&outer_decl, |this| {
                 let async_ret_ty = if let FnRetTy::Ty(ty) = &decl.output {
                     let itctx = ImplTraitContext::Disallowed(ImplTraitPosition::AsyncBlock);
-                    Some(hir::FnRetTy::Return(this.lower_ty(&ty, &itctx)))
+                    Some(hir::FnRetTy::Return(this.lower_ty(ty, &itctx)))
                 } else {
                     None
                 };
@@ -1066,7 +1174,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     async_ret_ty,
                     body.span,
                     hir::CoroutineSource::Closure,
-                    |this| this.with_new_scopes(|this| this.lower_expr_mut(body)),
+                    |this| this.with_new_scopes(fn_decl_span, |this| this.lower_expr_mut(body)),
                 );
                 let hir_id = this.lower_node_id(inner_closure_id);
                 this.maybe_forward_track_caller(body.span, closure_hir_id, hir_id);
@@ -1113,6 +1221,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 | ExprKind::Struct(..)
                 | ExprKind::Tup(..)
                 | ExprKind::Underscore => false,
+                // Check for unit struct constructor.
+                ExprKind::Path(..) => lower_ctx.extract_unit_struct_path(lhs).is_none(),
                 // Check for tuple struct constructor.
                 ExprKind::Call(callee, ..) => lower_ctx.extract_tuple_struct_path(callee).is_none(),
                 ExprKind::Paren(e) => {
@@ -1149,12 +1259,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         );
 
         // `a = lhs1; b = lhs2;`.
-        let stmts = self
-            .arena
-            .alloc_from_iter(std::iter::once(destructure_let).chain(assignments.into_iter()));
+        let stmts = self.arena.alloc_from_iter(std::iter::once(destructure_let).chain(assignments));
 
         // Wrap everything in a block.
-        hir::ExprKind::Block(&self.block_all(whole_span, stmts, None), None)
+        hir::ExprKind::Block(self.block_all(whole_span, stmts, None), None)
     }
 
     /// If the given expression is a path to a tuple struct, returns that path.
@@ -1377,8 +1485,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_expr_range_closed(&mut self, span: Span, e1: &Expr, e2: &Expr) -> hir::ExprKind<'hir> {
         let e1 = self.lower_expr_mut(e1);
         let e2 = self.lower_expr_mut(e2);
-        let fn_path =
-            hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span), None);
+        let fn_path = hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span));
         let fn_expr = self.arena.alloc(self.expr(span, hir::ExprKind::Path(fn_path)));
         hir::ExprKind::Call(fn_expr, arena_vec![self; e1, e2])
     }
@@ -1411,7 +1518,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let fields = self.arena.alloc_from_iter(
             e1.iter().map(|e| (sym::start, e)).chain(e2.iter().map(|e| (sym::end, e))).map(
                 |(s, e)| {
-                    let expr = self.lower_expr(&e);
+                    let expr = self.lower_expr(e);
                     let ident = Ident::new(s, self.lower_span(e.span));
                     self.expr_field(ident, expr, e.span)
                 },
@@ -1419,7 +1526,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         );
 
         hir::ExprKind::Struct(
-            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             fields,
             None,
         )
@@ -1504,10 +1611,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Gen(_)) => {}
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Gen(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Async(_)) => {
-                self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span });
+                return hir::ExprKind::Err(
+                    self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span }),
+                );
             }
             Some(hir::CoroutineKind::Coroutine) | None => {
                 if !self.tcx.features().coroutines {
@@ -1519,14 +1629,37 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     )
                     .emit();
                 }
-                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine)
+                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine);
+                false
             }
-        }
+        };
 
-        let expr =
+        let yielded =
             opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
 
-        hir::ExprKind::Yield(expr, hir::YieldSource::Yield)
+        if is_async_gen {
+            // `yield $expr` is transformed into `task_context = yield async_gen_ready($expr)`.
+            // This ensures that we store our resumed `ResumeContext` correctly, and also that
+            // the apparent value of the `yield` expression is `()`.
+            let wrapped_yielded = self.expr_call_lang_item_fn(
+                span,
+                hir::LangItem::AsyncGenReady,
+                std::slice::from_ref(yielded),
+            );
+            let yield_expr = self.arena.alloc(
+                self.expr(span, hir::ExprKind::Yield(wrapped_yielded, hir::YieldSource::Yield)),
+            );
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+            let task_context_ident = Ident::with_dummy_span(sym::_task_context);
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+
+            hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span))
+        } else {
+            hir::ExprKind::Yield(yielded, hir::YieldSource::Yield)
+        }
     }
 
     /// Desugar `ExprForLoop` from: `[opt_ident]: for <pat> in <head> <body>` into:
@@ -1588,7 +1721,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 head_span,
                 hir::LangItem::IteratorNext,
                 arena_vec![self; ref_mut_iter],
-                None,
             );
             let arms = arena_vec![self; none_arm, some_arm];
 
@@ -1617,7 +1749,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 head_span,
                 hir::LangItem::IntoIterIntoIter,
                 arena_vec![self; head],
-                None,
             )
         };
 
@@ -1655,13 +1786,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
         let try_span = self.tcx.sess.source_map().end_point(span);
         let try_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             try_span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         // `Try::branch(<expr>)`
@@ -1673,7 +1804,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 unstable_span,
                 hir::LangItem::TryTraitBranch,
                 arena_vec![self; sub_expr],
-                None,
             )
         };
 
@@ -1755,7 +1885,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::YeetExpr,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         let from_yeet_expr = self.wrap_in_try_constructor(
@@ -1878,9 +2008,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> hir::Expr<'hir> {
-        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item, hir_id));
+        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item));
         self.expr_call_mut(span, path, args)
     }
 
@@ -1889,21 +2018,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> &'hir hir::Expr<'hir> {
-        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args, hir_id))
+        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args))
     }
 
-    fn expr_lang_item_path(
-        &mut self,
-        span: Span,
-        lang_item: hir::LangItem,
-        hir_id: Option<hir::HirId>,
-    ) -> hir::Expr<'hir> {
-        self.expr(
-            span,
-            hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), hir_id)),
-        )
+    fn expr_lang_item_path(&mut self, span: Span, lang_item: hir::LangItem) -> hir::Expr<'hir> {
+        self.expr(span, hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))))
     }
 
     /// `<LangItem>::name`
@@ -1916,7 +2036,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let path = hir::ExprKind::Path(hir::QPath::TypeRelative(
             self.arena.alloc(self.ty(
                 span,
-                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             )),
             self.arena.alloc(hir::PathSegment::new(
                 Ident::new(name, span),