Merging upstream version 1.66.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 246 insertions, 124 deletions

diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index e1da3ecde..c8de60ccb 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -10,17 +10,19 @@ use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind
 use rustc_attr::{self as attr, Deprecation, Stability};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_data_structures::sync::{self, Lrc};
-use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan, PResult};
+use rustc_errors::{
+    Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, MultiSpan, PResult,
+};
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
 use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics};
 use rustc_parse::{self, parser, MACRO_ARGUMENTS};
-use rustc_session::{parse::ParseSess, Limit, Session, SessionDiagnostic};
+use rustc_session::{parse::ParseSess, Limit, Session};
 use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId};
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::{FileName, Span, DUMMY_SP};
+use rustc_span::{BytePos, FileName, RealFileName, Span, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
 
 use std::default::Default;
@@ -1109,12 +1111,12 @@ impl<'a> ExtCtxt<'a> {
 
     pub fn create_err(
         &self,
-        err: impl SessionDiagnostic<'a>,
+        err: impl IntoDiagnostic<'a>,
     ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         self.sess.create_err(err)
     }
 
-    pub fn emit_err(&self, err: impl SessionDiagnostic<'a>) -> ErrorGuaranteed {
+    pub fn emit_err(&self, err: impl IntoDiagnostic<'a>) -> ErrorGuaranteed {
         self.sess.emit_err(err)
     }
 
@@ -1226,8 +1228,9 @@ pub fn expr_to_spanned_string<'a>(
             ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)),
             ast::LitKind::ByteStr(_) => {
                 let mut err = cx.struct_span_err(l.span, err_msg);
+                let span = expr.span.shrink_to_lo();
                 err.span_suggestion(
-                    expr.span.shrink_to_lo(),
+                    span.with_hi(span.lo() + BytePos(1)),
                     "consider removing the leading `b`",
                     "",
                     Applicability::MaybeIncorrect,
@@ -1420,16 +1423,40 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &ParseSess) -> bool {
         if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
             if let [variant] = &*enum_def.variants {
                 if variant.ident.name == sym::Input {
-                    sess.buffer_lint_with_diagnostic(
-                        &PROC_MACRO_BACK_COMPAT,
-                        item.ident.span,
-                        ast::CRATE_NODE_ID,
-                        "using `procedural-masquerade` crate",
-                        BuiltinLintDiagnostics::ProcMacroBackCompat(
-                        "The `procedural-masquerade` crate has been unnecessary since Rust 1.30.0. \
-                        Versions of this crate below 0.1.7 will eventually stop compiling.".to_string())
-                    );
-                    return true;
+                    let filename = sess.source_map().span_to_filename(item.ident.span);
+                    if let FileName::Real(RealFileName::LocalPath(path)) = filename {
+                        if let Some(c) = path
+                            .components()
+                            .flat_map(|c| c.as_os_str().to_str())
+                            .find(|c| c.starts_with("rental") || c.starts_with("allsorts-rental"))
+                        {
+                            let crate_matches = if c.starts_with("allsorts-rental") {
+                                true
+                            } else {
+                                let mut version = c.trim_start_matches("rental-").split(".");
+                                version.next() == Some("0")
+                                    && version.next() == Some("5")
+                                    && version
+                                        .next()
+                                        .and_then(|c| c.parse::<u32>().ok())
+                                        .map_or(false, |v| v < 6)
+                            };
+
+                            if crate_matches {
+                                sess.buffer_lint_with_diagnostic(
+                                        &PROC_MACRO_BACK_COMPAT,
+                                        item.ident.span,
+                                        ast::CRATE_NODE_ID,
+                                        "using an old version of `rental`",
+                                        BuiltinLintDiagnostics::ProcMacroBackCompat(
+                                        "older versions of the `rental` crate will stop compiling in future versions of Rust; \
+                                        please update to `rental` v0.5.6, or switch to one of the `rental` alternatives".to_string()
+                                        )
+                                    );
+                                return true;
+                            }
+                        }
+                    }
                 }
             }
         }
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 50d2be3ce..0952e65cf 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -252,6 +252,10 @@ impl<'a> ExtCtxt<'a> {
         self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower))
     }
 
+    pub fn expr_field(&self, span: Span, expr: P<Expr>, field: Ident) -> P<ast::Expr> {
+        self.expr(span, ast::ExprKind::Field(expr, field))
+    }
+
     pub fn expr_binary(
         &self,
         sp: Span,
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 8d4e36407..1d2b1298a 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -469,6 +469,7 @@ impl<'a> StripUnconfigured<'a> {
     }
 
     /// If attributes are not allowed on expressions, emit an error for `attr`
+    #[instrument(level = "trace", skip(self))]
     pub(crate) fn maybe_emit_expr_attr_err(&self, attr: &Attribute) {
         if !self.features.map_or(true, |features| features.stmt_expr_attributes) {
             let mut err = feature_err(
@@ -486,9 +487,12 @@ impl<'a> StripUnconfigured<'a> {
         }
     }
 
-    pub fn configure_expr(&self, expr: &mut P<ast::Expr>) {
-        for attr in expr.attrs.iter() {
-            self.maybe_emit_expr_attr_err(attr);
+    #[instrument(level = "trace", skip(self))]
+    pub fn configure_expr(&self, expr: &mut P<ast::Expr>, method_receiver: bool) {
+        if !method_receiver {
+            for attr in expr.attrs.iter() {
+                self.maybe_emit_expr_attr_err(attr);
+            }
         }
 
         // If an expr is valid to cfg away it will have been removed by the
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index 0feae0deb..d383f4832 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -1,46 +1,46 @@
-use rustc_macros::SessionDiagnostic;
+use rustc_macros::Diagnostic;
 use rustc_span::symbol::MacroRulesNormalizedIdent;
 use rustc_span::Span;
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::expr_repeat_no_syntax_vars)]
+#[derive(Diagnostic)]
+#[diag(expand_expr_repeat_no_syntax_vars)]
 pub(crate) struct NoSyntaxVarsExprRepeat {
     #[primary_span]
     pub span: Span,
 }
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::must_repeat_once)]
+#[derive(Diagnostic)]
+#[diag(expand_must_repeat_once)]
 pub(crate) struct MustRepeatOnce {
     #[primary_span]
     pub span: Span,
 }
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::count_repetition_misplaced)]
+#[derive(Diagnostic)]
+#[diag(expand_count_repetition_misplaced)]
 pub(crate) struct CountRepetitionMisplaced {
     #[primary_span]
     pub span: Span,
 }
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::meta_var_expr_unrecognized_var)]
+#[derive(Diagnostic)]
+#[diag(expand_meta_var_expr_unrecognized_var)]
 pub(crate) struct MetaVarExprUnrecognizedVar {
     #[primary_span]
     pub span: Span,
     pub key: MacroRulesNormalizedIdent,
 }
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::var_still_repeating)]
+#[derive(Diagnostic)]
+#[diag(expand_var_still_repeating)]
 pub(crate) struct VarStillRepeating {
     #[primary_span]
     pub span: Span,
     pub ident: MacroRulesNormalizedIdent,
 }
 
-#[derive(SessionDiagnostic)]
-#[diag(expand::meta_var_dif_seq_matchers)]
+#[derive(Diagnostic)]
+#[diag(expand_meta_var_dif_seq_matchers)]
 pub(crate) struct MetaVarsDifSeqMatchers {
     #[primary_span]
     pub span: Span,
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index c2add852a..57713fb3c 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -50,6 +50,7 @@ macro_rules! ast_fragments {
         /// Can also serve as an input and intermediate result for macro expansion operations.
         pub enum AstFragment {
             OptExpr(Option<P<ast::Expr>>),
+            MethodReceiverExpr(P<ast::Expr>),
             $($Kind($AstTy),)*
         }
 
@@ -57,6 +58,7 @@ macro_rules! ast_fragments {
         #[derive(Copy, Clone, PartialEq, Eq)]
         pub enum AstFragmentKind {
             OptExpr,
+            MethodReceiverExpr,
             $($Kind,)*
         }
 
@@ -64,6 +66,7 @@ macro_rules! ast_fragments {
             pub fn name(self) -> &'static str {
                 match self {
                     AstFragmentKind::OptExpr => "expression",
+                    AstFragmentKind::MethodReceiverExpr => "expression",
                     $(AstFragmentKind::$Kind => $kind_name,)*
                 }
             }
@@ -72,6 +75,8 @@ macro_rules! ast_fragments {
                 match self {
                     AstFragmentKind::OptExpr =>
                         result.make_expr().map(Some).map(AstFragment::OptExpr),
+                    AstFragmentKind::MethodReceiverExpr =>
+                        result.make_expr().map(AstFragment::MethodReceiverExpr),
                     $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
                 }
             }
@@ -98,6 +103,13 @@ macro_rules! ast_fragments {
                 }
             }
 
+            pub fn make_method_receiver_expr(self) -> P<ast::Expr> {
+                match self {
+                    AstFragment::MethodReceiverExpr(expr) => expr,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            }
+
             $(pub fn $make_ast(self) -> $AstTy {
                 match self {
                     AstFragment::$Kind(ast) => ast,
@@ -120,6 +132,7 @@ macro_rules! ast_fragments {
                             }
                         });
                     }
+                    AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
                     $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
                     $($(AstFragment::$Kind(ast) =>
                         ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
@@ -130,6 +143,7 @@ macro_rules! ast_fragments {
                 match *self {
                     AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
                     AstFragment::OptExpr(None) => {}
+                    AstFragment::MethodReceiverExpr(ref expr) => visitor.visit_method_receiver_expr(expr),
                     $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)*
                     $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] {
                         visitor.$visit_ast_elt(ast_elt, $($args)*);
@@ -222,6 +236,7 @@ impl AstFragmentKind {
         match self {
             AstFragmentKind::OptExpr
             | AstFragmentKind::Expr
+            | AstFragmentKind::MethodReceiverExpr
             | AstFragmentKind::Stmts
             | AstFragmentKind::Ty
             | AstFragmentKind::Pat => SupportsMacroExpansion::Yes { supports_inner_attrs: false },
@@ -285,6 +300,9 @@ impl AstFragmentKind {
             AstFragmentKind::Expr => AstFragment::Expr(
                 items.next().expect("expected exactly one expression").expect_expr(),
             ),
+            AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(
+                items.next().expect("expected exactly one expression").expect_expr(),
+            ),
             AstFragmentKind::OptExpr => {
                 AstFragment::OptExpr(items.next().map(Annotatable::expect_expr))
             }
@@ -327,7 +345,7 @@ impl InvocationKind {
     fn placeholder_visibility(&self) -> Option<ast::Visibility> {
         // HACK: For unnamed fields placeholders should have the same visibility as the actual
         // fields because for tuple structs/variants resolve determines visibilities of their
-        // constructor using these field visibilities before attributes on them are are expanded.
+        // constructor using these field visibilities before attributes on them are expanded.
         // The assumption is that the attribute expansion cannot change field visibilities,
         // and it holds because only inert attributes are supported in this position.
         match self {
@@ -893,6 +911,7 @@ pub fn parse_ast_fragment<'a>(
             AstFragment::Stmts(stmts)
         }
         AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?),
+        AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(this.parse_expr()?),
         AstFragmentKind::OptExpr => {
             if this.token != token::Eof {
                 AstFragment::OptExpr(Some(this.parse_expr()?))
@@ -937,13 +956,12 @@ pub fn ensure_complete_parse<'a>(
             kind_name,
         );
         err.note(&msg);
-        let semi_span = this.sess.source_map().next_point(span);
 
-        let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span));
-        match this.sess.source_map().span_to_snippet(semi_full_span) {
+        let semi_span = this.sess.source_map().next_point(span);
+        match this.sess.source_map().span_to_snippet(semi_span) {
             Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
                 err.span_suggestion(
-                    semi_span,
+                    span.shrink_to_hi(),
                     "you might be missing a semicolon here",
                     ";",
                     Applicability::MaybeIncorrect,
@@ -1478,6 +1496,42 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> {
     }
 }
 
+/// This struct is a hack to workaround unstable of `stmt_expr_attributes`.
+/// It can be removed once that feature is stabilized.
+struct MethodReceiverTag;
+impl DummyAstNode for MethodReceiverTag {
+    fn dummy() -> MethodReceiverTag {
+        MethodReceiverTag
+    }
+}
+impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
+    type OutputTy = Self;
+    type AttrsTy = ast::AttrVec;
+    const KIND: AstFragmentKind = AstFragmentKind::MethodReceiverExpr;
+    fn descr() -> &'static str {
+        "an expression"
+    }
+    fn to_annotatable(self) -> Annotatable {
+        Annotatable::Expr(self.wrapped)
+    }
+    fn fragment_to_output(fragment: AstFragment) -> Self::OutputTy {
+        AstNodeWrapper::new(fragment.make_method_receiver_expr(), MethodReceiverTag)
+    }
+    fn noop_visit<V: MutVisitor>(&mut self, visitor: &mut V) {
+        noop_visit_expr(&mut self.wrapped, visitor)
+    }
+    fn is_mac_call(&self) -> bool {
+        matches!(self.wrapped.kind, ast::ExprKind::MacCall(..))
+    }
+    fn take_mac_call(self) -> (P<ast::MacCall>, Self::AttrsTy, AddSemicolon) {
+        let node = self.wrapped.into_inner();
+        match node.kind {
+            ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
+            _ => unreachable!(),
+        }
+    }
+}
+
 struct InvocationCollector<'a, 'b> {
     cx: &'a mut ExtCtxt<'b>,
     invocations: Vec<(Invocation, Option<Lrc<SyntaxExtension>>)>,
@@ -1841,6 +1895,14 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
         self.visit_node(node)
     }
 
+    fn visit_method_receiver_expr(&mut self, node: &mut P<ast::Expr>) {
+        visit_clobber(node, |node| {
+            let mut wrapper = AstNodeWrapper::new(node, MethodReceiverTag);
+            self.visit_node(&mut wrapper);
+            wrapper.wrapped
+        })
+    }
+
     fn filter_map_expr(&mut self, node: P<ast::Expr>) -> Option<P<ast::Expr>> {
         self.flat_map_node(AstNodeWrapper::new(node, OptExprTag))
     }
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index ffc9abe64..b34de94fb 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -3,7 +3,6 @@
 #![feature(associated_type_defaults)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
-#![cfg_attr(bootstrap, feature(let_else))]
 #![feature(macro_metavar_expr)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 7764ffd24..f6fe38174 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -14,7 +14,7 @@ use rustc_ast::{NodeId, DUMMY_NODE_ID};
 use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, TransparencyError};
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
-use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder};
+use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage};
 use rustc_feature::Features;
 use rustc_lint_defs::builtin::{
     RUST_2021_INCOMPATIBLE_OR_PATTERNS, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
@@ -68,19 +68,22 @@ fn emit_frag_parse_err(
     kind: AstFragmentKind,
 ) {
     // FIXME(davidtwco): avoid depending on the error message text
-    if parser.token == token::Eof && e.message[0].0.expect_str().ends_with(", found `<eof>`") {
-        if !e.span.is_dummy() {
-            // early end of macro arm (#52866)
-            e.replace_span_with(parser.sess.source_map().next_point(parser.token.span));
-        }
+    if parser.token == token::Eof
+        && let DiagnosticMessage::Str(message) = &e.message[0].0
+        && message.ends_with(", found `<eof>`")
+    {
         let msg = &e.message[0];
         e.message[0] = (
-            rustc_errors::DiagnosticMessage::Str(format!(
+            DiagnosticMessage::Str(format!(
                 "macro expansion ends with an incomplete expression: {}",
-                msg.0.expect_str().replace(", found `<eof>`", ""),
+                message.replace(", found `<eof>`", ""),
             )),
             msg.1,
         );
+        if !e.span.is_dummy() {
+            // early end of macro arm (#52866)
+            e.replace_span_with(parser.token.span.shrink_to_hi());
+        }
     }
     if e.span.is_dummy() {
         // Get around lack of span in error (#30128)
@@ -247,6 +250,7 @@ fn expand_macro<'cx>(
     // hacky, but speeds up the `html5ever` benchmark significantly. (Issue
     // 68836 suggests a more comprehensive but more complex change to deal with
     // this situation.)
+    // FIXME(Nilstrieb): Stop recovery from happening on this parser and retry later with recovery if the macro failed to match.
     let parser = parser_from_cx(sess, arg.clone());
 
     // Try each arm's matchers.
@@ -593,14 +597,14 @@ pub fn compile_declarative_macro(
     (mk_syn_ext(expander), rule_spans)
 }
 
-#[derive(SessionSubdiagnostic)]
+#[derive(Subdiagnostic)]
 enum ExplainDocComment {
-    #[label(expand::explain_doc_comment_inner)]
+    #[label(expand_explain_doc_comment_inner)]
     Inner {
         #[primary_span]
         span: Span,
     },
-    #[label(expand::explain_doc_comment_outer)]
+    #[label(expand_explain_doc_comment_outer)]
     Outer {
         #[primary_span]
         span: Span,
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 3b0d5ddb9..faaf3b3fe 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -55,6 +55,7 @@ pub fn placeholder(
         }),
         AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()),
         AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())),
+        AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(expr_placeholder()),
         AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item {
             id,
             span,
@@ -296,6 +297,13 @@ impl MutVisitor for PlaceholderExpander {
         }
     }
 
+    fn visit_method_receiver_expr(&mut self, expr: &mut P<ast::Expr>) {
+        match expr.kind {
+            ast::ExprKind::MacCall(_) => *expr = self.remove(expr.id).make_method_receiver_expr(),
+            _ => noop_visit_expr(expr, self),
+        }
+    }
+
     fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
         match expr.kind {
             ast::ExprKind::MacCall(_) => self.remove(expr.id).make_opt_expr(),
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 59a7b668a..cc2858d3f 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -1,5 +1,8 @@
 use crate::base::ExtCtxt;
-
+use pm::bridge::{
+    server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
+};
+use pm::{Delimiter, Level, LineColumn};
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
@@ -13,11 +16,7 @@ use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::symbol::{self, sym, Symbol};
 use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
-
-use pm::bridge::{
-    server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
-};
-use pm::{Delimiter, Level, LineColumn};
+use smallvec::{smallvec, SmallVec};
 use std::ops::Bound;
 
 trait FromInternal<T> {
@@ -110,10 +109,26 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                 tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
             };
 
+            // Split the operator into one or more `Punct`s, one per character.
+            // The final one inherits the jointness of the original token. Any
+            // before that get `joint = true`.
             let mut op = |s: &str| {
                 assert!(s.is_ascii());
-                trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| {
-                    TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span })
+                trees.extend(s.bytes().enumerate().map(|(i, ch)| {
+                    let is_final = i == s.len() - 1;
+                    // Split the token span into single chars. Unless the span
+                    // is an unusual one, e.g. due to proc macro expansion. We
+                    // determine this by assuming any span with a length that
+                    // matches the operator length is a normal one, and any
+                    // span with a different length is an unusual one.
+                    let span = if (span.hi() - span.lo()).to_usize() == s.len() {
+                        let lo = span.lo() + BytePos::from_usize(i);
+                        let hi = lo + BytePos::from_usize(1);
+                        span.with_lo(lo).with_hi(hi)
+                    } else {
+                        span
+                    };
+                    TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
                 }));
             };
 
@@ -237,23 +252,57 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
     }
 }
 
-impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>) {
-    fn to_internal(self) -> TokenStream {
+// We use a `SmallVec` because the output size is always one or two `TokenTree`s.
+impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
+    for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>)
+{
+    fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> {
         use rustc_ast::token::*;
 
         let (tree, rustc) = self;
-        let (ch, joint, span) = match tree {
-            TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
+        match tree {
+            TokenTree::Punct(Punct { ch, joint, span }) => {
+                let kind = match ch {
+                    b'=' => Eq,
+                    b'<' => Lt,
+                    b'>' => Gt,
+                    b'!' => Not,
+                    b'~' => Tilde,
+                    b'+' => BinOp(Plus),
+                    b'-' => BinOp(Minus),
+                    b'*' => BinOp(Star),
+                    b'/' => BinOp(Slash),
+                    b'%' => BinOp(Percent),
+                    b'^' => BinOp(Caret),
+                    b'&' => BinOp(And),
+                    b'|' => BinOp(Or),
+                    b'@' => At,
+                    b'.' => Dot,
+                    b',' => Comma,
+                    b';' => Semi,
+                    b':' => Colon,
+                    b'#' => Pound,
+                    b'$' => Dollar,
+                    b'?' => Question,
+                    b'\'' => SingleQuote,
+                    _ => unreachable!(),
+                };
+                smallvec![if joint {
+                    tokenstream::TokenTree::token_joint(kind, span)
+                } else {
+                    tokenstream::TokenTree::token_alone(kind, span)
+                }]
+            }
             TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
-                return tokenstream::TokenStream::delimited(
+                smallvec![tokenstream::TokenTree::Delimited(
                     tokenstream::DelimSpan { open, close },
                     delimiter.to_internal(),
                     stream.unwrap_or_default(),
-                );
+                )]
             }
             TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
                 rustc.sess().symbol_gallery.insert(sym, span);
-                return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span);
+                smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)]
             }
             TokenTree::Literal(self::Literal {
                 kind: self::LitKind::Integer,
@@ -266,7 +315,7 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
                 let integer = TokenKind::lit(token::Integer, symbol, suffix);
                 let a = tokenstream::TokenTree::token_alone(minus, span);
                 let b = tokenstream::TokenTree::token_alone(integer, span);
-                return [a, b].into_iter().collect();
+                smallvec![a, b]
             }
             TokenTree::Literal(self::Literal {
                 kind: self::LitKind::Float,
@@ -279,46 +328,14 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus
                 let float = TokenKind::lit(token::Float, symbol, suffix);
                 let a = tokenstream::TokenTree::token_alone(minus, span);
                 let b = tokenstream::TokenTree::token_alone(float, span);
-                return [a, b].into_iter().collect();
+                smallvec![a, b]
             }
             TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
-                return tokenstream::TokenStream::token_alone(
+                smallvec![tokenstream::TokenTree::token_alone(
                     TokenKind::lit(kind.to_internal(), symbol, suffix),
                     span,
-                );
+                )]
             }
-        };
-
-        let kind = match ch {
-            b'=' => Eq,
-            b'<' => Lt,
-            b'>' => Gt,
-            b'!' => Not,
-            b'~' => Tilde,
-            b'+' => BinOp(Plus),
-            b'-' => BinOp(Minus),
-            b'*' => BinOp(Star),
-            b'/' => BinOp(Slash),
-            b'%' => BinOp(Percent),
-            b'^' => BinOp(Caret),
-            b'&' => BinOp(And),
-            b'|' => BinOp(Or),
-            b'@' => At,
-            b'.' => Dot,
-            b',' => Comma,
-            b';' => Semi,
-            b':' => Colon,
-            b'#' => Pound,
-            b'$' => Dollar,
-            b'?' => Question,
-            b'\'' => SingleQuote,
-            _ => unreachable!(),
-        };
-
-        if joint {
-            tokenstream::TokenStream::token_joint(kind, span)
-        } else {
-            tokenstream::TokenStream::token_alone(kind, span)
         }
     }
 }
@@ -533,7 +550,7 @@ impl server::TokenStream for Rustc<'_, '_> {
         &mut self,
         tree: TokenTree<Self::TokenStream, Self::Span, Self::Symbol>,
     ) -> Self::TokenStream {
-        (tree, &mut *self).to_internal()
+        Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::<Vec<_>>())
     }
 
     fn concat_trees(
@@ -541,14 +558,14 @@ impl server::TokenStream for Rustc<'_, '_> {
         base: Option<Self::TokenStream>,
         trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>>,
     ) -> Self::TokenStream {
-        let mut builder = tokenstream::TokenStreamBuilder::new();
-        if let Some(base) = base {
-            builder.push(base);
-        }
+        let mut stream =
+            if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
         for tree in trees {
-            builder.push((tree, &mut *self).to_internal());
+            for tt in (tree, &mut *self).to_internal() {
+                stream.push_tree(tt);
+            }
         }
-        builder.build()
+        stream
     }
 
     fn concat_streams(
@@ -556,14 +573,12 @@ impl server::TokenStream for Rustc<'_, '_> {
         base: Option<Self::TokenStream>,
         streams: Vec<Self::TokenStream>,
     ) -> Self::TokenStream {
-        let mut builder = tokenstream::TokenStreamBuilder::new();
-        if let Some(base) = base {
-            builder.push(base);
+        let mut stream =
+            if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
+        for s in streams {
+            stream.push_stream(s);
         }
-        for stream in streams {
-            builder.push(stream);
-        }
-        builder.build()
+        stream
     }
 
     fn into_trees(
@@ -689,6 +704,7 @@ impl server::Span for Rustc<'_, '_> {
     fn source_text(&mut self, span: Self::Span) -> Option<String> {
         self.sess().source_map().span_to_snippet(span).ok()
     }
+
     /// Saves the provided span into the metadata of
     /// *the crate we are currently compiling*, which must
     /// be a proc-macro crate. This id can be passed to
diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs
index eed696810..91c4dd732 100644
--- a/compiler/rustc_expand/src/tokenstream/tests.rs
+++ b/compiler/rustc_expand/src/tokenstream/tests.rs
@@ -1,7 +1,7 @@
 use crate::tests::string_to_stream;
 
 use rustc_ast::token;
-use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
+use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_span::create_default_session_globals_then;
 use rustc_span::{BytePos, Span, Symbol};
 
@@ -19,10 +19,9 @@ fn test_concat() {
         let test_res = string_to_ts("foo::bar::baz");
         let test_fst = string_to_ts("foo::bar");
         let test_snd = string_to_ts("::baz");
-        let mut builder = TokenStreamBuilder::new();
-        builder.push(test_fst);
-        builder.push(test_snd);
-        let eq_res = builder.build();
+        let mut eq_res = TokenStream::default();
+        eq_res.push_stream(test_fst);
+        eq_res.push_stream(test_snd);
         assert_eq!(test_res.trees().count(), 5);
         assert_eq!(eq_res.trees().count(), 5);
         assert_eq!(test_res.eq_unspanned(&eq_res), true);
@@ -99,11 +98,10 @@ fn test_is_empty() {
 #[test]
 fn test_dotdotdot() {
     create_default_session_globals_then(|| {
-        let mut builder = TokenStreamBuilder::new();
-        builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
-        builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
-        builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
-        let stream = builder.build();
+        let mut stream = TokenStream::default();
+        stream.push_tree(TokenTree::token_joint(token::Dot, sp(0, 1)));
+        stream.push_tree(TokenTree::token_joint(token::Dot, sp(1, 2)));
+        stream.push_tree(TokenTree::token_alone(token::Dot, sp(2, 3)));
         assert!(stream.eq_unspanned(&string_to_ts("...")));
         assert_eq!(stream.trees().count(), 1);
     })