diff options
Diffstat (limited to 'compiler/rustc_expand/src')
| -rw-r--r-- | compiler/rustc_expand/src/base.rs | 59 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/build.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/config.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/errors.rs | 26 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/expand.rs | 72 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/lib.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/macro_rules.rs | 26 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/placeholders.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/proc_macro_server.rs | 146 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/tokenstream/tests.rs | 18 |
10 files changed, 246 insertions, 124 deletions
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index e1da3ecde..c8de60ccb 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -10,17 +10,19 @@ use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind use rustc_attr::{self as attr, Deprecation, Stability}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_data_structures::sync::{self, Lrc}; -use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan, PResult}; +use rustc_errors::{ + Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, MultiSpan, PResult, +}; use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT; use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics}; use rustc_parse::{self, parser, MACRO_ARGUMENTS}; -use rustc_session::{parse::ParseSess, Limit, Session, SessionDiagnostic}; +use rustc_session::{parse::ParseSess, Limit, Session}; use rustc_span::def_id::{CrateNum, DefId, LocalDefId}; use rustc_span::edition::Edition; use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId}; use rustc_span::source_map::SourceMap; use rustc_span::symbol::{kw, sym, Ident, Symbol}; -use rustc_span::{FileName, Span, DUMMY_SP}; +use rustc_span::{BytePos, FileName, RealFileName, Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; use std::default::Default; @@ -1109,12 +1111,12 @@ impl<'a> ExtCtxt<'a> { pub fn create_err( &self, - err: impl SessionDiagnostic<'a>, + err: impl IntoDiagnostic<'a>, ) -> DiagnosticBuilder<'a, ErrorGuaranteed> { self.sess.create_err(err) } - pub fn emit_err(&self, err: impl SessionDiagnostic<'a>) -> ErrorGuaranteed { + pub fn emit_err(&self, err: impl IntoDiagnostic<'a>) -> ErrorGuaranteed { self.sess.emit_err(err) } @@ -1226,8 +1228,9 @@ pub fn expr_to_spanned_string<'a>( ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), ast::LitKind::ByteStr(_) => { let mut err = cx.struct_span_err(l.span, err_msg); + let span = expr.span.shrink_to_lo(); err.span_suggestion( - expr.span.shrink_to_lo(), + span.with_hi(span.lo() + BytePos(1)), "consider removing the leading `b`", "", Applicability::MaybeIncorrect, @@ -1420,16 +1423,40 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &ParseSess) -> bool { if let ast::ItemKind::Enum(enum_def, _) = &item.kind { if let [variant] = &*enum_def.variants { if variant.ident.name == sym::Input { - sess.buffer_lint_with_diagnostic( - &PROC_MACRO_BACK_COMPAT, - item.ident.span, - ast::CRATE_NODE_ID, - "using `procedural-masquerade` crate", - BuiltinLintDiagnostics::ProcMacroBackCompat( - "The `procedural-masquerade` crate has been unnecessary since Rust 1.30.0. \ - Versions of this crate below 0.1.7 will eventually stop compiling.".to_string()) - ); - return true; + let filename = sess.source_map().span_to_filename(item.ident.span); + if let FileName::Real(RealFileName::LocalPath(path)) = filename { + if let Some(c) = path + .components() + .flat_map(|c| c.as_os_str().to_str()) + .find(|c| c.starts_with("rental") || c.starts_with("allsorts-rental")) + { + let crate_matches = if c.starts_with("allsorts-rental") { + true + } else { + let mut version = c.trim_start_matches("rental-").split("."); + version.next() == Some("0") + && version.next() == Some("5") + && version + .next() + .and_then(|c| c.parse::<u32>().ok()) + .map_or(false, |v| v < 6) + }; + + if crate_matches { + sess.buffer_lint_with_diagnostic( + &PROC_MACRO_BACK_COMPAT, + item.ident.span, + ast::CRATE_NODE_ID, + "using an old version of `rental`", + BuiltinLintDiagnostics::ProcMacroBackCompat( + "older versions of the `rental` crate will stop compiling in future versions of Rust; \ + please update to `rental` v0.5.6, or switch to one of the `rental` alternatives".to_string() + ) + ); + return true; + } + } + } } } } diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index 50d2be3ce..0952e65cf 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -252,6 +252,10 @@ impl<'a> ExtCtxt<'a> { self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) } + pub fn expr_field(&self, span: Span, expr: P<Expr>, field: Ident) -> P<ast::Expr> { + self.expr(span, ast::ExprKind::Field(expr, field)) + } + pub fn expr_binary( &self, sp: Span, diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 8d4e36407..1d2b1298a 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -469,6 +469,7 @@ impl<'a> StripUnconfigured<'a> { } /// If attributes are not allowed on expressions, emit an error for `attr` + #[instrument(level = "trace", skip(self))] pub(crate) fn maybe_emit_expr_attr_err(&self, attr: &Attribute) { if !self.features.map_or(true, |features| features.stmt_expr_attributes) { let mut err = feature_err( @@ -486,9 +487,12 @@ impl<'a> StripUnconfigured<'a> { } } - pub fn configure_expr(&self, expr: &mut P<ast::Expr>) { - for attr in expr.attrs.iter() { - self.maybe_emit_expr_attr_err(attr); + #[instrument(level = "trace", skip(self))] + pub fn configure_expr(&self, expr: &mut P<ast::Expr>, method_receiver: bool) { + if !method_receiver { + for attr in expr.attrs.iter() { + self.maybe_emit_expr_attr_err(attr); + } } // If an expr is valid to cfg away it will have been removed by the diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs index 0feae0deb..d383f4832 100644 --- a/compiler/rustc_expand/src/errors.rs +++ b/compiler/rustc_expand/src/errors.rs @@ -1,46 +1,46 @@ -use rustc_macros::SessionDiagnostic; +use rustc_macros::Diagnostic; use rustc_span::symbol::MacroRulesNormalizedIdent; use rustc_span::Span; -#[derive(SessionDiagnostic)] -#[diag(expand::expr_repeat_no_syntax_vars)] +#[derive(Diagnostic)] +#[diag(expand_expr_repeat_no_syntax_vars)] pub(crate) struct NoSyntaxVarsExprRepeat { #[primary_span] pub span: Span, } -#[derive(SessionDiagnostic)] -#[diag(expand::must_repeat_once)] +#[derive(Diagnostic)] +#[diag(expand_must_repeat_once)] pub(crate) struct MustRepeatOnce { #[primary_span] pub span: Span, } -#[derive(SessionDiagnostic)] -#[diag(expand::count_repetition_misplaced)] +#[derive(Diagnostic)] +#[diag(expand_count_repetition_misplaced)] pub(crate) struct CountRepetitionMisplaced { #[primary_span] pub span: Span, } -#[derive(SessionDiagnostic)] -#[diag(expand::meta_var_expr_unrecognized_var)] +#[derive(Diagnostic)] +#[diag(expand_meta_var_expr_unrecognized_var)] pub(crate) struct MetaVarExprUnrecognizedVar { #[primary_span] pub span: Span, pub key: MacroRulesNormalizedIdent, } -#[derive(SessionDiagnostic)] -#[diag(expand::var_still_repeating)] +#[derive(Diagnostic)] +#[diag(expand_var_still_repeating)] pub(crate) struct VarStillRepeating { #[primary_span] pub span: Span, pub ident: MacroRulesNormalizedIdent, } -#[derive(SessionDiagnostic)] -#[diag(expand::meta_var_dif_seq_matchers)] +#[derive(Diagnostic)] +#[diag(expand_meta_var_dif_seq_matchers)] pub(crate) struct MetaVarsDifSeqMatchers { #[primary_span] pub span: Span, diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index c2add852a..57713fb3c 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -50,6 +50,7 @@ macro_rules! ast_fragments { /// Can also serve as an input and intermediate result for macro expansion operations. pub enum AstFragment { OptExpr(Option<P<ast::Expr>>), + MethodReceiverExpr(P<ast::Expr>), $($Kind($AstTy),)* } @@ -57,6 +58,7 @@ macro_rules! ast_fragments { #[derive(Copy, Clone, PartialEq, Eq)] pub enum AstFragmentKind { OptExpr, + MethodReceiverExpr, $($Kind,)* } @@ -64,6 +66,7 @@ macro_rules! ast_fragments { pub fn name(self) -> &'static str { match self { AstFragmentKind::OptExpr => "expression", + AstFragmentKind::MethodReceiverExpr => "expression", $(AstFragmentKind::$Kind => $kind_name,)* } } @@ -72,6 +75,8 @@ macro_rules! ast_fragments { match self { AstFragmentKind::OptExpr => result.make_expr().map(Some).map(AstFragment::OptExpr), + AstFragmentKind::MethodReceiverExpr => + result.make_expr().map(AstFragment::MethodReceiverExpr), $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* } } @@ -98,6 +103,13 @@ macro_rules! ast_fragments { } } + pub fn make_method_receiver_expr(self) -> P<ast::Expr> { + match self { + AstFragment::MethodReceiverExpr(expr) => expr, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + } + $(pub fn $make_ast(self) -> $AstTy { match self { AstFragment::$Kind(ast) => ast, @@ -120,6 +132,7 @@ macro_rules! ast_fragments { } }); } + AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr), $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* $($(AstFragment::$Kind(ast) => ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* @@ -130,6 +143,7 @@ macro_rules! ast_fragments { match *self { AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), AstFragment::OptExpr(None) => {} + AstFragment::MethodReceiverExpr(ref expr) => visitor.visit_method_receiver_expr(expr), $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { visitor.$visit_ast_elt(ast_elt, $($args)*); @@ -222,6 +236,7 @@ impl AstFragmentKind { match self { AstFragmentKind::OptExpr | AstFragmentKind::Expr + | AstFragmentKind::MethodReceiverExpr | AstFragmentKind::Stmts | AstFragmentKind::Ty | AstFragmentKind::Pat => SupportsMacroExpansion::Yes { supports_inner_attrs: false }, @@ -285,6 +300,9 @@ impl AstFragmentKind { AstFragmentKind::Expr => AstFragment::Expr( items.next().expect("expected exactly one expression").expect_expr(), ), + AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr( + items.next().expect("expected exactly one expression").expect_expr(), + ), AstFragmentKind::OptExpr => { AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)) } @@ -327,7 +345,7 @@ impl InvocationKind { fn placeholder_visibility(&self) -> Option<ast::Visibility> { // HACK: For unnamed fields placeholders should have the same visibility as the actual // fields because for tuple structs/variants resolve determines visibilities of their - // constructor using these field visibilities before attributes on them are are expanded. + // constructor using these field visibilities before attributes on them are expanded. // The assumption is that the attribute expansion cannot change field visibilities, // and it holds because only inert attributes are supported in this position. match self { @@ -893,6 +911,7 @@ pub fn parse_ast_fragment<'a>( AstFragment::Stmts(stmts) } AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), + AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(this.parse_expr()?), AstFragmentKind::OptExpr => { if this.token != token::Eof { AstFragment::OptExpr(Some(this.parse_expr()?)) @@ -937,13 +956,12 @@ pub fn ensure_complete_parse<'a>( kind_name, ); err.note(&msg); - let semi_span = this.sess.source_map().next_point(span); - let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); - match this.sess.source_map().span_to_snippet(semi_full_span) { + let semi_span = this.sess.source_map().next_point(span); + match this.sess.source_map().span_to_snippet(semi_span) { Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { err.span_suggestion( - semi_span, + span.shrink_to_hi(), "you might be missing a semicolon here", ";", Applicability::MaybeIncorrect, @@ -1478,6 +1496,42 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> { } } +/// This struct is a hack to workaround unstable of `stmt_expr_attributes`. +/// It can be removed once that feature is stabilized. +struct MethodReceiverTag; +impl DummyAstNode for MethodReceiverTag { + fn dummy() -> MethodReceiverTag { + MethodReceiverTag + } +} +impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> { + type OutputTy = Self; + type AttrsTy = ast::AttrVec; + const KIND: AstFragmentKind = AstFragmentKind::MethodReceiverExpr; + fn descr() -> &'static str { + "an expression" + } + fn to_annotatable(self) -> Annotatable { + Annotatable::Expr(self.wrapped) + } + fn fragment_to_output(fragment: AstFragment) -> Self::OutputTy { + AstNodeWrapper::new(fragment.make_method_receiver_expr(), MethodReceiverTag) + } + fn noop_visit<V: MutVisitor>(&mut self, visitor: &mut V) { + noop_visit_expr(&mut self.wrapped, visitor) + } + fn is_mac_call(&self) -> bool { + matches!(self.wrapped.kind, ast::ExprKind::MacCall(..)) + } + fn take_mac_call(self) -> (P<ast::MacCall>, Self::AttrsTy, AddSemicolon) { + let node = self.wrapped.into_inner(); + match node.kind { + ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No), + _ => unreachable!(), + } + } +} + struct InvocationCollector<'a, 'b> { cx: &'a mut ExtCtxt<'b>, invocations: Vec<(Invocation, Option<Lrc<SyntaxExtension>>)>, @@ -1841,6 +1895,14 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { self.visit_node(node) } + fn visit_method_receiver_expr(&mut self, node: &mut P<ast::Expr>) { + visit_clobber(node, |node| { + let mut wrapper = AstNodeWrapper::new(node, MethodReceiverTag); + self.visit_node(&mut wrapper); + wrapper.wrapped + }) + } + fn filter_map_expr(&mut self, node: P<ast::Expr>) -> Option<P<ast::Expr>> { self.flat_map_node(AstNodeWrapper::new(node, OptExprTag)) } diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index ffc9abe64..b34de94fb 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -3,7 +3,6 @@ #![feature(associated_type_defaults)] #![feature(if_let_guard)] #![feature(let_chains)] -#![cfg_attr(bootstrap, feature(let_else))] #![feature(macro_metavar_expr)] #![feature(proc_macro_diagnostic)] #![feature(proc_macro_internals)] diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 7764ffd24..f6fe38174 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -14,7 +14,7 @@ use rustc_ast::{NodeId, DUMMY_NODE_ID}; use rustc_ast_pretty::pprust; use rustc_attr::{self as attr, TransparencyError}; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; -use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder}; +use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage}; use rustc_feature::Features; use rustc_lint_defs::builtin::{ RUST_2021_INCOMPATIBLE_OR_PATTERNS, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS, @@ -68,19 +68,22 @@ fn emit_frag_parse_err( kind: AstFragmentKind, ) { // FIXME(davidtwco): avoid depending on the error message text - if parser.token == token::Eof && e.message[0].0.expect_str().ends_with(", found `<eof>`") { - if !e.span.is_dummy() { - // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); - } + if parser.token == token::Eof + && let DiagnosticMessage::Str(message) = &e.message[0].0 + && message.ends_with(", found `<eof>`") + { let msg = &e.message[0]; e.message[0] = ( - rustc_errors::DiagnosticMessage::Str(format!( + DiagnosticMessage::Str(format!( "macro expansion ends with an incomplete expression: {}", - msg.0.expect_str().replace(", found `<eof>`", ""), + message.replace(", found `<eof>`", ""), )), msg.1, ); + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.token.span.shrink_to_hi()); + } } if e.span.is_dummy() { // Get around lack of span in error (#30128) @@ -247,6 +250,7 @@ fn expand_macro<'cx>( // hacky, but speeds up the `html5ever` benchmark significantly. (Issue // 68836 suggests a more comprehensive but more complex change to deal with // this situation.) + // FIXME(Nilstrieb): Stop recovery from happening on this parser and retry later with recovery if the macro failed to match. let parser = parser_from_cx(sess, arg.clone()); // Try each arm's matchers. @@ -593,14 +597,14 @@ pub fn compile_declarative_macro( (mk_syn_ext(expander), rule_spans) } -#[derive(SessionSubdiagnostic)] +#[derive(Subdiagnostic)] enum ExplainDocComment { - #[label(expand::explain_doc_comment_inner)] + #[label(expand_explain_doc_comment_inner)] Inner { #[primary_span] span: Span, }, - #[label(expand::explain_doc_comment_outer)] + #[label(expand_explain_doc_comment_outer)] Outer { #[primary_span] span: Span, diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index 3b0d5ddb9..faaf3b3fe 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -55,6 +55,7 @@ pub fn placeholder( }), AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), + AstFragmentKind::MethodReceiverExpr => AstFragment::MethodReceiverExpr(expr_placeholder()), AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { id, span, @@ -296,6 +297,13 @@ impl MutVisitor for PlaceholderExpander { } } + fn visit_method_receiver_expr(&mut self, expr: &mut P<ast::Expr>) { + match expr.kind { + ast::ExprKind::MacCall(_) => *expr = self.remove(expr.id).make_method_receiver_expr(), + _ => noop_visit_expr(expr, self), + } + } + fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> { match expr.kind { ast::ExprKind::MacCall(_) => self.remove(expr.id).make_opt_expr(), diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 59a7b668a..cc2858d3f 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -1,5 +1,8 @@ use crate::base::ExtCtxt; - +use pm::bridge::{ + server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, +}; +use pm::{Delimiter, Level, LineColumn}; use rustc_ast as ast; use rustc_ast::token; use rustc_ast::tokenstream::{self, Spacing::*, TokenStream}; @@ -13,11 +16,7 @@ use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; - -use pm::bridge::{ - server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, -}; -use pm::{Delimiter, Level, LineColumn}; +use smallvec::{smallvec, SmallVec}; use std::ops::Bound; trait FromInternal<T> { @@ -110,10 +109,26 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint), }; + // Split the operator into one or more `Punct`s, one per character. + // The final one inherits the jointness of the original token. Any + // before that get `joint = true`. let mut op = |s: &str| { assert!(s.is_ascii()); - trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| { - TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span }) + trees.extend(s.bytes().enumerate().map(|(i, ch)| { + let is_final = i == s.len() - 1; + // Split the token span into single chars. Unless the span + // is an unusual one, e.g. due to proc macro expansion. We + // determine this by assuming any span with a length that + // matches the operator length is a normal one, and any + // span with a different length is an unusual one. + let span = if (span.hi() - span.lo()).to_usize() == s.len() { + let lo = span.lo() + BytePos::from_usize(i); + let hi = lo + BytePos::from_usize(1); + span.with_lo(lo).with_hi(hi) + } else { + span + }; + TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span }) })); }; @@ -237,23 +252,57 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre } } -impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>) { - fn to_internal(self) -> TokenStream { +// We use a `SmallVec` because the output size is always one or two `TokenTree`s. +impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>> + for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>) +{ + fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { use rustc_ast::token::*; let (tree, rustc) = self; - let (ch, joint, span) = match tree { - TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + match tree { + TokenTree::Punct(Punct { ch, joint, span }) => { + let kind = match ch { + b'=' => Eq, + b'<' => Lt, + b'>' => Gt, + b'!' => Not, + b'~' => Tilde, + b'+' => BinOp(Plus), + b'-' => BinOp(Minus), + b'*' => BinOp(Star), + b'/' => BinOp(Slash), + b'%' => BinOp(Percent), + b'^' => BinOp(Caret), + b'&' => BinOp(And), + b'|' => BinOp(Or), + b'@' => At, + b'.' => Dot, + b',' => Comma, + b';' => Semi, + b':' => Colon, + b'#' => Pound, + b'$' => Dollar, + b'?' => Question, + b'\'' => SingleQuote, + _ => unreachable!(), + }; + smallvec![if joint { + tokenstream::TokenTree::token_joint(kind, span) + } else { + tokenstream::TokenTree::token_alone(kind, span) + }] + } TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { - return tokenstream::TokenStream::delimited( + smallvec![tokenstream::TokenTree::Delimited( tokenstream::DelimSpan { open, close }, delimiter.to_internal(), stream.unwrap_or_default(), - ); + )] } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { rustc.sess().symbol_gallery.insert(sym, span); - return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span); + smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)] } TokenTree::Literal(self::Literal { kind: self::LitKind::Integer, @@ -266,7 +315,7 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus let integer = TokenKind::lit(token::Integer, symbol, suffix); let a = tokenstream::TokenTree::token_alone(minus, span); let b = tokenstream::TokenTree::token_alone(integer, span); - return [a, b].into_iter().collect(); + smallvec![a, b] } TokenTree::Literal(self::Literal { kind: self::LitKind::Float, @@ -279,46 +328,14 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus let float = TokenKind::lit(token::Float, symbol, suffix); let a = tokenstream::TokenTree::token_alone(minus, span); let b = tokenstream::TokenTree::token_alone(float, span); - return [a, b].into_iter().collect(); + smallvec![a, b] } TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { - return tokenstream::TokenStream::token_alone( + smallvec![tokenstream::TokenTree::token_alone( TokenKind::lit(kind.to_internal(), symbol, suffix), span, - ); + )] } - }; - - let kind = match ch { - b'=' => Eq, - b'<' => Lt, - b'>' => Gt, - b'!' => Not, - b'~' => Tilde, - b'+' => BinOp(Plus), - b'-' => BinOp(Minus), - b'*' => BinOp(Star), - b'/' => BinOp(Slash), - b'%' => BinOp(Percent), - b'^' => BinOp(Caret), - b'&' => BinOp(And), - b'|' => BinOp(Or), - b'@' => At, - b'.' => Dot, - b',' => Comma, - b';' => Semi, - b':' => Colon, - b'#' => Pound, - b'$' => Dollar, - b'?' => Question, - b'\'' => SingleQuote, - _ => unreachable!(), - }; - - if joint { - tokenstream::TokenStream::token_joint(kind, span) - } else { - tokenstream::TokenStream::token_alone(kind, span) } } } @@ -533,7 +550,7 @@ impl server::TokenStream for Rustc<'_, '_> { &mut self, tree: TokenTree<Self::TokenStream, Self::Span, Self::Symbol>, ) -> Self::TokenStream { - (tree, &mut *self).to_internal() + Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::<Vec<_>>()) } fn concat_trees( @@ -541,14 +558,14 @@ impl server::TokenStream for Rustc<'_, '_> { base: Option<Self::TokenStream>, trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>>, ) -> Self::TokenStream { - let mut builder = tokenstream::TokenStreamBuilder::new(); - if let Some(base) = base { - builder.push(base); - } + let mut stream = + if let Some(base) = base { base } else { tokenstream::TokenStream::default() }; for tree in trees { - builder.push((tree, &mut *self).to_internal()); + for tt in (tree, &mut *self).to_internal() { + stream.push_tree(tt); + } } - builder.build() + stream } fn concat_streams( @@ -556,14 +573,12 @@ impl server::TokenStream for Rustc<'_, '_> { base: Option<Self::TokenStream>, streams: Vec<Self::TokenStream>, ) -> Self::TokenStream { - let mut builder = tokenstream::TokenStreamBuilder::new(); - if let Some(base) = base { - builder.push(base); + let mut stream = + if let Some(base) = base { base } else { tokenstream::TokenStream::default() }; + for s in streams { + stream.push_stream(s); } - for stream in streams { - builder.push(stream); - } - builder.build() + stream } fn into_trees( @@ -689,6 +704,7 @@ impl server::Span for Rustc<'_, '_> { fn source_text(&mut self, span: Self::Span) -> Option<String> { self.sess().source_map().span_to_snippet(span).ok() } + /// Saves the provided span into the metadata of /// *the crate we are currently compiling*, which must /// be a proc-macro crate. This id can be passed to diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index eed696810..91c4dd732 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -1,7 +1,7 @@ use crate::tests::string_to_stream; use rustc_ast::token; -use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder}; +use rustc_ast::tokenstream::{TokenStream, TokenTree}; use rustc_span::create_default_session_globals_then; use rustc_span::{BytePos, Span, Symbol}; @@ -19,10 +19,9 @@ fn test_concat() { let test_res = string_to_ts("foo::bar::baz"); let test_fst = string_to_ts("foo::bar"); let test_snd = string_to_ts("::baz"); - let mut builder = TokenStreamBuilder::new(); - builder.push(test_fst); - builder.push(test_snd); - let eq_res = builder.build(); + let mut eq_res = TokenStream::default(); + eq_res.push_stream(test_fst); + eq_res.push_stream(test_snd); assert_eq!(test_res.trees().count(), 5); assert_eq!(eq_res.trees().count(), 5); assert_eq!(test_res.eq_unspanned(&eq_res), true); @@ -99,11 +98,10 @@ fn test_is_empty() { #[test] fn test_dotdotdot() { create_default_session_globals_then(|| { - let mut builder = TokenStreamBuilder::new(); - builder.push(TokenStream::token_joint(token::Dot, sp(0, 1))); - builder.push(TokenStream::token_joint(token::Dot, sp(1, 2))); - builder.push(TokenStream::token_alone(token::Dot, sp(2, 3))); - let stream = builder.build(); + let mut stream = TokenStream::default(); + stream.push_tree(TokenTree::token_joint(token::Dot, sp(0, 1))); + stream.push_tree(TokenTree::token_joint(token::Dot, sp(1, 2))); + stream.push_tree(TokenTree::token_alone(token::Dot, sp(2, 3))); assert!(stream.eq_unspanned(&string_to_ts("..."))); assert_eq!(stream.trees().count(), 1); }) |