diff options
Diffstat (limited to 'src/tools/rust-analyzer/crates/mbe/src')
10 files changed, 300 insertions, 162 deletions
diff --git a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs index 4b7500250..894355fcb 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs @@ -9,7 +9,7 @@ use test_utils::{bench, bench_fixture, skip_slow_tests}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, DeclarativeMacro, + syntax_node_to_token_tree, tt, DeclarativeMacro, }; #[test] @@ -91,7 +91,14 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri // So we just skip any error cases and try again let mut try_cnt = 0; loop { - let mut subtree = tt::Subtree::default(); + let mut subtree = tt::Subtree { + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, + kind: tt::DelimiterKind::Invisible, + }, + token_trees: vec![], + }; for op in rule.lhs.iter() { collect_from_op(op, &mut subtree, &mut seed); } @@ -145,7 +152,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()), Op::Punct(puncts) => { for punct in puncts { - parent.token_trees.push(tt::Leaf::from(punct.clone()).into()); + parent.token_trees.push(tt::Leaf::from(*punct).into()); } } Op::Repeat { tokens, kind, separator } => { @@ -196,12 +203,15 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri *seed } fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) }) - .into() + tt::Leaf::Ident(tt::Ident { + span: tt::TokenId::unspecified(), + text: SmolStr::new(ident), + }) + .into() } fn make_punct(char: char) -> tt::TokenTree { tt::Leaf::Punct(tt::Punct { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), char, spacing: tt::Spacing::Alone, }) @@ -209,7 +219,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri } fn make_literal(lit: &str) -> tt::TokenTree { tt::Leaf::Literal(tt::Literal { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), text: SmolStr::new(lit), }) .into() @@ -219,7 +229,11 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri token_trees: Option<Vec<tt::TokenTree>>, ) -> tt::TokenTree { tt::Subtree { - delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), + kind, + }, token_trees: token_trees.unwrap_or_default(), } .into() diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander.rs b/src/tools/rust-analyzer/crates/mbe/src/expander.rs index 100ec6bfb..7537dc322 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander.rs @@ -8,7 +8,7 @@ mod transcriber; use rustc_hash::FxHashMap; use syntax::SmolStr; -use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; +use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult}; pub(crate) fn expand_rules( rules: &[crate::Rule], @@ -45,7 +45,10 @@ pub(crate) fn expand_rules( transcriber::transcribe(&rule.rhs, &match_.bindings); ExpandResult { value, err: match_.err.or(transcribe_err) } } else { - ExpandResult::only_err(ExpandError::NoMatchingRule) + ExpandResult::with_err( + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, + ExpandError::NoMatchingRule, + ) } } diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs index 88eae136f..f4ea9e5c8 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs @@ -67,6 +67,7 @@ use syntax::SmolStr; use crate::{ expander::{Binding, Bindings, ExpandResult, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, + tt, tt_iter::TtIter, ExpandError, MetaTemplate, ValueResult, }; @@ -75,7 +76,8 @@ impl Bindings { fn push_optional(&mut self, name: &SmolStr) { // FIXME: Do we have a better way to represent an empty token ? // Insert an empty subtree for empty token - let tt = tt::Subtree::default().into(); + let tt = + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into(); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); } @@ -462,9 +464,9 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Subtree { tokens, delimiter }) => { if let Ok(subtree) = src.clone().expect_subtree() { - if subtree.delimiter_kind() == delimiter.map(|it| it.kind) { + if subtree.delimiter.kind == delimiter.kind { item.stack.push(item.dot); - item.dot = tokens.iter_delimited(delimiter.as_ref()); + item.dot = tokens.iter_delimited(Some(delimiter)); cur_items.push(item); } } @@ -663,8 +665,8 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { } res.add_err(ExpandError::LeftoverTokens); - if let Some(error_reover_item) = error_recover_item { - res.bindings = bindings_builder.build(&error_reover_item); + if let Some(error_recover_item) = error_recover_item { + res.bindings = bindings_builder.build(&error_recover_item); } return res; } @@ -782,7 +784,7 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Opt match neg { None => lit.into(), Some(neg) => tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![neg, lit.into()], }), } @@ -810,7 +812,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) } impl MetaTemplate { fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> { - OpDelimitedIter { inner: &self.0, idx: 0, delimited } + OpDelimitedIter { + inner: &self.0, + idx: 0, + delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED), + } } } @@ -824,20 +830,21 @@ enum OpDelimited<'a> { #[derive(Debug, Clone, Copy)] struct OpDelimitedIter<'a> { inner: &'a [Op], - delimited: Option<&'a tt::Delimiter>, + delimited: &'a tt::Delimiter, idx: usize, } impl<'a> OpDelimitedIter<'a> { fn is_eof(&self) -> bool { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + let len = self.inner.len() + + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; self.idx >= len } fn peek(&self) -> Option<OpDelimited<'a>> { - match self.delimited { - None => self.inner.get(self.idx).map(OpDelimited::Op), - Some(_) => match self.idx { + match self.delimited.kind { + tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op), + _ => match self.idx { 0 => Some(OpDelimited::Open), i if i == self.inner.len() + 1 => Some(OpDelimited::Close), i => self.inner.get(i - 1).map(OpDelimited::Op), @@ -860,7 +867,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> { } fn size_hint(&self) -> (usize, Option<usize>) { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + let len = self.inner.len() + + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; let remain = len.saturating_sub(self.idx); (remain, Some(remain)) } @@ -904,7 +912,10 @@ impl<'a> TtIter<'a> { } else { let puncts = self.expect_glued_punct()?; let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect(); - Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees })) + Ok(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees, + })) } } else { self.next().ok_or(()).cloned() @@ -919,7 +930,7 @@ impl<'a> TtIter<'a> { let ident = self.expect_ident_or_underscore()?; Ok(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![ tt::Leaf::Punct(*punct).into(), tt::Leaf::Ident(ident.clone()).into(), diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs index db0d327bf..dffb40d4b 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs @@ -2,11 +2,11 @@ //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` use syntax::SmolStr; -use tt::{Delimiter, Subtree}; use crate::{ expander::{Binding, Bindings, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, + tt::{self, Delimiter}, ExpandError, ExpandResult, MetaTemplate, }; @@ -44,22 +44,23 @@ impl Bindings { Binding::Missing(it) => Ok(match it { MetaVarKind::Stmt => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), char: ';', spacing: tt::Spacing::Alone, }))) } MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Brace, - }), + }, token_trees: vec![], })), // FIXME: Meta and Item should get proper defaults MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![], })) } @@ -71,19 +72,19 @@ impl Bindings { | MetaVarKind::Ident => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("missing"), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } MetaVarKind::Lifetime => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("'missing"), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } MetaVarKind::Literal => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("\"missing\""), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } }), @@ -138,12 +139,12 @@ fn expand_subtree( Op::Ident(it) => arena.push(tt::Leaf::from(it.clone()).into()), Op::Punct(puncts) => { for punct in puncts { - arena.push(tt::Leaf::from(punct.clone()).into()); + arena.push(tt::Leaf::from(*punct).into()); } } Op::Subtree { tokens, delimiter } => { let ExpandResult { value: tt, err: e } = - expand_subtree(ctx, tokens, *delimiter, arena); + expand_subtree(ctx, tokens, Some(*delimiter), arena); err = err.or(e); arena.push(tt.into()); } @@ -170,7 +171,7 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: index.to_string().into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }) .into(), ); @@ -179,7 +180,13 @@ fn expand_subtree( } // drain the elements added in this instance of expand_subtree let tts = arena.drain(start_elements..).collect(); - ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err } + ExpandResult { + value: tt::Subtree { + delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified), + token_trees: tts, + }, + err, + } } fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> { @@ -201,18 +208,25 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe // ``` // We just treat it a normal tokens let tt = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(), - tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(), + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) + .into(), + tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), ], } .into(); ExpandResult::ok(Fragment::Tokens(tt)) } else { ctx.bindings.get(v, &mut ctx.nesting).map_or_else( - |e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, - |it| ExpandResult::ok(it), + |e| ExpandResult { + value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: vec![], + })), + err: Some(e), + }, + ExpandResult::ok, ) } } @@ -249,7 +263,10 @@ fn expand_repeat( ctx ); return ExpandResult { - value: Fragment::Tokens(Subtree::default().into()), + value: Fragment::Tokens( + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] } + .into(), + ), err: Some(ExpandError::LimitExceeded), }; } @@ -258,7 +275,7 @@ fn expand_repeat( continue; } - t.delimiter = None; + t.delimiter = tt::Delimiter::unspecified(); push_subtree(&mut buf, t); if let Some(sep) = separator { @@ -292,7 +309,7 @@ fn expand_repeat( // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { return ExpandResult { @@ -307,11 +324,12 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { - if tt.delimiter.is_none() { - tt.delimiter = Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + if tt.delimiter.kind == tt::DelimiterKind::Invisible { + tt.delimiter = tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, kind: tt::DelimiterKind::Parenthesis, - }) + }; } buf.push(tt.into()) } @@ -320,8 +338,8 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { } fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { - match tt.delimiter { - None => buf.extend(tt.token_trees), - Some(_) => buf.push(tt.into()), + match tt.delimiter.kind { + tt::DelimiterKind::Invisible => buf.extend(tt.token_trees), + _ => buf.push(tt.into()), } } diff --git a/src/tools/rust-analyzer/crates/mbe/src/lib.rs b/src/tools/rust-analyzer/crates/mbe/src/lib.rs index 2373db97a..ac107a0d6 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/lib.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/lib.rs @@ -18,6 +18,8 @@ mod to_parser_input; mod benchmark; mod token_map; +use ::tt::token_id as tt; + use std::fmt; use crate::{ @@ -26,8 +28,8 @@ use crate::{ }; // FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces +pub use self::tt::{Delimiter, DelimiterKind, Punct}; pub use ::parser::TopEntryPoint; -pub use tt::{Delimiter, DelimiterKind, Punct}; pub use crate::{ syntax_bridge::{ @@ -125,24 +127,26 @@ impl Shift { // Find the max token id inside a subtree fn max_id(subtree: &tt::Subtree) -> Option<u32> { - let filter = |tt: &_| match tt { - tt::TokenTree::Subtree(subtree) => { - let tree_id = max_id(subtree); - match subtree.delimiter { - Some(it) if it.id != tt::TokenId::unspecified() => { - Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) + let filter = + |tt: &_| match tt { + tt::TokenTree::Subtree(subtree) => { + let tree_id = max_id(subtree); + if subtree.delimiter.open != tt::TokenId::unspecified() { + Some(tree_id.map_or(subtree.delimiter.open.0, |t| { + t.max(subtree.delimiter.open.0) + })) + } else { + tree_id } - _ => tree_id, } - } - tt::TokenTree::Leaf(leaf) => { - let &(tt::Leaf::Ident(tt::Ident { id, .. }) - | tt::Leaf::Punct(tt::Punct { id, .. }) - | tt::Leaf::Literal(tt::Literal { id, .. })) = leaf; + tt::TokenTree::Leaf(leaf) => { + let &(tt::Leaf::Ident(tt::Ident { span, .. }) + | tt::Leaf::Punct(tt::Punct { span, .. }) + | tt::Leaf::Literal(tt::Literal { span, .. })) = leaf; - (id != tt::TokenId::unspecified()).then_some(id.0) - } - }; + (span != tt::TokenId::unspecified()).then_some(span.0) + } + }; subtree.token_trees.iter().filter_map(filter).max() } } @@ -152,14 +156,13 @@ impl Shift { for t in &mut tt.token_trees { match t { tt::TokenTree::Leaf( - tt::Leaf::Ident(tt::Ident { id, .. }) - | tt::Leaf::Punct(tt::Punct { id, .. }) - | tt::Leaf::Literal(tt::Literal { id, .. }), - ) => *id = self.shift(*id), + tt::Leaf::Ident(tt::Ident { span, .. }) + | tt::Leaf::Punct(tt::Punct { span, .. }) + | tt::Leaf::Literal(tt::Literal { span, .. }), + ) => *span = self.shift(*span), tt::TokenTree::Subtree(tt) => { - if let Some(it) = tt.delimiter.as_mut() { - it.id = self.shift(it.id); - } + tt.delimiter.open = self.shift(tt.delimiter.open); + tt.delimiter.close = self.shift(tt.delimiter.close); self.shift_all(tt) } } @@ -216,7 +219,7 @@ impl DeclarativeMacro { let mut src = TtIter::new(tt); let mut rules = Vec::new(); - if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() { + if tt::DelimiterKind::Brace == tt.delimiter.kind { cov_mark::hit!(parse_macro_def_rules); while src.len() > 0 { let rule = Rule::parse(&mut src, true)?; @@ -325,6 +328,10 @@ impl<T, E> ValueResult<T, E> { Self { value, err: None } } + pub fn with_err(value: T, err: E) -> Self { + Self { value, err: Some(err) } + } + pub fn only_err(err: E) -> Self where T: Default, diff --git a/src/tools/rust-analyzer/crates/mbe/src/parser.rs b/src/tools/rust-analyzer/crates/mbe/src/parser.rs index fad905e97..fd3d64719 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/parser.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/parser.rs @@ -4,7 +4,7 @@ use smallvec::{smallvec, SmallVec}; use syntax::SmolStr; -use crate::{tt_iter::TtIter, ParseError}; +use crate::{tt, tt_iter::TtIter, ParseError}; /// Consider /// @@ -54,7 +54,7 @@ pub(crate) enum Op { Ignore { name: SmolStr, id: tt::TokenId }, Index { depth: u32 }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> }, - Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> }, + Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter }, Literal(tt::Literal), Punct(SmallVec<[tt::Punct; 3]>), Ident(tt::Ident), @@ -126,17 +126,17 @@ fn next_op( src.next().expect("first token already peeked"); // Note that the '$' itself is a valid token inside macro_rules. let second = match src.next() { - None => return Ok(Op::Punct(smallvec![p.clone()])), + None => return Ok(Op::Punct(smallvec![*p])), Some(it) => it, }; match second { - tt::TokenTree::Subtree(subtree) => match subtree.delimiter_kind() { - Some(tt::DelimiterKind::Parenthesis) => { + tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => { let (separator, kind) = parse_repeat(src)?; let tokens = MetaTemplate::parse(subtree, mode)?; Op::Repeat { tokens, separator, kind } } - Some(tt::DelimiterKind::Brace) => match mode { + tt::DelimiterKind::Brace => match mode { Mode::Template => { parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| { ParseError::unexpected("invalid metavariable expression") @@ -157,18 +157,18 @@ fn next_op( tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) if ident.text == "crate" => { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. - Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id }) + Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span }) } tt::Leaf::Ident(ident) => { let kind = eat_fragment_kind(src, mode)?; let name = ident.text.clone(); - let id = ident.id; + let id = ident.span; Op::Var { name, kind, id } } tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { let kind = eat_fragment_kind(src, mode)?; let name = lit.text.clone(); - let id = lit.id; + let id = lit.span; Op::Var { name, kind, id } } tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode { @@ -284,7 +284,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> { let func = src.expect_ident()?; let args = src.expect_subtree()?; - if args.delimiter_kind() != Some(tt::DelimiterKind::Parenthesis) { + if args.delimiter.kind != tt::DelimiterKind::Parenthesis { return Err(()); } @@ -293,7 +293,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> { let op = match &*func.text { "ignore" => { let ident = args.expect_ident()?; - Op::Ignore { name: ident.text.clone(), id: ident.id } + Op::Ignore { name: ident.text.clone(), id: ident.span } } "index" => { let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? }; diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs index 5c9650556..fb5313401 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs @@ -8,9 +8,16 @@ use syntax::{ SyntaxKind::*, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; -use tt::buffer::{Cursor, TokenBuffer}; -use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; +use crate::{ + to_parser_input::to_parser_input, + tt::{ + self, + buffer::{Cursor, TokenBuffer}, + }, + tt_iter::TtIter, + TokenMap, +}; #[cfg(test)] mod tests; @@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node( entry_point: parser::TopEntryPoint, ) -> (Parse<SyntaxNode>, TokenMap) { let buffer = match tt { - tt::Subtree { delimiter: None, token_trees } => { - TokenBuffer::from_tokens(token_trees.as_slice()) - } + tt::Subtree { + delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. }, + token_trees, + } => TokenBuffer::from_tokens(token_trees.as_slice()), _ => TokenBuffer::from_subtree(tt), }; let parser_input = to_parser_input(&buffer); @@ -87,13 +95,15 @@ pub fn token_tree_to_syntax_node( parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => { tree_sink.token(kind, n_raw_tokens) } + parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => { + tree_sink.float_split(has_pseudo_dot) + } parser::Step::Enter { kind } => tree_sink.start_node(kind), parser::Step::Exit => tree_sink.finish_node(), parser::Step::Error { msg } => tree_sink.error(msg.to_string()), } } - let (parse, range_map) = tree_sink.finish(); - (parse, range_map) + tree_sink.finish() } /// Convert a string to a `TokenTree` @@ -132,7 +142,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> { res.push(match expanded.value { None => break, Some(tt @ tt::TokenTree::Leaf(_)) => { - tt::Subtree { delimiter: None, token_trees: vec![tt] } + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] } } Some(tt::TokenTree::Subtree(tt)) => tt, }); @@ -145,7 +155,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> { } if iter.peek_n(0).is_some() { - res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() }); + res.push(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: iter.cloned().collect(), + }); } res @@ -159,7 +172,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { } let entry = StackEntry { - subtree: tt::Subtree { delimiter: None, ..Default::default() }, + subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, // never used (delimiter is `None`) idx: !0, open_range: TextRange::empty(TextSize::of('.')), @@ -186,7 +199,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = sub.token_trees.get_mut(2) { - lit.id = id + lit.span = id } } tt @@ -199,13 +212,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { assert_eq!(range.len(), TextSize::of('.')); } - if let Some(delim) = subtree.delimiter { - let expected = match delim.kind { - tt::DelimiterKind::Parenthesis => T![')'], - tt::DelimiterKind::Brace => T!['}'], - tt::DelimiterKind::Bracket => T![']'], - }; + let expected = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + }; + if let Some(expected) = expected { if kind == expected { if let Some(entry) = stack.pop() { conv.id_alloc().close_delim(entry.idx, Some(range)); @@ -223,9 +237,11 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { }; if let Some(kind) = delim { - let mut subtree = tt::Subtree::default(); let (id, idx) = conv.id_alloc().open_delim(range, synth_id); - subtree.delimiter = Some(tt::Delimiter { id, kind }); + let subtree = tt::Subtree { + delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind }, + token_trees: vec![], + }; stack.push(StackEntry { subtree, idx, open_range: range }); continue; } @@ -240,13 +256,20 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { panic!("Token from lexer must be single char: token = {token:#?}"); } }; - tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) }) - .into() + tt::Leaf::from(tt::Punct { + char, + spacing, + span: conv.id_alloc().alloc(range, synth_id), + }) + .into() } else { macro_rules! make_leaf { ($i:ident) => { - tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) } - .into() + tt::$i { + span: conv.id_alloc().alloc(range, synth_id), + text: token.to_text(conv), + } + .into() }; } let leaf: tt::Leaf = match kind { @@ -261,14 +284,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { let apostrophe = tt::Leaf::from(tt::Punct { char: '\'', spacing: tt::Spacing::Joint, - id: conv.id_alloc().alloc(r, synth_id), + span: conv.id_alloc().alloc(r, synth_id), }); result.push(apostrophe.into()); let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); let ident = tt::Leaf::from(tt::Ident { text: SmolStr::new(&token.to_text(conv)[1..]), - id: conv.id_alloc().alloc(r, synth_id), + span: conv.id_alloc().alloc(r, synth_id), }); result.push(ident.into()); continue; @@ -289,11 +312,12 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { conv.id_alloc().close_delim(entry.idx, None); let leaf: tt::Leaf = tt::Punct { - id: conv.id_alloc().alloc(entry.open_range, None), - char: match entry.subtree.delimiter.unwrap().kind { + span: conv.id_alloc().alloc(entry.open_range, None), + char: match entry.subtree.delimiter.kind { tt::DelimiterKind::Parenthesis => '(', tt::DelimiterKind::Brace => '{', tt::DelimiterKind::Bracket => '[', + tt::DelimiterKind::Invisible => '$', }, spacing: tt::Spacing::Alone, } @@ -373,10 +397,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree> token_trees.push(mk_punct('!')); } token_trees.push(tt::TokenTree::from(tt::Subtree { - delimiter: Some(tt::Delimiter { + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, kind: tt::DelimiterKind::Bracket, - id: tt::TokenId::unspecified(), - }), + }, token_trees: meta_tkns, })); @@ -386,7 +411,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree> fn mk_ident(s: &str) -> tt::TokenTree { tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) } @@ -394,12 +419,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree> tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) } fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { - let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; + let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() }; tt::TokenTree::from(tt::Leaf::from(lit)) } @@ -761,18 +786,56 @@ impl<'a> TtTreeSink<'a> { } } -fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { +fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { let texts = match d { tt::DelimiterKind::Parenthesis => "()", tt::DelimiterKind::Brace => "{}", tt::DelimiterKind::Bracket => "[]", + tt::DelimiterKind::Invisible => return None, }; let idx = closing as usize; - &texts[idx..texts.len() - (1 - idx)] + Some(&texts[idx..texts.len() - (1 - idx)]) } impl<'a> TtTreeSink<'a> { + /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. + /// This occurs when a float literal is used as a field access. + fn float_split(&mut self, has_pseudo_dot: bool) { + let (text, _span) = match self.cursor.token_tree() { + Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => { + (lit.text.as_str(), lit.span) + } + _ => unreachable!(), + }; + match text.split_once('.') { + Some((left, right)) => { + assert!(!left.is_empty()); + self.inner.start_node(SyntaxKind::NAME_REF); + self.inner.token(SyntaxKind::INT_NUMBER, left); + self.inner.finish_node(); + + // here we move the exit up, the original exit has been deleted in process + self.inner.finish_node(); + + self.inner.token(SyntaxKind::DOT, "."); + + if has_pseudo_dot { + assert!(right.is_empty(), "{left}.{right}"); + } else { + self.inner.start_node(SyntaxKind::NAME_REF); + self.inner.token(SyntaxKind::INT_NUMBER, right); + self.inner.finish_node(); + + // the parser creates an unbalanced start node, we are required to close it here + self.inner.finish_node(); + } + } + None => unreachable!(), + } + self.cursor = self.cursor.bump(); + } + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { if kind == LIFETIME_IDENT { n_tokens = 2; @@ -790,13 +853,16 @@ impl<'a> TtTreeSink<'a> { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { // Mark the range if needed let (text, id) = match leaf { - tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id), + tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span), tt::Leaf::Punct(punct) => { assert!(punct.char.is_ascii()); tmp = punct.char as u8; - (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id) + ( + std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), + punct.span, + ) } - tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id), + tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span), }; let range = TextRange::at(self.text_pos, TextSize::of(text)); self.token_map.insert(id, range); @@ -805,10 +871,10 @@ impl<'a> TtTreeSink<'a> { } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { self.cursor = self.cursor.subtree().unwrap(); - match subtree.delimiter { - Some(d) => { - self.open_delims.insert(d.id, self.text_pos); - delim_to_str(d.kind, false) + match delim_to_str(subtree.delimiter.kind, false) { + Some(it) => { + self.open_delims.insert(subtree.delimiter.open, self.text_pos); + it } None => continue, } @@ -816,15 +882,21 @@ impl<'a> TtTreeSink<'a> { None => { let parent = self.cursor.end().unwrap(); self.cursor = self.cursor.bump(); - match parent.delimiter { - Some(d) => { - if let Some(open_delim) = self.open_delims.get(&d.id) { + match delim_to_str(parent.delimiter.kind, true) { + Some(it) => { + if let Some(open_delim) = + self.open_delims.get(&parent.delimiter.open) + { let open_range = TextRange::at(*open_delim, TextSize::of('(')); let close_range = TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim(d.id, open_range, close_range); + self.token_map.insert_delim( + parent.delimiter.open, + open_range, + close_range, + ); } - delim_to_str(d.kind, true) + it } None => continue, } diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs index c1a608365..fa0125f3e 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs @@ -29,8 +29,8 @@ fn check_punct_spacing(fixture: &str) { let mut cursor = buf.begin(); while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { - if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree { - if let Some(expected) = annotations.remove(id) { + if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree { + if let Some(expected) = annotations.remove(span) { assert_eq!(expected, *spacing); } } diff --git a/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs b/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs index 7013aa58b..051e20b3a 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/to_parser_input.rs @@ -2,7 +2,8 @@ //! format that works for our parser. use syntax::{SyntaxKind, SyntaxKind::*, T}; -use tt::buffer::TokenBuffer; + +use crate::tt::buffer::TokenBuffer; pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { let mut res = parser::Input::default(); @@ -44,6 +45,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); res.push(kind); + + if kind == FLOAT_NUMBER && !inner_text.ends_with('.') { + // Tag the token as joint if it is float with a fractional part + // we use this jointness to inform the parser about what token split + // event to emit when we encounter a float literal in a field access + res.was_joint(); + } } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), @@ -70,23 +78,25 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { cursor.bump() } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - if let Some(d) = subtree.delimiter_kind() { - res.push(match d { - tt::DelimiterKind::Parenthesis => T!['('], - tt::DelimiterKind::Brace => T!['{'], - tt::DelimiterKind::Bracket => T!['['], - }); + if let Some(kind) = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T!['(']), + tt::DelimiterKind::Brace => Some(T!['{']), + tt::DelimiterKind::Bracket => Some(T!['[']), + tt::DelimiterKind::Invisible => None, + } { + res.push(kind); } cursor.subtree().unwrap() } None => match cursor.end() { Some(subtree) => { - if let Some(d) = subtree.delimiter_kind() { - res.push(match d { - tt::DelimiterKind::Parenthesis => T![')'], - tt::DelimiterKind::Brace => T!['}'], - tt::DelimiterKind::Bracket => T![']'], - }) + if let Some(kind) = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + } { + res.push(kind); } cursor.bump() } diff --git a/src/tools/rust-analyzer/crates/mbe/src/tt_iter.rs b/src/tools/rust-analyzer/crates/mbe/src/tt_iter.rs index bee7b5de6..f744481f3 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/tt_iter.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/tt_iter.rs @@ -3,9 +3,8 @@ use smallvec::{smallvec, SmallVec}; use syntax::SyntaxKind; -use tt::buffer::TokenBuffer; -use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult}; +use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult}; #[derive(Debug, Clone)] pub(crate) struct TtIter<'a> { @@ -114,7 +113,7 @@ impl<'a> TtIter<'a> { ('.', '.', Some('.' | '=')) | ('<', '<', Some('=')) | ('>', '>', Some('=')) => { let _ = self.next().unwrap(); let _ = self.next().unwrap(); - Ok(smallvec![first, second.clone(), third.unwrap().clone()]) + Ok(smallvec![first, *second, *third.unwrap()]) } ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _) | ('-' | '=' | '>', '>', _) @@ -125,7 +124,7 @@ impl<'a> TtIter<'a> { | ('<', '<', _) | ('|', '|', _) => { let _ = self.next().unwrap(); - Ok(smallvec![first, second.clone()]) + Ok(smallvec![first, *second]) } _ => Ok(smallvec![first]), } @@ -135,7 +134,7 @@ impl<'a> TtIter<'a> { &mut self, entry_point: parser::PrefixEntryPoint, ) -> ExpandResult<Option<tt::TokenTree>> { - let buffer = TokenBuffer::from_tokens(self.inner.as_slice()); + let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice()); let parser_input = to_parser_input(&buffer); let tree_traversal = entry_point.parse(&parser_input); @@ -151,6 +150,11 @@ impl<'a> TtIter<'a> { cursor = cursor.bump_subtree(); } } + parser::Step::FloatSplit { .. } => { + // FIXME: We need to split the tree properly here, but mutating the token trees + // in the buffer is somewhat tricky to pull off. + cursor = cursor.bump_subtree(); + } parser::Step::Enter { .. } | parser::Step::Exit => (), parser::Step::Error { .. } => error = true, } @@ -167,19 +171,18 @@ impl<'a> TtIter<'a> { if cursor.is_root() { while curr != cursor { - if let Some(token) = curr.token_tree() { - res.push(token); - } + let Some(token) = curr.token_tree() else { break }; + res.push(token.cloned()); curr = curr.bump(); } } + self.inner = self.inner.as_slice()[res.len()..].iter(); let res = match res.len() { - 1 => Some(res[0].cloned()), - 0 => None, + 0 | 1 => res.pop(), _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, - token_trees: res.into_iter().map(|it| it.cloned()).collect(), + delimiter: tt::Delimiter::unspecified(), + token_trees: res, })), }; ExpandResult { value: res, err } |