summaryrefslogtreecommitdiffstats
path: root/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs')
-rw-r--r--src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs166
1 files changed, 119 insertions, 47 deletions
diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs
index 5c9650556..fb5313401 100644
--- a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs
+++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs
@@ -8,9 +8,16 @@ use syntax::{
SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
-use tt::buffer::{Cursor, TokenBuffer};
-use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
+use crate::{
+ to_parser_input::to_parser_input,
+ tt::{
+ self,
+ buffer::{Cursor, TokenBuffer},
+ },
+ tt_iter::TtIter,
+ TokenMap,
+};
#[cfg(test)]
mod tests;
@@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node(
entry_point: parser::TopEntryPoint,
) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt {
- tt::Subtree { delimiter: None, token_trees } => {
- TokenBuffer::from_tokens(token_trees.as_slice())
- }
+ tt::Subtree {
+ delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. },
+ token_trees,
+ } => TokenBuffer::from_tokens(token_trees.as_slice()),
_ => TokenBuffer::from_subtree(tt),
};
let parser_input = to_parser_input(&buffer);
@@ -87,13 +95,15 @@ pub fn token_tree_to_syntax_node(
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
tree_sink.token(kind, n_raw_tokens)
}
+ parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
+ tree_sink.float_split(has_pseudo_dot)
+ }
parser::Step::Enter { kind } => tree_sink.start_node(kind),
parser::Step::Exit => tree_sink.finish_node(),
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
}
}
- let (parse, range_map) = tree_sink.finish();
- (parse, range_map)
+ tree_sink.finish()
}
/// Convert a string to a `TokenTree`
@@ -132,7 +142,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
res.push(match expanded.value {
None => break,
Some(tt @ tt::TokenTree::Leaf(_)) => {
- tt::Subtree { delimiter: None, token_trees: vec![tt] }
+ tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] }
}
Some(tt::TokenTree::Subtree(tt)) => tt,
});
@@ -145,7 +155,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
}
if iter.peek_n(0).is_some() {
- res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() });
+ res.push(tt::Subtree {
+ delimiter: tt::Delimiter::unspecified(),
+ token_trees: iter.cloned().collect(),
+ });
}
res
@@ -159,7 +172,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
}
let entry = StackEntry {
- subtree: tt::Subtree { delimiter: None, ..Default::default() },
+ subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
// never used (delimiter is `None`)
idx: !0,
open_range: TextRange::empty(TextSize::of('.')),
@@ -186,7 +199,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
sub.token_trees.get_mut(2)
{
- lit.id = id
+ lit.span = id
}
}
tt
@@ -199,13 +212,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
assert_eq!(range.len(), TextSize::of('.'));
}
- if let Some(delim) = subtree.delimiter {
- let expected = match delim.kind {
- tt::DelimiterKind::Parenthesis => T![')'],
- tt::DelimiterKind::Brace => T!['}'],
- tt::DelimiterKind::Bracket => T![']'],
- };
+ let expected = match subtree.delimiter.kind {
+ tt::DelimiterKind::Parenthesis => Some(T![')']),
+ tt::DelimiterKind::Brace => Some(T!['}']),
+ tt::DelimiterKind::Bracket => Some(T![']']),
+ tt::DelimiterKind::Invisible => None,
+ };
+ if let Some(expected) = expected {
if kind == expected {
if let Some(entry) = stack.pop() {
conv.id_alloc().close_delim(entry.idx, Some(range));
@@ -223,9 +237,11 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
};
if let Some(kind) = delim {
- let mut subtree = tt::Subtree::default();
let (id, idx) = conv.id_alloc().open_delim(range, synth_id);
- subtree.delimiter = Some(tt::Delimiter { id, kind });
+ let subtree = tt::Subtree {
+ delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind },
+ token_trees: vec![],
+ };
stack.push(StackEntry { subtree, idx, open_range: range });
continue;
}
@@ -240,13 +256,20 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
panic!("Token from lexer must be single char: token = {token:#?}");
}
};
- tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) })
- .into()
+ tt::Leaf::from(tt::Punct {
+ char,
+ spacing,
+ span: conv.id_alloc().alloc(range, synth_id),
+ })
+ .into()
} else {
macro_rules! make_leaf {
($i:ident) => {
- tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) }
- .into()
+ tt::$i {
+ span: conv.id_alloc().alloc(range, synth_id),
+ text: token.to_text(conv),
+ }
+ .into()
};
}
let leaf: tt::Leaf = match kind {
@@ -261,14 +284,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
let apostrophe = tt::Leaf::from(tt::Punct {
char: '\'',
spacing: tt::Spacing::Joint,
- id: conv.id_alloc().alloc(r, synth_id),
+ span: conv.id_alloc().alloc(r, synth_id),
});
result.push(apostrophe.into());
let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
let ident = tt::Leaf::from(tt::Ident {
text: SmolStr::new(&token.to_text(conv)[1..]),
- id: conv.id_alloc().alloc(r, synth_id),
+ span: conv.id_alloc().alloc(r, synth_id),
});
result.push(ident.into());
continue;
@@ -289,11 +312,12 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
conv.id_alloc().close_delim(entry.idx, None);
let leaf: tt::Leaf = tt::Punct {
- id: conv.id_alloc().alloc(entry.open_range, None),
- char: match entry.subtree.delimiter.unwrap().kind {
+ span: conv.id_alloc().alloc(entry.open_range, None),
+ char: match entry.subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => '(',
tt::DelimiterKind::Brace => '{',
tt::DelimiterKind::Bracket => '[',
+ tt::DelimiterKind::Invisible => '$',
},
spacing: tt::Spacing::Alone,
}
@@ -373,10 +397,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
token_trees.push(mk_punct('!'));
}
token_trees.push(tt::TokenTree::from(tt::Subtree {
- delimiter: Some(tt::Delimiter {
+ delimiter: tt::Delimiter {
+ open: tt::TokenId::UNSPECIFIED,
+ close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Bracket,
- id: tt::TokenId::unspecified(),
- }),
+ },
token_trees: meta_tkns,
}));
@@ -386,7 +411,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
fn mk_ident(s: &str) -> tt::TokenTree {
tt::TokenTree::from(tt::Leaf::from(tt::Ident {
text: s.into(),
- id: tt::TokenId::unspecified(),
+ span: tt::TokenId::unspecified(),
}))
}
@@ -394,12 +419,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
tt::TokenTree::from(tt::Leaf::from(tt::Punct {
char: c,
spacing: tt::Spacing::Alone,
- id: tt::TokenId::unspecified(),
+ span: tt::TokenId::unspecified(),
}))
}
fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
- let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
+ let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() };
tt::TokenTree::from(tt::Leaf::from(lit))
}
@@ -761,18 +786,56 @@ impl<'a> TtTreeSink<'a> {
}
}
-fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
+fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
let texts = match d {
tt::DelimiterKind::Parenthesis => "()",
tt::DelimiterKind::Brace => "{}",
tt::DelimiterKind::Bracket => "[]",
+ tt::DelimiterKind::Invisible => return None,
};
let idx = closing as usize;
- &texts[idx..texts.len() - (1 - idx)]
+ Some(&texts[idx..texts.len() - (1 - idx)])
}
impl<'a> TtTreeSink<'a> {
+ /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween.
+ /// This occurs when a float literal is used as a field access.
+ fn float_split(&mut self, has_pseudo_dot: bool) {
+ let (text, _span) = match self.cursor.token_tree() {
+ Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => {
+ (lit.text.as_str(), lit.span)
+ }
+ _ => unreachable!(),
+ };
+ match text.split_once('.') {
+ Some((left, right)) => {
+ assert!(!left.is_empty());
+ self.inner.start_node(SyntaxKind::NAME_REF);
+ self.inner.token(SyntaxKind::INT_NUMBER, left);
+ self.inner.finish_node();
+
+ // here we move the exit up, the original exit has been deleted in process
+ self.inner.finish_node();
+
+ self.inner.token(SyntaxKind::DOT, ".");
+
+ if has_pseudo_dot {
+ assert!(right.is_empty(), "{left}.{right}");
+ } else {
+ self.inner.start_node(SyntaxKind::NAME_REF);
+ self.inner.token(SyntaxKind::INT_NUMBER, right);
+ self.inner.finish_node();
+
+ // the parser creates an unbalanced start node, we are required to close it here
+ self.inner.finish_node();
+ }
+ }
+ None => unreachable!(),
+ }
+ self.cursor = self.cursor.bump();
+ }
+
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == LIFETIME_IDENT {
n_tokens = 2;
@@ -790,13 +853,16 @@ impl<'a> TtTreeSink<'a> {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
// Mark the range if needed
let (text, id) = match leaf {
- tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
+ tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span),
tt::Leaf::Punct(punct) => {
assert!(punct.char.is_ascii());
tmp = punct.char as u8;
- (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
+ (
+ std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(),
+ punct.span,
+ )
}
- tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
+ tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span),
};
let range = TextRange::at(self.text_pos, TextSize::of(text));
self.token_map.insert(id, range);
@@ -805,10 +871,10 @@ impl<'a> TtTreeSink<'a> {
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cursor = self.cursor.subtree().unwrap();
- match subtree.delimiter {
- Some(d) => {
- self.open_delims.insert(d.id, self.text_pos);
- delim_to_str(d.kind, false)
+ match delim_to_str(subtree.delimiter.kind, false) {
+ Some(it) => {
+ self.open_delims.insert(subtree.delimiter.open, self.text_pos);
+ it
}
None => continue,
}
@@ -816,15 +882,21 @@ impl<'a> TtTreeSink<'a> {
None => {
let parent = self.cursor.end().unwrap();
self.cursor = self.cursor.bump();
- match parent.delimiter {
- Some(d) => {
- if let Some(open_delim) = self.open_delims.get(&d.id) {
+ match delim_to_str(parent.delimiter.kind, true) {
+ Some(it) => {
+ if let Some(open_delim) =
+ self.open_delims.get(&parent.delimiter.open)
+ {
let open_range = TextRange::at(*open_delim, TextSize::of('('));
let close_range =
TextRange::at(self.text_pos, TextSize::of('('));
- self.token_map.insert_delim(d.id, open_range, close_range);
+ self.token_map.insert_delim(
+ parent.delimiter.open,
+ open_range,
+ close_range,
+ );
}
- delim_to_str(d.kind, true)
+ it
}
None => continue,
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-21 12:33:21 +0000