From 4547b622d8d29df964fa2914213088b148c498fc Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:18:32 +0200 Subject: Merging upstream version 1.67.1+dfsg1. Signed-off-by: Daniel Baumann --- src/tools/rust-analyzer/crates/mbe/Cargo.toml | 2 +- .../rust-analyzer/crates/mbe/src/syntax_bridge.rs | 89 ++++++++++++++------- .../crates/mbe/src/syntax_bridge/tests.rs | 93 ++++++++++++++++++++++ 3 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs (limited to 'src/tools/rust-analyzer/crates/mbe') diff --git a/src/tools/rust-analyzer/crates/mbe/Cargo.toml b/src/tools/rust-analyzer/crates/mbe/Cargo.toml index 13cd89010..bce2fc9a7 100644 --- a/src/tools/rust-analyzer/crates/mbe/Cargo.toml +++ b/src/tools/rust-analyzer/crates/mbe/Cargo.toml @@ -4,7 +4,7 @@ version = "0.0.0" description = "TBD" license = "MIT OR Apache-2.0" edition = "2021" -rust-version = "1.57" +rust-version = "1.65" [lib] doctest = false diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs index e4c56565b..cf53c1672 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs @@ -12,6 +12,9 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; +#[cfg(test)] +mod tests; + /// Convert the syntax node to a `TokenTree` (what macro /// will consume). pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { @@ -35,7 +38,7 @@ pub fn syntax_node_to_token_tree_with_modifications( append: FxHashMap>, ) -> (tt::Subtree, TokenMap, u32) { let global_offset = node.text_range().start(); - let mut c = Convertor::new(node, global_offset, existing_token_map, next_id, replace, append); + let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append); let subtree = convert_tokens(&mut c); c.id_alloc.map.shrink_to_fit(); always!(c.replace.is_empty(), "replace: {:?}", c.replace); @@ -100,7 +103,7 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { return None; } - let mut conv = RawConvertor { + let mut conv = RawConverter { lexed, pos: 0, id_alloc: TokenIdAlloc { @@ -148,7 +151,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec { res } -fn convert_tokens(conv: &mut C) -> tt::Subtree { +fn convert_tokens(conv: &mut C) -> tt::Subtree { struct StackEntry { subtree: tt::Subtree, idx: usize, @@ -228,7 +231,7 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { } let spacing = match conv.peek().map(|next| next.kind(conv)) { - Some(kind) if !kind.is_trivia() => tt::Spacing::Joint, + Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, _ => tt::Spacing::Alone, }; let char = match token.to_char(conv) { @@ -307,6 +310,35 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { } } +fn is_single_token_op(kind: SyntaxKind) -> bool { + matches!( + kind, + EQ | L_ANGLE + | R_ANGLE + | BANG + | AMP + | PIPE + | TILDE + | AT + | DOT + | COMMA + | SEMICOLON + | COLON + | POUND + | DOLLAR + | QUESTION + | PLUS + | MINUS + | STAR + | SLASH + | PERCENT + | CARET + // LIFETIME_IDENT will be split into a sequence of `'` (a single quote) and an + // identifier. + | LIFETIME_IDENT + ) +} + /// Returns the textual content of a doc comment block as a quoted string /// That is, strips leading `///` (or `/**`, etc) /// and strips the ending `*/` @@ -425,8 +457,8 @@ impl TokenIdAlloc { } } -/// A raw token (straight from lexer) convertor -struct RawConvertor<'a> { +/// A raw token (straight from lexer) converter +struct RawConverter<'a> { lexed: parser::LexedStr<'a>, pos: usize, id_alloc: TokenIdAlloc, @@ -442,7 +474,7 @@ trait SrcToken: std::fmt::Debug { fn synthetic_id(&self, ctx: &Ctx) -> Option; } -trait TokenConvertor: Sized { +trait TokenConverter: Sized { type Token: SrcToken; fn convert_doc_comment(&self, token: &Self::Token) -> Option>; @@ -454,25 +486,25 @@ trait TokenConvertor: Sized { fn id_alloc(&mut self) -> &mut TokenIdAlloc; } -impl<'a> SrcToken> for usize { - fn kind(&self, ctx: &RawConvertor<'a>) -> SyntaxKind { +impl<'a> SrcToken> for usize { + fn kind(&self, ctx: &RawConverter<'a>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConvertor<'a>) -> Option { + fn to_char(&self, ctx: &RawConverter<'a>) -> Option { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConvertor<'_>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { ctx.lexed.text(*self).into() } - fn synthetic_id(&self, _ctx: &RawConvertor<'a>) -> Option { + fn synthetic_id(&self, _ctx: &RawConverter<'a>) -> Option { None } } -impl<'a> TokenConvertor for RawConvertor<'a> { +impl<'a> TokenConverter for RawConverter<'a> { type Token = usize; fn convert_doc_comment(&self, &token: &usize) -> Option> { @@ -504,7 +536,7 @@ impl<'a> TokenConvertor for RawConvertor<'a> { } } -struct Convertor { +struct Converter { id_alloc: TokenIdAlloc, current: Option, current_synthetic: Vec, @@ -515,7 +547,7 @@ struct Convertor { punct_offset: Option<(SyntaxToken, TextSize)>, } -impl Convertor { +impl Converter { fn new( node: &SyntaxNode, global_offset: TextSize, @@ -523,11 +555,11 @@ impl Convertor { next_id: u32, mut replace: FxHashMap>, mut append: FxHashMap>, - ) -> Convertor { + ) -> Converter { let range = node.text_range(); let mut preorder = node.preorder_with_tokens(); let (first, synthetic) = Self::next_token(&mut preorder, &mut replace, &mut append); - Convertor { + Converter { id_alloc: { TokenIdAlloc { map: existing_token_map, global_offset, next_id } }, current: first, current_synthetic: synthetic, @@ -590,15 +622,15 @@ impl SynToken { } } -impl SrcToken for SynToken { - fn kind(&self, _ctx: &Convertor) -> SyntaxKind { +impl SrcToken for SynToken { + fn kind(&self, ctx: &Converter) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), - SynToken::Punch(token, _) => token.kind(), + SynToken::Punch(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), SynToken::Synthetic(token) => token.kind, } } - fn to_char(&self, _ctx: &Convertor) -> Option { + fn to_char(&self, _ctx: &Converter) -> Option { match self { SynToken::Ordinary(_) => None, SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), @@ -606,7 +638,7 @@ impl SrcToken for SynToken { SynToken::Synthetic(_) => None, } } - fn to_text(&self, _ctx: &Convertor) -> SmolStr { + fn to_text(&self, _ctx: &Converter) -> SmolStr { match self { SynToken::Ordinary(token) => token.text().into(), SynToken::Punch(token, _) => token.text().into(), @@ -614,7 +646,7 @@ impl SrcToken for SynToken { } } - fn synthetic_id(&self, _ctx: &Convertor) -> Option { + fn synthetic_id(&self, _ctx: &Converter) -> Option { match self { SynToken::Synthetic(token) => Some(token.id), _ => None, @@ -622,7 +654,7 @@ impl SrcToken for SynToken { } } -impl TokenConvertor for Convertor { +impl TokenConverter for Converter { type Token = SynToken; fn convert_doc_comment(&self, token: &Self::Token) -> Option> { convert_doc_comment(token.token()?) @@ -651,7 +683,7 @@ impl TokenConvertor for Convertor { } let curr = self.current.clone()?; - if !&self.range.contains_range(curr.text_range()) { + if !self.range.contains_range(curr.text_range()) { return None; } let (new_current, new_synth) = @@ -809,12 +841,15 @@ impl<'a> TtTreeSink<'a> { let next = last.bump(); if let ( Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)), - Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(_), _)), + Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(next), _)), ) = (last.token_tree(), next.token_tree()) { // Note: We always assume the semi-colon would be the last token in // other parts of RA such that we don't add whitespace here. - if curr.spacing == tt::Spacing::Alone && curr.char != ';' { + // + // When `next` is a `Punct` of `'`, that's a part of a lifetime identifier so we don't + // need to add whitespace either. + if curr.spacing == tt::Spacing::Alone && curr.char != ';' && next.char != '\'' { self.inner.token(WHITESPACE, " "); self.text_pos += TextSize::of(' '); } diff --git a/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs new file mode 100644 index 000000000..4e04d2bc1 --- /dev/null +++ b/src/tools/rust-analyzer/crates/mbe/src/syntax_bridge/tests.rs @@ -0,0 +1,93 @@ +use std::collections::HashMap; + +use syntax::{ast, AstNode}; +use test_utils::extract_annotations; +use tt::{ + buffer::{TokenBuffer, TokenTreeRef}, + Leaf, Punct, Spacing, +}; + +use super::syntax_node_to_token_tree; + +fn check_punct_spacing(fixture: &str) { + let source_file = ast::SourceFile::parse(fixture).ok().unwrap(); + let (subtree, token_map) = syntax_node_to_token_tree(source_file.syntax()); + let mut annotations: HashMap<_, _> = extract_annotations(fixture) + .into_iter() + .map(|(range, annotation)| { + let token = token_map.token_by_range(range).expect("no token found"); + let spacing = match annotation.as_str() { + "Alone" => Spacing::Alone, + "Joint" => Spacing::Joint, + a => panic!("unknown annotation: {}", a), + }; + (token, spacing) + }) + .collect(); + + let buf = TokenBuffer::from_subtree(&subtree); + let mut cursor = buf.begin(); + while !cursor.eof() { + while let Some(token_tree) = cursor.token_tree() { + if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree { + if let Some(expected) = annotations.remove(&id) { + assert_eq!(expected, *spacing); + } + } + cursor = cursor.bump_subtree(); + } + cursor = cursor.bump(); + } + + assert!(annotations.is_empty(), "unchecked annotations: {:?}", annotations); +} + +#[test] +fn punct_spacing() { + check_punct_spacing( + r#" +fn main() { + 0+0; + //^ Alone + 0+(0); + //^ Alone + 0<=0; + //^ Joint + // ^ Alone + 0<=(0); + // ^ Alone + a=0; + //^ Alone + a=(0); + //^ Alone + a+=0; + //^ Joint + // ^ Alone + a+=(0); + // ^ Alone + a&&b; + //^ Joint + // ^ Alone + a&&(b); + // ^ Alone + foo::bar; + // ^ Joint + // ^ Alone + use foo::{bar,baz,}; + // ^ Alone + // ^ Alone + // ^ Alone + struct Struct<'a> {}; + // ^ Joint + // ^ Joint + Struct::<0>; + // ^ Alone + Struct::<{0}>; + // ^ Alone + ;; + //^ Joint + // ^ Alone +} + "#, + ); +} -- cgit v1.2.3