//! To make attribute macros work reliably when typing, we need to take care to //! fix up syntax errors in the code we're passing to them. use std::mem; use mbe::{SyntheticToken, SyntheticTokenId, TokenMap}; use rustc_hash::FxHashMap; use smallvec::SmallVec; use syntax::{ ast::{self, AstNode, HasLoopBody}, match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, }; use tt::Subtree; /// The result of calculating fixes for a syntax node -- a bunch of changes /// (appending to and replacing nodes), the information that is needed to /// reverse those changes afterwards, and a token map. #[derive(Debug)] pub(crate) struct SyntaxFixups { pub(crate) append: FxHashMap>, pub(crate) replace: FxHashMap>, pub(crate) undo_info: SyntaxFixupUndoInfo, pub(crate) token_map: TokenMap, pub(crate) next_id: u32, } /// This is the information needed to reverse the fixups. #[derive(Debug, PartialEq, Eq)] pub struct SyntaxFixupUndoInfo { original: Vec, } const EMPTY_ID: SyntheticTokenId = SyntheticTokenId(!0); pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { let mut append = FxHashMap::::default(); let mut replace = FxHashMap::::default(); let mut preorder = node.preorder(); let mut original = Vec::new(); let mut token_map = TokenMap::default(); let mut next_id = 0; while let Some(event) = preorder.next() { let node = match event { syntax::WalkEvent::Enter(node) => node, syntax::WalkEvent::Leave(_) => continue, }; if can_handle_error(&node) && has_error_to_handle(&node) { // the node contains an error node, we have to completely replace it by something valid let (original_tree, new_tmap, new_next_id) = mbe::syntax_node_to_token_tree_with_modifications( &node, mem::take(&mut token_map), next_id, Default::default(), Default::default(), ); token_map = new_tmap; next_id = new_next_id; let idx = original.len() as u32; original.push(original_tree); let replacement = SyntheticToken { kind: SyntaxKind::IDENT, text: "__ra_fixup".into(), range: node.text_range(), id: SyntheticTokenId(idx), }; replace.insert(node.clone().into(), vec![replacement]); preorder.skip_subtree(); continue; } // In some other situations, we can fix things by just appending some tokens. let end_range = TextRange::empty(node.text_range().end()); match_ast! { match node { ast::FieldExpr(it) => { if it.name_ref().is_none() { // incomplete field access: some_expr.| append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::IDENT, text: "__ra_fixup".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::ExprStmt(it) => { if it.semicolon_token().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::SEMICOLON, text: ";".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::LetStmt(it) => { if it.semicolon_token().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::SEMICOLON, text: ";".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::IfExpr(it) => { if it.condition().is_none() { // insert placeholder token after the if token let if_token = match it.if_token() { Some(t) => t, None => continue, }; append.insert(if_token.into(), vec![ SyntheticToken { kind: SyntaxKind::IDENT, text: "__ra_fixup".into(), range: end_range, id: EMPTY_ID, }, ]); } if it.then_branch().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::L_CURLY, text: "{".into(), range: end_range, id: EMPTY_ID, }, SyntheticToken { kind: SyntaxKind::R_CURLY, text: "}".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::WhileExpr(it) => { if it.condition().is_none() { // insert placeholder token after the while token let while_token = match it.while_token() { Some(t) => t, None => continue, }; append.insert(while_token.into(), vec![ SyntheticToken { kind: SyntaxKind::IDENT, text: "__ra_fixup".into(), range: end_range, id: EMPTY_ID, }, ]); } if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::L_CURLY, text: "{".into(), range: end_range, id: EMPTY_ID, }, SyntheticToken { kind: SyntaxKind::R_CURLY, text: "}".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::LoopExpr(it) => { if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::L_CURLY, text: "{".into(), range: end_range, id: EMPTY_ID, }, SyntheticToken { kind: SyntaxKind::R_CURLY, text: "}".into(), range: end_range, id: EMPTY_ID, }, ]); } }, // FIXME: foo:: ast::MatchExpr(it) => { if it.expr().is_none() { let match_token = match it.match_token() { Some(t) => t, None => continue }; append.insert(match_token.into(), vec![ SyntheticToken { kind: SyntaxKind::IDENT, text: "__ra_fixup".into(), range: end_range, id: EMPTY_ID }, ]); } if it.match_arm_list().is_none() { // No match arms append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::L_CURLY, text: "{".into(), range: end_range, id: EMPTY_ID, }, SyntheticToken { kind: SyntaxKind::R_CURLY, text: "}".into(), range: end_range, id: EMPTY_ID, }, ]); } }, ast::ForExpr(it) => { let for_token = match it.for_token() { Some(token) => token, None => continue }; let [pat, in_token, iter] = [ (SyntaxKind::UNDERSCORE, "_"), (SyntaxKind::IN_KW, "in"), (SyntaxKind::IDENT, "__ra_fixup") ].map(|(kind, text)| SyntheticToken { kind, text: text.into(), range: end_range, id: EMPTY_ID}); if it.pat().is_none() && it.in_token().is_none() && it.iterable().is_none() { append.insert(for_token.into(), vec![pat, in_token, iter]); // does something funky -- see test case for_no_pat } else if it.pat().is_none() { append.insert(for_token.into(), vec![pat]); } if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ SyntheticToken { kind: SyntaxKind::L_CURLY, text: "{".into(), range: end_range, id: EMPTY_ID, }, SyntheticToken { kind: SyntaxKind::R_CURLY, text: "}".into(), range: end_range, id: EMPTY_ID, }, ]); } }, _ => (), } } } SyntaxFixups { append, replace, token_map, next_id, undo_info: SyntaxFixupUndoInfo { original }, } } fn has_error(node: &SyntaxNode) -> bool { node.children().any(|c| c.kind() == SyntaxKind::ERROR) } fn can_handle_error(node: &SyntaxNode) -> bool { ast::Expr::can_cast(node.kind()) } fn has_error_to_handle(node: &SyntaxNode) -> bool { has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c)) } pub(crate) fn reverse_fixups( tt: &mut Subtree, token_map: &TokenMap, undo_info: &SyntaxFixupUndoInfo, ) { let tts = std::mem::take(&mut tt.token_trees); tt.token_trees = tts .into_iter() .filter(|tt| match tt { tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID), tt::TokenTree::Subtree(st) => { st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID)) } }) .flat_map(|tt| match tt { tt::TokenTree::Subtree(mut tt) => { reverse_fixups(&mut tt, token_map, undo_info); SmallVec::from_const([tt.into()]) } tt::TokenTree::Leaf(leaf) => { if let Some(id) = token_map.synthetic_token_id(leaf.id()) { let original = undo_info.original[id.0 as usize].clone(); if original.delimiter.is_none() { original.token_trees.into() } else { SmallVec::from_const([original.into()]) } } else { SmallVec::from_const([leaf.into()]) } } }) .collect(); } #[cfg(test)] mod tests { use expect_test::{expect, Expect}; use super::reverse_fixups; // The following three functions are only meant to check partial structural equivalence of // `TokenTree`s, see the last assertion in `check()`. fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool { match (a, b) { (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text == b.text, (tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char, (tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.text == b.text, _ => false, } } fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool { a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind) && a.token_trees.len() == b.token_trees.len() && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b)) } fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool { match (a, b) { (tt::TokenTree::Leaf(a), tt::TokenTree::Leaf(b)) => check_leaf_eq(a, b), (tt::TokenTree::Subtree(a), tt::TokenTree::Subtree(b)) => check_subtree_eq(a, b), _ => false, } } #[track_caller] fn check(ra_fixture: &str, mut expect: Expect) { let parsed = syntax::SourceFile::parse(ra_fixture); let fixups = super::fixup_syntax(&parsed.syntax_node()); let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( &parsed.syntax_node(), fixups.token_map, fixups.next_id, fixups.replace, fixups.append, ); let actual = format!("{tt}\n"); expect.indent(false); expect.assert_eq(&actual); // the fixed-up tree should be syntactically valid let (parse, _) = mbe::token_tree_to_syntax_node(&tt, ::mbe::TopEntryPoint::MacroItems); assert!( parse.errors().is_empty(), "parse has syntax errors. parse tree:\n{:#?}", parse.syntax_node() ); reverse_fixups(&mut tt, &tmap, &fixups.undo_info); // the fixed-up + reversed version should be equivalent to the original input // modulo token IDs and `Punct`s' spacing. let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node()); assert!( check_subtree_eq(&tt, &original_as_tt), "different token tree: {tt:?}, {original_as_tt:?}" ); } #[test] fn just_for_token() { check( r#" fn foo() { for } "#, expect![[r#" fn foo () {for _ in __ra_fixup {}} "#]], ) } #[test] fn for_no_iter_pattern() { check( r#" fn foo() { for {} } "#, expect![[r#" fn foo () {for _ in __ra_fixup {}} "#]], ) } #[test] fn for_no_body() { check( r#" fn foo() { for bar in qux } "#, expect![[r#" fn foo () {for bar in qux {}} "#]], ) } // FIXME: https://github.com/rust-lang/rust-analyzer/pull/12937#discussion_r937633695 #[test] fn for_no_pat() { check( r#" fn foo() { for in qux { } } "#, expect![[r#" fn foo () {__ra_fixup} "#]], ) } #[test] fn match_no_expr_no_arms() { check( r#" fn foo() { match } "#, expect![[r#" fn foo () {match __ra_fixup {}} "#]], ) } #[test] fn match_expr_no_arms() { check( r#" fn foo() { match x { } } "#, expect![[r#" fn foo () {match x {}} "#]], ) } #[test] fn match_no_expr() { check( r#" fn foo() { match { _ => {} } } "#, expect![[r#" fn foo () {match __ra_fixup {}} "#]], ) } #[test] fn incomplete_field_expr_1() { check( r#" fn foo() { a. } "#, expect![[r#" fn foo () {a . __ra_fixup} "#]], ) } #[test] fn incomplete_field_expr_2() { check( r#" fn foo() { a.; } "#, expect![[r#" fn foo () {a . __ra_fixup ;} "#]], ) } #[test] fn incomplete_field_expr_3() { check( r#" fn foo() { a.; bar(); } "#, expect![[r#" fn foo () {a . __ra_fixup ; bar () ;} "#]], ) } #[test] fn incomplete_let() { check( r#" fn foo() { let x = a } "#, expect![[r#" fn foo () {let x = a ;} "#]], ) } #[test] fn incomplete_field_expr_in_let() { check( r#" fn foo() { let x = a. } "#, expect![[r#" fn foo () {let x = a . __ra_fixup ;} "#]], ) } #[test] fn field_expr_before_call() { // another case that easily happens while typing check( r#" fn foo() { a.b bar(); } "#, expect![[r#" fn foo () {a . b ; bar () ;} "#]], ) } #[test] fn extraneous_comma() { check( r#" fn foo() { bar(,); } "#, expect![[r#" fn foo () {__ra_fixup ;} "#]], ) } #[test] fn fixup_if_1() { check( r#" fn foo() { if a } "#, expect![[r#" fn foo () {if a {}} "#]], ) } #[test] fn fixup_if_2() { check( r#" fn foo() { if } "#, expect![[r#" fn foo () {if __ra_fixup {}} "#]], ) } #[test] fn fixup_if_3() { check( r#" fn foo() { if {} } "#, // the {} gets parsed as the condition, I think? expect![[r#" fn foo () {if {} {}} "#]], ) } #[test] fn fixup_while_1() { check( r#" fn foo() { while } "#, expect![[r#" fn foo () {while __ra_fixup {}} "#]], ) } #[test] fn fixup_while_2() { check( r#" fn foo() { while foo } "#, expect![[r#" fn foo () {while foo {}} "#]], ) } #[test] fn fixup_while_3() { check( r#" fn foo() { while {} } "#, expect![[r#" fn foo () {while __ra_fixup {}} "#]], ) } #[test] fn fixup_loop() { check( r#" fn foo() { loop } "#, expect![[r#" fn foo () {loop {}} "#]], ) } }