diff options
Diffstat (limited to 'src/tools/rust-analyzer/crates/parser/src/shortcuts.rs')
| -rw-r--r-- | src/tools/rust-analyzer/crates/parser/src/shortcuts.rs | 215 |
1 files changed, 215 insertions, 0 deletions
diff --git a/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs b/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs new file mode 100644 index 000000000..4b805fadd --- /dev/null +++ b/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs @@ -0,0 +1,215 @@ +//! Shortcuts that span lexer/parser abstraction. +//! +//! The way Rust works, parser doesn't necessary parse text, and you might +//! tokenize text without parsing it further. So, it makes sense to keep +//! abstract token parsing, and string tokenization as completely separate +//! layers. +//! +//! However, often you do pares text into syntax trees and the glue code for +//! that needs to live somewhere. Rather than putting it to lexer or parser, we +//! use a separate shortcuts module for that. + +use std::mem; + +use crate::{ + LexedStr, Step, + SyntaxKind::{self, *}, +}; + +#[derive(Debug)] +pub enum StrStep<'a> { + Token { kind: SyntaxKind, text: &'a str }, + Enter { kind: SyntaxKind }, + Exit, + Error { msg: &'a str, pos: usize }, +} + +impl<'a> LexedStr<'a> { + pub fn to_input(&self) -> crate::Input { + let mut res = crate::Input::default(); + let mut was_joint = false; + for i in 0..self.len() { + let kind = self.kind(i); + if kind.is_trivia() { + was_joint = false + } else { + if kind == SyntaxKind::IDENT { + let token_text = self.text(i); + let contextual_kw = SyntaxKind::from_contextual_keyword(token_text) + .unwrap_or(SyntaxKind::IDENT); + res.push_ident(contextual_kw); + } else { + if was_joint { + res.was_joint(); + } + res.push(kind); + } + was_joint = true; + } + } + res + } + + /// NB: only valid to call with Output from Reparser/TopLevelEntry. + pub fn intersperse_trivia( + &self, + output: &crate::Output, + sink: &mut dyn FnMut(StrStep<'_>), + ) -> bool { + let mut builder = Builder { lexed: self, pos: 0, state: State::PendingEnter, sink }; + + for event in output.iter() { + match event { + Step::Token { kind, n_input_tokens: n_raw_tokens } => { + builder.token(kind, n_raw_tokens) + } + Step::Enter { kind } => builder.enter(kind), + Step::Exit => builder.exit(), + Step::Error { msg } => { + let text_pos = builder.lexed.text_start(builder.pos); + (builder.sink)(StrStep::Error { msg, pos: text_pos }); + } + } + } + + match mem::replace(&mut builder.state, State::Normal) { + State::PendingExit => { + builder.eat_trivias(); + (builder.sink)(StrStep::Exit); + } + State::PendingEnter | State::Normal => unreachable!(), + } + + let is_eof = builder.pos == builder.lexed.len(); + is_eof + } +} + +struct Builder<'a, 'b> { + lexed: &'a LexedStr<'a>, + pos: usize, + state: State, + sink: &'b mut dyn FnMut(StrStep<'_>), +} + +enum State { + PendingEnter, + Normal, + PendingExit, +} + +impl Builder<'_, '_> { + fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingEnter => unreachable!(), + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + self.eat_trivias(); + self.do_token(kind, n_tokens as usize); + } + + fn enter(&mut self, kind: SyntaxKind) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingEnter => { + (self.sink)(StrStep::Enter { kind }); + // No need to attach trivias to previous node: there is no + // previous node. + return; + } + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + + let n_trivias = + (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count(); + let leading_trivias = self.pos..self.pos + n_trivias; + let n_attached_trivias = n_attached_trivias( + kind, + leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))), + ); + self.eat_n_trivias(n_trivias - n_attached_trivias); + (self.sink)(StrStep::Enter { kind }); + self.eat_n_trivias(n_attached_trivias); + } + + fn exit(&mut self) { + match mem::replace(&mut self.state, State::PendingExit) { + State::PendingEnter => unreachable!(), + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + } + + fn eat_trivias(&mut self) { + while self.pos < self.lexed.len() { + let kind = self.lexed.kind(self.pos); + if !kind.is_trivia() { + break; + } + self.do_token(kind, 1); + } + } + + fn eat_n_trivias(&mut self, n: usize) { + for _ in 0..n { + let kind = self.lexed.kind(self.pos); + assert!(kind.is_trivia()); + self.do_token(kind, 1); + } + } + + fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) { + let text = &self.lexed.range_text(self.pos..self.pos + n_tokens); + self.pos += n_tokens; + (self.sink)(StrStep::Token { kind, text }); + } +} + +fn n_attached_trivias<'a>( + kind: SyntaxKind, + trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, +) -> usize { + match kind { + CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD + | STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => { + let mut res = 0; + let mut trivias = trivias.enumerate().peekable(); + + while let Some((i, (kind, text))) = trivias.next() { + match kind { + WHITESPACE if text.contains("\n\n") => { + // we check whether the next token is a doc-comment + // and skip the whitespace in this case + if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) { + if is_outer(peek_text) { + continue; + } + } + break; + } + COMMENT => { + if is_inner(text) { + break; + } + res = i + 1; + } + _ => (), + } + } + res + } + _ => 0, + } +} + +fn is_outer(text: &str) -> bool { + if text.starts_with("////") || text.starts_with("/***") { + return false; + } + text.starts_with("///") || text.starts_with("/**") +} + +fn is_inner(text: &str) -> bool { + text.starts_with("//!") || text.starts_with("/*!") +} |