diff options
Diffstat (limited to 'vendor/ra-ap-rustc_lexer')
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/.cargo-checksum.json | 1 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/Cargo.toml | 36 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/src/cursor.rs | 93 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/src/lib.rs | 893 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/src/tests.rs | 298 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/src/unescape.rs | 419 | ||||
| -rw-r--r-- | vendor/ra-ap-rustc_lexer/src/unescape/tests.rs | 286 |
7 files changed, 2026 insertions, 0 deletions
diff --git a/vendor/ra-ap-rustc_lexer/.cargo-checksum.json b/vendor/ra-ap-rustc_lexer/.cargo-checksum.json new file mode 100644 index 000000000..090276c27 --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"4999bf349766c3a72dc8e2b760ca5ad8ae37ef42d8fe6131023dd5c7e475eda3","src/cursor.rs":"669ee4cf5b0bde41c18dcfe206b224cef0bbaa4e9400adaa0e37cb9e942f855b","src/lib.rs":"6823810b92d70b72b86c9a278dfc6f2e60610759505cbf9395b6cf5be43c8f6f","src/tests.rs":"bab7446fa7eb773d6e6b8bdb48213390167e83ee1ca008ae4630a5e85336cfbf","src/unescape.rs":"d6dabd9d7aabdac9cd1f648de5df96a25766757ca63e8b2e8a6c39ca6f288827","src/unescape/tests.rs":"5da0043e4a48f27b910d19c28ca443a5eb7d59d2aa16e8f08d4273d4f3754244"},"package":"e1c145702ed3f237918e512685185dc8a4d0edc3a5326c63d20361d8ba9b45b3"}
\ No newline at end of file diff --git a/vendor/ra-ap-rustc_lexer/Cargo.toml b/vendor/ra-ap-rustc_lexer/Cargo.toml new file mode 100644 index 000000000..662b3cea4 --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/Cargo.toml @@ -0,0 +1,36 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "ra-ap-rustc_lexer" +version = "0.1.0" +description = """ + +Automatically published version of the package `rustc_lexer` +in the rust-lang/rust repository from commit 77c836e1ae582661924d3b6ec4d57a2de120f59f + +The publishing script for this crate lives at: +https://github.com/rust-analyzer/rustc-auto-publish +""" +license = "MIT / Apache-2.0" +repository = "https://github.com/rust-lang/rust" + +[lib] + +[dependencies.unic-emoji-char] +version = "0.9.0" + +[dependencies.unicode-xid] +version = "0.2.0" + +[dev-dependencies.expect-test] +version = "1.4.0" diff --git a/vendor/ra-ap-rustc_lexer/src/cursor.rs b/vendor/ra-ap-rustc_lexer/src/cursor.rs new file mode 100644 index 000000000..eceef5980 --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/src/cursor.rs @@ -0,0 +1,93 @@ +use std::str::Chars; + +/// Peekable iterator over a char sequence. +/// +/// Next characters can be peeked via `first` method, +/// and position can be shifted forward via `bump` method. +pub struct Cursor<'a> { + len_remaining: usize, + /// Iterator over chars. Slightly faster than a &str. + chars: Chars<'a>, + #[cfg(debug_assertions)] + prev: char, +} + +pub(crate) const EOF_CHAR: char = '\0'; + +impl<'a> Cursor<'a> { + pub fn new(input: &'a str) -> Cursor<'a> { + Cursor { + len_remaining: input.len(), + chars: input.chars(), + #[cfg(debug_assertions)] + prev: EOF_CHAR, + } + } + + /// Returns the last eaten symbol (or `'\0'` in release builds). + /// (For debug assertions only.) + pub(crate) fn prev(&self) -> char { + #[cfg(debug_assertions)] + { + self.prev + } + + #[cfg(not(debug_assertions))] + { + EOF_CHAR + } + } + + /// Peeks the next symbol from the input stream without consuming it. + /// If requested position doesn't exist, `EOF_CHAR` is returned. + /// However, getting `EOF_CHAR` doesn't always mean actual end of file, + /// it should be checked with `is_eof` method. + pub(crate) fn first(&self) -> char { + // `.next()` optimizes better than `.nth(0)` + self.chars.clone().next().unwrap_or(EOF_CHAR) + } + + /// Peeks the second symbol from the input stream without consuming it. + pub(crate) fn second(&self) -> char { + // `.next()` optimizes better than `.nth(1)` + let mut iter = self.chars.clone(); + iter.next(); + iter.next().unwrap_or(EOF_CHAR) + } + + /// Checks if there is nothing more to consume. + pub(crate) fn is_eof(&self) -> bool { + self.chars.as_str().is_empty() + } + + /// Returns amount of already consumed symbols. + pub(crate) fn pos_within_token(&self) -> u32 { + (self.len_remaining - self.chars.as_str().len()) as u32 + } + + /// Resets the number of bytes consumed to 0. + pub(crate) fn reset_pos_within_token(&mut self) { + self.len_remaining = self.chars.as_str().len(); + } + + /// Moves to the next character. + pub(crate) fn bump(&mut self) -> Option<char> { + let c = self.chars.next()?; + + #[cfg(debug_assertions)] + { + self.prev = c; + } + + Some(c) + } + + /// Eats symbols while predicate returns true or until the end of file is reached. + pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) { + // It was tried making optimized version of this for eg. line comments, but + // LLVM can inline all of this and compile it down to fast iteration over bytes. + while predicate(self.first()) && !self.is_eof() { + self.bump(); + } + } +} diff --git a/vendor/ra-ap-rustc_lexer/src/lib.rs b/vendor/ra-ap-rustc_lexer/src/lib.rs new file mode 100644 index 000000000..d511d2b12 --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/src/lib.rs @@ -0,0 +1,893 @@ +//! Low-level Rust lexer. +//! +//! The idea with `rustc_lexer` is to make a reusable library, +//! by separating out pure lexing and rustc-specific concerns, like spans, +//! error reporting, and interning. So, rustc_lexer operates directly on `&str`, +//! produces simple tokens which are a pair of type-tag and a bit of original text, +//! and does not report errors, instead storing them as flags on the token. +//! +//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax. +//! For that see [`rustc_parse::lexer`], which converts this basic token stream +//! into wide tokens used by actual parser. +//! +//! The purpose of this crate is to convert raw sources into a labeled sequence +//! of well-known token types, so building an actual Rust token stream will +//! be easier. +//! +//! The main entity of this crate is the [`TokenKind`] enum which represents common +//! lexeme types. +//! +//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html +#![deny(rustc::untranslatable_diagnostic)] +#![deny(rustc::diagnostic_outside_of_impl)] +// We want to be able to build this crate with a stable compiler, so no +// `#![feature]` attributes should be added. + +mod cursor; +pub mod unescape; + +#[cfg(test)] +mod tests; + +pub use crate::cursor::Cursor; + +use self::LiteralKind::*; +use self::TokenKind::*; +use crate::cursor::EOF_CHAR; + +/// Parsed token. +/// It doesn't contain information about data that has been parsed, +/// only the type of the token and its size. +#[derive(Debug)] +pub struct Token { + pub kind: TokenKind, + pub len: u32, +} + +impl Token { + fn new(kind: TokenKind, len: u32) -> Token { + Token { kind, len } + } +} + +/// Enum representing common lexeme types. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TokenKind { + // Multi-char tokens: + /// "// comment" + LineComment { doc_style: Option<DocStyle> }, + + /// `/* block comment */` + /// + /// Block comments can be recursive, so a sequence like `/* /* */` + /// will not be considered terminated and will result in a parsing error. + BlockComment { doc_style: Option<DocStyle>, terminated: bool }, + + /// Any whitespace character sequence. + Whitespace, + + /// "ident" or "continue" + /// + /// At this step, keywords are also considered identifiers. + Ident, + + /// Like the above, but containing invalid unicode codepoints. + InvalidIdent, + + /// "r#ident" + RawIdent, + + /// An unknown prefix, like `foo#`, `foo'`, `foo"`. + /// + /// Note that only the + /// prefix (`foo`) is included in the token, not the separator (which is + /// lexed as its own distinct token). In Rust 2021 and later, reserved + /// prefixes are reported as errors; in earlier editions, they result in a + /// (allowed by default) lint, and are treated as regular identifier + /// tokens. + UnknownPrefix, + + /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid + /// suffix, but may be present here on string and float literals. Users of + /// this type will need to check for and reject that case. + /// + /// See [LiteralKind] for more details. + Literal { kind: LiteralKind, suffix_start: u32 }, + + /// "'a" + Lifetime { starts_with_number: bool }, + + // One-char tokens: + /// ";" + Semi, + /// "," + Comma, + /// "." + Dot, + /// "(" + OpenParen, + /// ")" + CloseParen, + /// "{" + OpenBrace, + /// "}" + CloseBrace, + /// "[" + OpenBracket, + /// "]" + CloseBracket, + /// "@" + At, + /// "#" + Pound, + /// "~" + Tilde, + /// "?" + Question, + /// ":" + Colon, + /// "$" + Dollar, + /// "=" + Eq, + /// "!" + Bang, + /// "<" + Lt, + /// ">" + Gt, + /// "-" + Minus, + /// "&" + And, + /// "|" + Or, + /// "+" + Plus, + /// "*" + Star, + /// "/" + Slash, + /// "^" + Caret, + /// "%" + Percent, + + /// Unknown token, not expected by the lexer, e.g. "№" + Unknown, + + /// End of input. + Eof, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DocStyle { + Outer, + Inner, +} + +/// Enum representing the literal types supported by the lexer. +/// +/// Note that the suffix is *not* considered when deciding the `LiteralKind` in +/// this type. This means that float literals like `1f32` are classified by this +/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and +/// `rustc_ast::ast::LitKind`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum LiteralKind { + /// "12_u8", "0o100", "0b120i99", "1f32". + Int { base: Base, empty_int: bool }, + /// "12.34f32", "1e3", but not "1f32". + Float { base: Base, empty_exponent: bool }, + /// "'a'", "'\\'", "'''", "';" + Char { terminated: bool }, + /// "b'a'", "b'\\'", "b'''", "b';" + Byte { terminated: bool }, + /// ""abc"", ""abc" + Str { terminated: bool }, + /// "b"abc"", "b"abc" + ByteStr { terminated: bool }, + /// `c"abc"`, `c"abc` + CStr { terminated: bool }, + /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates + /// an invalid literal. + RawStr { n_hashes: Option<u8> }, + /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None` + /// indicates an invalid literal. + RawByteStr { n_hashes: Option<u8> }, + /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal. + RawCStr { n_hashes: Option<u8> }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum RawStrError { + /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##` + InvalidStarter { bad_char: char }, + /// The string was not terminated, e.g. `r###"abcde"##`. + /// `possible_terminator_offset` is the number of characters after `r` or + /// `br` where they may have intended to terminate it. + NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> }, + /// More than 255 `#`s exist. + TooManyDelimiters { found: u32 }, +} + +/// Base of numeric literal encoding according to its prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Base { + /// Literal starts with "0b". + Binary = 2, + /// Literal starts with "0o". + Octal = 8, + /// Literal doesn't contain a prefix. + Decimal = 10, + /// Literal starts with "0x". + Hexadecimal = 16, +} + +/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun", +/// but shebang isn't a part of rust syntax. +pub fn strip_shebang(input: &str) -> Option<usize> { + // Shebang must start with `#!` literally, without any preceding whitespace. + // For simplicity we consider any line starting with `#!` a shebang, + // regardless of restrictions put on shebangs by specific platforms. + if let Some(input_tail) = input.strip_prefix("#!") { + // Ok, this is a shebang but if the next non-whitespace token is `[`, + // then it may be valid Rust code, so consider it Rust code. + let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| { + !matches!( + tok, + TokenKind::Whitespace + | TokenKind::LineComment { doc_style: None } + | TokenKind::BlockComment { doc_style: None, .. } + ) + }); + if next_non_whitespace_token != Some(TokenKind::OpenBracket) { + // No other choice than to consider this a shebang. + return Some(2 + input_tail.lines().next().unwrap_or_default().len()); + } + } + None +} + +/// Validates a raw string literal. Used for getting more information about a +/// problem with a `RawStr`/`RawByteStr` with a `None` field. +#[inline] +pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> { + debug_assert!(!input.is_empty()); + let mut cursor = Cursor::new(input); + // Move past the leading `r` or `br`. + for _ in 0..prefix_len { + cursor.bump().unwrap(); + } + cursor.raw_double_quoted_string(prefix_len).map(|_| ()) +} + +/// Creates an iterator that produces tokens from the input string. +pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ { + let mut cursor = Cursor::new(input); + std::iter::from_fn(move || { + let token = cursor.advance_token(); + if token.kind != TokenKind::Eof { Some(token) } else { None } + }) +} + +/// True if `c` is considered a whitespace according to Rust language definition. +/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html) +/// for definitions of these classes. +pub fn is_whitespace(c: char) -> bool { + // This is Pattern_White_Space. + // + // Note that this set is stable (ie, it doesn't change with different + // Unicode versions), so it's ok to just hard-code the values. + + matches!( + c, + // Usual ASCII suspects + '\u{0009}' // \t + | '\u{000A}' // \n + | '\u{000B}' // vertical tab + | '\u{000C}' // form feed + | '\u{000D}' // \r + | '\u{0020}' // space + + // NEXT LINE from latin1 + | '\u{0085}' + + // Bidi markers + | '\u{200E}' // LEFT-TO-RIGHT MARK + | '\u{200F}' // RIGHT-TO-LEFT MARK + + // Dedicated whitespace characters from Unicode + | '\u{2028}' // LINE SEPARATOR + | '\u{2029}' // PARAGRAPH SEPARATOR + ) +} + +/// True if `c` is valid as a first character of an identifier. +/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for +/// a formal definition of valid identifier name. +pub fn is_id_start(c: char) -> bool { + // This is XID_Start OR '_' (which formally is not a XID_Start). + c == '_' || unicode_xid::UnicodeXID::is_xid_start(c) +} + +/// True if `c` is valid as a non-first character of an identifier. +/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for +/// a formal definition of valid identifier name. +pub fn is_id_continue(c: char) -> bool { + unicode_xid::UnicodeXID::is_xid_continue(c) +} + +/// The passed string is lexically an identifier. +pub fn is_ident(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + is_id_start(start) && chars.all(is_id_continue) + } else { + false + } +} + +impl Cursor<'_> { + /// Parses a token from the input string. + pub fn advance_token(&mut self) -> Token { + let first_char = match self.bump() { + Some(c) => c, + None => return Token::new(TokenKind::Eof, 0), + }; + let token_kind = match first_char { + // Slash, comment or block comment. + '/' => match self.first() { + '/' => self.line_comment(), + '*' => self.block_comment(), + _ => Slash, + }, + + // Whitespace sequence. + c if is_whitespace(c) => self.whitespace(), + + // Raw identifier, raw string literal or identifier. + 'r' => match (self.first(), self.second()) { + ('#', c1) if is_id_start(c1) => self.raw_ident(), + ('#', _) | ('"', _) => { + let res = self.raw_double_quoted_string(1); + let suffix_start = self.pos_within_token(); + if res.is_ok() { + self.eat_literal_suffix(); + } + let kind = RawStr { n_hashes: res.ok() }; + Literal { kind, suffix_start } + } + _ => self.ident_or_unknown_prefix(), + }, + + // Byte literal, byte string literal, raw byte string literal or identifier. + 'b' => self.c_or_byte_string( + |terminated| ByteStr { terminated }, + |n_hashes| RawByteStr { n_hashes }, + Some(|terminated| Byte { terminated }), + ), + + // c-string literal, raw c-string literal or identifier. + 'c' => self.c_or_byte_string( + |terminated| CStr { terminated }, + |n_hashes| RawCStr { n_hashes }, + None, + ), + + // Identifier (this should be checked after other variant that can + // start as identifier). + c if is_id_start(c) => self.ident_or_unknown_prefix(), + + // Numeric literal. + c @ '0'..='9' => { + let literal_kind = self.number(c); + let suffix_start = self.pos_within_token(); + self.eat_literal_suffix(); + TokenKind::Literal { kind: literal_kind, suffix_start } + } + + // One-symbol tokens. + ';' => Semi, + ',' => Comma, + '.' => Dot, + '(' => OpenParen, + ')' => CloseParen, + '{' => OpenBrace, + '}' => CloseBrace, + '[' => OpenBracket, + ']' => CloseBracket, + '@' => At, + '#' => Pound, + '~' => Tilde, + '?' => Question, + ':' => Colon, + '$' => Dollar, + '=' => Eq, + '!' => Bang, + '<' => Lt, + '>' => Gt, + '-' => Minus, + '&' => And, + '|' => Or, + '+' => Plus, + '*' => Star, + '^' => Caret, + '%' => Percent, + + // Lifetime or character literal. + '\'' => self.lifetime_or_char(), + + // String literal. + '"' => { + let terminated = self.double_quoted_string(); + let suffix_start = self.pos_within_token(); + if terminated { + self.eat_literal_suffix(); + } + let kind = Str { terminated }; + Literal { kind, suffix_start } + } + // Identifier starting with an emoji. Only lexed for graceful error recovery. + c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => { + self.fake_ident_or_unknown_prefix() + } + _ => Unknown, + }; + let res = Token::new(token_kind, self.pos_within_token()); + self.reset_pos_within_token(); + res + } + + fn line_comment(&mut self) -> TokenKind { + debug_assert!(self.prev() == '/' && self.first() == '/'); + self.bump(); + + let doc_style = match self.first() { + // `//!` is an inner line doc comment. + '!' => Some(DocStyle::Inner), + // `////` (more than 3 slashes) is not considered a doc comment. + '/' if self.second() != '/' => Some(DocStyle::Outer), + _ => None, + }; + + self.eat_while(|c| c != '\n'); + LineComment { doc_style } + } + + fn block_comment(&mut self) -> TokenKind { + debug_assert!(self.prev() == '/' && self.first() == '*'); + self.bump(); + + let doc_style = match self.first() { + // `/*!` is an inner block doc comment. + '!' => Some(DocStyle::Inner), + // `/***` (more than 2 stars) is not considered a doc comment. + // `/**/` is not considered a doc comment. + '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer), + _ => None, + }; + + let mut depth = 1usize; + while let Some(c) = self.bump() { + match c { + '/' if self.first() == '*' => { + self.bump(); + depth += 1; + } + '*' if self.first() == '/' => { + self.bump(); + depth -= 1; + if depth == 0 { + // This block comment is closed, so for a construction like "/* */ */" + // there will be a successfully parsed block comment "/* */" + // and " */" will be processed separately. + break; + } + } + _ => (), + } + } + + BlockComment { doc_style, terminated: depth == 0 } + } + + fn whitespace(&mut self) -> TokenKind { + debug_assert!(is_whitespace(self.prev())); + self.eat_while(is_whitespace); + Whitespace + } + + fn raw_ident(&mut self) -> TokenKind { + debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second())); + // Eat "#" symbol. + self.bump(); + // Eat the identifier part of RawIdent. + self.eat_identifier(); + RawIdent + } + + fn ident_or_unknown_prefix(&mut self) -> TokenKind { + debug_assert!(is_id_start(self.prev())); + // Start is already eaten, eat the rest of identifier. + self.eat_while(is_id_continue); + // Known prefixes must have been handled earlier. So if + // we see a prefix here, it is definitely an unknown prefix. + match self.first() { + '#' | '"' | '\'' => UnknownPrefix, + c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => { + self.fake_ident_or_unknown_prefix() + } + _ => Ident, + } + } + + fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind { + // Start is already eaten, eat the rest of identifier. + self.eat_while(|c| { + unicode_xid::UnicodeXID::is_xid_continue(c) + || (!c.is_ascii() && unic_emoji_char::is_emoji(c)) + || c == '\u{200d}' + }); + // Known prefixes must have been handled earlier. So if + // we see a prefix here, it is definitely an unknown prefix. + match self.first() { + '#' | '"' | '\'' => UnknownPrefix, + _ => InvalidIdent, + } + } + + fn c_or_byte_string( + &mut self, + mk_kind: impl FnOnce(bool) -> LiteralKind, + mk_kind_raw: impl FnOnce(Option<u8>) -> LiteralKind, + single_quoted: Option<fn(bool) -> LiteralKind>, + ) -> TokenKind { + match (self.first(), self.second(), single_quoted) { + ('\'', _, Some(mk_kind)) => { + self.bump(); + let terminated = self.single_quoted_string(); + let suffix_start = self.pos_within_token(); + if terminated { + self.eat_literal_suffix(); + } + let kind = mk_kind(terminated); + Literal { kind, suffix_start } + } + ('"', _, _) => { + self.bump(); + let terminated = self.double_quoted_string(); + let suffix_start = self.pos_within_token(); + if terminated { + self.eat_literal_suffix(); + } + let kind = mk_kind(terminated); + Literal { kind, suffix_start } + } + ('r', '"', _) | ('r', '#', _) => { + self.bump(); + let res = self.raw_double_quoted_string(2); + let suffix_start = self.pos_within_token(); + if res.is_ok() { + self.eat_literal_suffix(); + } + let kind = mk_kind_raw(res.ok()); + Literal { kind, suffix_start } + } + _ => self.ident_or_unknown_prefix(), + } + } + + fn number(&mut self, first_digit: char) -> LiteralKind { + debug_assert!('0' <= self.prev() && self.prev() <= '9'); + let mut base = Base::Decimal; + if first_digit == '0' { + // Attempt to parse encoding base. + match self.first() { + 'b' => { + base = Base::Binary; + self.bump(); + if !self.eat_decimal_digits() { + return Int { base, empty_int: true }; + } + } + 'o' => { + base = Base::Octal; + self.bump(); + if !self.eat_decimal_digits() { + return Int { base, empty_int: true }; + } + } + 'x' => { + base = Base::Hexadecimal; + self.bump(); + if !self.eat_hexadecimal_digits() { + return Int { base, empty_int: true }; + } + } + // Not a base prefix; consume additional digits. + '0'..='9' | '_' => { + self.eat_decimal_digits(); + } + + // Also not a base prefix; nothing more to do here. + '.' | 'e' | 'E' => {} + + // Just a 0. + _ => return Int { base, empty_int: false }, + } + } else { + // No base prefix, parse number in the usual way. + self.eat_decimal_digits(); + }; + + match self.first() { + // Don't be greedy if this is actually an + // integer literal followed by field/method access or a range pattern + // (`0..2` and `12.foo()`) + '.' if self.second() != '.' && !is_id_start(self.second()) => { + // might have stuff after the ., and if it does, it needs to start + // with a number + self.bump(); + let mut empty_exponent = false; + if self.first().is_digit(10) { + self.eat_decimal_digits(); + match self.first() { + 'e' | 'E' => { + self.bump(); + empty_exponent = !self.eat_float_exponent(); + } + _ => (), + } + } + Float { base, empty_exponent } + } + 'e' | 'E' => { + self.bump(); + let empty_exponent = !self.eat_float_exponent(); + Float { base, empty_exponent } + } + _ => Int { base, empty_int: false }, + } + } + + fn lifetime_or_char(&mut self) -> TokenKind { + debug_assert!(self.prev() == '\''); + + let can_be_a_lifetime = if self.second() == '\'' { + // It's surely not a lifetime. + false + } else { + // If the first symbol is valid for identifier, it can be a lifetime. + // Also check if it's a number for a better error reporting (so '0 will + // be reported as invalid lifetime and not as unterminated char literal). + is_id_start(self.first()) || self.first().is_digit(10) + }; + + if !can_be_a_lifetime { + let terminated = self.single_quoted_string(); + let suffix_start = self.pos_within_token(); + if terminated { + self.eat_literal_suffix(); + } + let kind = Char { terminated }; + return Literal { kind, suffix_start }; + } + + // Either a lifetime or a character literal with + // length greater than 1. + + let starts_with_number = self.first().is_digit(10); + + // Skip the literal contents. + // First symbol can be a number (which isn't a valid identifier start), + // so skip it without any checks. + self.bump(); + self.eat_while(is_id_continue); + + // Check if after skipping literal contents we've met a closing + // single quote (which means that user attempted to create a + // string with single quotes). + if self.first() == '\'' { + self.bump(); + let kind = Char { terminated: true }; + Literal { kind, suffix_start: self.pos_within_token() } + } else { + Lifetime { starts_with_number } + } + } + + fn single_quoted_string(&mut self) -> bool { + debug_assert!(self.prev() == '\''); + // Check if it's a one-symbol literal. + if self.second() == '\'' && self.first() != '\\' { + self.bump(); + self.bump(); + return true; + } + + // Literal has more than one symbol. + + // Parse until either quotes are terminated or error is detected. + loop { + match self.first() { + // Quotes are terminated, finish parsing. + '\'' => { + self.bump(); + return true; + } + // Probably beginning of the comment, which we don't want to include + // to the error report. + '/' => break, + // Newline without following '\'' means unclosed quote, stop parsing. + '\n' if self.second() != '\'' => break, + // End of file, stop parsing. + EOF_CHAR if self.is_eof() => break, + // Escaped slash is considered one character, so bump twice. + '\\' => { + self.bump(); + self.bump(); + } + // Skip the character. + _ => { + self.bump(); + } + } + } + // String was not terminated. + false + } + + /// Eats double-quoted string and returns true + /// if string is terminated. + fn double_quoted_string(&mut self) -> bool { + debug_assert!(self.prev() == '"'); + while let Some(c) = self.bump() { + match c { + '"' => { + return true; + } + '\\' if self.first() == '\\' || self.first() == '"' => { + // Bump again to skip escaped character. + self.bump(); + } + _ => (), + } + } + // End of file reached. + false + } + + /// Eats the double-quoted string and returns `n_hashes` and an error if encountered. + fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> { + // Wrap the actual function to handle the error with too many hashes. + // This way, it eats the whole raw string. + let n_hashes = self.raw_string_unvalidated(prefix_len)?; + // Only up to 255 `#`s are allowed in raw strings + match u8::try_from(n_hashes) { + Ok(num) => Ok(num), + Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }), + } + } + + fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> { + debug_assert!(self.prev() == 'r'); + let start_pos = self.pos_within_token(); + let mut possible_terminator_offset = None; + let mut max_hashes = 0; + + // Count opening '#' symbols. + let mut eaten = 0; + while self.first() == '#' { + eaten += 1; + self.bump(); + } + let n_start_hashes = eaten; + + // Check that string is started. + match self.bump() { + Some('"') => (), + c => { + let c = c.unwrap_or(EOF_CHAR); + return Err(RawStrError::InvalidStarter { bad_char: c }); + } + } + + // Skip the string contents and on each '#' character met, check if this is + // a raw string termination. + loop { + self.eat_while(|c| c != '"'); + + if self.is_eof() { + return Err(RawStrError::NoTerminator { + expected: n_start_hashes, + found: max_hashes, + possible_terminator_offset, + }); + } + + // Eat closing double quote. + self.bump(); + + // Check that amount of closing '#' symbols + // is equal to the amount of opening ones. + // Note that this will not consume extra trailing `#` characters: + // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }` + // followed by a `#` token. + let mut n_end_hashes = 0; + while self.first() == '#' && n_end_hashes < n_start_hashes { + n_end_hashes += 1; + self.bump(); + } + + if n_end_hashes == n_start_hashes { + return Ok(n_start_hashes); + } else if n_end_hashes > max_hashes { + // Keep track of possible terminators to give a hint about + // where there might be a missing terminator + possible_terminator_offset = + Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len); + max_hashes = n_end_hashes; + } + } + } + + fn eat_decimal_digits(&mut self) -> bool { + let mut has_digits = false; + loop { + match self.first() { + '_' => { + self.bump(); + } + '0'..='9' => { + has_digits = true; + self.bump(); + } + _ => break, + } + } + has_digits + } + + fn eat_hexadecimal_digits(&mut self) -> bool { + let mut has_digits = false; + loop { + match self.first() { + '_' => { + self.bump(); + } + '0'..='9' | 'a'..='f' | 'A'..='F' => { + has_digits = true; + self.bump(); + } + _ => break, + } + } + has_digits + } + + /// Eats the float exponent. Returns true if at least one digit was met, + /// and returns false otherwise. + fn eat_float_exponent(&mut self) -> bool { + debug_assert!(self.prev() == 'e' || self.prev() == 'E'); + if self.first() == '-' || self.first() == '+' { + self.bump(); + } + self.eat_decimal_digits() + } + + // Eats the suffix of the literal, e.g. "u8". + fn eat_literal_suffix(&mut self) { + self.eat_identifier(); + } + + // Eats the identifier. Note: succeeds on `_`, which isn't a valid + // identifier. + fn eat_identifier(&mut self) { + if !is_id_start(self.first()) { + return; + } + self.bump(); + + self.eat_while(is_id_continue); + } +} diff --git a/vendor/ra-ap-rustc_lexer/src/tests.rs b/vendor/ra-ap-rustc_lexer/src/tests.rs new file mode 100644 index 000000000..e4c1787f2 --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/src/tests.rs @@ -0,0 +1,298 @@ +use super::*; + +use expect_test::{expect, Expect}; + +fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) { + let s = &format!("r{}", s); + let mut cursor = Cursor::new(s); + cursor.bump(); + let res = cursor.raw_double_quoted_string(0); + assert_eq!(res, expected); +} + +#[test] +fn test_naked_raw_str() { + check_raw_str(r#""abc""#, Ok(0)); +} + +#[test] +fn test_raw_no_start() { + check_raw_str(r##""abc"#"##, Ok(0)); +} + +#[test] +fn test_too_many_terminators() { + // this error is handled in the parser later + check_raw_str(r###"#"abc"##"###, Ok(1)); +} + +#[test] +fn test_unterminated() { + check_raw_str( + r#"#"abc"#, + Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }), + ); + check_raw_str( + r###"##"abc"#"###, + Err(RawStrError::NoTerminator { + expected: 2, + found: 1, + possible_terminator_offset: Some(7), + }), + ); + // We're looking for "# not just any # + check_raw_str( + r###"##"abc#"###, + Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }), + ) +} + +#[test] +fn test_invalid_start() { + check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' })); +} + +#[test] +fn test_unterminated_no_pound() { + // https://github.com/rust-lang/rust/issues/70677 + check_raw_str( + r#"""#, + Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }), + ); +} + +#[test] +fn test_too_many_hashes() { + let max_count = u8::MAX; + let hashes1 = "#".repeat(max_count as usize); + let hashes2 = "#".repeat(max_count as usize + 1); + let middle = "\"abc\""; + let s1 = [&hashes1, middle, &hashes1].join(""); + let s2 = [&hashes2, middle, &hashes2].join(""); + + // Valid number of hashes (255 = 2^8 - 1 = u8::MAX). + check_raw_str(&s1, Ok(255)); + + // One more hash sign (256 = 2^8) becomes too many. + check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 })); +} + +#[test] +fn test_valid_shebang() { + // https://github.com/rust-lang/rust/issues/70528 + let input = "#!/usr/bin/rustrun\nlet x = 5;"; + assert_eq!(strip_shebang(input), Some(18)); +} + +#[test] +fn test_invalid_shebang_valid_rust_syntax() { + // https://github.com/rust-lang/rust/issues/70528 + let input = "#! [bad_attribute]"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_shebang_second_line() { + // Because shebangs are interpreted by the kernel, they must be on the first line + let input = "\n#!/bin/bash"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_shebang_space() { + let input = "#! /bin/bash"; + assert_eq!(strip_shebang(input), Some(input.len())); +} + +#[test] +fn test_shebang_empty_shebang() { + let input = "#! \n[attribute(foo)]"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_invalid_shebang_comment() { + let input = "#!//bin/ami/a/comment\n["; + assert_eq!(strip_shebang(input), None) +} + +#[test] +fn test_invalid_shebang_another_comment() { + let input = "#!/*bin/ami/a/comment*/\n[attribute"; + assert_eq!(strip_shebang(input), None) +} + +#[test] +fn test_shebang_valid_rust_after() { + let input = "#!/*bin/ami/a/comment*/\npub fn main() {}"; + assert_eq!(strip_shebang(input), Some(23)) +} + +#[test] +fn test_shebang_followed_by_attrib() { + let input = "#!/bin/rust-scripts\n#![allow_unused(true)]"; + assert_eq!(strip_shebang(input), Some(19)); +} + +fn check_lexing(src: &str, expect: Expect) { + let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect(); + expect.assert_eq(&actual) +} + +#[test] +fn smoke_test() { + check_lexing( + "/* my source file */ fn main() { println!(\"zebra\"); }\n", + expect![[r#" + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 2 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 4 } + Token { kind: OpenParen, len: 1 } + Token { kind: CloseParen, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: OpenBrace, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 7 } + Token { kind: Bang, len: 1 } + Token { kind: OpenParen, len: 1 } + Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 } + Token { kind: CloseParen, len: 1 } + Token { kind: Semi, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: CloseBrace, len: 1 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} + +#[test] +fn comment_flavors() { + check_lexing( + r" +// line +//// line as well +/// outer doc line +//! inner doc line +/* block */ +/**/ +/*** also block */ +/** outer doc block */ +/*! inner doc block */ +", + expect![[r#" + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 7 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 17 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} + +#[test] +fn nested_block_comments() { + check_lexing( + "/* /* */ */'a'", + expect![[r#" + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + "#]], + ) +} + +#[test] +fn characters() { + check_lexing( + "'a' ' ' '\\n'", + expect![[r#" + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 } + "#]], + ); +} + +#[test] +fn lifetime() { + check_lexing( + "'abc", + expect![[r#" + Token { kind: Lifetime { starts_with_number: false }, len: 4 } + "#]], + ); +} + +#[test] +fn raw_string() { + check_lexing( + "r###\"\"#a\\b\x00c\"\"###", + expect![[r#" + Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 } + "#]], + ) +} + +#[test] +fn literal_suffixes() { + check_lexing( + r####" +'a' +b'a' +"a" +b"a" +1234 +0b101 +0xABC +1.0 +1.0e10 +2us +r###"raw"###suffix +br###"raw"###suffix +"####, + expect![[r#" + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} diff --git a/vendor/ra-ap-rustc_lexer/src/unescape.rs b/vendor/ra-ap-rustc_lexer/src/unescape.rs new file mode 100644 index 000000000..c9ad54d8d --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/src/unescape.rs @@ -0,0 +1,419 @@ +//! Utilities for validating string and char literals and turning them into +//! values they represent. + +use std::ops::Range; +use std::str::Chars; + +#[cfg(test)] +mod tests; + +/// Errors and warnings that can occur during string unescaping. +#[derive(Debug, PartialEq, Eq)] +pub enum EscapeError { + /// Expected 1 char, but 0 were found. + ZeroChars, + /// Expected 1 char, but more than 1 were found. + MoreThanOneChar, + + /// Escaped '\' character without continuation. + LoneSlash, + /// Invalid escape character (e.g. '\z'). + InvalidEscape, + /// Raw '\r' encountered. + BareCarriageReturn, + /// Raw '\r' encountered in raw string. + BareCarriageReturnInRawString, + /// Unescaped character that was expected to be escaped (e.g. raw '\t'). + EscapeOnlyChar, + + /// Numeric character escape is too short (e.g. '\x1'). + TooShortHexEscape, + /// Invalid character in numeric escape (e.g. '\xz') + InvalidCharInHexEscape, + /// Character code in numeric escape is non-ascii (e.g. '\xFF'). + OutOfRangeHexEscape, + + /// '\u' not followed by '{'. + NoBraceInUnicodeEscape, + /// Non-hexadecimal value in '\u{..}'. + InvalidCharInUnicodeEscape, + /// '\u{}' + EmptyUnicodeEscape, + /// No closing brace in '\u{..}', e.g. '\u{12'. + UnclosedUnicodeEscape, + /// '\u{_12}' + LeadingUnderscoreUnicodeEscape, + /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}' + OverlongUnicodeEscape, + /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'. + LoneSurrogateUnicodeEscape, + /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'. + OutOfRangeUnicodeEscape, + + /// Unicode escape code in byte literal. + UnicodeEscapeInByte, + /// Non-ascii character in byte literal, byte string literal, or raw byte string literal. + NonAsciiCharInByte, + + /// After a line ending with '\', the next line contains whitespace + /// characters that are not skipped. + UnskippedWhitespaceWarning, + + /// After a line ending with '\', multiple lines are skipped. + MultipleSkippedLinesWarning, +} + +impl EscapeError { + /// Returns true for actual errors, as opposed to warnings. + pub fn is_fatal(&self) -> bool { + !matches!( + self, + EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning + ) + } +} + +/// Takes a contents of a literal (without quotes) and produces a +/// sequence of escaped characters or errors. +/// Values are returned through invoking of the provided callback. +pub fn unescape_literal<F>(src: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range<usize>, Result<char, EscapeError>), +{ + match mode { + Mode::Char | Mode::Byte => { + let mut chars = src.chars(); + let res = unescape_char_or_byte(&mut chars, mode == Mode::Byte); + callback(0..(src.len() - chars.as_str().len()), res); + } + Mode::Str | Mode::ByteStr => unescape_str_common(src, mode, callback), + + Mode::RawStr | Mode::RawByteStr => { + unescape_raw_str_or_raw_byte_str(src, mode == Mode::RawByteStr, callback) + } + Mode::CStr | Mode::RawCStr => unreachable!(), + } +} + +/// A unit within CStr. Must not be a nul character. +pub enum CStrUnit { + Byte(u8), + Char(char), +} + +impl From<u8> for CStrUnit { + fn from(value: u8) -> Self { + CStrUnit::Byte(value) + } +} + +impl From<char> for CStrUnit { + fn from(value: char) -> Self { + CStrUnit::Char(value) + } +} + +pub fn unescape_c_string<F>(src: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range<usize>, Result<CStrUnit, EscapeError>), +{ + if mode == Mode::RawCStr { + unescape_raw_str_or_raw_byte_str( + src, + mode.characters_should_be_ascii(), + &mut |r, result| callback(r, result.map(CStrUnit::Char)), + ); + } else { + unescape_str_common(src, mode, callback); + } +} + +/// Takes a contents of a char literal (without quotes), and returns an +/// unescaped char or an error. +pub fn unescape_char(src: &str) -> Result<char, EscapeError> { + unescape_char_or_byte(&mut src.chars(), false) +} + +/// Takes a contents of a byte literal (without quotes), and returns an +/// unescaped byte or an error. +pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> { + unescape_char_or_byte(&mut src.chars(), true).map(byte_from_char) +} + +/// What kind of literal do we parse. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Mode { + Char, + Str, + Byte, + ByteStr, + RawStr, + RawByteStr, + CStr, + RawCStr, +} + +impl Mode { + pub fn in_double_quotes(self) -> bool { + match self { + Mode::Str + | Mode::ByteStr + | Mode::RawStr + | Mode::RawByteStr + | Mode::CStr + | Mode::RawCStr => true, + Mode::Char | Mode::Byte => false, + } + } + + /// Non-byte literals should have `\xXX` escapes that are within the ASCII range. + pub fn ascii_escapes_should_be_ascii(self) -> bool { + match self { + Mode::Char | Mode::Str | Mode::RawStr => true, + Mode::Byte | Mode::ByteStr | Mode::RawByteStr | Mode::CStr | Mode::RawCStr => false, + } + } + + /// Whether characters within the literal must be within the ASCII range + pub fn characters_should_be_ascii(self) -> bool { + match self { + Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true, + Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false, + } + } + + /// Byte literals do not allow unicode escape. + pub fn is_unicode_escape_disallowed(self) -> bool { + match self { + Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true, + Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false, + } + } + + pub fn prefix_noraw(self) -> &'static str { + match self { + Mode::Byte | Mode::ByteStr | Mode::RawByteStr => "b", + Mode::CStr | Mode::RawCStr => "c", + Mode::Char | Mode::Str | Mode::RawStr => "", + } + } +} + +fn scan_escape<T: From<u8> + From<char>>( + chars: &mut Chars<'_>, + mode: Mode, +) -> Result<T, EscapeError> { + // Previous character was '\\', unescape what follows. + let res = match chars.next().ok_or(EscapeError::LoneSlash)? { + '"' => b'"', + 'n' => b'\n', + 'r' => b'\r', + 't' => b'\t', + '\\' => b'\\', + '\'' => b'\'', + '0' => b'\0', + + 'x' => { + // Parse hexadecimal character code. + + let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?; + let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?; + + let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?; + let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?; + + let value = hi * 16 + lo; + + if mode.ascii_escapes_should_be_ascii() && !is_ascii(value) { + return Err(EscapeError::OutOfRangeHexEscape); + } + + value as u8 + } + + 'u' => return scan_unicode(chars, mode.is_unicode_escape_disallowed()).map(Into::into), + _ => return Err(EscapeError::InvalidEscape), + }; + Ok(res.into()) +} + +fn scan_unicode( + chars: &mut Chars<'_>, + is_unicode_escape_disallowed: bool, +) -> Result<char, EscapeError> { + // We've parsed '\u', now we have to parse '{..}'. + + if chars.next() != Some('{') { + return Err(EscapeError::NoBraceInUnicodeEscape); + } + + // First character must be a hexadecimal digit. + let mut n_digits = 1; + let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? { + '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape), + '}' => return Err(EscapeError::EmptyUnicodeEscape), + c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?, + }; + + // First character is valid, now parse the rest of the number + // and closing brace. + loop { + match chars.next() { + None => return Err(EscapeError::UnclosedUnicodeEscape), + Some('_') => continue, + Some('}') => { + if n_digits > 6 { + return Err(EscapeError::OverlongUnicodeEscape); + } + + // Incorrect syntax has higher priority for error reporting + // than unallowed value for a literal. + if is_unicode_escape_disallowed { + return Err(EscapeError::UnicodeEscapeInByte); + } + + break std::char::from_u32(value).ok_or_else(|| { + if value > 0x10FFFF { + EscapeError::OutOfRangeUnicodeEscape + } else { + EscapeError::LoneSurrogateUnicodeEscape + } + }); + } + Some(c) => { + let digit: u32 = c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?; + n_digits += 1; + if n_digits > 6 { + // Stop updating value since we're sure that it's incorrect already. + continue; + } + value = value * 16 + digit; + } + }; + } +} + +#[inline] +fn ascii_check(c: char, characters_should_be_ascii: bool) -> Result<char, EscapeError> { + if characters_should_be_ascii && !c.is_ascii() { + // Byte literal can't be a non-ascii character. + Err(EscapeError::NonAsciiCharInByte) + } else { + Ok(c) + } +} + +fn unescape_char_or_byte(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, EscapeError> { + let c = chars.next().ok_or(EscapeError::ZeroChars)?; + let res = match c { + '\\' => scan_escape(chars, if is_byte { Mode::Byte } else { Mode::Char }), + '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar), + '\r' => Err(EscapeError::BareCarriageReturn), + _ => ascii_check(c, is_byte), + }?; + if chars.next().is_some() { + return Err(EscapeError::MoreThanOneChar); + } + Ok(res) +} + +/// Takes a contents of a string literal (without quotes) and produces a +/// sequence of escaped characters or errors. +fn unescape_str_common<F, T: From<u8> + From<char>>(src: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range<usize>, Result<T, EscapeError>), +{ + let mut chars = src.chars(); + + // The `start` and `end` computation here is complicated because + // `skip_ascii_whitespace` makes us to skip over chars without counting + // them in the range computation. + while let Some(c) = chars.next() { + let start = src.len() - chars.as_str().len() - c.len_utf8(); + let res = match c { + '\\' => { + match chars.clone().next() { + Some('\n') => { + // Rust language specification requires us to skip whitespaces + // if unescaped '\' character is followed by '\n'. + // For details see [Rust language reference] + // (https://doc.rust-lang.org/reference/tokens.html#string-literals). + skip_ascii_whitespace(&mut chars, start, &mut |range, err| { + callback(range, Err(err)) + }); + continue; + } + _ => scan_escape::<T>(&mut chars, mode), + } + } + '\n' => Ok(b'\n'.into()), + '\t' => Ok(b'\t'.into()), + '"' => Err(EscapeError::EscapeOnlyChar), + '\r' => Err(EscapeError::BareCarriageReturn), + _ => ascii_check(c, mode.characters_should_be_ascii()).map(Into::into), + }; + let end = src.len() - chars.as_str().len(); + callback(start..end, res.map(Into::into)); + } +} + +fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F) +where + F: FnMut(Range<usize>, EscapeError), +{ + let tail = chars.as_str(); + let first_non_space = tail + .bytes() + .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r') + .unwrap_or(tail.len()); + if tail[1..first_non_space].contains('\n') { + // The +1 accounts for the escaping slash. + let end = start + first_non_space + 1; + callback(start..end, EscapeError::MultipleSkippedLinesWarning); + } + let tail = &tail[first_non_space..]; + if let Some(c) = tail.chars().nth(0) { + if c.is_whitespace() { + // For error reporting, we would like the span to contain the character that was not + // skipped. The +1 is necessary to account for the leading \ that started the escape. + let end = start + first_non_space + c.len_utf8() + 1; + callback(start..end, EscapeError::UnskippedWhitespaceWarning); + } + } + *chars = tail.chars(); +} + +/// Takes a contents of a string literal (without quotes) and produces a +/// sequence of characters or errors. +/// NOTE: Raw strings do not perform any explicit character escaping, here we +/// only produce errors on bare CR. +fn unescape_raw_str_or_raw_byte_str<F>(src: &str, is_byte: bool, callback: &mut F) +where + F: FnMut(Range<usize>, Result<char, EscapeError>), +{ + let mut chars = src.chars(); + + // The `start` and `end` computation here matches the one in + // `unescape_str_or_byte_str` for consistency, even though this function + // doesn't have to worry about skipping any chars. + while let Some(c) = chars.next() { + let start = src.len() - chars.as_str().len() - c.len_utf8(); + let res = match c { + '\r' => Err(EscapeError::BareCarriageReturnInRawString), + _ => ascii_check(c, is_byte), + }; + let end = src.len() - chars.as_str().len(); + callback(start..end, res); + } +} + +#[inline] +pub fn byte_from_char(c: char) -> u8 { + let res = c as u32; + debug_assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr"); + res as u8 +} + +fn is_ascii(x: u32) -> bool { + x <= 0x7F +} diff --git a/vendor/ra-ap-rustc_lexer/src/unescape/tests.rs b/vendor/ra-ap-rustc_lexer/src/unescape/tests.rs new file mode 100644 index 000000000..1c25b03fd --- /dev/null +++ b/vendor/ra-ap-rustc_lexer/src/unescape/tests.rs @@ -0,0 +1,286 @@ +use super::*; + +#[test] +fn test_unescape_char_bad() { + fn check(literal_text: &str, expected_error: EscapeError) { + assert_eq!(unescape_char(literal_text), Err(expected_error)); + } + + check("", EscapeError::ZeroChars); + check(r"\", EscapeError::LoneSlash); + + check("\n", EscapeError::EscapeOnlyChar); + check("\t", EscapeError::EscapeOnlyChar); + check("'", EscapeError::EscapeOnlyChar); + check("\r", EscapeError::BareCarriageReturn); + + check("spam", EscapeError::MoreThanOneChar); + check(r"\x0ff", EscapeError::MoreThanOneChar); + check(r#"\"a"#, EscapeError::MoreThanOneChar); + check(r"\na", EscapeError::MoreThanOneChar); + check(r"\ra", EscapeError::MoreThanOneChar); + check(r"\ta", EscapeError::MoreThanOneChar); + check(r"\\a", EscapeError::MoreThanOneChar); + check(r"\'a", EscapeError::MoreThanOneChar); + check(r"\0a", EscapeError::MoreThanOneChar); + check(r"\u{0}x", EscapeError::MoreThanOneChar); + check(r"\u{1F63b}}", EscapeError::MoreThanOneChar); + + check(r"\v", EscapeError::InvalidEscape); + check(r"\💩", EscapeError::InvalidEscape); + check(r"\●", EscapeError::InvalidEscape); + check("\\\r", EscapeError::InvalidEscape); + + check(r"\x", EscapeError::TooShortHexEscape); + check(r"\x0", EscapeError::TooShortHexEscape); + check(r"\xf", EscapeError::TooShortHexEscape); + check(r"\xa", EscapeError::TooShortHexEscape); + check(r"\xx", EscapeError::InvalidCharInHexEscape); + check(r"\xы", EscapeError::InvalidCharInHexEscape); + check(r"\x🦀", EscapeError::InvalidCharInHexEscape); + check(r"\xtt", EscapeError::InvalidCharInHexEscape); + check(r"\xff", EscapeError::OutOfRangeHexEscape); + check(r"\xFF", EscapeError::OutOfRangeHexEscape); + check(r"\x80", EscapeError::OutOfRangeHexEscape); + + check(r"\u", EscapeError::NoBraceInUnicodeEscape); + check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape); + check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape); + check(r"\u{", EscapeError::UnclosedUnicodeEscape); + check(r"\u{0000", EscapeError::UnclosedUnicodeEscape); + check(r"\u{}", EscapeError::EmptyUnicodeEscape); + check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape); + check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape); + check(r"\u{FFFFFF}", EscapeError::OutOfRangeUnicodeEscape); + check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape); + check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape); + + check(r"\u{DC00}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DDDD}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DFFF}", EscapeError::LoneSurrogateUnicodeEscape); + + check(r"\u{D800}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DAAA}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DBFF}", EscapeError::LoneSurrogateUnicodeEscape); +} + +#[test] +fn test_unescape_char_good() { + fn check(literal_text: &str, expected_char: char) { + assert_eq!(unescape_char(literal_text), Ok(expected_char)); + } + + check("a", 'a'); + check("ы", 'ы'); + check("🦀", '🦀'); + + check(r#"\""#, '"'); + check(r"\n", '\n'); + check(r"\r", '\r'); + check(r"\t", '\t'); + check(r"\\", '\\'); + check(r"\'", '\''); + check(r"\0", '\0'); + + check(r"\x00", '\0'); + check(r"\x5a", 'Z'); + check(r"\x5A", 'Z'); + check(r"\x7f", 127 as char); + + check(r"\u{0}", '\0'); + check(r"\u{000000}", '\0'); + check(r"\u{41}", 'A'); + check(r"\u{0041}", 'A'); + check(r"\u{00_41}", 'A'); + check(r"\u{4__1__}", 'A'); + check(r"\u{1F63b}", '😻'); +} + +#[test] +fn test_unescape_str_warn() { + fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_literal(literal, Mode::Str, &mut |range, res| unescaped.push((range, res))); + assert_eq!(unescaped, expected); + } + + // Check we can handle escaped newlines at the end of a file. + check("\\\n", &[]); + check("\\\n ", &[]); + + check( + "\\\n \u{a0} x", + &[ + (0..5, Err(EscapeError::UnskippedWhitespaceWarning)), + (3..5, Ok('\u{a0}')), + (5..6, Ok(' ')), + (6..7, Ok('x')), + ], + ); + check("\\\n \n x", &[(0..7, Err(EscapeError::MultipleSkippedLinesWarning)), (7..8, Ok('x'))]); +} + +#[test] +fn test_unescape_str_good() { + fn check(literal_text: &str, expected: &str) { + let mut buf = Ok(String::with_capacity(literal_text.len())); + unescape_literal(literal_text, Mode::Str, &mut |range, c| { + if let Ok(b) = &mut buf { + match c { + Ok(c) => b.push(c), + Err(e) => buf = Err((range, e)), + } + } + }); + assert_eq!(buf.as_deref(), Ok(expected)) + } + + check("foo", "foo"); + check("", ""); + check(" \t\n", " \t\n"); + + check("hello \\\n world", "hello world"); + check("thread's", "thread's") +} + +#[test] +fn test_unescape_byte_bad() { + fn check(literal_text: &str, expected_error: EscapeError) { + assert_eq!(unescape_byte(literal_text), Err(expected_error)); + } + + check("", EscapeError::ZeroChars); + check(r"\", EscapeError::LoneSlash); + + check("\n", EscapeError::EscapeOnlyChar); + check("\t", EscapeError::EscapeOnlyChar); + check("'", EscapeError::EscapeOnlyChar); + check("\r", EscapeError::BareCarriageReturn); + + check("spam", EscapeError::MoreThanOneChar); + check(r"\x0ff", EscapeError::MoreThanOneChar); + check(r#"\"a"#, EscapeError::MoreThanOneChar); + check(r"\na", EscapeError::MoreThanOneChar); + check(r"\ra", EscapeError::MoreThanOneChar); + check(r"\ta", EscapeError::MoreThanOneChar); + check(r"\\a", EscapeError::MoreThanOneChar); + check(r"\'a", EscapeError::MoreThanOneChar); + check(r"\0a", EscapeError::MoreThanOneChar); + + check(r"\v", EscapeError::InvalidEscape); + check(r"\💩", EscapeError::InvalidEscape); + check(r"\●", EscapeError::InvalidEscape); + + check(r"\x", EscapeError::TooShortHexEscape); + check(r"\x0", EscapeError::TooShortHexEscape); + check(r"\xa", EscapeError::TooShortHexEscape); + check(r"\xf", EscapeError::TooShortHexEscape); + check(r"\xx", EscapeError::InvalidCharInHexEscape); + check(r"\xы", EscapeError::InvalidCharInHexEscape); + check(r"\x🦀", EscapeError::InvalidCharInHexEscape); + check(r"\xtt", EscapeError::InvalidCharInHexEscape); + + check(r"\u", EscapeError::NoBraceInUnicodeEscape); + check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape); + check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape); + check(r"\u{", EscapeError::UnclosedUnicodeEscape); + check(r"\u{0000", EscapeError::UnclosedUnicodeEscape); + check(r"\u{}", EscapeError::EmptyUnicodeEscape); + check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape); + check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape); + + check("ы", EscapeError::NonAsciiCharInByte); + check("🦀", EscapeError::NonAsciiCharInByte); + + check(r"\u{0}", EscapeError::UnicodeEscapeInByte); + check(r"\u{000000}", EscapeError::UnicodeEscapeInByte); + check(r"\u{41}", EscapeError::UnicodeEscapeInByte); + check(r"\u{0041}", EscapeError::UnicodeEscapeInByte); + check(r"\u{00_41}", EscapeError::UnicodeEscapeInByte); + check(r"\u{4__1__}", EscapeError::UnicodeEscapeInByte); + check(r"\u{1F63b}", EscapeError::UnicodeEscapeInByte); + check(r"\u{0}x", EscapeError::UnicodeEscapeInByte); + check(r"\u{1F63b}}", EscapeError::UnicodeEscapeInByte); + check(r"\u{FFFFFF}", EscapeError::UnicodeEscapeInByte); + check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte); + check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DC00}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DDDD}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DFFF}", EscapeError::UnicodeEscapeInByte); + check(r"\u{D800}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DAAA}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DBFF}", EscapeError::UnicodeEscapeInByte); +} + +#[test] +fn test_unescape_byte_good() { + fn check(literal_text: &str, expected_byte: u8) { + assert_eq!(unescape_byte(literal_text), Ok(expected_byte)); + } + + check("a", b'a'); + + check(r#"\""#, b'"'); + check(r"\n", b'\n'); + check(r"\r", b'\r'); + check(r"\t", b'\t'); + check(r"\\", b'\\'); + check(r"\'", b'\''); + check(r"\0", b'\0'); + + check(r"\x00", b'\0'); + check(r"\x5a", b'Z'); + check(r"\x5A", b'Z'); + check(r"\x7f", 127); + check(r"\x80", 128); + check(r"\xff", 255); + check(r"\xFF", 255); +} + +#[test] +fn test_unescape_byte_str_good() { + fn check(literal_text: &str, expected: &[u8]) { + let mut buf = Ok(Vec::with_capacity(literal_text.len())); + unescape_literal(literal_text, Mode::ByteStr, &mut |range, c| { + if let Ok(b) = &mut buf { + match c { + Ok(c) => b.push(byte_from_char(c)), + Err(e) => buf = Err((range, e)), + } + } + }); + assert_eq!(buf.as_deref(), Ok(expected)) + } + + check("foo", b"foo"); + check("", b""); + check(" \t\n", b" \t\n"); + + check("hello \\\n world", b"hello world"); + check("thread's", b"thread's") +} + +#[test] +fn test_unescape_raw_str() { + fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_literal(literal, Mode::RawStr, &mut |range, res| unescaped.push((range, res))); + assert_eq!(unescaped, expected); + } + + check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]); + check("\rx", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString)), (1..2, Ok('x'))]); +} + +#[test] +fn test_unescape_raw_byte_str() { + fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_literal(literal, Mode::RawByteStr, &mut |range, res| unescaped.push((range, res))); + assert_eq!(unescaped, expected); + } + + check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]); + check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByte))]); + check("🦀a", &[(0..4, Err(EscapeError::NonAsciiCharInByte)), (4..5, Ok('a'))]); +} |