From 698f8c2f01ea549d77d7dc3338a12e04c11057b9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:02:58 +0200 Subject: Adding upstream version 1.64.0+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_lexer/Cargo.toml | 23 + compiler/rustc_lexer/src/cursor.rs | 93 ++++ compiler/rustc_lexer/src/lib.rs | 843 +++++++++++++++++++++++++++++ compiler/rustc_lexer/src/tests.rs | 298 ++++++++++ compiler/rustc_lexer/src/unescape.rs | 377 +++++++++++++ compiler/rustc_lexer/src/unescape/tests.rs | 297 ++++++++++ 6 files changed, 1931 insertions(+) create mode 100644 compiler/rustc_lexer/Cargo.toml create mode 100644 compiler/rustc_lexer/src/cursor.rs create mode 100644 compiler/rustc_lexer/src/lib.rs create mode 100644 compiler/rustc_lexer/src/tests.rs create mode 100644 compiler/rustc_lexer/src/unescape.rs create mode 100644 compiler/rustc_lexer/src/unescape/tests.rs (limited to 'compiler/rustc_lexer') diff --git a/compiler/rustc_lexer/Cargo.toml b/compiler/rustc_lexer/Cargo.toml new file mode 100644 index 000000000..35af11053 --- /dev/null +++ b/compiler/rustc_lexer/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rustc_lexer" +version = "0.1.0" +license = "MIT OR Apache-2.0" +edition = "2021" + +repository = "https://github.com/rust-lang/rust/" +description = """ +Rust lexer used by rustc. No stability guarantees are provided. +""" + +# Note: do not remove this blank `[lib]` section. +# This will be used when publishing this crate as `rustc-ap-rustc_lexer`. +[lib] +doctest = false + +# Note that this crate purposefully does not depend on other rustc crates +[dependencies] +unicode-xid = "0.2.0" +unic-emoji-char = "0.9.0" + +[dev-dependencies] +expect-test = "1.0" diff --git a/compiler/rustc_lexer/src/cursor.rs b/compiler/rustc_lexer/src/cursor.rs new file mode 100644 index 000000000..21557a9c8 --- /dev/null +++ b/compiler/rustc_lexer/src/cursor.rs @@ -0,0 +1,93 @@ +use std::str::Chars; + +/// Peekable iterator over a char sequence. +/// +/// Next characters can be peeked via `first` method, +/// and position can be shifted forward via `bump` method. +pub(crate) struct Cursor<'a> { + initial_len: usize, + /// Iterator over chars. Slightly faster than a &str. + chars: Chars<'a>, + #[cfg(debug_assertions)] + prev: char, +} + +pub(crate) const EOF_CHAR: char = '\0'; + +impl<'a> Cursor<'a> { + pub(crate) fn new(input: &'a str) -> Cursor<'a> { + Cursor { + initial_len: input.len(), + chars: input.chars(), + #[cfg(debug_assertions)] + prev: EOF_CHAR, + } + } + + /// Returns the last eaten symbol (or `'\0'` in release builds). + /// (For debug assertions only.) + pub(crate) fn prev(&self) -> char { + #[cfg(debug_assertions)] + { + self.prev + } + + #[cfg(not(debug_assertions))] + { + EOF_CHAR + } + } + + /// Peeks the next symbol from the input stream without consuming it. + /// If requested position doesn't exist, `EOF_CHAR` is returned. + /// However, getting `EOF_CHAR` doesn't always mean actual end of file, + /// it should be checked with `is_eof` method. + pub(crate) fn first(&self) -> char { + // `.next()` optimizes better than `.nth(0)` + self.chars.clone().next().unwrap_or(EOF_CHAR) + } + + /// Peeks the second symbol from the input stream without consuming it. + pub(crate) fn second(&self) -> char { + // `.next()` optimizes better than `.nth(1)` + let mut iter = self.chars.clone(); + iter.next(); + iter.next().unwrap_or(EOF_CHAR) + } + + /// Checks if there is nothing more to consume. + pub(crate) fn is_eof(&self) -> bool { + self.chars.as_str().is_empty() + } + + /// Returns amount of already consumed symbols. + pub(crate) fn len_consumed(&self) -> u32 { + (self.initial_len - self.chars.as_str().len()) as u32 + } + + /// Resets the number of bytes consumed to 0. + pub(crate) fn reset_len_consumed(&mut self) { + self.initial_len = self.chars.as_str().len(); + } + + /// Moves to the next character. + pub(crate) fn bump(&mut self) -> Option { + let c = self.chars.next()?; + + #[cfg(debug_assertions)] + { + self.prev = c; + } + + Some(c) + } + + /// Eats symbols while predicate returns true or until the end of file is reached. + pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) { + // It was tried making optimized version of this for eg. line comments, but + // LLVM can inline all of this and compile it down to fast iteration over bytes. + while predicate(self.first()) && !self.is_eof() { + self.bump(); + } + } +} diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs new file mode 100644 index 000000000..6d311af90 --- /dev/null +++ b/compiler/rustc_lexer/src/lib.rs @@ -0,0 +1,843 @@ +//! Low-level Rust lexer. +//! +//! The idea with `rustc_lexer` is to make a reusable library, +//! by separating out pure lexing and rustc-specific concerns, like spans, +//! error reporting, and interning. So, rustc_lexer operates directly on `&str`, +//! produces simple tokens which are a pair of type-tag and a bit of original text, +//! and does not report errors, instead storing them as flags on the token. +//! +//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax. +//! For that see [`rustc_parse::lexer`], which converts this basic token stream +//! into wide tokens used by actual parser. +//! +//! The purpose of this crate is to convert raw sources into a labeled sequence +//! of well-known token types, so building an actual Rust token stream will +//! be easier. +//! +//! The main entity of this crate is the [`TokenKind`] enum which represents common +//! lexeme types. +//! +//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html +// We want to be able to build this crate with a stable compiler, so no +// `#![feature]` attributes should be added. + +mod cursor; +pub mod unescape; + +#[cfg(test)] +mod tests; + +use self::LiteralKind::*; +use self::TokenKind::*; +use crate::cursor::{Cursor, EOF_CHAR}; +use std::convert::TryFrom; + +/// Parsed token. +/// It doesn't contain information about data that has been parsed, +/// only the type of the token and its size. +#[derive(Debug)] +pub struct Token { + pub kind: TokenKind, + pub len: u32, +} + +impl Token { + fn new(kind: TokenKind, len: u32) -> Token { + Token { kind, len } + } +} + +/// Enum representing common lexeme types. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TokenKind { + // Multi-char tokens: + /// "// comment" + LineComment { doc_style: Option }, + /// `/* block comment */` + /// + /// Block comments can be recursive, so the sequence like `/* /* */` + /// will not be considered terminated and will result in a parsing error. + BlockComment { doc_style: Option, terminated: bool }, + /// Any whitespace characters sequence. + Whitespace, + /// "ident" or "continue" + /// At this step keywords are also considered identifiers. + Ident, + /// Like the above, but containing invalid unicode codepoints. + InvalidIdent, + /// "r#ident" + RawIdent, + /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the + /// prefix (`foo`) is included in the token, not the separator (which is + /// lexed as its own distinct token). In Rust 2021 and later, reserved + /// prefixes are reported as errors; in earlier editions, they result in a + /// (allowed by default) lint, and are treated as regular identifier + /// tokens. + UnknownPrefix, + /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details. + Literal { kind: LiteralKind, suffix_start: u32 }, + /// "'a" + Lifetime { starts_with_number: bool }, + + // One-char tokens: + /// ";" + Semi, + /// "," + Comma, + /// "." + Dot, + /// "(" + OpenParen, + /// ")" + CloseParen, + /// "{" + OpenBrace, + /// "}" + CloseBrace, + /// "[" + OpenBracket, + /// "]" + CloseBracket, + /// "@" + At, + /// "#" + Pound, + /// "~" + Tilde, + /// "?" + Question, + /// ":" + Colon, + /// "$" + Dollar, + /// "=" + Eq, + /// "!" + Bang, + /// "<" + Lt, + /// ">" + Gt, + /// "-" + Minus, + /// "&" + And, + /// "|" + Or, + /// "+" + Plus, + /// "*" + Star, + /// "/" + Slash, + /// "^" + Caret, + /// "%" + Percent, + + /// Unknown token, not expected by the lexer, e.g. "№" + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DocStyle { + Outer, + Inner, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum LiteralKind { + /// "12_u8", "0o100", "0b120i99" + Int { base: Base, empty_int: bool }, + /// "12.34f32", "0b100.100" + Float { base: Base, empty_exponent: bool }, + /// "'a'", "'\\'", "'''", "';" + Char { terminated: bool }, + /// "b'a'", "b'\\'", "b'''", "b';" + Byte { terminated: bool }, + /// ""abc"", ""abc" + Str { terminated: bool }, + /// "b"abc"", "b"abc" + ByteStr { terminated: bool }, + /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates + /// an invalid literal. + RawStr { n_hashes: Option }, + /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None` + /// indicates an invalid literal. + RawByteStr { n_hashes: Option }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum RawStrError { + /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##` + InvalidStarter { bad_char: char }, + /// The string was not terminated, e.g. `r###"abcde"##`. + /// `possible_terminator_offset` is the number of characters after `r` or + /// `br` where they may have intended to terminate it. + NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option }, + /// More than 255 `#`s exist. + TooManyDelimiters { found: u32 }, +} + +/// Base of numeric literal encoding according to its prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Base { + /// Literal starts with "0b". + Binary, + /// Literal starts with "0o". + Octal, + /// Literal starts with "0x". + Hexadecimal, + /// Literal doesn't contain a prefix. + Decimal, +} + +/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun", +/// but shebang isn't a part of rust syntax. +pub fn strip_shebang(input: &str) -> Option { + // Shebang must start with `#!` literally, without any preceding whitespace. + // For simplicity we consider any line starting with `#!` a shebang, + // regardless of restrictions put on shebangs by specific platforms. + if let Some(input_tail) = input.strip_prefix("#!") { + // Ok, this is a shebang but if the next non-whitespace token is `[`, + // then it may be valid Rust code, so consider it Rust code. + let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| { + !matches!( + tok, + TokenKind::Whitespace + | TokenKind::LineComment { doc_style: None } + | TokenKind::BlockComment { doc_style: None, .. } + ) + }); + if next_non_whitespace_token != Some(TokenKind::OpenBracket) { + // No other choice than to consider this a shebang. + return Some(2 + input_tail.lines().next().unwrap_or_default().len()); + } + } + None +} + +/// Parses the first token from the provided input string. +#[inline] +pub fn first_token(input: &str) -> Token { + debug_assert!(!input.is_empty()); + Cursor::new(input).advance_token() +} + +/// Validates a raw string literal. Used for getting more information about a +/// problem with a `RawStr`/`RawByteStr` with a `None` field. +#[inline] +pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> { + debug_assert!(!input.is_empty()); + let mut cursor = Cursor::new(input); + // Move past the leading `r` or `br`. + for _ in 0..prefix_len { + cursor.bump().unwrap(); + } + cursor.raw_double_quoted_string(prefix_len).map(|_| ()) +} + +/// Creates an iterator that produces tokens from the input string. +pub fn tokenize(input: &str) -> impl Iterator + '_ { + let mut cursor = Cursor::new(input); + std::iter::from_fn(move || { + if cursor.is_eof() { + None + } else { + cursor.reset_len_consumed(); + Some(cursor.advance_token()) + } + }) +} + +/// True if `c` is considered a whitespace according to Rust language definition. +/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html) +/// for definitions of these classes. +pub fn is_whitespace(c: char) -> bool { + // This is Pattern_White_Space. + // + // Note that this set is stable (ie, it doesn't change with different + // Unicode versions), so it's ok to just hard-code the values. + + matches!( + c, + // Usual ASCII suspects + '\u{0009}' // \t + | '\u{000A}' // \n + | '\u{000B}' // vertical tab + | '\u{000C}' // form feed + | '\u{000D}' // \r + | '\u{0020}' // space + + // NEXT LINE from latin1 + | '\u{0085}' + + // Bidi markers + | '\u{200E}' // LEFT-TO-RIGHT MARK + | '\u{200F}' // RIGHT-TO-LEFT MARK + + // Dedicated whitespace characters from Unicode + | '\u{2028}' // LINE SEPARATOR + | '\u{2029}' // PARAGRAPH SEPARATOR + ) +} + +/// True if `c` is valid as a first character of an identifier. +/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for +/// a formal definition of valid identifier name. +pub fn is_id_start(c: char) -> bool { + // This is XID_Start OR '_' (which formally is not a XID_Start). + c == '_' || unicode_xid::UnicodeXID::is_xid_start(c) +} + +/// True if `c` is valid as a non-first character of an identifier. +/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for +/// a formal definition of valid identifier name. +pub fn is_id_continue(c: char) -> bool { + unicode_xid::UnicodeXID::is_xid_continue(c) +} + +/// The passed string is lexically an identifier. +pub fn is_ident(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + is_id_start(start) && chars.all(is_id_continue) + } else { + false + } +} + +impl Cursor<'_> { + /// Parses a token from the input string. + fn advance_token(&mut self) -> Token { + let first_char = self.bump().unwrap(); + let token_kind = match first_char { + // Slash, comment or block comment. + '/' => match self.first() { + '/' => self.line_comment(), + '*' => self.block_comment(), + _ => Slash, + }, + + // Whitespace sequence. + c if is_whitespace(c) => self.whitespace(), + + // Raw identifier, raw string literal or identifier. + 'r' => match (self.first(), self.second()) { + ('#', c1) if is_id_start(c1) => self.raw_ident(), + ('#', _) | ('"', _) => { + let res = self.raw_double_quoted_string(1); + let suffix_start = self.len_consumed(); + if res.is_ok() { + self.eat_literal_suffix(); + } + let kind = RawStr { n_hashes: res.ok() }; + Literal { kind, suffix_start } + } + _ => self.ident_or_unknown_prefix(), + }, + + // Byte literal, byte string literal, raw byte string literal or identifier. + 'b' => match (self.first(), self.second()) { + ('\'', _) => { + self.bump(); + let terminated = self.single_quoted_string(); + let suffix_start = self.len_consumed(); + if terminated { + self.eat_literal_suffix(); + } + let kind = Byte { terminated }; + Literal { kind, suffix_start } + } + ('"', _) => { + self.bump(); + let terminated = self.double_quoted_string(); + let suffix_start = self.len_consumed(); + if terminated { + self.eat_literal_suffix(); + } + let kind = ByteStr { terminated }; + Literal { kind, suffix_start } + } + ('r', '"') | ('r', '#') => { + self.bump(); + let res = self.raw_double_quoted_string(2); + let suffix_start = self.len_consumed(); + if res.is_ok() { + self.eat_literal_suffix(); + } + let kind = RawByteStr { n_hashes: res.ok() }; + Literal { kind, suffix_start } + } + _ => self.ident_or_unknown_prefix(), + }, + + // Identifier (this should be checked after other variant that can + // start as identifier). + c if is_id_start(c) => self.ident_or_unknown_prefix(), + + // Numeric literal. + c @ '0'..='9' => { + let literal_kind = self.number(c); + let suffix_start = self.len_consumed(); + self.eat_literal_suffix(); + TokenKind::Literal { kind: literal_kind, suffix_start } + } + + // One-symbol tokens. + ';' => Semi, + ',' => Comma, + '.' => Dot, + '(' => OpenParen, + ')' => CloseParen, + '{' => OpenBrace, + '}' => CloseBrace, + '[' => OpenBracket, + ']' => CloseBracket, + '@' => At, + '#' => Pound, + '~' => Tilde, + '?' => Question, + ':' => Colon, + '$' => Dollar, + '=' => Eq, + '!' => Bang, + '<' => Lt, + '>' => Gt, + '-' => Minus, + '&' => And, + '|' => Or, + '+' => Plus, + '*' => Star, + '^' => Caret, + '%' => Percent, + + // Lifetime or character literal. + '\'' => self.lifetime_or_char(), + + // String literal. + '"' => { + let terminated = self.double_quoted_string(); + let suffix_start = self.len_consumed(); + if terminated { + self.eat_literal_suffix(); + } + let kind = Str { terminated }; + Literal { kind, suffix_start } + } + // Identifier starting with an emoji. Only lexed for graceful error recovery. + c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => { + self.fake_ident_or_unknown_prefix() + } + _ => Unknown, + }; + Token::new(token_kind, self.len_consumed()) + } + + fn line_comment(&mut self) -> TokenKind { + debug_assert!(self.prev() == '/' && self.first() == '/'); + self.bump(); + + let doc_style = match self.first() { + // `//!` is an inner line doc comment. + '!' => Some(DocStyle::Inner), + // `////` (more than 3 slashes) is not considered a doc comment. + '/' if self.second() != '/' => Some(DocStyle::Outer), + _ => None, + }; + + self.eat_while(|c| c != '\n'); + LineComment { doc_style } + } + + fn block_comment(&mut self) -> TokenKind { + debug_assert!(self.prev() == '/' && self.first() == '*'); + self.bump(); + + let doc_style = match self.first() { + // `/*!` is an inner block doc comment. + '!' => Some(DocStyle::Inner), + // `/***` (more than 2 stars) is not considered a doc comment. + // `/**/` is not considered a doc comment. + '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer), + _ => None, + }; + + let mut depth = 1usize; + while let Some(c) = self.bump() { + match c { + '/' if self.first() == '*' => { + self.bump(); + depth += 1; + } + '*' if self.first() == '/' => { + self.bump(); + depth -= 1; + if depth == 0 { + // This block comment is closed, so for a construction like "/* */ */" + // there will be a successfully parsed block comment "/* */" + // and " */" will be processed separately. + break; + } + } + _ => (), + } + } + + BlockComment { doc_style, terminated: depth == 0 } + } + + fn whitespace(&mut self) -> TokenKind { + debug_assert!(is_whitespace(self.prev())); + self.eat_while(is_whitespace); + Whitespace + } + + fn raw_ident(&mut self) -> TokenKind { + debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second())); + // Eat "#" symbol. + self.bump(); + // Eat the identifier part of RawIdent. + self.eat_identifier(); + RawIdent + } + + fn ident_or_unknown_prefix(&mut self) -> TokenKind { + debug_assert!(is_id_start(self.prev())); + // Start is already eaten, eat the rest of identifier. + self.eat_while(is_id_continue); + // Known prefixes must have been handled earlier. So if + // we see a prefix here, it is definitely an unknown prefix. + match self.first() { + '#' | '"' | '\'' => UnknownPrefix, + c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => { + self.fake_ident_or_unknown_prefix() + } + _ => Ident, + } + } + + fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind { + // Start is already eaten, eat the rest of identifier. + self.eat_while(|c| { + unicode_xid::UnicodeXID::is_xid_continue(c) + || (!c.is_ascii() && unic_emoji_char::is_emoji(c)) + || c == '\u{200d}' + }); + // Known prefixes must have been handled earlier. So if + // we see a prefix here, it is definitely an unknown prefix. + match self.first() { + '#' | '"' | '\'' => UnknownPrefix, + _ => InvalidIdent, + } + } + + fn number(&mut self, first_digit: char) -> LiteralKind { + debug_assert!('0' <= self.prev() && self.prev() <= '9'); + let mut base = Base::Decimal; + if first_digit == '0' { + // Attempt to parse encoding base. + let has_digits = match self.first() { + 'b' => { + base = Base::Binary; + self.bump(); + self.eat_decimal_digits() + } + 'o' => { + base = Base::Octal; + self.bump(); + self.eat_decimal_digits() + } + 'x' => { + base = Base::Hexadecimal; + self.bump(); + self.eat_hexadecimal_digits() + } + // Not a base prefix. + '0'..='9' | '_' | '.' | 'e' | 'E' => { + self.eat_decimal_digits(); + true + } + // Just a 0. + _ => return Int { base, empty_int: false }, + }; + // Base prefix was provided, but there were no digits + // after it, e.g. "0x". + if !has_digits { + return Int { base, empty_int: true }; + } + } else { + // No base prefix, parse number in the usual way. + self.eat_decimal_digits(); + }; + + match self.first() { + // Don't be greedy if this is actually an + // integer literal followed by field/method access or a range pattern + // (`0..2` and `12.foo()`) + '.' if self.second() != '.' && !is_id_start(self.second()) => { + // might have stuff after the ., and if it does, it needs to start + // with a number + self.bump(); + let mut empty_exponent = false; + if self.first().is_digit(10) { + self.eat_decimal_digits(); + match self.first() { + 'e' | 'E' => { + self.bump(); + empty_exponent = !self.eat_float_exponent(); + } + _ => (), + } + } + Float { base, empty_exponent } + } + 'e' | 'E' => { + self.bump(); + let empty_exponent = !self.eat_float_exponent(); + Float { base, empty_exponent } + } + _ => Int { base, empty_int: false }, + } + } + + fn lifetime_or_char(&mut self) -> TokenKind { + debug_assert!(self.prev() == '\''); + + let can_be_a_lifetime = if self.second() == '\'' { + // It's surely not a lifetime. + false + } else { + // If the first symbol is valid for identifier, it can be a lifetime. + // Also check if it's a number for a better error reporting (so '0 will + // be reported as invalid lifetime and not as unterminated char literal). + is_id_start(self.first()) || self.first().is_digit(10) + }; + + if !can_be_a_lifetime { + let terminated = self.single_quoted_string(); + let suffix_start = self.len_consumed(); + if terminated { + self.eat_literal_suffix(); + } + let kind = Char { terminated }; + return Literal { kind, suffix_start }; + } + + // Either a lifetime or a character literal with + // length greater than 1. + + let starts_with_number = self.first().is_digit(10); + + // Skip the literal contents. + // First symbol can be a number (which isn't a valid identifier start), + // so skip it without any checks. + self.bump(); + self.eat_while(is_id_continue); + + // Check if after skipping literal contents we've met a closing + // single quote (which means that user attempted to create a + // string with single quotes). + if self.first() == '\'' { + self.bump(); + let kind = Char { terminated: true }; + Literal { kind, suffix_start: self.len_consumed() } + } else { + Lifetime { starts_with_number } + } + } + + fn single_quoted_string(&mut self) -> bool { + debug_assert!(self.prev() == '\''); + // Check if it's a one-symbol literal. + if self.second() == '\'' && self.first() != '\\' { + self.bump(); + self.bump(); + return true; + } + + // Literal has more than one symbol. + + // Parse until either quotes are terminated or error is detected. + loop { + match self.first() { + // Quotes are terminated, finish parsing. + '\'' => { + self.bump(); + return true; + } + // Probably beginning of the comment, which we don't want to include + // to the error report. + '/' => break, + // Newline without following '\'' means unclosed quote, stop parsing. + '\n' if self.second() != '\'' => break, + // End of file, stop parsing. + EOF_CHAR if self.is_eof() => break, + // Escaped slash is considered one character, so bump twice. + '\\' => { + self.bump(); + self.bump(); + } + // Skip the character. + _ => { + self.bump(); + } + } + } + // String was not terminated. + false + } + + /// Eats double-quoted string and returns true + /// if string is terminated. + fn double_quoted_string(&mut self) -> bool { + debug_assert!(self.prev() == '"'); + while let Some(c) = self.bump() { + match c { + '"' => { + return true; + } + '\\' if self.first() == '\\' || self.first() == '"' => { + // Bump again to skip escaped character. + self.bump(); + } + _ => (), + } + } + // End of file reached. + false + } + + /// Eats the double-quoted string and returns `n_hashes` and an error if encountered. + fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result { + // Wrap the actual function to handle the error with too many hashes. + // This way, it eats the whole raw string. + let n_hashes = self.raw_string_unvalidated(prefix_len)?; + // Only up to 255 `#`s are allowed in raw strings + match u8::try_from(n_hashes) { + Ok(num) => Ok(num), + Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }), + } + } + + fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result { + debug_assert!(self.prev() == 'r'); + let start_pos = self.len_consumed(); + let mut possible_terminator_offset = None; + let mut max_hashes = 0; + + // Count opening '#' symbols. + let mut eaten = 0; + while self.first() == '#' { + eaten += 1; + self.bump(); + } + let n_start_hashes = eaten; + + // Check that string is started. + match self.bump() { + Some('"') => (), + c => { + let c = c.unwrap_or(EOF_CHAR); + return Err(RawStrError::InvalidStarter { bad_char: c }); + } + } + + // Skip the string contents and on each '#' character met, check if this is + // a raw string termination. + loop { + self.eat_while(|c| c != '"'); + + if self.is_eof() { + return Err(RawStrError::NoTerminator { + expected: n_start_hashes, + found: max_hashes, + possible_terminator_offset, + }); + } + + // Eat closing double quote. + self.bump(); + + // Check that amount of closing '#' symbols + // is equal to the amount of opening ones. + // Note that this will not consume extra trailing `#` characters: + // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }` + // followed by a `#` token. + let mut n_end_hashes = 0; + while self.first() == '#' && n_end_hashes < n_start_hashes { + n_end_hashes += 1; + self.bump(); + } + + if n_end_hashes == n_start_hashes { + return Ok(n_start_hashes); + } else if n_end_hashes > max_hashes { + // Keep track of possible terminators to give a hint about + // where there might be a missing terminator + possible_terminator_offset = + Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len); + max_hashes = n_end_hashes; + } + } + } + + fn eat_decimal_digits(&mut self) -> bool { + let mut has_digits = false; + loop { + match self.first() { + '_' => { + self.bump(); + } + '0'..='9' => { + has_digits = true; + self.bump(); + } + _ => break, + } + } + has_digits + } + + fn eat_hexadecimal_digits(&mut self) -> bool { + let mut has_digits = false; + loop { + match self.first() { + '_' => { + self.bump(); + } + '0'..='9' | 'a'..='f' | 'A'..='F' => { + has_digits = true; + self.bump(); + } + _ => break, + } + } + has_digits + } + + /// Eats the float exponent. Returns true if at least one digit was met, + /// and returns false otherwise. + fn eat_float_exponent(&mut self) -> bool { + debug_assert!(self.prev() == 'e' || self.prev() == 'E'); + if self.first() == '-' || self.first() == '+' { + self.bump(); + } + self.eat_decimal_digits() + } + + // Eats the suffix of the literal, e.g. "_u8". + fn eat_literal_suffix(&mut self) { + self.eat_identifier(); + } + + // Eats the identifier. + fn eat_identifier(&mut self) { + if !is_id_start(self.first()) { + return; + } + self.bump(); + + self.eat_while(is_id_continue); + } +} diff --git a/compiler/rustc_lexer/src/tests.rs b/compiler/rustc_lexer/src/tests.rs new file mode 100644 index 000000000..e4c1787f2 --- /dev/null +++ b/compiler/rustc_lexer/src/tests.rs @@ -0,0 +1,298 @@ +use super::*; + +use expect_test::{expect, Expect}; + +fn check_raw_str(s: &str, expected: Result) { + let s = &format!("r{}", s); + let mut cursor = Cursor::new(s); + cursor.bump(); + let res = cursor.raw_double_quoted_string(0); + assert_eq!(res, expected); +} + +#[test] +fn test_naked_raw_str() { + check_raw_str(r#""abc""#, Ok(0)); +} + +#[test] +fn test_raw_no_start() { + check_raw_str(r##""abc"#"##, Ok(0)); +} + +#[test] +fn test_too_many_terminators() { + // this error is handled in the parser later + check_raw_str(r###"#"abc"##"###, Ok(1)); +} + +#[test] +fn test_unterminated() { + check_raw_str( + r#"#"abc"#, + Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }), + ); + check_raw_str( + r###"##"abc"#"###, + Err(RawStrError::NoTerminator { + expected: 2, + found: 1, + possible_terminator_offset: Some(7), + }), + ); + // We're looking for "# not just any # + check_raw_str( + r###"##"abc#"###, + Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }), + ) +} + +#[test] +fn test_invalid_start() { + check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' })); +} + +#[test] +fn test_unterminated_no_pound() { + // https://github.com/rust-lang/rust/issues/70677 + check_raw_str( + r#"""#, + Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }), + ); +} + +#[test] +fn test_too_many_hashes() { + let max_count = u8::MAX; + let hashes1 = "#".repeat(max_count as usize); + let hashes2 = "#".repeat(max_count as usize + 1); + let middle = "\"abc\""; + let s1 = [&hashes1, middle, &hashes1].join(""); + let s2 = [&hashes2, middle, &hashes2].join(""); + + // Valid number of hashes (255 = 2^8 - 1 = u8::MAX). + check_raw_str(&s1, Ok(255)); + + // One more hash sign (256 = 2^8) becomes too many. + check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 })); +} + +#[test] +fn test_valid_shebang() { + // https://github.com/rust-lang/rust/issues/70528 + let input = "#!/usr/bin/rustrun\nlet x = 5;"; + assert_eq!(strip_shebang(input), Some(18)); +} + +#[test] +fn test_invalid_shebang_valid_rust_syntax() { + // https://github.com/rust-lang/rust/issues/70528 + let input = "#! [bad_attribute]"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_shebang_second_line() { + // Because shebangs are interpreted by the kernel, they must be on the first line + let input = "\n#!/bin/bash"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_shebang_space() { + let input = "#! /bin/bash"; + assert_eq!(strip_shebang(input), Some(input.len())); +} + +#[test] +fn test_shebang_empty_shebang() { + let input = "#! \n[attribute(foo)]"; + assert_eq!(strip_shebang(input), None); +} + +#[test] +fn test_invalid_shebang_comment() { + let input = "#!//bin/ami/a/comment\n["; + assert_eq!(strip_shebang(input), None) +} + +#[test] +fn test_invalid_shebang_another_comment() { + let input = "#!/*bin/ami/a/comment*/\n[attribute"; + assert_eq!(strip_shebang(input), None) +} + +#[test] +fn test_shebang_valid_rust_after() { + let input = "#!/*bin/ami/a/comment*/\npub fn main() {}"; + assert_eq!(strip_shebang(input), Some(23)) +} + +#[test] +fn test_shebang_followed_by_attrib() { + let input = "#!/bin/rust-scripts\n#![allow_unused(true)]"; + assert_eq!(strip_shebang(input), Some(19)); +} + +fn check_lexing(src: &str, expect: Expect) { + let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect(); + expect.assert_eq(&actual) +} + +#[test] +fn smoke_test() { + check_lexing( + "/* my source file */ fn main() { println!(\"zebra\"); }\n", + expect![[r#" + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 2 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 4 } + Token { kind: OpenParen, len: 1 } + Token { kind: CloseParen, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: OpenBrace, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: Ident, len: 7 } + Token { kind: Bang, len: 1 } + Token { kind: OpenParen, len: 1 } + Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 } + Token { kind: CloseParen, len: 1 } + Token { kind: Semi, len: 1 } + Token { kind: Whitespace, len: 1 } + Token { kind: CloseBrace, len: 1 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} + +#[test] +fn comment_flavors() { + check_lexing( + r" +// line +//// line as well +/// outer doc line +//! inner doc line +/* block */ +/**/ +/*** also block */ +/** outer doc block */ +/*! inner doc block */ +", + expect![[r#" + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 7 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 17 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} + +#[test] +fn nested_block_comments() { + check_lexing( + "/* /* */ */'a'", + expect![[r#" + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + "#]], + ) +} + +#[test] +fn characters() { + check_lexing( + "'a' ' ' '\\n'", + expect![[r#" + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 } + "#]], + ); +} + +#[test] +fn lifetime() { + check_lexing( + "'abc", + expect![[r#" + Token { kind: Lifetime { starts_with_number: false }, len: 4 } + "#]], + ); +} + +#[test] +fn raw_string() { + check_lexing( + "r###\"\"#a\\b\x00c\"\"###", + expect![[r#" + Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 } + "#]], + ) +} + +#[test] +fn literal_suffixes() { + check_lexing( + r####" +'a' +b'a' +"a" +b"a" +1234 +0b101 +0xABC +1.0 +1.0e10 +2us +r###"raw"###suffix +br###"raw"###suffix +"####, + expect![[r#" + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs new file mode 100644 index 000000000..3da6bc146 --- /dev/null +++ b/compiler/rustc_lexer/src/unescape.rs @@ -0,0 +1,377 @@ +//! Utilities for validating string and char literals and turning them into +//! values they represent. + +use std::ops::Range; +use std::str::Chars; + +#[cfg(test)] +mod tests; + +/// Errors and warnings that can occur during string unescaping. +#[derive(Debug, PartialEq, Eq)] +pub enum EscapeError { + /// Expected 1 char, but 0 were found. + ZeroChars, + /// Expected 1 char, but more than 1 were found. + MoreThanOneChar, + + /// Escaped '\' character without continuation. + LoneSlash, + /// Invalid escape character (e.g. '\z'). + InvalidEscape, + /// Raw '\r' encountered. + BareCarriageReturn, + /// Raw '\r' encountered in raw string. + BareCarriageReturnInRawString, + /// Unescaped character that was expected to be escaped (e.g. raw '\t'). + EscapeOnlyChar, + + /// Numeric character escape is too short (e.g. '\x1'). + TooShortHexEscape, + /// Invalid character in numeric escape (e.g. '\xz') + InvalidCharInHexEscape, + /// Character code in numeric escape is non-ascii (e.g. '\xFF'). + OutOfRangeHexEscape, + + /// '\u' not followed by '{'. + NoBraceInUnicodeEscape, + /// Non-hexadecimal value in '\u{..}'. + InvalidCharInUnicodeEscape, + /// '\u{}' + EmptyUnicodeEscape, + /// No closing brace in '\u{..}', e.g. '\u{12'. + UnclosedUnicodeEscape, + /// '\u{_12}' + LeadingUnderscoreUnicodeEscape, + /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}' + OverlongUnicodeEscape, + /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'. + LoneSurrogateUnicodeEscape, + /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'. + OutOfRangeUnicodeEscape, + + /// Unicode escape code in byte literal. + UnicodeEscapeInByte, + /// Non-ascii character in byte literal. + NonAsciiCharInByte, + /// Non-ascii character in byte string literal. + NonAsciiCharInByteString, + + /// After a line ending with '\', the next line contains whitespace + /// characters that are not skipped. + UnskippedWhitespaceWarning, + + /// After a line ending with '\', multiple lines are skipped. + MultipleSkippedLinesWarning, +} + +impl EscapeError { + /// Returns true for actual errors, as opposed to warnings. + pub fn is_fatal(&self) -> bool { + !matches!( + self, + EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning + ) + } +} + +/// Takes a contents of a literal (without quotes) and produces a +/// sequence of escaped characters or errors. +/// Values are returned through invoking of the provided callback. +pub fn unescape_literal(literal_text: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range, Result), +{ + match mode { + Mode::Char | Mode::Byte => { + let mut chars = literal_text.chars(); + let result = unescape_char_or_byte(&mut chars, mode); + // The Chars iterator moved forward. + callback(0..(literal_text.len() - chars.as_str().len()), result); + } + Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(literal_text, mode, callback), + // NOTE: Raw strings do not perform any explicit character escaping, here we + // only translate CRLF to LF and produce errors on bare CR. + Mode::RawStr | Mode::RawByteStr => { + unescape_raw_str_or_byte_str(literal_text, mode, callback) + } + } +} + +/// Takes a contents of a byte, byte string or raw byte string (without quotes) +/// and produces a sequence of bytes or errors. +/// Values are returned through invoking of the provided callback. +pub fn unescape_byte_literal(literal_text: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range, Result), +{ + assert!(mode.is_bytes()); + unescape_literal(literal_text, mode, &mut |range, result| { + callback(range, result.map(byte_from_char)); + }) +} + +/// Takes a contents of a char literal (without quotes), and returns an +/// unescaped char or an error +pub fn unescape_char(literal_text: &str) -> Result { + let mut chars = literal_text.chars(); + unescape_char_or_byte(&mut chars, Mode::Char) + .map_err(|err| (literal_text.len() - chars.as_str().len(), err)) +} + +/// Takes a contents of a byte literal (without quotes), and returns an +/// unescaped byte or an error. +pub fn unescape_byte(literal_text: &str) -> Result { + let mut chars = literal_text.chars(); + unescape_char_or_byte(&mut chars, Mode::Byte) + .map(byte_from_char) + .map_err(|err| (literal_text.len() - chars.as_str().len(), err)) +} + +/// What kind of literal do we parse. +#[derive(Debug, Clone, Copy)] +pub enum Mode { + Char, + Str, + Byte, + ByteStr, + RawStr, + RawByteStr, +} + +impl Mode { + pub fn in_single_quotes(self) -> bool { + match self { + Mode::Char | Mode::Byte => true, + Mode::Str | Mode::ByteStr | Mode::RawStr | Mode::RawByteStr => false, + } + } + + pub fn in_double_quotes(self) -> bool { + !self.in_single_quotes() + } + + pub fn is_bytes(self) -> bool { + match self { + Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true, + Mode::Char | Mode::Str | Mode::RawStr => false, + } + } +} + +fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result { + // Previous character was '\\', unescape what follows. + + let second_char = chars.next().ok_or(EscapeError::LoneSlash)?; + + let res = match second_char { + '"' => '"', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '\\' => '\\', + '\'' => '\'', + '0' => '\0', + + 'x' => { + // Parse hexadecimal character code. + + let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?; + let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?; + + let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?; + let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?; + + let value = hi * 16 + lo; + + // For a byte literal verify that it is within ASCII range. + if !mode.is_bytes() && !is_ascii(value) { + return Err(EscapeError::OutOfRangeHexEscape); + } + let value = value as u8; + + value as char + } + + 'u' => { + // We've parsed '\u', now we have to parse '{..}'. + + if chars.next() != Some('{') { + return Err(EscapeError::NoBraceInUnicodeEscape); + } + + // First character must be a hexadecimal digit. + let mut n_digits = 1; + let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? { + '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape), + '}' => return Err(EscapeError::EmptyUnicodeEscape), + c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?, + }; + + // First character is valid, now parse the rest of the number + // and closing brace. + loop { + match chars.next() { + None => return Err(EscapeError::UnclosedUnicodeEscape), + Some('_') => continue, + Some('}') => { + if n_digits > 6 { + return Err(EscapeError::OverlongUnicodeEscape); + } + + // Incorrect syntax has higher priority for error reporting + // than unallowed value for a literal. + if mode.is_bytes() { + return Err(EscapeError::UnicodeEscapeInByte); + } + + break std::char::from_u32(value).ok_or_else(|| { + if value > 0x10FFFF { + EscapeError::OutOfRangeUnicodeEscape + } else { + EscapeError::LoneSurrogateUnicodeEscape + } + })?; + } + Some(c) => { + let digit = + c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?; + n_digits += 1; + if n_digits > 6 { + // Stop updating value since we're sure that it's incorrect already. + continue; + } + let digit = digit as u32; + value = value * 16 + digit; + } + }; + } + } + _ => return Err(EscapeError::InvalidEscape), + }; + Ok(res) +} + +#[inline] +fn ascii_check(first_char: char, mode: Mode) -> Result { + if mode.is_bytes() && !first_char.is_ascii() { + // Byte literal can't be a non-ascii character. + Err(EscapeError::NonAsciiCharInByte) + } else { + Ok(first_char) + } +} + +fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result { + let first_char = chars.next().ok_or(EscapeError::ZeroChars)?; + let res = match first_char { + '\\' => scan_escape(chars, mode), + '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar), + '\r' => Err(EscapeError::BareCarriageReturn), + _ => ascii_check(first_char, mode), + }?; + if chars.next().is_some() { + return Err(EscapeError::MoreThanOneChar); + } + Ok(res) +} + +/// Takes a contents of a string literal (without quotes) and produces a +/// sequence of escaped characters or errors. +fn unescape_str_or_byte_str(src: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range, Result), +{ + assert!(mode.in_double_quotes()); + let initial_len = src.len(); + let mut chars = src.chars(); + while let Some(first_char) = chars.next() { + let start = initial_len - chars.as_str().len() - first_char.len_utf8(); + + let unescaped_char = match first_char { + '\\' => { + let second_char = chars.clone().next(); + match second_char { + Some('\n') => { + // Rust language specification requires us to skip whitespaces + // if unescaped '\' character is followed by '\n'. + // For details see [Rust language reference] + // (https://doc.rust-lang.org/reference/tokens.html#string-literals). + skip_ascii_whitespace(&mut chars, start, callback); + continue; + } + _ => scan_escape(&mut chars, mode), + } + } + '\n' => Ok('\n'), + '\t' => Ok('\t'), + '"' => Err(EscapeError::EscapeOnlyChar), + '\r' => Err(EscapeError::BareCarriageReturn), + _ => ascii_check(first_char, mode), + }; + let end = initial_len - chars.as_str().len(); + callback(start..end, unescaped_char); + } + + fn skip_ascii_whitespace(chars: &mut Chars<'_>, start: usize, callback: &mut F) + where + F: FnMut(Range, Result), + { + let tail = chars.as_str(); + let first_non_space = tail + .bytes() + .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r') + .unwrap_or(tail.len()); + if tail[1..first_non_space].contains('\n') { + // The +1 accounts for the escaping slash. + let end = start + first_non_space + 1; + callback(start..end, Err(EscapeError::MultipleSkippedLinesWarning)); + } + let tail = &tail[first_non_space..]; + if let Some(c) = tail.chars().nth(0) { + // For error reporting, we would like the span to contain the character that was not + // skipped. The +1 is necessary to account for the leading \ that started the escape. + let end = start + first_non_space + c.len_utf8() + 1; + if c.is_whitespace() { + callback(start..end, Err(EscapeError::UnskippedWhitespaceWarning)); + } + } + *chars = tail.chars(); + } +} + +/// Takes a contents of a string literal (without quotes) and produces a +/// sequence of characters or errors. +/// NOTE: Raw strings do not perform any explicit character escaping, here we +/// only translate CRLF to LF and produce errors on bare CR. +fn unescape_raw_str_or_byte_str(literal_text: &str, mode: Mode, callback: &mut F) +where + F: FnMut(Range, Result), +{ + assert!(mode.in_double_quotes()); + let initial_len = literal_text.len(); + + let mut chars = literal_text.chars(); + while let Some(curr) = chars.next() { + let start = initial_len - chars.as_str().len() - curr.len_utf8(); + + let result = match curr { + '\r' => Err(EscapeError::BareCarriageReturnInRawString), + c if mode.is_bytes() && !c.is_ascii() => Err(EscapeError::NonAsciiCharInByteString), + c => Ok(c), + }; + let end = initial_len - chars.as_str().len(); + + callback(start..end, result); + } +} + +fn byte_from_char(c: char) -> u8 { + let res = c as u32; + assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr"); + res as u8 +} + +fn is_ascii(x: u32) -> bool { + x <= 0x7F +} diff --git a/compiler/rustc_lexer/src/unescape/tests.rs b/compiler/rustc_lexer/src/unescape/tests.rs new file mode 100644 index 000000000..fa61554af --- /dev/null +++ b/compiler/rustc_lexer/src/unescape/tests.rs @@ -0,0 +1,297 @@ +use super::*; + +#[test] +fn test_unescape_char_bad() { + fn check(literal_text: &str, expected_error: EscapeError) { + let actual_result = unescape_char(literal_text).map_err(|(_offset, err)| err); + assert_eq!(actual_result, Err(expected_error)); + } + + check("", EscapeError::ZeroChars); + check(r"\", EscapeError::LoneSlash); + + check("\n", EscapeError::EscapeOnlyChar); + check("\t", EscapeError::EscapeOnlyChar); + check("'", EscapeError::EscapeOnlyChar); + check("\r", EscapeError::BareCarriageReturn); + + check("spam", EscapeError::MoreThanOneChar); + check(r"\x0ff", EscapeError::MoreThanOneChar); + check(r#"\"a"#, EscapeError::MoreThanOneChar); + check(r"\na", EscapeError::MoreThanOneChar); + check(r"\ra", EscapeError::MoreThanOneChar); + check(r"\ta", EscapeError::MoreThanOneChar); + check(r"\\a", EscapeError::MoreThanOneChar); + check(r"\'a", EscapeError::MoreThanOneChar); + check(r"\0a", EscapeError::MoreThanOneChar); + check(r"\u{0}x", EscapeError::MoreThanOneChar); + check(r"\u{1F63b}}", EscapeError::MoreThanOneChar); + + check(r"\v", EscapeError::InvalidEscape); + check(r"\💩", EscapeError::InvalidEscape); + check(r"\●", EscapeError::InvalidEscape); + check("\\\r", EscapeError::InvalidEscape); + + check(r"\x", EscapeError::TooShortHexEscape); + check(r"\x0", EscapeError::TooShortHexEscape); + check(r"\xf", EscapeError::TooShortHexEscape); + check(r"\xa", EscapeError::TooShortHexEscape); + check(r"\xx", EscapeError::InvalidCharInHexEscape); + check(r"\xы", EscapeError::InvalidCharInHexEscape); + check(r"\x🦀", EscapeError::InvalidCharInHexEscape); + check(r"\xtt", EscapeError::InvalidCharInHexEscape); + check(r"\xff", EscapeError::OutOfRangeHexEscape); + check(r"\xFF", EscapeError::OutOfRangeHexEscape); + check(r"\x80", EscapeError::OutOfRangeHexEscape); + + check(r"\u", EscapeError::NoBraceInUnicodeEscape); + check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape); + check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape); + check(r"\u{", EscapeError::UnclosedUnicodeEscape); + check(r"\u{0000", EscapeError::UnclosedUnicodeEscape); + check(r"\u{}", EscapeError::EmptyUnicodeEscape); + check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape); + check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape); + check(r"\u{FFFFFF}", EscapeError::OutOfRangeUnicodeEscape); + check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape); + check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape); + + check(r"\u{DC00}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DDDD}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DFFF}", EscapeError::LoneSurrogateUnicodeEscape); + + check(r"\u{D800}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DAAA}", EscapeError::LoneSurrogateUnicodeEscape); + check(r"\u{DBFF}", EscapeError::LoneSurrogateUnicodeEscape); +} + +#[test] +fn test_unescape_char_good() { + fn check(literal_text: &str, expected_char: char) { + let actual_result = unescape_char(literal_text); + assert_eq!(actual_result, Ok(expected_char)); + } + + check("a", 'a'); + check("ы", 'ы'); + check("🦀", '🦀'); + + check(r#"\""#, '"'); + check(r"\n", '\n'); + check(r"\r", '\r'); + check(r"\t", '\t'); + check(r"\\", '\\'); + check(r"\'", '\''); + check(r"\0", '\0'); + + check(r"\x00", '\0'); + check(r"\x5a", 'Z'); + check(r"\x5A", 'Z'); + check(r"\x7f", 127 as char); + + check(r"\u{0}", '\0'); + check(r"\u{000000}", '\0'); + check(r"\u{41}", 'A'); + check(r"\u{0041}", 'A'); + check(r"\u{00_41}", 'A'); + check(r"\u{4__1__}", 'A'); + check(r"\u{1F63b}", '😻'); +} + +#[test] +fn test_unescape_str_warn() { + fn check(literal: &str, expected: &[(Range, Result)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_literal(literal, Mode::Str, &mut |range, res| unescaped.push((range, res))); + assert_eq!(unescaped, expected); + } + + // Check we can handle escaped newlines at the end of a file. + check("\\\n", &[]); + check("\\\n ", &[]); + + check( + "\\\n \u{a0} x", + &[ + (0..5, Err(EscapeError::UnskippedWhitespaceWarning)), + (3..5, Ok('\u{a0}')), + (5..6, Ok(' ')), + (6..7, Ok('x')), + ], + ); + check("\\\n \n x", &[(0..7, Err(EscapeError::MultipleSkippedLinesWarning)), (7..8, Ok('x'))]); +} + +#[test] +fn test_unescape_str_good() { + fn check(literal_text: &str, expected: &str) { + let mut buf = Ok(String::with_capacity(literal_text.len())); + unescape_literal(literal_text, Mode::Str, &mut |range, c| { + if let Ok(b) = &mut buf { + match c { + Ok(c) => b.push(c), + Err(e) => buf = Err((range, e)), + } + } + }); + let buf = buf.as_ref().map(|it| it.as_ref()); + assert_eq!(buf, Ok(expected)) + } + + check("foo", "foo"); + check("", ""); + check(" \t\n", " \t\n"); + + check("hello \\\n world", "hello world"); + check("thread's", "thread's") +} + +#[test] +fn test_unescape_byte_bad() { + fn check(literal_text: &str, expected_error: EscapeError) { + let actual_result = unescape_byte(literal_text).map_err(|(_offset, err)| err); + assert_eq!(actual_result, Err(expected_error)); + } + + check("", EscapeError::ZeroChars); + check(r"\", EscapeError::LoneSlash); + + check("\n", EscapeError::EscapeOnlyChar); + check("\t", EscapeError::EscapeOnlyChar); + check("'", EscapeError::EscapeOnlyChar); + check("\r", EscapeError::BareCarriageReturn); + + check("spam", EscapeError::MoreThanOneChar); + check(r"\x0ff", EscapeError::MoreThanOneChar); + check(r#"\"a"#, EscapeError::MoreThanOneChar); + check(r"\na", EscapeError::MoreThanOneChar); + check(r"\ra", EscapeError::MoreThanOneChar); + check(r"\ta", EscapeError::MoreThanOneChar); + check(r"\\a", EscapeError::MoreThanOneChar); + check(r"\'a", EscapeError::MoreThanOneChar); + check(r"\0a", EscapeError::MoreThanOneChar); + + check(r"\v", EscapeError::InvalidEscape); + check(r"\💩", EscapeError::InvalidEscape); + check(r"\●", EscapeError::InvalidEscape); + + check(r"\x", EscapeError::TooShortHexEscape); + check(r"\x0", EscapeError::TooShortHexEscape); + check(r"\xa", EscapeError::TooShortHexEscape); + check(r"\xf", EscapeError::TooShortHexEscape); + check(r"\xx", EscapeError::InvalidCharInHexEscape); + check(r"\xы", EscapeError::InvalidCharInHexEscape); + check(r"\x🦀", EscapeError::InvalidCharInHexEscape); + check(r"\xtt", EscapeError::InvalidCharInHexEscape); + + check(r"\u", EscapeError::NoBraceInUnicodeEscape); + check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape); + check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape); + check(r"\u{", EscapeError::UnclosedUnicodeEscape); + check(r"\u{0000", EscapeError::UnclosedUnicodeEscape); + check(r"\u{}", EscapeError::EmptyUnicodeEscape); + check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape); + check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape); + + check("ы", EscapeError::NonAsciiCharInByte); + check("🦀", EscapeError::NonAsciiCharInByte); + + check(r"\u{0}", EscapeError::UnicodeEscapeInByte); + check(r"\u{000000}", EscapeError::UnicodeEscapeInByte); + check(r"\u{41}", EscapeError::UnicodeEscapeInByte); + check(r"\u{0041}", EscapeError::UnicodeEscapeInByte); + check(r"\u{00_41}", EscapeError::UnicodeEscapeInByte); + check(r"\u{4__1__}", EscapeError::UnicodeEscapeInByte); + check(r"\u{1F63b}", EscapeError::UnicodeEscapeInByte); + check(r"\u{0}x", EscapeError::UnicodeEscapeInByte); + check(r"\u{1F63b}}", EscapeError::UnicodeEscapeInByte); + check(r"\u{FFFFFF}", EscapeError::UnicodeEscapeInByte); + check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte); + check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DC00}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DDDD}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DFFF}", EscapeError::UnicodeEscapeInByte); + check(r"\u{D800}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DAAA}", EscapeError::UnicodeEscapeInByte); + check(r"\u{DBFF}", EscapeError::UnicodeEscapeInByte); +} + +#[test] +fn test_unescape_byte_good() { + fn check(literal_text: &str, expected_byte: u8) { + let actual_result = unescape_byte(literal_text); + assert_eq!(actual_result, Ok(expected_byte)); + } + + check("a", b'a'); + + check(r#"\""#, b'"'); + check(r"\n", b'\n'); + check(r"\r", b'\r'); + check(r"\t", b'\t'); + check(r"\\", b'\\'); + check(r"\'", b'\''); + check(r"\0", b'\0'); + + check(r"\x00", b'\0'); + check(r"\x5a", b'Z'); + check(r"\x5A", b'Z'); + check(r"\x7f", 127); + check(r"\x80", 128); + check(r"\xff", 255); + check(r"\xFF", 255); +} + +#[test] +fn test_unescape_byte_str_good() { + fn check(literal_text: &str, expected: &[u8]) { + let mut buf = Ok(Vec::with_capacity(literal_text.len())); + unescape_byte_literal(literal_text, Mode::ByteStr, &mut |range, c| { + if let Ok(b) = &mut buf { + match c { + Ok(c) => b.push(c), + Err(e) => buf = Err((range, e)), + } + } + }); + let buf = buf.as_ref().map(|it| it.as_ref()); + assert_eq!(buf, Ok(expected)) + } + + check("foo", b"foo"); + check("", b""); + check(" \t\n", b" \t\n"); + + check("hello \\\n world", b"hello world"); + check("thread's", b"thread's") +} + +#[test] +fn test_unescape_raw_str() { + fn check(literal: &str, expected: &[(Range, Result)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_literal(literal, Mode::RawStr, &mut |range, res| unescaped.push((range, res))); + assert_eq!(unescaped, expected); + } + + check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]); + check("\rx", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString)), (1..2, Ok('x'))]); +} + +#[test] +fn test_unescape_raw_byte_str() { + fn check(literal: &str, expected: &[(Range, Result)]) { + let mut unescaped = Vec::with_capacity(literal.len()); + unescape_byte_literal(literal, Mode::RawByteStr, &mut |range, res| { + unescaped.push((range, res)) + }); + assert_eq!(unescaped, expected); + } + + check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]); + check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByteString))]); + check( + "🦀a", + &[(0..4, Err(EscapeError::NonAsciiCharInByteString)), (4..5, Ok(byte_from_char('a')))], + ); +} -- cgit v1.2.3