summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_lexer/src
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_lexer/src')
-rw-r--r--compiler/rustc_lexer/src/cursor.rs93
-rw-r--r--compiler/rustc_lexer/src/lib.rs843
-rw-r--r--compiler/rustc_lexer/src/tests.rs298
-rw-r--r--compiler/rustc_lexer/src/unescape.rs377
-rw-r--r--compiler/rustc_lexer/src/unescape/tests.rs297
5 files changed, 1908 insertions, 0 deletions
diff --git a/compiler/rustc_lexer/src/cursor.rs b/compiler/rustc_lexer/src/cursor.rs
new file mode 100644
index 000000000..21557a9c8
--- /dev/null
+++ b/compiler/rustc_lexer/src/cursor.rs
@@ -0,0 +1,93 @@
+use std::str::Chars;
+
+/// Peekable iterator over a char sequence.
+///
+/// Next characters can be peeked via `first` method,
+/// and position can be shifted forward via `bump` method.
+pub(crate) struct Cursor<'a> {
+ initial_len: usize,
+ /// Iterator over chars. Slightly faster than a &str.
+ chars: Chars<'a>,
+ #[cfg(debug_assertions)]
+ prev: char,
+}
+
+pub(crate) const EOF_CHAR: char = '\0';
+
+impl<'a> Cursor<'a> {
+ pub(crate) fn new(input: &'a str) -> Cursor<'a> {
+ Cursor {
+ initial_len: input.len(),
+ chars: input.chars(),
+ #[cfg(debug_assertions)]
+ prev: EOF_CHAR,
+ }
+ }
+
+ /// Returns the last eaten symbol (or `'\0'` in release builds).
+ /// (For debug assertions only.)
+ pub(crate) fn prev(&self) -> char {
+ #[cfg(debug_assertions)]
+ {
+ self.prev
+ }
+
+ #[cfg(not(debug_assertions))]
+ {
+ EOF_CHAR
+ }
+ }
+
+ /// Peeks the next symbol from the input stream without consuming it.
+ /// If requested position doesn't exist, `EOF_CHAR` is returned.
+ /// However, getting `EOF_CHAR` doesn't always mean actual end of file,
+ /// it should be checked with `is_eof` method.
+ pub(crate) fn first(&self) -> char {
+ // `.next()` optimizes better than `.nth(0)`
+ self.chars.clone().next().unwrap_or(EOF_CHAR)
+ }
+
+ /// Peeks the second symbol from the input stream without consuming it.
+ pub(crate) fn second(&self) -> char {
+ // `.next()` optimizes better than `.nth(1)`
+ let mut iter = self.chars.clone();
+ iter.next();
+ iter.next().unwrap_or(EOF_CHAR)
+ }
+
+ /// Checks if there is nothing more to consume.
+ pub(crate) fn is_eof(&self) -> bool {
+ self.chars.as_str().is_empty()
+ }
+
+ /// Returns amount of already consumed symbols.
+ pub(crate) fn len_consumed(&self) -> u32 {
+ (self.initial_len - self.chars.as_str().len()) as u32
+ }
+
+ /// Resets the number of bytes consumed to 0.
+ pub(crate) fn reset_len_consumed(&mut self) {
+ self.initial_len = self.chars.as_str().len();
+ }
+
+ /// Moves to the next character.
+ pub(crate) fn bump(&mut self) -> Option<char> {
+ let c = self.chars.next()?;
+
+ #[cfg(debug_assertions)]
+ {
+ self.prev = c;
+ }
+
+ Some(c)
+ }
+
+ /// Eats symbols while predicate returns true or until the end of file is reached.
+ pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
+ // It was tried making optimized version of this for eg. line comments, but
+ // LLVM can inline all of this and compile it down to fast iteration over bytes.
+ while predicate(self.first()) && !self.is_eof() {
+ self.bump();
+ }
+ }
+}
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
new file mode 100644
index 000000000..6d311af90
--- /dev/null
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -0,0 +1,843 @@
+//! Low-level Rust lexer.
+//!
+//! The idea with `rustc_lexer` is to make a reusable library,
+//! by separating out pure lexing and rustc-specific concerns, like spans,
+//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
+//! produces simple tokens which are a pair of type-tag and a bit of original text,
+//! and does not report errors, instead storing them as flags on the token.
+//!
+//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
+//! For that see [`rustc_parse::lexer`], which converts this basic token stream
+//! into wide tokens used by actual parser.
+//!
+//! The purpose of this crate is to convert raw sources into a labeled sequence
+//! of well-known token types, so building an actual Rust token stream will
+//! be easier.
+//!
+//! The main entity of this crate is the [`TokenKind`] enum which represents common
+//! lexeme types.
+//!
+//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
+// We want to be able to build this crate with a stable compiler, so no
+// `#![feature]` attributes should be added.
+
+mod cursor;
+pub mod unescape;
+
+#[cfg(test)]
+mod tests;
+
+use self::LiteralKind::*;
+use self::TokenKind::*;
+use crate::cursor::{Cursor, EOF_CHAR};
+use std::convert::TryFrom;
+
+/// Parsed token.
+/// It doesn't contain information about data that has been parsed,
+/// only the type of the token and its size.
+#[derive(Debug)]
+pub struct Token {
+ pub kind: TokenKind,
+ pub len: u32,
+}
+
+impl Token {
+ fn new(kind: TokenKind, len: u32) -> Token {
+ Token { kind, len }
+ }
+}
+
+/// Enum representing common lexeme types.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TokenKind {
+ // Multi-char tokens:
+ /// "// comment"
+ LineComment { doc_style: Option<DocStyle> },
+ /// `/* block comment */`
+ ///
+ /// Block comments can be recursive, so the sequence like `/* /* */`
+ /// will not be considered terminated and will result in a parsing error.
+ BlockComment { doc_style: Option<DocStyle>, terminated: bool },
+ /// Any whitespace characters sequence.
+ Whitespace,
+ /// "ident" or "continue"
+ /// At this step keywords are also considered identifiers.
+ Ident,
+ /// Like the above, but containing invalid unicode codepoints.
+ InvalidIdent,
+ /// "r#ident"
+ RawIdent,
+ /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
+ /// prefix (`foo`) is included in the token, not the separator (which is
+ /// lexed as its own distinct token). In Rust 2021 and later, reserved
+ /// prefixes are reported as errors; in earlier editions, they result in a
+ /// (allowed by default) lint, and are treated as regular identifier
+ /// tokens.
+ UnknownPrefix,
+ /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
+ Literal { kind: LiteralKind, suffix_start: u32 },
+ /// "'a"
+ Lifetime { starts_with_number: bool },
+
+ // One-char tokens:
+ /// ";"
+ Semi,
+ /// ","
+ Comma,
+ /// "."
+ Dot,
+ /// "("
+ OpenParen,
+ /// ")"
+ CloseParen,
+ /// "{"
+ OpenBrace,
+ /// "}"
+ CloseBrace,
+ /// "["
+ OpenBracket,
+ /// "]"
+ CloseBracket,
+ /// "@"
+ At,
+ /// "#"
+ Pound,
+ /// "~"
+ Tilde,
+ /// "?"
+ Question,
+ /// ":"
+ Colon,
+ /// "$"
+ Dollar,
+ /// "="
+ Eq,
+ /// "!"
+ Bang,
+ /// "<"
+ Lt,
+ /// ">"
+ Gt,
+ /// "-"
+ Minus,
+ /// "&"
+ And,
+ /// "|"
+ Or,
+ /// "+"
+ Plus,
+ /// "*"
+ Star,
+ /// "/"
+ Slash,
+ /// "^"
+ Caret,
+ /// "%"
+ Percent,
+
+ /// Unknown token, not expected by the lexer, e.g. "№"
+ Unknown,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum DocStyle {
+ Outer,
+ Inner,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum LiteralKind {
+ /// "12_u8", "0o100", "0b120i99"
+ Int { base: Base, empty_int: bool },
+ /// "12.34f32", "0b100.100"
+ Float { base: Base, empty_exponent: bool },
+ /// "'a'", "'\\'", "'''", "';"
+ Char { terminated: bool },
+ /// "b'a'", "b'\\'", "b'''", "b';"
+ Byte { terminated: bool },
+ /// ""abc"", ""abc"
+ Str { terminated: bool },
+ /// "b"abc"", "b"abc"
+ ByteStr { terminated: bool },
+ /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates
+ /// an invalid literal.
+ RawStr { n_hashes: Option<u8> },
+ /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None`
+ /// indicates an invalid literal.
+ RawByteStr { n_hashes: Option<u8> },
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RawStrError {
+ /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
+ InvalidStarter { bad_char: char },
+ /// The string was not terminated, e.g. `r###"abcde"##`.
+ /// `possible_terminator_offset` is the number of characters after `r` or
+ /// `br` where they may have intended to terminate it.
+ NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
+ /// More than 255 `#`s exist.
+ TooManyDelimiters { found: u32 },
+}
+
+/// Base of numeric literal encoding according to its prefix.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Base {
+ /// Literal starts with "0b".
+ Binary,
+ /// Literal starts with "0o".
+ Octal,
+ /// Literal starts with "0x".
+ Hexadecimal,
+ /// Literal doesn't contain a prefix.
+ Decimal,
+}
+
+/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
+/// but shebang isn't a part of rust syntax.
+pub fn strip_shebang(input: &str) -> Option<usize> {
+ // Shebang must start with `#!` literally, without any preceding whitespace.
+ // For simplicity we consider any line starting with `#!` a shebang,
+ // regardless of restrictions put on shebangs by specific platforms.
+ if let Some(input_tail) = input.strip_prefix("#!") {
+ // Ok, this is a shebang but if the next non-whitespace token is `[`,
+ // then it may be valid Rust code, so consider it Rust code.
+ let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
+ !matches!(
+ tok,
+ TokenKind::Whitespace
+ | TokenKind::LineComment { doc_style: None }
+ | TokenKind::BlockComment { doc_style: None, .. }
+ )
+ });
+ if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
+ // No other choice than to consider this a shebang.
+ return Some(2 + input_tail.lines().next().unwrap_or_default().len());
+ }
+ }
+ None
+}
+
+/// Parses the first token from the provided input string.
+#[inline]
+pub fn first_token(input: &str) -> Token {
+ debug_assert!(!input.is_empty());
+ Cursor::new(input).advance_token()
+}
+
+/// Validates a raw string literal. Used for getting more information about a
+/// problem with a `RawStr`/`RawByteStr` with a `None` field.
+#[inline]
+pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
+ debug_assert!(!input.is_empty());
+ let mut cursor = Cursor::new(input);
+ // Move past the leading `r` or `br`.
+ for _ in 0..prefix_len {
+ cursor.bump().unwrap();
+ }
+ cursor.raw_double_quoted_string(prefix_len).map(|_| ())
+}
+
+/// Creates an iterator that produces tokens from the input string.
+pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
+ let mut cursor = Cursor::new(input);
+ std::iter::from_fn(move || {
+ if cursor.is_eof() {
+ None
+ } else {
+ cursor.reset_len_consumed();
+ Some(cursor.advance_token())
+ }
+ })
+}
+
+/// True if `c` is considered a whitespace according to Rust language definition.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
+/// for definitions of these classes.
+pub fn is_whitespace(c: char) -> bool {
+ // This is Pattern_White_Space.
+ //
+ // Note that this set is stable (ie, it doesn't change with different
+ // Unicode versions), so it's ok to just hard-code the values.
+
+ matches!(
+ c,
+ // Usual ASCII suspects
+ '\u{0009}' // \t
+ | '\u{000A}' // \n
+ | '\u{000B}' // vertical tab
+ | '\u{000C}' // form feed
+ | '\u{000D}' // \r
+ | '\u{0020}' // space
+
+ // NEXT LINE from latin1
+ | '\u{0085}'
+
+ // Bidi markers
+ | '\u{200E}' // LEFT-TO-RIGHT MARK
+ | '\u{200F}' // RIGHT-TO-LEFT MARK
+
+ // Dedicated whitespace characters from Unicode
+ | '\u{2028}' // LINE SEPARATOR
+ | '\u{2029}' // PARAGRAPH SEPARATOR
+ )
+}
+
+/// True if `c` is valid as a first character of an identifier.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+/// a formal definition of valid identifier name.
+pub fn is_id_start(c: char) -> bool {
+ // This is XID_Start OR '_' (which formally is not a XID_Start).
+ c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
+}
+
+/// True if `c` is valid as a non-first character of an identifier.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+/// a formal definition of valid identifier name.
+pub fn is_id_continue(c: char) -> bool {
+ unicode_xid::UnicodeXID::is_xid_continue(c)
+}
+
+/// The passed string is lexically an identifier.
+pub fn is_ident(string: &str) -> bool {
+ let mut chars = string.chars();
+ if let Some(start) = chars.next() {
+ is_id_start(start) && chars.all(is_id_continue)
+ } else {
+ false
+ }
+}
+
+impl Cursor<'_> {
+ /// Parses a token from the input string.
+ fn advance_token(&mut self) -> Token {
+ let first_char = self.bump().unwrap();
+ let token_kind = match first_char {
+ // Slash, comment or block comment.
+ '/' => match self.first() {
+ '/' => self.line_comment(),
+ '*' => self.block_comment(),
+ _ => Slash,
+ },
+
+ // Whitespace sequence.
+ c if is_whitespace(c) => self.whitespace(),
+
+ // Raw identifier, raw string literal or identifier.
+ 'r' => match (self.first(), self.second()) {
+ ('#', c1) if is_id_start(c1) => self.raw_ident(),
+ ('#', _) | ('"', _) => {
+ let res = self.raw_double_quoted_string(1);
+ let suffix_start = self.len_consumed();
+ if res.is_ok() {
+ self.eat_literal_suffix();
+ }
+ let kind = RawStr { n_hashes: res.ok() };
+ Literal { kind, suffix_start }
+ }
+ _ => self.ident_or_unknown_prefix(),
+ },
+
+ // Byte literal, byte string literal, raw byte string literal or identifier.
+ 'b' => match (self.first(), self.second()) {
+ ('\'', _) => {
+ self.bump();
+ let terminated = self.single_quoted_string();
+ let suffix_start = self.len_consumed();
+ if terminated {
+ self.eat_literal_suffix();
+ }
+ let kind = Byte { terminated };
+ Literal { kind, suffix_start }
+ }
+ ('"', _) => {
+ self.bump();
+ let terminated = self.double_quoted_string();
+ let suffix_start = self.len_consumed();
+ if terminated {
+ self.eat_literal_suffix();
+ }
+ let kind = ByteStr { terminated };
+ Literal { kind, suffix_start }
+ }
+ ('r', '"') | ('r', '#') => {
+ self.bump();
+ let res = self.raw_double_quoted_string(2);
+ let suffix_start = self.len_consumed();
+ if res.is_ok() {
+ self.eat_literal_suffix();
+ }
+ let kind = RawByteStr { n_hashes: res.ok() };
+ Literal { kind, suffix_start }
+ }
+ _ => self.ident_or_unknown_prefix(),
+ },
+
+ // Identifier (this should be checked after other variant that can
+ // start as identifier).
+ c if is_id_start(c) => self.ident_or_unknown_prefix(),
+
+ // Numeric literal.
+ c @ '0'..='9' => {
+ let literal_kind = self.number(c);
+ let suffix_start = self.len_consumed();
+ self.eat_literal_suffix();
+ TokenKind::Literal { kind: literal_kind, suffix_start }
+ }
+
+ // One-symbol tokens.
+ ';' => Semi,
+ ',' => Comma,
+ '.' => Dot,
+ '(' => OpenParen,
+ ')' => CloseParen,
+ '{' => OpenBrace,
+ '}' => CloseBrace,
+ '[' => OpenBracket,
+ ']' => CloseBracket,
+ '@' => At,
+ '#' => Pound,
+ '~' => Tilde,
+ '?' => Question,
+ ':' => Colon,
+ '$' => Dollar,
+ '=' => Eq,
+ '!' => Bang,
+ '<' => Lt,
+ '>' => Gt,
+ '-' => Minus,
+ '&' => And,
+ '|' => Or,
+ '+' => Plus,
+ '*' => Star,
+ '^' => Caret,
+ '%' => Percent,
+
+ // Lifetime or character literal.
+ '\'' => self.lifetime_or_char(),
+
+ // String literal.
+ '"' => {
+ let terminated = self.double_quoted_string();
+ let suffix_start = self.len_consumed();
+ if terminated {
+ self.eat_literal_suffix();
+ }
+ let kind = Str { terminated };
+ Literal { kind, suffix_start }
+ }
+ // Identifier starting with an emoji. Only lexed for graceful error recovery.
+ c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
+ self.fake_ident_or_unknown_prefix()
+ }
+ _ => Unknown,
+ };
+ Token::new(token_kind, self.len_consumed())
+ }
+
+ fn line_comment(&mut self) -> TokenKind {
+ debug_assert!(self.prev() == '/' && self.first() == '/');
+ self.bump();
+
+ let doc_style = match self.first() {
+ // `//!` is an inner line doc comment.
+ '!' => Some(DocStyle::Inner),
+ // `////` (more than 3 slashes) is not considered a doc comment.
+ '/' if self.second() != '/' => Some(DocStyle::Outer),
+ _ => None,
+ };
+
+ self.eat_while(|c| c != '\n');
+ LineComment { doc_style }
+ }
+
+ fn block_comment(&mut self) -> TokenKind {
+ debug_assert!(self.prev() == '/' && self.first() == '*');
+ self.bump();
+
+ let doc_style = match self.first() {
+ // `/*!` is an inner block doc comment.
+ '!' => Some(DocStyle::Inner),
+ // `/***` (more than 2 stars) is not considered a doc comment.
+ // `/**/` is not considered a doc comment.
+ '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
+ _ => None,
+ };
+
+ let mut depth = 1usize;
+ while let Some(c) = self.bump() {
+ match c {
+ '/' if self.first() == '*' => {
+ self.bump();
+ depth += 1;
+ }
+ '*' if self.first() == '/' => {
+ self.bump();
+ depth -= 1;
+ if depth == 0 {
+ // This block comment is closed, so for a construction like "/* */ */"
+ // there will be a successfully parsed block comment "/* */"
+ // and " */" will be processed separately.
+ break;
+ }
+ }
+ _ => (),
+ }
+ }
+
+ BlockComment { doc_style, terminated: depth == 0 }
+ }
+
+ fn whitespace(&mut self) -> TokenKind {
+ debug_assert!(is_whitespace(self.prev()));
+ self.eat_while(is_whitespace);
+ Whitespace
+ }
+
+ fn raw_ident(&mut self) -> TokenKind {
+ debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
+ // Eat "#" symbol.
+ self.bump();
+ // Eat the identifier part of RawIdent.
+ self.eat_identifier();
+ RawIdent
+ }
+
+ fn ident_or_unknown_prefix(&mut self) -> TokenKind {
+ debug_assert!(is_id_start(self.prev()));
+ // Start is already eaten, eat the rest of identifier.
+ self.eat_while(is_id_continue);
+ // Known prefixes must have been handled earlier. So if
+ // we see a prefix here, it is definitely an unknown prefix.
+ match self.first() {
+ '#' | '"' | '\'' => UnknownPrefix,
+ c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
+ self.fake_ident_or_unknown_prefix()
+ }
+ _ => Ident,
+ }
+ }
+
+ fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
+ // Start is already eaten, eat the rest of identifier.
+ self.eat_while(|c| {
+ unicode_xid::UnicodeXID::is_xid_continue(c)
+ || (!c.is_ascii() && unic_emoji_char::is_emoji(c))
+ || c == '\u{200d}'
+ });
+ // Known prefixes must have been handled earlier. So if
+ // we see a prefix here, it is definitely an unknown prefix.
+ match self.first() {
+ '#' | '"' | '\'' => UnknownPrefix,
+ _ => InvalidIdent,
+ }
+ }
+
+ fn number(&mut self, first_digit: char) -> LiteralKind {
+ debug_assert!('0' <= self.prev() && self.prev() <= '9');
+ let mut base = Base::Decimal;
+ if first_digit == '0' {
+ // Attempt to parse encoding base.
+ let has_digits = match self.first() {
+ 'b' => {
+ base = Base::Binary;
+ self.bump();
+ self.eat_decimal_digits()
+ }
+ 'o' => {
+ base = Base::Octal;
+ self.bump();
+ self.eat_decimal_digits()
+ }
+ 'x' => {
+ base = Base::Hexadecimal;
+ self.bump();
+ self.eat_hexadecimal_digits()
+ }
+ // Not a base prefix.
+ '0'..='9' | '_' | '.' | 'e' | 'E' => {
+ self.eat_decimal_digits();
+ true
+ }
+ // Just a 0.
+ _ => return Int { base, empty_int: false },
+ };
+ // Base prefix was provided, but there were no digits
+ // after it, e.g. "0x".
+ if !has_digits {
+ return Int { base, empty_int: true };
+ }
+ } else {
+ // No base prefix, parse number in the usual way.
+ self.eat_decimal_digits();
+ };
+
+ match self.first() {
+ // Don't be greedy if this is actually an
+ // integer literal followed by field/method access or a range pattern
+ // (`0..2` and `12.foo()`)
+ '.' if self.second() != '.' && !is_id_start(self.second()) => {
+ // might have stuff after the ., and if it does, it needs to start
+ // with a number
+ self.bump();
+ let mut empty_exponent = false;
+ if self.first().is_digit(10) {
+ self.eat_decimal_digits();
+ match self.first() {
+ 'e' | 'E' => {
+ self.bump();
+ empty_exponent = !self.eat_float_exponent();
+ }
+ _ => (),
+ }
+ }
+ Float { base, empty_exponent }
+ }
+ 'e' | 'E' => {
+ self.bump();
+ let empty_exponent = !self.eat_float_exponent();
+ Float { base, empty_exponent }
+ }
+ _ => Int { base, empty_int: false },
+ }
+ }
+
+ fn lifetime_or_char(&mut self) -> TokenKind {
+ debug_assert!(self.prev() == '\'');
+
+ let can_be_a_lifetime = if self.second() == '\'' {
+ // It's surely not a lifetime.
+ false
+ } else {
+ // If the first symbol is valid for identifier, it can be a lifetime.
+ // Also check if it's a number for a better error reporting (so '0 will
+ // be reported as invalid lifetime and not as unterminated char literal).
+ is_id_start(self.first()) || self.first().is_digit(10)
+ };
+
+ if !can_be_a_lifetime {
+ let terminated = self.single_quoted_string();
+ let suffix_start = self.len_consumed();
+ if terminated {
+ self.eat_literal_suffix();
+ }
+ let kind = Char { terminated };
+ return Literal { kind, suffix_start };
+ }
+
+ // Either a lifetime or a character literal with
+ // length greater than 1.
+
+ let starts_with_number = self.first().is_digit(10);
+
+ // Skip the literal contents.
+ // First symbol can be a number (which isn't a valid identifier start),
+ // so skip it without any checks.
+ self.bump();
+ self.eat_while(is_id_continue);
+
+ // Check if after skipping literal contents we've met a closing
+ // single quote (which means that user attempted to create a
+ // string with single quotes).
+ if self.first() == '\'' {
+ self.bump();
+ let kind = Char { terminated: true };
+ Literal { kind, suffix_start: self.len_consumed() }
+ } else {
+ Lifetime { starts_with_number }
+ }
+ }
+
+ fn single_quoted_string(&mut self) -> bool {
+ debug_assert!(self.prev() == '\'');
+ // Check if it's a one-symbol literal.
+ if self.second() == '\'' && self.first() != '\\' {
+ self.bump();
+ self.bump();
+ return true;
+ }
+
+ // Literal has more than one symbol.
+
+ // Parse until either quotes are terminated or error is detected.
+ loop {
+ match self.first() {
+ // Quotes are terminated, finish parsing.
+ '\'' => {
+ self.bump();
+ return true;
+ }
+ // Probably beginning of the comment, which we don't want to include
+ // to the error report.
+ '/' => break,
+ // Newline without following '\'' means unclosed quote, stop parsing.
+ '\n' if self.second() != '\'' => break,
+ // End of file, stop parsing.
+ EOF_CHAR if self.is_eof() => break,
+ // Escaped slash is considered one character, so bump twice.
+ '\\' => {
+ self.bump();
+ self.bump();
+ }
+ // Skip the character.
+ _ => {
+ self.bump();
+ }
+ }
+ }
+ // String was not terminated.
+ false
+ }
+
+ /// Eats double-quoted string and returns true
+ /// if string is terminated.
+ fn double_quoted_string(&mut self) -> bool {
+ debug_assert!(self.prev() == '"');
+ while let Some(c) = self.bump() {
+ match c {
+ '"' => {
+ return true;
+ }
+ '\\' if self.first() == '\\' || self.first() == '"' => {
+ // Bump again to skip escaped character.
+ self.bump();
+ }
+ _ => (),
+ }
+ }
+ // End of file reached.
+ false
+ }
+
+ /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
+ fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
+ // Wrap the actual function to handle the error with too many hashes.
+ // This way, it eats the whole raw string.
+ let n_hashes = self.raw_string_unvalidated(prefix_len)?;
+ // Only up to 255 `#`s are allowed in raw strings
+ match u8::try_from(n_hashes) {
+ Ok(num) => Ok(num),
+ Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
+ }
+ }
+
+ fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
+ debug_assert!(self.prev() == 'r');
+ let start_pos = self.len_consumed();
+ let mut possible_terminator_offset = None;
+ let mut max_hashes = 0;
+
+ // Count opening '#' symbols.
+ let mut eaten = 0;
+ while self.first() == '#' {
+ eaten += 1;
+ self.bump();
+ }
+ let n_start_hashes = eaten;
+
+ // Check that string is started.
+ match self.bump() {
+ Some('"') => (),
+ c => {
+ let c = c.unwrap_or(EOF_CHAR);
+ return Err(RawStrError::InvalidStarter { bad_char: c });
+ }
+ }
+
+ // Skip the string contents and on each '#' character met, check if this is
+ // a raw string termination.
+ loop {
+ self.eat_while(|c| c != '"');
+
+ if self.is_eof() {
+ return Err(RawStrError::NoTerminator {
+ expected: n_start_hashes,
+ found: max_hashes,
+ possible_terminator_offset,
+ });
+ }
+
+ // Eat closing double quote.
+ self.bump();
+
+ // Check that amount of closing '#' symbols
+ // is equal to the amount of opening ones.
+ // Note that this will not consume extra trailing `#` characters:
+ // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
+ // followed by a `#` token.
+ let mut n_end_hashes = 0;
+ while self.first() == '#' && n_end_hashes < n_start_hashes {
+ n_end_hashes += 1;
+ self.bump();
+ }
+
+ if n_end_hashes == n_start_hashes {
+ return Ok(n_start_hashes);
+ } else if n_end_hashes > max_hashes {
+ // Keep track of possible terminators to give a hint about
+ // where there might be a missing terminator
+ possible_terminator_offset =
+ Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
+ max_hashes = n_end_hashes;
+ }
+ }
+ }
+
+ fn eat_decimal_digits(&mut self) -> bool {
+ let mut has_digits = false;
+ loop {
+ match self.first() {
+ '_' => {
+ self.bump();
+ }
+ '0'..='9' => {
+ has_digits = true;
+ self.bump();
+ }
+ _ => break,
+ }
+ }
+ has_digits
+ }
+
+ fn eat_hexadecimal_digits(&mut self) -> bool {
+ let mut has_digits = false;
+ loop {
+ match self.first() {
+ '_' => {
+ self.bump();
+ }
+ '0'..='9' | 'a'..='f' | 'A'..='F' => {
+ has_digits = true;
+ self.bump();
+ }
+ _ => break,
+ }
+ }
+ has_digits
+ }
+
+ /// Eats the float exponent. Returns true if at least one digit was met,
+ /// and returns false otherwise.
+ fn eat_float_exponent(&mut self) -> bool {
+ debug_assert!(self.prev() == 'e' || self.prev() == 'E');
+ if self.first() == '-' || self.first() == '+' {
+ self.bump();
+ }
+ self.eat_decimal_digits()
+ }
+
+ // Eats the suffix of the literal, e.g. "_u8".
+ fn eat_literal_suffix(&mut self) {
+ self.eat_identifier();
+ }
+
+ // Eats the identifier.
+ fn eat_identifier(&mut self) {
+ if !is_id_start(self.first()) {
+ return;
+ }
+ self.bump();
+
+ self.eat_while(is_id_continue);
+ }
+}
diff --git a/compiler/rustc_lexer/src/tests.rs b/compiler/rustc_lexer/src/tests.rs
new file mode 100644
index 000000000..e4c1787f2
--- /dev/null
+++ b/compiler/rustc_lexer/src/tests.rs
@@ -0,0 +1,298 @@
+use super::*;
+
+use expect_test::{expect, Expect};
+
+fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) {
+ let s = &format!("r{}", s);
+ let mut cursor = Cursor::new(s);
+ cursor.bump();
+ let res = cursor.raw_double_quoted_string(0);
+ assert_eq!(res, expected);
+}
+
+#[test]
+fn test_naked_raw_str() {
+ check_raw_str(r#""abc""#, Ok(0));
+}
+
+#[test]
+fn test_raw_no_start() {
+ check_raw_str(r##""abc"#"##, Ok(0));
+}
+
+#[test]
+fn test_too_many_terminators() {
+ // this error is handled in the parser later
+ check_raw_str(r###"#"abc"##"###, Ok(1));
+}
+
+#[test]
+fn test_unterminated() {
+ check_raw_str(
+ r#"#"abc"#,
+ Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }),
+ );
+ check_raw_str(
+ r###"##"abc"#"###,
+ Err(RawStrError::NoTerminator {
+ expected: 2,
+ found: 1,
+ possible_terminator_offset: Some(7),
+ }),
+ );
+ // We're looking for "# not just any #
+ check_raw_str(
+ r###"##"abc#"###,
+ Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }),
+ )
+}
+
+#[test]
+fn test_invalid_start() {
+ check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' }));
+}
+
+#[test]
+fn test_unterminated_no_pound() {
+ // https://github.com/rust-lang/rust/issues/70677
+ check_raw_str(
+ r#"""#,
+ Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }),
+ );
+}
+
+#[test]
+fn test_too_many_hashes() {
+ let max_count = u8::MAX;
+ let hashes1 = "#".repeat(max_count as usize);
+ let hashes2 = "#".repeat(max_count as usize + 1);
+ let middle = "\"abc\"";
+ let s1 = [&hashes1, middle, &hashes1].join("");
+ let s2 = [&hashes2, middle, &hashes2].join("");
+
+ // Valid number of hashes (255 = 2^8 - 1 = u8::MAX).
+ check_raw_str(&s1, Ok(255));
+
+ // One more hash sign (256 = 2^8) becomes too many.
+ check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 }));
+}
+
+#[test]
+fn test_valid_shebang() {
+ // https://github.com/rust-lang/rust/issues/70528
+ let input = "#!/usr/bin/rustrun\nlet x = 5;";
+ assert_eq!(strip_shebang(input), Some(18));
+}
+
+#[test]
+fn test_invalid_shebang_valid_rust_syntax() {
+ // https://github.com/rust-lang/rust/issues/70528
+ let input = "#! [bad_attribute]";
+ assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_shebang_second_line() {
+ // Because shebangs are interpreted by the kernel, they must be on the first line
+ let input = "\n#!/bin/bash";
+ assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_shebang_space() {
+ let input = "#! /bin/bash";
+ assert_eq!(strip_shebang(input), Some(input.len()));
+}
+
+#[test]
+fn test_shebang_empty_shebang() {
+ let input = "#! \n[attribute(foo)]";
+ assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_invalid_shebang_comment() {
+ let input = "#!//bin/ami/a/comment\n[";
+ assert_eq!(strip_shebang(input), None)
+}
+
+#[test]
+fn test_invalid_shebang_another_comment() {
+ let input = "#!/*bin/ami/a/comment*/\n[attribute";
+ assert_eq!(strip_shebang(input), None)
+}
+
+#[test]
+fn test_shebang_valid_rust_after() {
+ let input = "#!/*bin/ami/a/comment*/\npub fn main() {}";
+ assert_eq!(strip_shebang(input), Some(23))
+}
+
+#[test]
+fn test_shebang_followed_by_attrib() {
+ let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
+ assert_eq!(strip_shebang(input), Some(19));
+}
+
+fn check_lexing(src: &str, expect: Expect) {
+ let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
+ expect.assert_eq(&actual)
+}
+
+#[test]
+fn smoke_test() {
+ check_lexing(
+ "/* my source file */ fn main() { println!(\"zebra\"); }\n",
+ expect![[r#"
+ Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Ident, len: 2 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Ident, len: 4 }
+ Token { kind: OpenParen, len: 1 }
+ Token { kind: CloseParen, len: 1 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: OpenBrace, len: 1 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Ident, len: 7 }
+ Token { kind: Bang, len: 1 }
+ Token { kind: OpenParen, len: 1 }
+ Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 }
+ Token { kind: CloseParen, len: 1 }
+ Token { kind: Semi, len: 1 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: CloseBrace, len: 1 }
+ Token { kind: Whitespace, len: 1 }
+ "#]],
+ )
+}
+
+#[test]
+fn comment_flavors() {
+ check_lexing(
+ r"
+// line
+//// line as well
+/// outer doc line
+//! inner doc line
+/* block */
+/**/
+/*** also block */
+/** outer doc block */
+/*! inner doc block */
+",
+ expect![[r#"
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: LineComment { doc_style: None }, len: 7 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: LineComment { doc_style: None }, len: 17 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
+ Token { kind: Whitespace, len: 1 }
+ "#]],
+ )
+}
+
+#[test]
+fn nested_block_comments() {
+ check_lexing(
+ "/* /* */ */'a'",
+ expect![[r#"
+ Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
+ Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
+ "#]],
+ )
+}
+
+#[test]
+fn characters() {
+ check_lexing(
+ "'a' ' ' '\\n'",
+ expect![[r#"
+ Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
+ "#]],
+ );
+}
+
+#[test]
+fn lifetime() {
+ check_lexing(
+ "'abc",
+ expect![[r#"
+ Token { kind: Lifetime { starts_with_number: false }, len: 4 }
+ "#]],
+ );
+}
+
+#[test]
+fn raw_string() {
+ check_lexing(
+ "r###\"\"#a\\b\x00c\"\"###",
+ expect![[r#"
+ Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 }
+ "#]],
+ )
+}
+
+#[test]
+fn literal_suffixes() {
+ check_lexing(
+ r####"
+'a'
+b'a'
+"a"
+b"a"
+1234
+0b101
+0xABC
+1.0
+1.0e10
+2us
+r###"raw"###suffix
+br###"raw"###suffix
+"####,
+ expect![[r#"
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 }
+ Token { kind: Whitespace, len: 1 }
+ Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 }
+ Token { kind: Whitespace, len: 1 }
+ "#]],
+ )
+}
diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
new file mode 100644
index 000000000..3da6bc146
--- /dev/null
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -0,0 +1,377 @@
+//! Utilities for validating string and char literals and turning them into
+//! values they represent.
+
+use std::ops::Range;
+use std::str::Chars;
+
+#[cfg(test)]
+mod tests;
+
+/// Errors and warnings that can occur during string unescaping.
+#[derive(Debug, PartialEq, Eq)]
+pub enum EscapeError {
+ /// Expected 1 char, but 0 were found.
+ ZeroChars,
+ /// Expected 1 char, but more than 1 were found.
+ MoreThanOneChar,
+
+ /// Escaped '\' character without continuation.
+ LoneSlash,
+ /// Invalid escape character (e.g. '\z').
+ InvalidEscape,
+ /// Raw '\r' encountered.
+ BareCarriageReturn,
+ /// Raw '\r' encountered in raw string.
+ BareCarriageReturnInRawString,
+ /// Unescaped character that was expected to be escaped (e.g. raw '\t').
+ EscapeOnlyChar,
+
+ /// Numeric character escape is too short (e.g. '\x1').
+ TooShortHexEscape,
+ /// Invalid character in numeric escape (e.g. '\xz')
+ InvalidCharInHexEscape,
+ /// Character code in numeric escape is non-ascii (e.g. '\xFF').
+ OutOfRangeHexEscape,
+
+ /// '\u' not followed by '{'.
+ NoBraceInUnicodeEscape,
+ /// Non-hexadecimal value in '\u{..}'.
+ InvalidCharInUnicodeEscape,
+ /// '\u{}'
+ EmptyUnicodeEscape,
+ /// No closing brace in '\u{..}', e.g. '\u{12'.
+ UnclosedUnicodeEscape,
+ /// '\u{_12}'
+ LeadingUnderscoreUnicodeEscape,
+ /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}'
+ OverlongUnicodeEscape,
+ /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'.
+ LoneSurrogateUnicodeEscape,
+ /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'.
+ OutOfRangeUnicodeEscape,
+
+ /// Unicode escape code in byte literal.
+ UnicodeEscapeInByte,
+ /// Non-ascii character in byte literal.
+ NonAsciiCharInByte,
+ /// Non-ascii character in byte string literal.
+ NonAsciiCharInByteString,
+
+ /// After a line ending with '\', the next line contains whitespace
+ /// characters that are not skipped.
+ UnskippedWhitespaceWarning,
+
+ /// After a line ending with '\', multiple lines are skipped.
+ MultipleSkippedLinesWarning,
+}
+
+impl EscapeError {
+ /// Returns true for actual errors, as opposed to warnings.
+ pub fn is_fatal(&self) -> bool {
+ !matches!(
+ self,
+ EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning
+ )
+ }
+}
+
+/// Takes a contents of a literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ match mode {
+ Mode::Char | Mode::Byte => {
+ let mut chars = literal_text.chars();
+ let result = unescape_char_or_byte(&mut chars, mode);
+ // The Chars iterator moved forward.
+ callback(0..(literal_text.len() - chars.as_str().len()), result);
+ }
+ Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(literal_text, mode, callback),
+ // NOTE: Raw strings do not perform any explicit character escaping, here we
+ // only translate CRLF to LF and produce errors on bare CR.
+ Mode::RawStr | Mode::RawByteStr => {
+ unescape_raw_str_or_byte_str(literal_text, mode, callback)
+ }
+ }
+}
+
+/// Takes a contents of a byte, byte string or raw byte string (without quotes)
+/// and produces a sequence of bytes or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_byte_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<u8, EscapeError>),
+{
+ assert!(mode.is_bytes());
+ unescape_literal(literal_text, mode, &mut |range, result| {
+ callback(range, result.map(byte_from_char));
+ })
+}
+
+/// Takes a contents of a char literal (without quotes), and returns an
+/// unescaped char or an error
+pub fn unescape_char(literal_text: &str) -> Result<char, (usize, EscapeError)> {
+ let mut chars = literal_text.chars();
+ unescape_char_or_byte(&mut chars, Mode::Char)
+ .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// Takes a contents of a byte literal (without quotes), and returns an
+/// unescaped byte or an error.
+pub fn unescape_byte(literal_text: &str) -> Result<u8, (usize, EscapeError)> {
+ let mut chars = literal_text.chars();
+ unescape_char_or_byte(&mut chars, Mode::Byte)
+ .map(byte_from_char)
+ .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// What kind of literal do we parse.
+#[derive(Debug, Clone, Copy)]
+pub enum Mode {
+ Char,
+ Str,
+ Byte,
+ ByteStr,
+ RawStr,
+ RawByteStr,
+}
+
+impl Mode {
+ pub fn in_single_quotes(self) -> bool {
+ match self {
+ Mode::Char | Mode::Byte => true,
+ Mode::Str | Mode::ByteStr | Mode::RawStr | Mode::RawByteStr => false,
+ }
+ }
+
+ pub fn in_double_quotes(self) -> bool {
+ !self.in_single_quotes()
+ }
+
+ pub fn is_bytes(self) -> bool {
+ match self {
+ Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
+ Mode::Char | Mode::Str | Mode::RawStr => false,
+ }
+ }
+}
+
+fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+ // Previous character was '\\', unescape what follows.
+
+ let second_char = chars.next().ok_or(EscapeError::LoneSlash)?;
+
+ let res = match second_char {
+ '"' => '"',
+ 'n' => '\n',
+ 'r' => '\r',
+ 't' => '\t',
+ '\\' => '\\',
+ '\'' => '\'',
+ '0' => '\0',
+
+ 'x' => {
+ // Parse hexadecimal character code.
+
+ let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+ let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+ let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+ let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+ let value = hi * 16 + lo;
+
+ // For a byte literal verify that it is within ASCII range.
+ if !mode.is_bytes() && !is_ascii(value) {
+ return Err(EscapeError::OutOfRangeHexEscape);
+ }
+ let value = value as u8;
+
+ value as char
+ }
+
+ 'u' => {
+ // We've parsed '\u', now we have to parse '{..}'.
+
+ if chars.next() != Some('{') {
+ return Err(EscapeError::NoBraceInUnicodeEscape);
+ }
+
+ // First character must be a hexadecimal digit.
+ let mut n_digits = 1;
+ let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
+ '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
+ '}' => return Err(EscapeError::EmptyUnicodeEscape),
+ c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
+ };
+
+ // First character is valid, now parse the rest of the number
+ // and closing brace.
+ loop {
+ match chars.next() {
+ None => return Err(EscapeError::UnclosedUnicodeEscape),
+ Some('_') => continue,
+ Some('}') => {
+ if n_digits > 6 {
+ return Err(EscapeError::OverlongUnicodeEscape);
+ }
+
+ // Incorrect syntax has higher priority for error reporting
+ // than unallowed value for a literal.
+ if mode.is_bytes() {
+ return Err(EscapeError::UnicodeEscapeInByte);
+ }
+
+ break std::char::from_u32(value).ok_or_else(|| {
+ if value > 0x10FFFF {
+ EscapeError::OutOfRangeUnicodeEscape
+ } else {
+ EscapeError::LoneSurrogateUnicodeEscape
+ }
+ })?;
+ }
+ Some(c) => {
+ let digit =
+ c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
+ n_digits += 1;
+ if n_digits > 6 {
+ // Stop updating value since we're sure that it's incorrect already.
+ continue;
+ }
+ let digit = digit as u32;
+ value = value * 16 + digit;
+ }
+ };
+ }
+ }
+ _ => return Err(EscapeError::InvalidEscape),
+ };
+ Ok(res)
+}
+
+#[inline]
+fn ascii_check(first_char: char, mode: Mode) -> Result<char, EscapeError> {
+ if mode.is_bytes() && !first_char.is_ascii() {
+ // Byte literal can't be a non-ascii character.
+ Err(EscapeError::NonAsciiCharInByte)
+ } else {
+ Ok(first_char)
+ }
+}
+
+fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+ let first_char = chars.next().ok_or(EscapeError::ZeroChars)?;
+ let res = match first_char {
+ '\\' => scan_escape(chars, mode),
+ '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
+ '\r' => Err(EscapeError::BareCarriageReturn),
+ _ => ascii_check(first_char, mode),
+ }?;
+ if chars.next().is_some() {
+ return Err(EscapeError::MoreThanOneChar);
+ }
+ Ok(res)
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ assert!(mode.in_double_quotes());
+ let initial_len = src.len();
+ let mut chars = src.chars();
+ while let Some(first_char) = chars.next() {
+ let start = initial_len - chars.as_str().len() - first_char.len_utf8();
+
+ let unescaped_char = match first_char {
+ '\\' => {
+ let second_char = chars.clone().next();
+ match second_char {
+ Some('\n') => {
+ // Rust language specification requires us to skip whitespaces
+ // if unescaped '\' character is followed by '\n'.
+ // For details see [Rust language reference]
+ // (https://doc.rust-lang.org/reference/tokens.html#string-literals).
+ skip_ascii_whitespace(&mut chars, start, callback);
+ continue;
+ }
+ _ => scan_escape(&mut chars, mode),
+ }
+ }
+ '\n' => Ok('\n'),
+ '\t' => Ok('\t'),
+ '"' => Err(EscapeError::EscapeOnlyChar),
+ '\r' => Err(EscapeError::BareCarriageReturn),
+ _ => ascii_check(first_char, mode),
+ };
+ let end = initial_len - chars.as_str().len();
+ callback(start..end, unescaped_char);
+ }
+
+ fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
+ where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+ {
+ let tail = chars.as_str();
+ let first_non_space = tail
+ .bytes()
+ .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
+ .unwrap_or(tail.len());
+ if tail[1..first_non_space].contains('\n') {
+ // The +1 accounts for the escaping slash.
+ let end = start + first_non_space + 1;
+ callback(start..end, Err(EscapeError::MultipleSkippedLinesWarning));
+ }
+ let tail = &tail[first_non_space..];
+ if let Some(c) = tail.chars().nth(0) {
+ // For error reporting, we would like the span to contain the character that was not
+ // skipped. The +1 is necessary to account for the leading \ that started the escape.
+ let end = start + first_non_space + c.len_utf8() + 1;
+ if c.is_whitespace() {
+ callback(start..end, Err(EscapeError::UnskippedWhitespaceWarning));
+ }
+ }
+ *chars = tail.chars();
+ }
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of characters or errors.
+/// NOTE: Raw strings do not perform any explicit character escaping, here we
+/// only translate CRLF to LF and produce errors on bare CR.
+fn unescape_raw_str_or_byte_str<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ assert!(mode.in_double_quotes());
+ let initial_len = literal_text.len();
+
+ let mut chars = literal_text.chars();
+ while let Some(curr) = chars.next() {
+ let start = initial_len - chars.as_str().len() - curr.len_utf8();
+
+ let result = match curr {
+ '\r' => Err(EscapeError::BareCarriageReturnInRawString),
+ c if mode.is_bytes() && !c.is_ascii() => Err(EscapeError::NonAsciiCharInByteString),
+ c => Ok(c),
+ };
+ let end = initial_len - chars.as_str().len();
+
+ callback(start..end, result);
+ }
+}
+
+fn byte_from_char(c: char) -> u8 {
+ let res = c as u32;
+ assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
+ res as u8
+}
+
+fn is_ascii(x: u32) -> bool {
+ x <= 0x7F
+}
diff --git a/compiler/rustc_lexer/src/unescape/tests.rs b/compiler/rustc_lexer/src/unescape/tests.rs
new file mode 100644
index 000000000..fa61554af
--- /dev/null
+++ b/compiler/rustc_lexer/src/unescape/tests.rs
@@ -0,0 +1,297 @@
+use super::*;
+
+#[test]
+fn test_unescape_char_bad() {
+ fn check(literal_text: &str, expected_error: EscapeError) {
+ let actual_result = unescape_char(literal_text).map_err(|(_offset, err)| err);
+ assert_eq!(actual_result, Err(expected_error));
+ }
+
+ check("", EscapeError::ZeroChars);
+ check(r"\", EscapeError::LoneSlash);
+
+ check("\n", EscapeError::EscapeOnlyChar);
+ check("\t", EscapeError::EscapeOnlyChar);
+ check("'", EscapeError::EscapeOnlyChar);
+ check("\r", EscapeError::BareCarriageReturn);
+
+ check("spam", EscapeError::MoreThanOneChar);
+ check(r"\x0ff", EscapeError::MoreThanOneChar);
+ check(r#"\"a"#, EscapeError::MoreThanOneChar);
+ check(r"\na", EscapeError::MoreThanOneChar);
+ check(r"\ra", EscapeError::MoreThanOneChar);
+ check(r"\ta", EscapeError::MoreThanOneChar);
+ check(r"\\a", EscapeError::MoreThanOneChar);
+ check(r"\'a", EscapeError::MoreThanOneChar);
+ check(r"\0a", EscapeError::MoreThanOneChar);
+ check(r"\u{0}x", EscapeError::MoreThanOneChar);
+ check(r"\u{1F63b}}", EscapeError::MoreThanOneChar);
+
+ check(r"\v", EscapeError::InvalidEscape);
+ check(r"\💩", EscapeError::InvalidEscape);
+ check(r"\●", EscapeError::InvalidEscape);
+ check("\\\r", EscapeError::InvalidEscape);
+
+ check(r"\x", EscapeError::TooShortHexEscape);
+ check(r"\x0", EscapeError::TooShortHexEscape);
+ check(r"\xf", EscapeError::TooShortHexEscape);
+ check(r"\xa", EscapeError::TooShortHexEscape);
+ check(r"\xx", EscapeError::InvalidCharInHexEscape);
+ check(r"\xы", EscapeError::InvalidCharInHexEscape);
+ check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
+ check(r"\xtt", EscapeError::InvalidCharInHexEscape);
+ check(r"\xff", EscapeError::OutOfRangeHexEscape);
+ check(r"\xFF", EscapeError::OutOfRangeHexEscape);
+ check(r"\x80", EscapeError::OutOfRangeHexEscape);
+
+ check(r"\u", EscapeError::NoBraceInUnicodeEscape);
+ check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
+ check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
+ check(r"\u{", EscapeError::UnclosedUnicodeEscape);
+ check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
+ check(r"\u{}", EscapeError::EmptyUnicodeEscape);
+ check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
+ check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
+ check(r"\u{FFFFFF}", EscapeError::OutOfRangeUnicodeEscape);
+ check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
+ check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
+
+ check(r"\u{DC00}", EscapeError::LoneSurrogateUnicodeEscape);
+ check(r"\u{DDDD}", EscapeError::LoneSurrogateUnicodeEscape);
+ check(r"\u{DFFF}", EscapeError::LoneSurrogateUnicodeEscape);
+
+ check(r"\u{D800}", EscapeError::LoneSurrogateUnicodeEscape);
+ check(r"\u{DAAA}", EscapeError::LoneSurrogateUnicodeEscape);
+ check(r"\u{DBFF}", EscapeError::LoneSurrogateUnicodeEscape);
+}
+
+#[test]
+fn test_unescape_char_good() {
+ fn check(literal_text: &str, expected_char: char) {
+ let actual_result = unescape_char(literal_text);
+ assert_eq!(actual_result, Ok(expected_char));
+ }
+
+ check("a", 'a');
+ check("ы", 'ы');
+ check("🦀", '🦀');
+
+ check(r#"\""#, '"');
+ check(r"\n", '\n');
+ check(r"\r", '\r');
+ check(r"\t", '\t');
+ check(r"\\", '\\');
+ check(r"\'", '\'');
+ check(r"\0", '\0');
+
+ check(r"\x00", '\0');
+ check(r"\x5a", 'Z');
+ check(r"\x5A", 'Z');
+ check(r"\x7f", 127 as char);
+
+ check(r"\u{0}", '\0');
+ check(r"\u{000000}", '\0');
+ check(r"\u{41}", 'A');
+ check(r"\u{0041}", 'A');
+ check(r"\u{00_41}", 'A');
+ check(r"\u{4__1__}", 'A');
+ check(r"\u{1F63b}", '😻');
+}
+
+#[test]
+fn test_unescape_str_warn() {
+ fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) {
+ let mut unescaped = Vec::with_capacity(literal.len());
+ unescape_literal(literal, Mode::Str, &mut |range, res| unescaped.push((range, res)));
+ assert_eq!(unescaped, expected);
+ }
+
+ // Check we can handle escaped newlines at the end of a file.
+ check("\\\n", &[]);
+ check("\\\n ", &[]);
+
+ check(
+ "\\\n \u{a0} x",
+ &[
+ (0..5, Err(EscapeError::UnskippedWhitespaceWarning)),
+ (3..5, Ok('\u{a0}')),
+ (5..6, Ok(' ')),
+ (6..7, Ok('x')),
+ ],
+ );
+ check("\\\n \n x", &[(0..7, Err(EscapeError::MultipleSkippedLinesWarning)), (7..8, Ok('x'))]);
+}
+
+#[test]
+fn test_unescape_str_good() {
+ fn check(literal_text: &str, expected: &str) {
+ let mut buf = Ok(String::with_capacity(literal_text.len()));
+ unescape_literal(literal_text, Mode::Str, &mut |range, c| {
+ if let Ok(b) = &mut buf {
+ match c {
+ Ok(c) => b.push(c),
+ Err(e) => buf = Err((range, e)),
+ }
+ }
+ });
+ let buf = buf.as_ref().map(|it| it.as_ref());
+ assert_eq!(buf, Ok(expected))
+ }
+
+ check("foo", "foo");
+ check("", "");
+ check(" \t\n", " \t\n");
+
+ check("hello \\\n world", "hello world");
+ check("thread's", "thread's")
+}
+
+#[test]
+fn test_unescape_byte_bad() {
+ fn check(literal_text: &str, expected_error: EscapeError) {
+ let actual_result = unescape_byte(literal_text).map_err(|(_offset, err)| err);
+ assert_eq!(actual_result, Err(expected_error));
+ }
+
+ check("", EscapeError::ZeroChars);
+ check(r"\", EscapeError::LoneSlash);
+
+ check("\n", EscapeError::EscapeOnlyChar);
+ check("\t", EscapeError::EscapeOnlyChar);
+ check("'", EscapeError::EscapeOnlyChar);
+ check("\r", EscapeError::BareCarriageReturn);
+
+ check("spam", EscapeError::MoreThanOneChar);
+ check(r"\x0ff", EscapeError::MoreThanOneChar);
+ check(r#"\"a"#, EscapeError::MoreThanOneChar);
+ check(r"\na", EscapeError::MoreThanOneChar);
+ check(r"\ra", EscapeError::MoreThanOneChar);
+ check(r"\ta", EscapeError::MoreThanOneChar);
+ check(r"\\a", EscapeError::MoreThanOneChar);
+ check(r"\'a", EscapeError::MoreThanOneChar);
+ check(r"\0a", EscapeError::MoreThanOneChar);
+
+ check(r"\v", EscapeError::InvalidEscape);
+ check(r"\💩", EscapeError::InvalidEscape);
+ check(r"\●", EscapeError::InvalidEscape);
+
+ check(r"\x", EscapeError::TooShortHexEscape);
+ check(r"\x0", EscapeError::TooShortHexEscape);
+ check(r"\xa", EscapeError::TooShortHexEscape);
+ check(r"\xf", EscapeError::TooShortHexEscape);
+ check(r"\xx", EscapeError::InvalidCharInHexEscape);
+ check(r"\xы", EscapeError::InvalidCharInHexEscape);
+ check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
+ check(r"\xtt", EscapeError::InvalidCharInHexEscape);
+
+ check(r"\u", EscapeError::NoBraceInUnicodeEscape);
+ check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
+ check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
+ check(r"\u{", EscapeError::UnclosedUnicodeEscape);
+ check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
+ check(r"\u{}", EscapeError::EmptyUnicodeEscape);
+ check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
+ check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
+
+ check("ы", EscapeError::NonAsciiCharInByte);
+ check("🦀", EscapeError::NonAsciiCharInByte);
+
+ check(r"\u{0}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{000000}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{41}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{0041}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{00_41}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{4__1__}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{1F63b}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{0}x", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{1F63b}}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{FFFFFF}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{DC00}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{DDDD}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{DFFF}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{D800}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{DAAA}", EscapeError::UnicodeEscapeInByte);
+ check(r"\u{DBFF}", EscapeError::UnicodeEscapeInByte);
+}
+
+#[test]
+fn test_unescape_byte_good() {
+ fn check(literal_text: &str, expected_byte: u8) {
+ let actual_result = unescape_byte(literal_text);
+ assert_eq!(actual_result, Ok(expected_byte));
+ }
+
+ check("a", b'a');
+
+ check(r#"\""#, b'"');
+ check(r"\n", b'\n');
+ check(r"\r", b'\r');
+ check(r"\t", b'\t');
+ check(r"\\", b'\\');
+ check(r"\'", b'\'');
+ check(r"\0", b'\0');
+
+ check(r"\x00", b'\0');
+ check(r"\x5a", b'Z');
+ check(r"\x5A", b'Z');
+ check(r"\x7f", 127);
+ check(r"\x80", 128);
+ check(r"\xff", 255);
+ check(r"\xFF", 255);
+}
+
+#[test]
+fn test_unescape_byte_str_good() {
+ fn check(literal_text: &str, expected: &[u8]) {
+ let mut buf = Ok(Vec::with_capacity(literal_text.len()));
+ unescape_byte_literal(literal_text, Mode::ByteStr, &mut |range, c| {
+ if let Ok(b) = &mut buf {
+ match c {
+ Ok(c) => b.push(c),
+ Err(e) => buf = Err((range, e)),
+ }
+ }
+ });
+ let buf = buf.as_ref().map(|it| it.as_ref());
+ assert_eq!(buf, Ok(expected))
+ }
+
+ check("foo", b"foo");
+ check("", b"");
+ check(" \t\n", b" \t\n");
+
+ check("hello \\\n world", b"hello world");
+ check("thread's", b"thread's")
+}
+
+#[test]
+fn test_unescape_raw_str() {
+ fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) {
+ let mut unescaped = Vec::with_capacity(literal.len());
+ unescape_literal(literal, Mode::RawStr, &mut |range, res| unescaped.push((range, res)));
+ assert_eq!(unescaped, expected);
+ }
+
+ check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]);
+ check("\rx", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString)), (1..2, Ok('x'))]);
+}
+
+#[test]
+fn test_unescape_raw_byte_str() {
+ fn check(literal: &str, expected: &[(Range<usize>, Result<u8, EscapeError>)]) {
+ let mut unescaped = Vec::with_capacity(literal.len());
+ unescape_byte_literal(literal, Mode::RawByteStr, &mut |range, res| {
+ unescaped.push((range, res))
+ });
+ assert_eq!(unescaped, expected);
+ }
+
+ check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]);
+ check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByteString))]);
+ check(
+ "🦀a",
+ &[(0..4, Err(EscapeError::NonAsciiCharInByteString)), (4..5, Ok(byte_from_char('a')))],
+ );
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-17 23:12:48 +0000