summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_lexer/src/unescape.rs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--compiler/rustc_lexer/src/unescape.rs377
1 files changed, 377 insertions, 0 deletions
diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
new file mode 100644
index 000000000..3da6bc146
--- /dev/null
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -0,0 +1,377 @@
+//! Utilities for validating string and char literals and turning them into
+//! values they represent.
+
+use std::ops::Range;
+use std::str::Chars;
+
+#[cfg(test)]
+mod tests;
+
+/// Errors and warnings that can occur during string unescaping.
+#[derive(Debug, PartialEq, Eq)]
+pub enum EscapeError {
+ /// Expected 1 char, but 0 were found.
+ ZeroChars,
+ /// Expected 1 char, but more than 1 were found.
+ MoreThanOneChar,
+
+ /// Escaped '\' character without continuation.
+ LoneSlash,
+ /// Invalid escape character (e.g. '\z').
+ InvalidEscape,
+ /// Raw '\r' encountered.
+ BareCarriageReturn,
+ /// Raw '\r' encountered in raw string.
+ BareCarriageReturnInRawString,
+ /// Unescaped character that was expected to be escaped (e.g. raw '\t').
+ EscapeOnlyChar,
+
+ /// Numeric character escape is too short (e.g. '\x1').
+ TooShortHexEscape,
+ /// Invalid character in numeric escape (e.g. '\xz')
+ InvalidCharInHexEscape,
+ /// Character code in numeric escape is non-ascii (e.g. '\xFF').
+ OutOfRangeHexEscape,
+
+ /// '\u' not followed by '{'.
+ NoBraceInUnicodeEscape,
+ /// Non-hexadecimal value in '\u{..}'.
+ InvalidCharInUnicodeEscape,
+ /// '\u{}'
+ EmptyUnicodeEscape,
+ /// No closing brace in '\u{..}', e.g. '\u{12'.
+ UnclosedUnicodeEscape,
+ /// '\u{_12}'
+ LeadingUnderscoreUnicodeEscape,
+ /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}'
+ OverlongUnicodeEscape,
+ /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'.
+ LoneSurrogateUnicodeEscape,
+ /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'.
+ OutOfRangeUnicodeEscape,
+
+ /// Unicode escape code in byte literal.
+ UnicodeEscapeInByte,
+ /// Non-ascii character in byte literal.
+ NonAsciiCharInByte,
+ /// Non-ascii character in byte string literal.
+ NonAsciiCharInByteString,
+
+ /// After a line ending with '\', the next line contains whitespace
+ /// characters that are not skipped.
+ UnskippedWhitespaceWarning,
+
+ /// After a line ending with '\', multiple lines are skipped.
+ MultipleSkippedLinesWarning,
+}
+
+impl EscapeError {
+ /// Returns true for actual errors, as opposed to warnings.
+ pub fn is_fatal(&self) -> bool {
+ !matches!(
+ self,
+ EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning
+ )
+ }
+}
+
+/// Takes a contents of a literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ match mode {
+ Mode::Char | Mode::Byte => {
+ let mut chars = literal_text.chars();
+ let result = unescape_char_or_byte(&mut chars, mode);
+ // The Chars iterator moved forward.
+ callback(0..(literal_text.len() - chars.as_str().len()), result);
+ }
+ Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(literal_text, mode, callback),
+ // NOTE: Raw strings do not perform any explicit character escaping, here we
+ // only translate CRLF to LF and produce errors on bare CR.
+ Mode::RawStr | Mode::RawByteStr => {
+ unescape_raw_str_or_byte_str(literal_text, mode, callback)
+ }
+ }
+}
+
+/// Takes a contents of a byte, byte string or raw byte string (without quotes)
+/// and produces a sequence of bytes or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_byte_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<u8, EscapeError>),
+{
+ assert!(mode.is_bytes());
+ unescape_literal(literal_text, mode, &mut |range, result| {
+ callback(range, result.map(byte_from_char));
+ })
+}
+
+/// Takes a contents of a char literal (without quotes), and returns an
+/// unescaped char or an error
+pub fn unescape_char(literal_text: &str) -> Result<char, (usize, EscapeError)> {
+ let mut chars = literal_text.chars();
+ unescape_char_or_byte(&mut chars, Mode::Char)
+ .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// Takes a contents of a byte literal (without quotes), and returns an
+/// unescaped byte or an error.
+pub fn unescape_byte(literal_text: &str) -> Result<u8, (usize, EscapeError)> {
+ let mut chars = literal_text.chars();
+ unescape_char_or_byte(&mut chars, Mode::Byte)
+ .map(byte_from_char)
+ .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// What kind of literal do we parse.
+#[derive(Debug, Clone, Copy)]
+pub enum Mode {
+ Char,
+ Str,
+ Byte,
+ ByteStr,
+ RawStr,
+ RawByteStr,
+}
+
+impl Mode {
+ pub fn in_single_quotes(self) -> bool {
+ match self {
+ Mode::Char | Mode::Byte => true,
+ Mode::Str | Mode::ByteStr | Mode::RawStr | Mode::RawByteStr => false,
+ }
+ }
+
+ pub fn in_double_quotes(self) -> bool {
+ !self.in_single_quotes()
+ }
+
+ pub fn is_bytes(self) -> bool {
+ match self {
+ Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
+ Mode::Char | Mode::Str | Mode::RawStr => false,
+ }
+ }
+}
+
+fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+ // Previous character was '\\', unescape what follows.
+
+ let second_char = chars.next().ok_or(EscapeError::LoneSlash)?;
+
+ let res = match second_char {
+ '"' => '"',
+ 'n' => '\n',
+ 'r' => '\r',
+ 't' => '\t',
+ '\\' => '\\',
+ '\'' => '\'',
+ '0' => '\0',
+
+ 'x' => {
+ // Parse hexadecimal character code.
+
+ let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+ let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+ let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+ let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+ let value = hi * 16 + lo;
+
+ // For a byte literal verify that it is within ASCII range.
+ if !mode.is_bytes() && !is_ascii(value) {
+ return Err(EscapeError::OutOfRangeHexEscape);
+ }
+ let value = value as u8;
+
+ value as char
+ }
+
+ 'u' => {
+ // We've parsed '\u', now we have to parse '{..}'.
+
+ if chars.next() != Some('{') {
+ return Err(EscapeError::NoBraceInUnicodeEscape);
+ }
+
+ // First character must be a hexadecimal digit.
+ let mut n_digits = 1;
+ let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
+ '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
+ '}' => return Err(EscapeError::EmptyUnicodeEscape),
+ c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
+ };
+
+ // First character is valid, now parse the rest of the number
+ // and closing brace.
+ loop {
+ match chars.next() {
+ None => return Err(EscapeError::UnclosedUnicodeEscape),
+ Some('_') => continue,
+ Some('}') => {
+ if n_digits > 6 {
+ return Err(EscapeError::OverlongUnicodeEscape);
+ }
+
+ // Incorrect syntax has higher priority for error reporting
+ // than unallowed value for a literal.
+ if mode.is_bytes() {
+ return Err(EscapeError::UnicodeEscapeInByte);
+ }
+
+ break std::char::from_u32(value).ok_or_else(|| {
+ if value > 0x10FFFF {
+ EscapeError::OutOfRangeUnicodeEscape
+ } else {
+ EscapeError::LoneSurrogateUnicodeEscape
+ }
+ })?;
+ }
+ Some(c) => {
+ let digit =
+ c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
+ n_digits += 1;
+ if n_digits > 6 {
+ // Stop updating value since we're sure that it's incorrect already.
+ continue;
+ }
+ let digit = digit as u32;
+ value = value * 16 + digit;
+ }
+ };
+ }
+ }
+ _ => return Err(EscapeError::InvalidEscape),
+ };
+ Ok(res)
+}
+
+#[inline]
+fn ascii_check(first_char: char, mode: Mode) -> Result<char, EscapeError> {
+ if mode.is_bytes() && !first_char.is_ascii() {
+ // Byte literal can't be a non-ascii character.
+ Err(EscapeError::NonAsciiCharInByte)
+ } else {
+ Ok(first_char)
+ }
+}
+
+fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+ let first_char = chars.next().ok_or(EscapeError::ZeroChars)?;
+ let res = match first_char {
+ '\\' => scan_escape(chars, mode),
+ '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
+ '\r' => Err(EscapeError::BareCarriageReturn),
+ _ => ascii_check(first_char, mode),
+ }?;
+ if chars.next().is_some() {
+ return Err(EscapeError::MoreThanOneChar);
+ }
+ Ok(res)
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ assert!(mode.in_double_quotes());
+ let initial_len = src.len();
+ let mut chars = src.chars();
+ while let Some(first_char) = chars.next() {
+ let start = initial_len - chars.as_str().len() - first_char.len_utf8();
+
+ let unescaped_char = match first_char {
+ '\\' => {
+ let second_char = chars.clone().next();
+ match second_char {
+ Some('\n') => {
+ // Rust language specification requires us to skip whitespaces
+ // if unescaped '\' character is followed by '\n'.
+ // For details see [Rust language reference]
+ // (https://doc.rust-lang.org/reference/tokens.html#string-literals).
+ skip_ascii_whitespace(&mut chars, start, callback);
+ continue;
+ }
+ _ => scan_escape(&mut chars, mode),
+ }
+ }
+ '\n' => Ok('\n'),
+ '\t' => Ok('\t'),
+ '"' => Err(EscapeError::EscapeOnlyChar),
+ '\r' => Err(EscapeError::BareCarriageReturn),
+ _ => ascii_check(first_char, mode),
+ };
+ let end = initial_len - chars.as_str().len();
+ callback(start..end, unescaped_char);
+ }
+
+ fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
+ where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+ {
+ let tail = chars.as_str();
+ let first_non_space = tail
+ .bytes()
+ .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
+ .unwrap_or(tail.len());
+ if tail[1..first_non_space].contains('\n') {
+ // The +1 accounts for the escaping slash.
+ let end = start + first_non_space + 1;
+ callback(start..end, Err(EscapeError::MultipleSkippedLinesWarning));
+ }
+ let tail = &tail[first_non_space..];
+ if let Some(c) = tail.chars().nth(0) {
+ // For error reporting, we would like the span to contain the character that was not
+ // skipped. The +1 is necessary to account for the leading \ that started the escape.
+ let end = start + first_non_space + c.len_utf8() + 1;
+ if c.is_whitespace() {
+ callback(start..end, Err(EscapeError::UnskippedWhitespaceWarning));
+ }
+ }
+ *chars = tail.chars();
+ }
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of characters or errors.
+/// NOTE: Raw strings do not perform any explicit character escaping, here we
+/// only translate CRLF to LF and produce errors on bare CR.
+fn unescape_raw_str_or_byte_str<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+ F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+ assert!(mode.in_double_quotes());
+ let initial_len = literal_text.len();
+
+ let mut chars = literal_text.chars();
+ while let Some(curr) = chars.next() {
+ let start = initial_len - chars.as_str().len() - curr.len_utf8();
+
+ let result = match curr {
+ '\r' => Err(EscapeError::BareCarriageReturnInRawString),
+ c if mode.is_bytes() && !c.is_ascii() => Err(EscapeError::NonAsciiCharInByteString),
+ c => Ok(c),
+ };
+ let end = initial_len - chars.as_str().len();
+
+ callback(start..end, result);
+ }
+}
+
+fn byte_from_char(c: char) -> u8 {
+ let res = c as u32;
+ assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
+ res as u8
+}
+
+fn is_ascii(x: u32) -> bool {
+ x <= 0x7F
+}