summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_lexer
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_lexer')
-rw-r--r--compiler/rustc_lexer/Cargo.toml2
-rw-r--r--compiler/rustc_lexer/src/lib.rs27
-rw-r--r--compiler/rustc_lexer/src/unescape.rs130
-rw-r--r--compiler/rustc_lexer/src/unescape/tests.rs35
4 files changed, 81 insertions, 113 deletions
diff --git a/compiler/rustc_lexer/Cargo.toml b/compiler/rustc_lexer/Cargo.toml
index ad685c2ad..23294dc2e 100644
--- a/compiler/rustc_lexer/Cargo.toml
+++ b/compiler/rustc_lexer/Cargo.toml
@@ -19,4 +19,4 @@ unicode-xid = "0.2.0"
unic-emoji-char = "0.9.0"
[dev-dependencies]
-expect-test = "1.0"
+expect-test = "1.4.0"
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 51515976e..3fbabbc63 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -88,7 +88,9 @@ pub enum TokenKind {
/// tokens.
UnknownPrefix,
- /// Examples: `"12_u8"`, `"1.0e-40"`, `b"123`.
+ /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
+ /// suffix, but may be present here on string and float literals. Users of
+ /// this type will need to check for and reject that case.
///
/// See [LiteralKind] for more details.
Literal { kind: LiteralKind, suffix_start: u32 },
@@ -165,11 +167,15 @@ pub enum DocStyle {
Inner,
}
+// Note that the suffix is *not* considered when deciding the `LiteralKind` in
+// this type. This means that float literals like `1f32` are classified by this
+// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
+// `rustc_ast::ast::LitKind.)
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum LiteralKind {
- /// "12_u8", "0o100", "0b120i99"
+ /// "12_u8", "0o100", "0b120i99", "1f32".
Int { base: Base, empty_int: bool },
- /// "12.34f32", "0b100.100"
+ /// "12.34f32", "1e3", but not "1f32`.
Float { base: Base, empty_exponent: bool },
/// "'a'", "'\\'", "'''", "';"
Char { terminated: bool },
@@ -203,13 +209,13 @@ pub enum RawStrError {
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum Base {
/// Literal starts with "0b".
- Binary,
+ Binary = 2,
/// Literal starts with "0o".
- Octal,
- /// Literal starts with "0x".
- Hexadecimal,
+ Octal = 8,
/// Literal doesn't contain a prefix.
- Decimal,
+ Decimal = 10,
+ /// Literal starts with "0x".
+ Hexadecimal = 16,
}
/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
@@ -840,12 +846,13 @@ impl Cursor<'_> {
self.eat_decimal_digits()
}
- // Eats the suffix of the literal, e.g. "_u8".
+ // Eats the suffix of the literal, e.g. "u8".
fn eat_literal_suffix(&mut self) {
self.eat_identifier();
}
- // Eats the identifier.
+ // Eats the identifier. Note: succeeds on `_`, which isn't a valid
+ // identifer.
fn eat_identifier(&mut self) {
if !is_id_start(self.first()) {
return;
diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 8f64b5f51..e405013dc 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -52,10 +52,8 @@ pub enum EscapeError {
/// Unicode escape code in byte literal.
UnicodeEscapeInByte,
- /// Non-ascii character in byte literal.
+ /// Non-ascii character in byte literal, byte string literal, or raw byte string literal.
NonAsciiCharInByte,
- /// Non-ascii character in byte string literal.
- NonAsciiCharInByteString,
/// After a line ending with '\', the next line contains whitespace
/// characters that are not skipped.
@@ -78,54 +76,33 @@ impl EscapeError {
/// Takes a contents of a literal (without quotes) and produces a
/// sequence of escaped characters or errors.
/// Values are returned through invoking of the provided callback.
-pub fn unescape_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+pub fn unescape_literal<F>(src: &str, mode: Mode, callback: &mut F)
where
F: FnMut(Range<usize>, Result<char, EscapeError>),
{
match mode {
Mode::Char | Mode::Byte => {
- let mut chars = literal_text.chars();
- let result = unescape_char_or_byte(&mut chars, mode);
- // The Chars iterator moved forward.
- callback(0..(literal_text.len() - chars.as_str().len()), result);
+ let mut chars = src.chars();
+ let res = unescape_char_or_byte(&mut chars, mode == Mode::Byte);
+ callback(0..(src.len() - chars.as_str().len()), res);
}
- Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(literal_text, mode, callback),
- // NOTE: Raw strings do not perform any explicit character escaping, here we
- // only translate CRLF to LF and produce errors on bare CR.
+ Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(src, mode == Mode::ByteStr, callback),
Mode::RawStr | Mode::RawByteStr => {
- unescape_raw_str_or_raw_byte_str(literal_text, mode, callback)
+ unescape_raw_str_or_raw_byte_str(src, mode == Mode::RawByteStr, callback)
}
}
}
-/// Takes a contents of a byte, byte string or raw byte string (without quotes)
-/// and produces a sequence of bytes or errors.
-/// Values are returned through invoking of the provided callback.
-pub fn unescape_byte_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
-where
- F: FnMut(Range<usize>, Result<u8, EscapeError>),
-{
- debug_assert!(mode.is_bytes());
- unescape_literal(literal_text, mode, &mut |range, result| {
- callback(range, result.map(byte_from_char));
- })
-}
-
/// Takes a contents of a char literal (without quotes), and returns an
-/// unescaped char or an error
-pub fn unescape_char(literal_text: &str) -> Result<char, (usize, EscapeError)> {
- let mut chars = literal_text.chars();
- unescape_char_or_byte(&mut chars, Mode::Char)
- .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+/// unescaped char or an error.
+pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
+ unescape_char_or_byte(&mut src.chars(), false)
}
/// Takes a contents of a byte literal (without quotes), and returns an
/// unescaped byte or an error.
-pub fn unescape_byte(literal_text: &str) -> Result<u8, (usize, EscapeError)> {
- let mut chars = literal_text.chars();
- unescape_char_or_byte(&mut chars, Mode::Byte)
- .map(byte_from_char)
- .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
+ unescape_char_or_byte(&mut src.chars(), true).map(byte_from_char)
}
/// What kind of literal do we parse.
@@ -147,7 +124,7 @@ impl Mode {
}
}
- pub fn is_bytes(self) -> bool {
+ pub fn is_byte(self) -> bool {
match self {
Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
Mode::Char | Mode::Str | Mode::RawStr => false,
@@ -155,12 +132,9 @@ impl Mode {
}
}
-fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+fn scan_escape(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, EscapeError> {
// Previous character was '\\', unescape what follows.
-
- let second_char = chars.next().ok_or(EscapeError::LoneSlash)?;
-
- let res = match second_char {
+ let res = match chars.next().ok_or(EscapeError::LoneSlash)? {
'"' => '"',
'n' => '\n',
'r' => '\r',
@@ -181,7 +155,7 @@ fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
let value = hi * 16 + lo;
// For a non-byte literal verify that it is within ASCII range.
- if !mode.is_bytes() && !is_ascii(value) {
+ if !is_byte && !is_ascii(value) {
return Err(EscapeError::OutOfRangeHexEscape);
}
let value = value as u8;
@@ -217,7 +191,7 @@ fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
// Incorrect syntax has higher priority for error reporting
// than unallowed value for a literal.
- if mode.is_bytes() {
+ if is_byte {
return Err(EscapeError::UnicodeEscapeInByte);
}
@@ -249,23 +223,22 @@ fn scan_escape(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
}
#[inline]
-fn ascii_check(first_char: char, mode: Mode) -> Result<char, EscapeError> {
- if mode.is_bytes() && !first_char.is_ascii() {
+fn ascii_check(c: char, is_byte: bool) -> Result<char, EscapeError> {
+ if is_byte && !c.is_ascii() {
// Byte literal can't be a non-ascii character.
Err(EscapeError::NonAsciiCharInByte)
} else {
- Ok(first_char)
+ Ok(c)
}
}
-fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
- debug_assert!(mode == Mode::Char || mode == Mode::Byte);
- let first_char = chars.next().ok_or(EscapeError::ZeroChars)?;
- let res = match first_char {
- '\\' => scan_escape(chars, mode),
+fn unescape_char_or_byte(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, EscapeError> {
+ let c = chars.next().ok_or(EscapeError::ZeroChars)?;
+ let res = match c {
+ '\\' => scan_escape(chars, is_byte),
'\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
'\r' => Err(EscapeError::BareCarriageReturn),
- _ => ascii_check(first_char, mode),
+ _ => ascii_check(c, is_byte),
}?;
if chars.next().is_some() {
return Err(EscapeError::MoreThanOneChar);
@@ -275,20 +248,20 @@ fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, Esca
/// Takes a contents of a string literal (without quotes) and produces a
/// sequence of escaped characters or errors.
-fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
+fn unescape_str_or_byte_str<F>(src: &str, is_byte: bool, callback: &mut F)
where
F: FnMut(Range<usize>, Result<char, EscapeError>),
{
- debug_assert!(mode == Mode::Str || mode == Mode::ByteStr);
- let initial_len = src.len();
let mut chars = src.chars();
- while let Some(first_char) = chars.next() {
- let start = initial_len - chars.as_str().len() - first_char.len_utf8();
- let unescaped_char = match first_char {
+ // The `start` and `end` computation here is complicated because
+ // `skip_ascii_whitespace` makes us to skip over chars without counting
+ // them in the range computation.
+ while let Some(c) = chars.next() {
+ let start = src.len() - chars.as_str().len() - c.len_utf8();
+ let res = match c {
'\\' => {
- let second_char = chars.clone().next();
- match second_char {
+ match chars.clone().next() {
Some('\n') => {
// Rust language specification requires us to skip whitespaces
// if unescaped '\' character is followed by '\n'.
@@ -297,17 +270,17 @@ where
skip_ascii_whitespace(&mut chars, start, callback);
continue;
}
- _ => scan_escape(&mut chars, mode),
+ _ => scan_escape(&mut chars, is_byte),
}
}
'\n' => Ok('\n'),
'\t' => Ok('\t'),
'"' => Err(EscapeError::EscapeOnlyChar),
'\r' => Err(EscapeError::BareCarriageReturn),
- _ => ascii_check(first_char, mode),
+ _ => ascii_check(c, is_byte),
};
- let end = initial_len - chars.as_str().len();
- callback(start..end, unescaped_char);
+ let end = src.len() - chars.as_str().len();
+ callback(start..end, res);
}
fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
@@ -340,30 +313,29 @@ where
/// Takes a contents of a string literal (without quotes) and produces a
/// sequence of characters or errors.
/// NOTE: Raw strings do not perform any explicit character escaping, here we
-/// only translate CRLF to LF and produce errors on bare CR.
-fn unescape_raw_str_or_raw_byte_str<F>(literal_text: &str, mode: Mode, callback: &mut F)
+/// only produce errors on bare CR.
+fn unescape_raw_str_or_raw_byte_str<F>(src: &str, is_byte: bool, callback: &mut F)
where
F: FnMut(Range<usize>, Result<char, EscapeError>),
{
- debug_assert!(mode == Mode::RawStr || mode == Mode::RawByteStr);
- let initial_len = literal_text.len();
-
- let mut chars = literal_text.chars();
- while let Some(curr) = chars.next() {
- let start = initial_len - chars.as_str().len() - curr.len_utf8();
+ let mut chars = src.chars();
- let result = match curr {
+ // The `start` and `end` computation here matches the one in
+ // `unescape_str_or_byte_str` for consistency, even though this function
+ // doesn't have to worry about skipping any chars.
+ while let Some(c) = chars.next() {
+ let start = src.len() - chars.as_str().len() - c.len_utf8();
+ let res = match c {
'\r' => Err(EscapeError::BareCarriageReturnInRawString),
- c if mode.is_bytes() && !c.is_ascii() => Err(EscapeError::NonAsciiCharInByteString),
- c => Ok(c),
+ _ => ascii_check(c, is_byte),
};
- let end = initial_len - chars.as_str().len();
-
- callback(start..end, result);
+ let end = src.len() - chars.as_str().len();
+ callback(start..end, res);
}
}
-fn byte_from_char(c: char) -> u8 {
+#[inline]
+pub fn byte_from_char(c: char) -> u8 {
let res = c as u32;
debug_assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
res as u8
diff --git a/compiler/rustc_lexer/src/unescape/tests.rs b/compiler/rustc_lexer/src/unescape/tests.rs
index fa61554af..1c25b03fd 100644
--- a/compiler/rustc_lexer/src/unescape/tests.rs
+++ b/compiler/rustc_lexer/src/unescape/tests.rs
@@ -3,8 +3,7 @@ use super::*;
#[test]
fn test_unescape_char_bad() {
fn check(literal_text: &str, expected_error: EscapeError) {
- let actual_result = unescape_char(literal_text).map_err(|(_offset, err)| err);
- assert_eq!(actual_result, Err(expected_error));
+ assert_eq!(unescape_char(literal_text), Err(expected_error));
}
check("", EscapeError::ZeroChars);
@@ -68,8 +67,7 @@ fn test_unescape_char_bad() {
#[test]
fn test_unescape_char_good() {
fn check(literal_text: &str, expected_char: char) {
- let actual_result = unescape_char(literal_text);
- assert_eq!(actual_result, Ok(expected_char));
+ assert_eq!(unescape_char(literal_text), Ok(expected_char));
}
check("a", 'a');
@@ -134,8 +132,7 @@ fn test_unescape_str_good() {
}
}
});
- let buf = buf.as_ref().map(|it| it.as_ref());
- assert_eq!(buf, Ok(expected))
+ assert_eq!(buf.as_deref(), Ok(expected))
}
check("foo", "foo");
@@ -149,8 +146,7 @@ fn test_unescape_str_good() {
#[test]
fn test_unescape_byte_bad() {
fn check(literal_text: &str, expected_error: EscapeError) {
- let actual_result = unescape_byte(literal_text).map_err(|(_offset, err)| err);
- assert_eq!(actual_result, Err(expected_error));
+ assert_eq!(unescape_byte(literal_text), Err(expected_error));
}
check("", EscapeError::ZeroChars);
@@ -219,8 +215,7 @@ fn test_unescape_byte_bad() {
#[test]
fn test_unescape_byte_good() {
fn check(literal_text: &str, expected_byte: u8) {
- let actual_result = unescape_byte(literal_text);
- assert_eq!(actual_result, Ok(expected_byte));
+ assert_eq!(unescape_byte(literal_text), Ok(expected_byte));
}
check("a", b'a');
@@ -246,16 +241,15 @@ fn test_unescape_byte_good() {
fn test_unescape_byte_str_good() {
fn check(literal_text: &str, expected: &[u8]) {
let mut buf = Ok(Vec::with_capacity(literal_text.len()));
- unescape_byte_literal(literal_text, Mode::ByteStr, &mut |range, c| {
+ unescape_literal(literal_text, Mode::ByteStr, &mut |range, c| {
if let Ok(b) = &mut buf {
match c {
- Ok(c) => b.push(c),
+ Ok(c) => b.push(byte_from_char(c)),
Err(e) => buf = Err((range, e)),
}
}
});
- let buf = buf.as_ref().map(|it| it.as_ref());
- assert_eq!(buf, Ok(expected))
+ assert_eq!(buf.as_deref(), Ok(expected))
}
check("foo", b"foo");
@@ -280,18 +274,13 @@ fn test_unescape_raw_str() {
#[test]
fn test_unescape_raw_byte_str() {
- fn check(literal: &str, expected: &[(Range<usize>, Result<u8, EscapeError>)]) {
+ fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) {
let mut unescaped = Vec::with_capacity(literal.len());
- unescape_byte_literal(literal, Mode::RawByteStr, &mut |range, res| {
- unescaped.push((range, res))
- });
+ unescape_literal(literal, Mode::RawByteStr, &mut |range, res| unescaped.push((range, res)));
assert_eq!(unescaped, expected);
}
check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]);
- check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByteString))]);
- check(
- "🦀a",
- &[(0..4, Err(EscapeError::NonAsciiCharInByteString)), (4..5, Ok(byte_from_char('a')))],
- );
+ check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByte))]);
+ check("🦀a", &[(0..4, Err(EscapeError::NonAsciiCharInByte)), (4..5, Ok('a'))]);
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-17 11:06:51 +0000