summaryrefslogtreecommitdiffstats
path: root/vendor/rustc-ap-rustc_lexer/src/lib.rs
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/rustc-ap-rustc_lexer/src/lib.rs')
-rw-r--r--vendor/rustc-ap-rustc_lexer/src/lib.rs22
1 files changed, 17 insertions, 5 deletions
diff --git a/vendor/rustc-ap-rustc_lexer/src/lib.rs b/vendor/rustc-ap-rustc_lexer/src/lib.rs
index b9781581f..4cb2a6ca5 100644
--- a/vendor/rustc-ap-rustc_lexer/src/lib.rs
+++ b/vendor/rustc-ap-rustc_lexer/src/lib.rs
@@ -66,6 +66,13 @@ pub enum TokenKind {
Ident,
/// "r#ident"
RawIdent,
+ /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
+ /// prefix (`foo`) is included in the token, not the separator (which is
+ /// lexed as its own distinct token). In Rust 2021 and later, reserved
+ /// prefixes are reported as errors; in earlier editions, they result in a
+ /// (allowed by default) lint, and are treated as regular identifier
+ /// tokens.
+ UnknownPrefix,
/// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
Literal { kind: LiteralKind, suffix_start: usize },
/// "'a"
@@ -323,7 +330,7 @@ impl Cursor<'_> {
let kind = RawStr { n_hashes, err };
Literal { kind, suffix_start }
}
- _ => self.ident(),
+ _ => self.ident_or_unknown_prefix(),
},
// Byte literal, byte string literal, raw byte string literal or identifier.
@@ -358,12 +365,12 @@ impl Cursor<'_> {
let kind = RawByteStr { n_hashes, err };
Literal { kind, suffix_start }
}
- _ => self.ident(),
+ _ => self.ident_or_unknown_prefix(),
},
// Identifier (this should be checked after other variant that can
// start as identifier).
- c if is_id_start(c) => self.ident(),
+ c if is_id_start(c) => self.ident_or_unknown_prefix(),
// Numeric literal.
c @ '0'..='9' => {
@@ -487,11 +494,16 @@ impl Cursor<'_> {
RawIdent
}
- fn ident(&mut self) -> TokenKind {
+ fn ident_or_unknown_prefix(&mut self) -> TokenKind {
debug_assert!(is_id_start(self.prev()));
// Start is already eaten, eat the rest of identifier.
self.eat_while(is_id_continue);
- Ident
+ // Known prefixes must have been handled earlier. So if
+ // we see a prefix here, it is definitely a unknown prefix.
+ match self.first() {
+ '#' | '"' | '\'' => UnknownPrefix,
+ _ => Ident,
+ }
}
fn number(&mut self, first_digit: char) -> LiteralKind {