Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 1780 insertions, 0 deletions

diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
new file mode 100644
index 000000000..848e142e5
--- /dev/null
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -0,0 +1,717 @@
+use crate::lexer::unicode_chars::UNICODE_ARRAY;
+use rustc_ast::ast::{self, AttrStyle};
+use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind};
+use rustc_ast::tokenstream::{Spacing, TokenStream};
+use rustc_ast::util::unicode::contains_text_flow_control_chars;
+use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
+use rustc_lexer::unescape::{self, Mode};
+use rustc_lexer::{Base, DocStyle, RawStrError};
+use rustc_session::lint::builtin::{
+    RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+};
+use rustc_session::lint::BuiltinLintDiagnostics;
+use rustc_session::parse::ParseSess;
+use rustc_span::symbol::{sym, Symbol};
+use rustc_span::{edition::Edition, BytePos, Pos, Span};
+
+use tracing::debug;
+
+mod tokentrees;
+mod unescape_error_reporting;
+mod unicode_chars;
+
+use unescape_error_reporting::{emit_unescape_error, escaped_char};
+
+// This type is used a lot. Make sure it doesn't unintentionally get bigger.
+//
+// This assertion is in this crate, rather than in `rustc_lexer`, because that
+// crate cannot depend on `rustc_data_structures`.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+rustc_data_structures::static_assert_size!(rustc_lexer::Token, 12);
+
+#[derive(Clone, Debug)]
+pub struct UnmatchedBrace {
+    pub expected_delim: Delimiter,
+    pub found_delim: Option<Delimiter>,
+    pub found_span: Span,
+    pub unclosed_span: Option<Span>,
+    pub candidate_span: Option<Span>,
+}
+
+pub(crate) fn parse_token_trees<'a>(
+    sess: &'a ParseSess,
+    src: &'a str,
+    start_pos: BytePos,
+    override_span: Option<Span>,
+) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
+    StringReader { sess, start_pos, pos: start_pos, src, override_span }.into_token_trees()
+}
+
+struct StringReader<'a> {
+    sess: &'a ParseSess,
+    /// Initial position, read-only.
+    start_pos: BytePos,
+    /// The absolute offset within the source_map of the current character.
+    pos: BytePos,
+    /// Source text to tokenize.
+    src: &'a str,
+    override_span: Option<Span>,
+}
+
+impl<'a> StringReader<'a> {
+    fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
+        self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi))
+    }
+
+    /// Returns the next token, and info about preceding whitespace, if any.
+    fn next_token(&mut self) -> (Spacing, Token) {
+        let mut spacing = Spacing::Joint;
+
+        // Skip `#!` at the start of the file
+        if self.pos == self.start_pos
+            && let Some(shebang_len) = rustc_lexer::strip_shebang(self.src)
+        {
+            self.pos = self.pos + BytePos::from_usize(shebang_len);
+            spacing = Spacing::Alone;
+        }
+
+        // Skip trivial (whitespace & comments) tokens
+        loop {
+            let start_src_index = self.src_index(self.pos);
+            let text: &str = &self.src[start_src_index..];
+
+            if text.is_empty() {
+                let span = self.mk_sp(self.pos, self.pos);
+                return (spacing, Token::new(token::Eof, span));
+            }
+
+            let token = rustc_lexer::first_token(text);
+
+            let start = self.pos;
+            self.pos = self.pos + BytePos(token.len);
+
+            debug!("next_token: {:?}({:?})", token.kind, self.str_from(start));
+
+            match self.cook_lexer_token(token.kind, start) {
+                Some(kind) => {
+                    let span = self.mk_sp(start, self.pos);
+                    return (spacing, Token::new(kind, span));
+                }
+                None => spacing = Spacing::Alone,
+            }
+        }
+    }
+
+    /// Report a fatal lexical error with a given span.
+    fn fatal_span(&self, sp: Span, m: &str) -> ! {
+        self.sess.span_diagnostic.span_fatal(sp, m)
+    }
+
+    /// Report a lexical error with a given span.
+    fn err_span(&self, sp: Span, m: &str) {
+        self.sess.span_diagnostic.struct_span_err(sp, m).emit();
+    }
+
+    /// Report a fatal error spanning [`from_pos`, `to_pos`).
+    fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> ! {
+        self.fatal_span(self.mk_sp(from_pos, to_pos), m)
+    }
+
+    /// Report a lexical error spanning [`from_pos`, `to_pos`).
+    fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) {
+        self.err_span(self.mk_sp(from_pos, to_pos), m)
+    }
+
+    fn struct_fatal_span_char(
+        &self,
+        from_pos: BytePos,
+        to_pos: BytePos,
+        m: &str,
+        c: char,
+    ) -> DiagnosticBuilder<'a, !> {
+        self.sess
+            .span_diagnostic
+            .struct_span_fatal(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c)))
+    }
+
+    fn struct_err_span_char(
+        &self,
+        from_pos: BytePos,
+        to_pos: BytePos,
+        m: &str,
+        c: char,
+    ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        self.sess
+            .span_diagnostic
+            .struct_span_err(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c)))
+    }
+
+    /// Detect usages of Unicode codepoints changing the direction of the text on screen and loudly
+    /// complain about it.
+    fn lint_unicode_text_flow(&self, start: BytePos) {
+        // Opening delimiter of the length 2 is not included into the comment text.
+        let content_start = start + BytePos(2);
+        let content = self.str_from(content_start);
+        if contains_text_flow_control_chars(content) {
+            let span = self.mk_sp(start, self.pos);
+            self.sess.buffer_lint_with_diagnostic(
+                &TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+                span,
+                ast::CRATE_NODE_ID,
+                "unicode codepoint changing visible direction of text present in comment",
+                BuiltinLintDiagnostics::UnicodeTextFlow(span, content.to_string()),
+            );
+        }
+    }
+
+    /// Turns simple `rustc_lexer::TokenKind` enum into a rich
+    /// `rustc_ast::TokenKind`. This turns strings into interned
+    /// symbols and runs additional validation.
+    fn cook_lexer_token(&self, token: rustc_lexer::TokenKind, start: BytePos) -> Option<TokenKind> {
+        Some(match token {
+            rustc_lexer::TokenKind::LineComment { doc_style } => {
+                // Skip non-doc comments
+                let Some(doc_style) = doc_style else {
+                    self.lint_unicode_text_flow(start);
+                    return None;
+                };
+
+                // Opening delimiter of the length 3 is not included into the symbol.
+                let content_start = start + BytePos(3);
+                let content = self.str_from(content_start);
+                self.cook_doc_comment(content_start, content, CommentKind::Line, doc_style)
+            }
+            rustc_lexer::TokenKind::BlockComment { doc_style, terminated } => {
+                if !terminated {
+                    self.report_unterminated_block_comment(start, doc_style);
+                }
+
+                // Skip non-doc comments
+                let Some(doc_style) = doc_style else {
+                    self.lint_unicode_text_flow(start);
+                    return None;
+                };
+
+                // Opening delimiter of the length 3 and closing delimiter of the length 2
+                // are not included into the symbol.
+                let content_start = start + BytePos(3);
+                let content_end = self.pos - BytePos(if terminated { 2 } else { 0 });
+                let content = self.str_from_to(content_start, content_end);
+                self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style)
+            }
+            rustc_lexer::TokenKind::Whitespace => return None,
+            rustc_lexer::TokenKind::Ident
+            | rustc_lexer::TokenKind::RawIdent
+            | rustc_lexer::TokenKind::UnknownPrefix => {
+                let is_raw_ident = token == rustc_lexer::TokenKind::RawIdent;
+                let is_unknown_prefix = token == rustc_lexer::TokenKind::UnknownPrefix;
+                let mut ident_start = start;
+                if is_raw_ident {
+                    ident_start = ident_start + BytePos(2);
+                }
+                if is_unknown_prefix {
+                    self.report_unknown_prefix(start);
+                }
+                let sym = nfc_normalize(self.str_from(ident_start));
+                let span = self.mk_sp(start, self.pos);
+                self.sess.symbol_gallery.insert(sym, span);
+                if is_raw_ident {
+                    if !sym.can_be_raw() {
+                        self.err_span(span, &format!("`{}` cannot be a raw identifier", sym));
+                    }
+                    self.sess.raw_identifier_spans.borrow_mut().push(span);
+                }
+                token::Ident(sym, is_raw_ident)
+            }
+            rustc_lexer::TokenKind::InvalidIdent
+                // Do not recover an identifier with emoji if the codepoint is a confusable
+                // with a recoverable substitution token, like `➖`.
+                if !UNICODE_ARRAY
+                    .iter()
+                    .any(|&(c, _, _)| {
+                        let sym = self.str_from(start);
+                        sym.chars().count() == 1 && c == sym.chars().next().unwrap()
+                    })
+                     =>
+            {
+                let sym = nfc_normalize(self.str_from(start));
+                let span = self.mk_sp(start, self.pos);
+                self.sess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default().push(span);
+                token::Ident(sym, false)
+            }
+            rustc_lexer::TokenKind::Literal { kind, suffix_start } => {
+                let suffix_start = start + BytePos(suffix_start);
+                let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind);
+                let suffix = if suffix_start < self.pos {
+                    let string = self.str_from(suffix_start);
+                    if string == "_" {
+                        self.sess
+                            .span_diagnostic
+                            .struct_span_warn(
+                                self.mk_sp(suffix_start, self.pos),
+                                "underscore literal suffix is not allowed",
+                            )
+                            .warn(
+                                "this was previously accepted by the compiler but is \
+                                   being phased out; it will become a hard error in \
+                                   a future release!",
+                            )
+                            .note(
+                                "see issue #42326 \
+                                 <https://github.com/rust-lang/rust/issues/42326> \
+                                 for more information",
+                            )
+                            .emit();
+                        None
+                    } else {
+                        Some(Symbol::intern(string))
+                    }
+                } else {
+                    None
+                };
+                token::Literal(token::Lit { kind, symbol, suffix })
+            }
+            rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
+                // Include the leading `'` in the real identifier, for macro
+                // expansion purposes. See #12512 for the gory details of why
+                // this is necessary.
+                let lifetime_name = self.str_from(start);
+                if starts_with_number {
+                    self.err_span_(start, self.pos, "lifetimes cannot start with a number");
+                }
+                let ident = Symbol::intern(lifetime_name);
+                token::Lifetime(ident)
+            }
+            rustc_lexer::TokenKind::Semi => token::Semi,
+            rustc_lexer::TokenKind::Comma => token::Comma,
+            rustc_lexer::TokenKind::Dot => token::Dot,
+            rustc_lexer::TokenKind::OpenParen => token::OpenDelim(Delimiter::Parenthesis),
+            rustc_lexer::TokenKind::CloseParen => token::CloseDelim(Delimiter::Parenthesis),
+            rustc_lexer::TokenKind::OpenBrace => token::OpenDelim(Delimiter::Brace),
+            rustc_lexer::TokenKind::CloseBrace => token::CloseDelim(Delimiter::Brace),
+            rustc_lexer::TokenKind::OpenBracket => token::OpenDelim(Delimiter::Bracket),
+            rustc_lexer::TokenKind::CloseBracket => token::CloseDelim(Delimiter::Bracket),
+            rustc_lexer::TokenKind::At => token::At,
+            rustc_lexer::TokenKind::Pound => token::Pound,
+            rustc_lexer::TokenKind::Tilde => token::Tilde,
+            rustc_lexer::TokenKind::Question => token::Question,
+            rustc_lexer::TokenKind::Colon => token::Colon,
+            rustc_lexer::TokenKind::Dollar => token::Dollar,
+            rustc_lexer::TokenKind::Eq => token::Eq,
+            rustc_lexer::TokenKind::Bang => token::Not,
+            rustc_lexer::TokenKind::Lt => token::Lt,
+            rustc_lexer::TokenKind::Gt => token::Gt,
+            rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
+            rustc_lexer::TokenKind::And => token::BinOp(token::And),
+            rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
+            rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
+            rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
+            rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
+            rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
+            rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
+
+            rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
+                let c = self.str_from(start).chars().next().unwrap();
+                let mut err =
+                    self.struct_err_span_char(start, self.pos, "unknown start of token", c);
+                // FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs,
+                // instead of keeping a table in `check_for_substitution`into the token. Ideally,
+                // this should be inside `rustc_lexer`. However, we should first remove compound
+                // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it,
+                // as there will be less overall work to do this way.
+                let token = unicode_chars::check_for_substitution(self, start, c, &mut err);
+                if c == '\x00' {
+                    err.help("source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used");
+                }
+                err.emit();
+                token?
+            }
+        })
+    }
+
+    fn cook_doc_comment(
+        &self,
+        content_start: BytePos,
+        content: &str,
+        comment_kind: CommentKind,
+        doc_style: DocStyle,
+    ) -> TokenKind {
+        if content.contains('\r') {
+            for (idx, _) in content.char_indices().filter(|&(_, c)| c == '\r') {
+                self.err_span_(
+                    content_start + BytePos(idx as u32),
+                    content_start + BytePos(idx as u32 + 1),
+                    match comment_kind {
+                        CommentKind::Line => "bare CR not allowed in doc-comment",
+                        CommentKind::Block => "bare CR not allowed in block doc-comment",
+                    },
+                );
+            }
+        }
+
+        let attr_style = match doc_style {
+            DocStyle::Outer => AttrStyle::Outer,
+            DocStyle::Inner => AttrStyle::Inner,
+        };
+
+        token::DocComment(comment_kind, attr_style, Symbol::intern(content))
+    }
+
+    fn cook_lexer_literal(
+        &self,
+        start: BytePos,
+        suffix_start: BytePos,
+        kind: rustc_lexer::LiteralKind,
+    ) -> (token::LitKind, Symbol) {
+        // prefix means `"` or `br"` or `r###"`, ...
+        let (lit_kind, mode, prefix_len, postfix_len) = match kind {
+            rustc_lexer::LiteralKind::Char { terminated } => {
+                if !terminated {
+                    self.sess.span_diagnostic.span_fatal_with_code(
+                        self.mk_sp(start, suffix_start),
+                        "unterminated character literal",
+                        error_code!(E0762),
+                    )
+                }
+                (token::Char, Mode::Char, 1, 1) // ' '
+            }
+            rustc_lexer::LiteralKind::Byte { terminated } => {
+                if !terminated {
+                    self.sess.span_diagnostic.span_fatal_with_code(
+                        self.mk_sp(start + BytePos(1), suffix_start),
+                        "unterminated byte constant",
+                        error_code!(E0763),
+                    )
+                }
+                (token::Byte, Mode::Byte, 2, 1) // b' '
+            }
+            rustc_lexer::LiteralKind::Str { terminated } => {
+                if !terminated {
+                    self.sess.span_diagnostic.span_fatal_with_code(
+                        self.mk_sp(start, suffix_start),
+                        "unterminated double quote string",
+                        error_code!(E0765),
+                    )
+                }
+                (token::Str, Mode::Str, 1, 1) // " "
+            }
+            rustc_lexer::LiteralKind::ByteStr { terminated } => {
+                if !terminated {
+                    self.sess.span_diagnostic.span_fatal_with_code(
+                        self.mk_sp(start + BytePos(1), suffix_start),
+                        "unterminated double quote byte string",
+                        error_code!(E0766),
+                    )
+                }
+                (token::ByteStr, Mode::ByteStr, 2, 1) // b" "
+            }
+            rustc_lexer::LiteralKind::RawStr { n_hashes } => {
+                if let Some(n_hashes) = n_hashes {
+                    let n = u32::from(n_hashes);
+                    (token::StrRaw(n_hashes), Mode::RawStr, 2 + n, 1 + n) // r##" "##
+                } else {
+                    self.report_raw_str_error(start, 1);
+                }
+            }
+            rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
+                if let Some(n_hashes) = n_hashes {
+                    let n = u32::from(n_hashes);
+                    (token::ByteStrRaw(n_hashes), Mode::RawByteStr, 3 + n, 1 + n) // br##" "##
+                } else {
+                    self.report_raw_str_error(start, 2);
+                }
+            }
+            rustc_lexer::LiteralKind::Int { base, empty_int } => {
+                return if empty_int {
+                    self.sess
+                        .span_diagnostic
+                        .struct_span_err_with_code(
+                            self.mk_sp(start, suffix_start),
+                            "no valid digits found for number",
+                            error_code!(E0768),
+                        )
+                        .emit();
+                    (token::Integer, sym::integer(0))
+                } else {
+                    self.validate_int_literal(base, start, suffix_start);
+                    (token::Integer, self.symbol_from_to(start, suffix_start))
+                };
+            }
+            rustc_lexer::LiteralKind::Float { base, empty_exponent } => {
+                if empty_exponent {
+                    self.err_span_(start, self.pos, "expected at least one digit in exponent");
+                }
+
+                match base {
+                    Base::Hexadecimal => self.err_span_(
+                        start,
+                        suffix_start,
+                        "hexadecimal float literal is not supported",
+                    ),
+                    Base::Octal => {
+                        self.err_span_(start, suffix_start, "octal float literal is not supported")
+                    }
+                    Base::Binary => {
+                        self.err_span_(start, suffix_start, "binary float literal is not supported")
+                    }
+                    _ => (),
+                }
+
+                let id = self.symbol_from_to(start, suffix_start);
+                return (token::Float, id);
+            }
+        };
+        let content_start = start + BytePos(prefix_len);
+        let content_end = suffix_start - BytePos(postfix_len);
+        let id = self.symbol_from_to(content_start, content_end);
+        self.validate_literal_escape(mode, content_start, content_end, prefix_len, postfix_len);
+        (lit_kind, id)
+    }
+
+    #[inline]
+    fn src_index(&self, pos: BytePos) -> usize {
+        (pos - self.start_pos).to_usize()
+    }
+
+    /// Slice of the source text from `start` up to but excluding `self.pos`,
+    /// meaning the slice does not include the character `self.ch`.
+    fn str_from(&self, start: BytePos) -> &str {
+        self.str_from_to(start, self.pos)
+    }
+
+    /// As symbol_from, with an explicit endpoint.
+    fn symbol_from_to(&self, start: BytePos, end: BytePos) -> Symbol {
+        debug!("taking an ident from {:?} to {:?}", start, end);
+        Symbol::intern(self.str_from_to(start, end))
+    }
+
+    /// Slice of the source text spanning from `start` up to but excluding `end`.
+    fn str_from_to(&self, start: BytePos, end: BytePos) -> &str {
+        &self.src[self.src_index(start)..self.src_index(end)]
+    }
+
+    fn report_raw_str_error(&self, start: BytePos, prefix_len: u32) -> ! {
+        match rustc_lexer::validate_raw_str(self.str_from(start), prefix_len) {
+            Err(RawStrError::InvalidStarter { bad_char }) => {
+                self.report_non_started_raw_string(start, bad_char)
+            }
+            Err(RawStrError::NoTerminator { expected, found, possible_terminator_offset }) => self
+                .report_unterminated_raw_string(start, expected, possible_terminator_offset, found),
+            Err(RawStrError::TooManyDelimiters { found }) => {
+                self.report_too_many_hashes(start, found)
+            }
+            Ok(()) => panic!("no error found for supposedly invalid raw string literal"),
+        }
+    }
+
+    fn report_non_started_raw_string(&self, start: BytePos, bad_char: char) -> ! {
+        self.struct_fatal_span_char(
+            start,
+            self.pos,
+            "found invalid character; only `#` is allowed in raw string delimitation",
+            bad_char,
+        )
+        .emit()
+    }
+
+    fn report_unterminated_raw_string(
+        &self,
+        start: BytePos,
+        n_hashes: u32,
+        possible_offset: Option<u32>,
+        found_terminators: u32,
+    ) -> ! {
+        let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code(
+            self.mk_sp(start, start),
+            "unterminated raw string",
+            error_code!(E0748),
+        );
+
+        err.span_label(self.mk_sp(start, start), "unterminated raw string");
+
+        if n_hashes > 0 {
+            err.note(&format!(
+                "this raw string should be terminated with `\"{}`",
+                "#".repeat(n_hashes as usize)
+            ));
+        }
+
+        if let Some(possible_offset) = possible_offset {
+            let lo = start + BytePos(possible_offset as u32);
+            let hi = lo + BytePos(found_terminators as u32);
+            let span = self.mk_sp(lo, hi);
+            err.span_suggestion(
+                span,
+                "consider terminating the string here",
+                "#".repeat(n_hashes as usize),
+                Applicability::MaybeIncorrect,
+            );
+        }
+
+        err.emit()
+    }
+
+    fn report_unterminated_block_comment(&self, start: BytePos, doc_style: Option<DocStyle>) {
+        let msg = match doc_style {
+            Some(_) => "unterminated block doc-comment",
+            None => "unterminated block comment",
+        };
+        let last_bpos = self.pos;
+        let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code(
+            self.mk_sp(start, last_bpos),
+            msg,
+            error_code!(E0758),
+        );
+        let mut nested_block_comment_open_idxs = vec![];
+        let mut last_nested_block_comment_idxs = None;
+        let mut content_chars = self.str_from(start).char_indices().peekable();
+
+        while let Some((idx, current_char)) = content_chars.next() {
+            match content_chars.peek() {
+                Some((_, '*')) if current_char == '/' => {
+                    nested_block_comment_open_idxs.push(idx);
+                }
+                Some((_, '/')) if current_char == '*' => {
+                    last_nested_block_comment_idxs =
+                        nested_block_comment_open_idxs.pop().map(|open_idx| (open_idx, idx));
+                }
+                _ => {}
+            };
+        }
+
+        if let Some((nested_open_idx, nested_close_idx)) = last_nested_block_comment_idxs {
+            err.span_label(self.mk_sp(start, start + BytePos(2)), msg)
+                .span_label(
+                    self.mk_sp(
+                        start + BytePos(nested_open_idx as u32),
+                        start + BytePos(nested_open_idx as u32 + 2),
+                    ),
+                    "...as last nested comment starts here, maybe you want to close this instead?",
+                )
+                .span_label(
+                    self.mk_sp(
+                        start + BytePos(nested_close_idx as u32),
+                        start + BytePos(nested_close_idx as u32 + 2),
+                    ),
+                    "...and last nested comment terminates here.",
+                );
+        }
+
+        err.emit();
+    }
+
+    // RFC 3101 introduced the idea of (reserved) prefixes. As of Rust 2021,
+    // using a (unknown) prefix is an error. In earlier editions, however, they
+    // only result in a (allowed by default) lint, and are treated as regular
+    // identifier tokens.
+    fn report_unknown_prefix(&self, start: BytePos) {
+        let prefix_span = self.mk_sp(start, self.pos);
+        let prefix_str = self.str_from_to(start, self.pos);
+        let msg = format!("prefix `{}` is unknown", prefix_str);
+
+        let expn_data = prefix_span.ctxt().outer_expn_data();
+
+        if expn_data.edition >= Edition::Edition2021 {
+            // In Rust 2021, this is a hard error.
+            let mut err = self.sess.span_diagnostic.struct_span_err(prefix_span, &msg);
+            err.span_label(prefix_span, "unknown prefix");
+            if prefix_str == "rb" {
+                err.span_suggestion_verbose(
+                    prefix_span,
+                    "use `br` for a raw byte string",
+                    "br",
+                    Applicability::MaybeIncorrect,
+                );
+            } else if expn_data.is_root() {
+                err.span_suggestion_verbose(
+                    prefix_span.shrink_to_hi(),
+                    "consider inserting whitespace here",
+                    " ",
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            err.note("prefixed identifiers and literals are reserved since Rust 2021");
+            err.emit();
+        } else {
+            // Before Rust 2021, only emit a lint for migration.
+            self.sess.buffer_lint_with_diagnostic(
+                &RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
+                prefix_span,
+                ast::CRATE_NODE_ID,
+                &msg,
+                BuiltinLintDiagnostics::ReservedPrefix(prefix_span),
+            );
+        }
+    }
+
+    fn report_too_many_hashes(&self, start: BytePos, found: u32) -> ! {
+        self.fatal_span_(
+            start,
+            self.pos,
+            &format!(
+                "too many `#` symbols: raw strings may be delimited \
+                by up to 255 `#` symbols, but found {}",
+                found
+            ),
+        )
+    }
+
+    fn validate_literal_escape(
+        &self,
+        mode: Mode,
+        content_start: BytePos,
+        content_end: BytePos,
+        prefix_len: u32,
+        postfix_len: u32,
+    ) {
+        let lit_content = self.str_from_to(content_start, content_end);
+        unescape::unescape_literal(lit_content, mode, &mut |range, result| {
+            // Here we only check for errors. The actual unescaping is done later.
+            if let Err(err) = result {
+                let span_with_quotes = self
+                    .mk_sp(content_start - BytePos(prefix_len), content_end + BytePos(postfix_len));
+                let (start, end) = (range.start as u32, range.end as u32);
+                let lo = content_start + BytePos(start);
+                let hi = lo + BytePos(end - start);
+                let span = self.mk_sp(lo, hi);
+                emit_unescape_error(
+                    &self.sess.span_diagnostic,
+                    lit_content,
+                    span_with_quotes,
+                    span,
+                    mode,
+                    range,
+                    err,
+                );
+            }
+        });
+    }
+
+    fn validate_int_literal(&self, base: Base, content_start: BytePos, content_end: BytePos) {
+        let base = match base {
+            Base::Binary => 2,
+            Base::Octal => 8,
+            _ => return,
+        };
+        let s = self.str_from_to(content_start + BytePos(2), content_end);
+        for (idx, c) in s.char_indices() {
+            let idx = idx as u32;
+            if c != '_' && c.to_digit(base).is_none() {
+                let lo = content_start + BytePos(2 + idx);
+                let hi = content_start + BytePos(2 + idx + c.len_utf8() as u32);
+                self.err_span_(lo, hi, &format!("invalid digit for a base {} literal", base));
+            }
+        }
+    }
+}
+
+pub fn nfc_normalize(string: &str) -> Symbol {
+    use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
+    match is_nfc_quick(string.chars()) {
+        IsNormalized::Yes => Symbol::intern(string),
+        _ => {
+            let normalized_str: String = string.chars().nfc().collect();
+            Symbol::intern(&normalized_str)
+        }
+    }
+}
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
new file mode 100644
index 000000000..aa70912dc
--- /dev/null
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -0,0 +1,296 @@
+use super::{StringReader, UnmatchedBrace};
+
+use rustc_ast::token::{self, Delimiter, Token};
+use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
+use rustc_ast_pretty::pprust::token_to_string;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_errors::PResult;
+use rustc_span::Span;
+
+impl<'a> StringReader<'a> {
+    pub(super) fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
+        let mut tt_reader = TokenTreesReader {
+            string_reader: self,
+            token: Token::dummy(),
+            open_braces: Vec::new(),
+            unmatched_braces: Vec::new(),
+            matching_delim_spans: Vec::new(),
+            last_unclosed_found_span: None,
+            last_delim_empty_block_spans: FxHashMap::default(),
+            matching_block_spans: Vec::new(),
+        };
+        let res = tt_reader.parse_all_token_trees();
+        (res, tt_reader.unmatched_braces)
+    }
+}
+
+struct TokenTreesReader<'a> {
+    string_reader: StringReader<'a>,
+    token: Token,
+    /// Stack of open delimiters and their spans. Used for error message.
+    open_braces: Vec<(Delimiter, Span)>,
+    unmatched_braces: Vec<UnmatchedBrace>,
+    /// The type and spans for all braces
+    ///
+    /// Used only for error recovery when arriving to EOF with mismatched braces.
+    matching_delim_spans: Vec<(Delimiter, Span, Span)>,
+    last_unclosed_found_span: Option<Span>,
+    /// Collect empty block spans that might have been auto-inserted by editors.
+    last_delim_empty_block_spans: FxHashMap<Delimiter, Span>,
+    /// Collect the spans of braces (Open, Close). Used only
+    /// for detecting if blocks are empty and only braces.
+    matching_block_spans: Vec<(Span, Span)>,
+}
+
+impl<'a> TokenTreesReader<'a> {
+    // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
+    fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
+        let mut buf = TokenStreamBuilder::default();
+
+        self.bump();
+        while self.token != token::Eof {
+            buf.push(self.parse_token_tree()?);
+        }
+
+        Ok(buf.into_token_stream())
+    }
+
+    // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
+    fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
+        let mut buf = TokenStreamBuilder::default();
+        loop {
+            if let token::CloseDelim(..) = self.token.kind {
+                return buf.into_token_stream();
+            }
+
+            match self.parse_token_tree() {
+                Ok(tree) => buf.push(tree),
+                Err(mut e) => {
+                    e.emit();
+                    return buf.into_token_stream();
+                }
+            }
+        }
+    }
+
+    fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
+        let sm = self.string_reader.sess.source_map();
+
+        match self.token.kind {
+            token::Eof => {
+                let msg = "this file contains an unclosed delimiter";
+                let mut err =
+                    self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
+                for &(_, sp) in &self.open_braces {
+                    err.span_label(sp, "unclosed delimiter");
+                    self.unmatched_braces.push(UnmatchedBrace {
+                        expected_delim: Delimiter::Brace,
+                        found_delim: None,
+                        found_span: self.token.span,
+                        unclosed_span: Some(sp),
+                        candidate_span: None,
+                    });
+                }
+
+                if let Some((delim, _)) = self.open_braces.last() {
+                    if let Some((_, open_sp, close_sp)) =
+                        self.matching_delim_spans.iter().find(|(d, open_sp, close_sp)| {
+                            if let Some(close_padding) = sm.span_to_margin(*close_sp) {
+                                if let Some(open_padding) = sm.span_to_margin(*open_sp) {
+                                    return delim == d && close_padding != open_padding;
+                                }
+                            }
+                            false
+                        })
+                    // these are in reverse order as they get inserted on close, but
+                    {
+                        // we want the last open/first close
+                        err.span_label(*open_sp, "this delimiter might not be properly closed...");
+                        err.span_label(
+                            *close_sp,
+                            "...as it matches this but it has different indentation",
+                        );
+                    }
+                }
+                Err(err)
+            }
+            token::OpenDelim(delim) => {
+                // The span for beginning of the delimited section
+                let pre_span = self.token.span;
+
+                // Parse the open delimiter.
+                self.open_braces.push((delim, self.token.span));
+                self.bump();
+
+                // Parse the token trees within the delimiters.
+                // We stop at any delimiter so we can try to recover if the user
+                // uses an incorrect delimiter.
+                let tts = self.parse_token_trees_until_close_delim();
+
+                // Expand to cover the entire delimited token tree
+                let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
+
+                match self.token.kind {
+                    // Correct delimiter.
+                    token::CloseDelim(d) if d == delim => {
+                        let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
+                        let close_brace_span = self.token.span;
+
+                        if tts.is_empty() {
+                            let empty_block_span = open_brace_span.to(close_brace_span);
+                            if !sm.is_multiline(empty_block_span) {
+                                // Only track if the block is in the form of `{}`, otherwise it is
+                                // likely that it was written on purpose.
+                                self.last_delim_empty_block_spans.insert(delim, empty_block_span);
+                            }
+                        }
+
+                        //only add braces
+                        if let (Delimiter::Brace, Delimiter::Brace) = (open_brace, delim) {
+                            self.matching_block_spans.push((open_brace_span, close_brace_span));
+                        }
+
+                        if self.open_braces.is_empty() {
+                            // Clear up these spans to avoid suggesting them as we've found
+                            // properly matched delimiters so far for an entire block.
+                            self.matching_delim_spans.clear();
+                        } else {
+                            self.matching_delim_spans.push((
+                                open_brace,
+                                open_brace_span,
+                                close_brace_span,
+                            ));
+                        }
+                        // Parse the closing delimiter.
+                        self.bump();
+                    }
+                    // Incorrect delimiter.
+                    token::CloseDelim(other) => {
+                        let mut unclosed_delimiter = None;
+                        let mut candidate = None;
+
+                        if self.last_unclosed_found_span != Some(self.token.span) {
+                            // do not complain about the same unclosed delimiter multiple times
+                            self.last_unclosed_found_span = Some(self.token.span);
+                            // This is a conservative error: only report the last unclosed
+                            // delimiter. The previous unclosed delimiters could actually be
+                            // closed! The parser just hasn't gotten to them yet.
+                            if let Some(&(_, sp)) = self.open_braces.last() {
+                                unclosed_delimiter = Some(sp);
+                            };
+                            if let Some(current_padding) = sm.span_to_margin(self.token.span) {
+                                for (brace, brace_span) in &self.open_braces {
+                                    if let Some(padding) = sm.span_to_margin(*brace_span) {
+                                        // high likelihood of these two corresponding
+                                        if current_padding == padding && brace == &other {
+                                            candidate = Some(*brace_span);
+                                        }
+                                    }
+                                }
+                            }
+                            let (tok, _) = self.open_braces.pop().unwrap();
+                            self.unmatched_braces.push(UnmatchedBrace {
+                                expected_delim: tok,
+                                found_delim: Some(other),
+                                found_span: self.token.span,
+                                unclosed_span: unclosed_delimiter,
+                                candidate_span: candidate,
+                            });
+                        } else {
+                            self.open_braces.pop();
+                        }
+
+                        // If the incorrect delimiter matches an earlier opening
+                        // delimiter, then don't consume it (it can be used to
+                        // close the earlier one). Otherwise, consume it.
+                        // E.g., we try to recover from:
+                        // fn foo() {
+                        //     bar(baz(
+                        // }  // Incorrect delimiter but matches the earlier `{`
+                        if !self.open_braces.iter().any(|&(b, _)| b == other) {
+                            self.bump();
+                        }
+                    }
+                    token::Eof => {
+                        // Silently recover, the EOF token will be seen again
+                        // and an error emitted then. Thus we don't pop from
+                        // self.open_braces here.
+                    }
+                    _ => {}
+                }
+
+                Ok(TokenTree::Delimited(delim_span, delim, tts))
+            }
+            token::CloseDelim(delim) => {
+                // An unexpected closing delimiter (i.e., there is no
+                // matching opening delimiter).
+                let token_str = token_to_string(&self.token);
+                let msg = format!("unexpected closing delimiter: `{}`", token_str);
+                let mut err =
+                    self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
+
+                // Braces are added at the end, so the last element is the biggest block
+                if let Some(parent) = self.matching_block_spans.last() {
+                    if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
+                        // Check if the (empty block) is in the last properly closed block
+                        if (parent.0.to(parent.1)).contains(span) {
+                            err.span_label(
+                                span,
+                                "block is empty, you might have not meant to close it",
+                            );
+                        } else {
+                            err.span_label(parent.0, "this opening brace...");
+
+                            err.span_label(parent.1, "...matches this closing brace");
+                        }
+                    } else {
+                        err.span_label(parent.0, "this opening brace...");
+
+                        err.span_label(parent.1, "...matches this closing brace");
+                    }
+                }
+
+                err.span_label(self.token.span, "unexpected closing delimiter");
+                Err(err)
+            }
+            _ => {
+                let tok = self.token.take();
+                let mut spacing = self.bump();
+                if !self.token.is_op() {
+                    spacing = Spacing::Alone;
+                }
+                Ok(TokenTree::Token(tok, spacing))
+            }
+        }
+    }
+
+    fn bump(&mut self) -> Spacing {
+        let (spacing, token) = self.string_reader.next_token();
+        self.token = token;
+        spacing
+    }
+}
+
+#[derive(Default)]
+struct TokenStreamBuilder {
+    buf: Vec<TokenTree>,
+}
+
+impl TokenStreamBuilder {
+    #[inline(always)]
+    fn push(&mut self, tree: TokenTree) {
+        if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
+            && let TokenTree::Token(token, joint) = &tree
+            && let Some(glued) = prev_token.glue(token)
+        {
+            self.buf.pop();
+            self.buf.push(TokenTree::Token(glued, *joint));
+        } else {
+            self.buf.push(tree)
+        }
+    }
+
+    fn into_token_stream(self) -> TokenStream {
+        TokenStream::new(self.buf)
+    }
+}
diff --git a/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
new file mode 100644
index 000000000..273827864
--- /dev/null
+++ b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
@@ -0,0 +1,381 @@
+//! Utilities for rendering escape sequence errors as diagnostics.
+
+use std::iter::once;
+use std::ops::Range;
+
+use rustc_errors::{pluralize, Applicability, Handler};
+use rustc_lexer::unescape::{EscapeError, Mode};
+use rustc_span::{BytePos, Span};
+
+pub(crate) fn emit_unescape_error(
+    handler: &Handler,
+    // interior part of the literal, without quotes
+    lit: &str,
+    // full span of the literal, including quotes
+    span_with_quotes: Span,
+    // interior span of the literal, without quotes
+    span: Span,
+    mode: Mode,
+    // range of the error inside `lit`
+    range: Range<usize>,
+    error: EscapeError,
+) {
+    tracing::debug!(
+        "emit_unescape_error: {:?}, {:?}, {:?}, {:?}, {:?}",
+        lit,
+        span_with_quotes,
+        mode,
+        range,
+        error
+    );
+    let last_char = || {
+        let c = lit[range.clone()].chars().rev().next().unwrap();
+        let span = span.with_lo(span.hi() - BytePos(c.len_utf8() as u32));
+        (c, span)
+    };
+    match error {
+        EscapeError::LoneSurrogateUnicodeEscape => {
+            handler
+                .struct_span_err(span, "invalid unicode character escape")
+                .span_label(span, "invalid escape")
+                .help("unicode escape must not be a surrogate")
+                .emit();
+        }
+        EscapeError::OutOfRangeUnicodeEscape => {
+            handler
+                .struct_span_err(span, "invalid unicode character escape")
+                .span_label(span, "invalid escape")
+                .help("unicode escape must be at most 10FFFF")
+                .emit();
+        }
+        EscapeError::MoreThanOneChar => {
+            use unicode_normalization::{char::is_combining_mark, UnicodeNormalization};
+
+            let mut has_help = false;
+            let mut handler = handler.struct_span_err(
+                span_with_quotes,
+                "character literal may only contain one codepoint",
+            );
+
+            if lit.chars().skip(1).all(|c| is_combining_mark(c)) {
+                let escaped_marks =
+                    lit.chars().skip(1).map(|c| c.escape_default().to_string()).collect::<Vec<_>>();
+                handler.span_note(
+                    span,
+                    &format!(
+                        "this `{}` is followed by the combining mark{} `{}`",
+                        lit.chars().next().unwrap(),
+                        pluralize!(escaped_marks.len()),
+                        escaped_marks.join(""),
+                    ),
+                );
+                let normalized = lit.nfc().to_string();
+                if normalized.chars().count() == 1 {
+                    has_help = true;
+                    handler.span_suggestion(
+                        span,
+                        &format!(
+                            "consider using the normalized form `{}` of this character",
+                            normalized.chars().next().unwrap().escape_default()
+                        ),
+                        normalized,
+                        Applicability::MachineApplicable,
+                    );
+                }
+            } else {
+                let printable: Vec<char> = lit
+                    .chars()
+                    .filter(|&x| {
+                        unicode_width::UnicodeWidthChar::width(x).unwrap_or(0) != 0
+                            && !x.is_whitespace()
+                    })
+                    .collect();
+
+                if let [ch] = printable.as_slice() {
+                    has_help = true;
+
+                    handler.span_note(
+                        span,
+                        &format!(
+                            "there are non-printing characters, the full sequence is `{}`",
+                            lit.escape_default(),
+                        ),
+                    );
+
+                    handler.span_suggestion(
+                        span,
+                        "consider removing the non-printing characters",
+                        ch,
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+            }
+
+            if !has_help {
+                let (prefix, msg) = if mode.is_bytes() {
+                    ("b", "if you meant to write a byte string literal, use double quotes")
+                } else {
+                    ("", "if you meant to write a `str` literal, use double quotes")
+                };
+
+                handler.span_suggestion(
+                    span_with_quotes,
+                    msg,
+                    format!("{}\"{}\"", prefix, lit),
+                    Applicability::MachineApplicable,
+                );
+            }
+
+            handler.emit();
+        }
+        EscapeError::EscapeOnlyChar => {
+            let (c, char_span) = last_char();
+
+            let msg = if mode.is_bytes() {
+                "byte constant must be escaped"
+            } else {
+                "character constant must be escaped"
+            };
+            handler
+                .struct_span_err(span, &format!("{}: `{}`", msg, escaped_char(c)))
+                .span_suggestion(
+                    char_span,
+                    "escape the character",
+                    c.escape_default(),
+                    Applicability::MachineApplicable,
+                )
+                .emit();
+        }
+        EscapeError::BareCarriageReturn => {
+            let msg = if mode.in_double_quotes() {
+                "bare CR not allowed in string, use `\\r` instead"
+            } else {
+                "character constant must be escaped: `\\r`"
+            };
+            handler
+                .struct_span_err(span, msg)
+                .span_suggestion(
+                    span,
+                    "escape the character",
+                    "\\r",
+                    Applicability::MachineApplicable,
+                )
+                .emit();
+        }
+        EscapeError::BareCarriageReturnInRawString => {
+            assert!(mode.in_double_quotes());
+            let msg = "bare CR not allowed in raw string";
+            handler.span_err(span, msg);
+        }
+        EscapeError::InvalidEscape => {
+            let (c, span) = last_char();
+
+            let label =
+                if mode.is_bytes() { "unknown byte escape" } else { "unknown character escape" };
+            let ec = escaped_char(c);
+            let mut diag = handler.struct_span_err(span, &format!("{}: `{}`", label, ec));
+            diag.span_label(span, label);
+            if c == '{' || c == '}' && !mode.is_bytes() {
+                diag.help(
+                    "if used in a formatting string, curly braces are escaped with `{{` and `}}`",
+                );
+            } else if c == '\r' {
+                diag.help(
+                    "this is an isolated carriage return; consider checking your editor and \
+                     version control settings",
+                );
+            } else {
+                if !mode.is_bytes() {
+                    diag.span_suggestion(
+                        span_with_quotes,
+                        "if you meant to write a literal backslash (perhaps escaping in a regular expression), consider a raw string literal",
+                        format!("r\"{}\"", lit),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+
+                diag.help(
+                    "for more information, visit \
+                     <https://static.rust-lang.org/doc/master/reference.html#literals>",
+                );
+            }
+            diag.emit();
+        }
+        EscapeError::TooShortHexEscape => {
+            handler.span_err(span, "numeric character escape is too short");
+        }
+        EscapeError::InvalidCharInHexEscape | EscapeError::InvalidCharInUnicodeEscape => {
+            let (c, span) = last_char();
+
+            let msg = if error == EscapeError::InvalidCharInHexEscape {
+                "invalid character in numeric character escape"
+            } else {
+                "invalid character in unicode escape"
+            };
+            let c = escaped_char(c);
+
+            handler
+                .struct_span_err(span, &format!("{}: `{}`", msg, c))
+                .span_label(span, msg)
+                .emit();
+        }
+        EscapeError::NonAsciiCharInByte => {
+            assert!(mode.is_bytes());
+            let (c, span) = last_char();
+            let mut err = handler.struct_span_err(span, "non-ASCII character in byte constant");
+            let postfix = if unicode_width::UnicodeWidthChar::width(c).unwrap_or(1) == 0 {
+                format!(" but is {:?}", c)
+            } else {
+                String::new()
+            };
+            err.span_label(span, &format!("byte constant must be ASCII{}", postfix));
+            if (c as u32) <= 0xFF {
+                err.span_suggestion(
+                    span,
+                    &format!(
+                        "if you meant to use the unicode code point for {:?}, use a \\xHH escape",
+                        c
+                    ),
+                    format!("\\x{:X}", c as u32),
+                    Applicability::MaybeIncorrect,
+                );
+            } else if matches!(mode, Mode::Byte) {
+                err.span_label(span, "this multibyte character does not fit into a single byte");
+            } else if matches!(mode, Mode::ByteStr) {
+                let mut utf8 = String::new();
+                utf8.push(c);
+                err.span_suggestion(
+                    span,
+                    &format!(
+                        "if you meant to use the UTF-8 encoding of {:?}, use \\xHH escapes",
+                        c
+                    ),
+                    utf8.as_bytes()
+                        .iter()
+                        .map(|b: &u8| format!("\\x{:X}", *b))
+                        .fold("".to_string(), |a, c| a + &c),
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            err.emit();
+        }
+        EscapeError::NonAsciiCharInByteString => {
+            assert!(mode.is_bytes());
+            let (c, span) = last_char();
+            let postfix = if unicode_width::UnicodeWidthChar::width(c).unwrap_or(1) == 0 {
+                format!(" but is {:?}", c)
+            } else {
+                String::new()
+            };
+            handler
+                .struct_span_err(span, "raw byte string must be ASCII")
+                .span_label(span, &format!("must be ASCII{}", postfix))
+                .emit();
+        }
+        EscapeError::OutOfRangeHexEscape => {
+            handler
+                .struct_span_err(span, "out of range hex escape")
+                .span_label(span, "must be a character in the range [\\x00-\\x7f]")
+                .emit();
+        }
+        EscapeError::LeadingUnderscoreUnicodeEscape => {
+            let (c, span) = last_char();
+            let msg = "invalid start of unicode escape";
+            handler
+                .struct_span_err(span, &format!("{}: `{}`", msg, c))
+                .span_label(span, msg)
+                .emit();
+        }
+        EscapeError::OverlongUnicodeEscape => {
+            handler
+                .struct_span_err(span, "overlong unicode escape")
+                .span_label(span, "must have at most 6 hex digits")
+                .emit();
+        }
+        EscapeError::UnclosedUnicodeEscape => {
+            handler
+                .struct_span_err(span, "unterminated unicode escape")
+                .span_label(span, "missing a closing `}`")
+                .span_suggestion_verbose(
+                    span.shrink_to_hi(),
+                    "terminate the unicode escape",
+                    "}",
+                    Applicability::MaybeIncorrect,
+                )
+                .emit();
+        }
+        EscapeError::NoBraceInUnicodeEscape => {
+            let msg = "incorrect unicode escape sequence";
+            let mut diag = handler.struct_span_err(span, msg);
+
+            let mut suggestion = "\\u{".to_owned();
+            let mut suggestion_len = 0;
+            let (c, char_span) = last_char();
+            let chars = once(c).chain(lit[range.end..].chars());
+            for c in chars.take(6).take_while(|c| c.is_digit(16)) {
+                suggestion.push(c);
+                suggestion_len += c.len_utf8();
+            }
+
+            if suggestion_len > 0 {
+                suggestion.push('}');
+                let hi = char_span.lo() + BytePos(suggestion_len as u32);
+                diag.span_suggestion(
+                    span.with_hi(hi),
+                    "format of unicode escape sequences uses braces",
+                    suggestion,
+                    Applicability::MaybeIncorrect,
+                );
+            } else {
+                diag.span_label(span, msg);
+                diag.help("format of unicode escape sequences is `\\u{...}`");
+            }
+
+            diag.emit();
+        }
+        EscapeError::UnicodeEscapeInByte => {
+            let msg = "unicode escape in byte string";
+            handler
+                .struct_span_err(span, msg)
+                .span_label(span, msg)
+                .help("unicode escape sequences cannot be used as a byte or in a byte string")
+                .emit();
+        }
+        EscapeError::EmptyUnicodeEscape => {
+            handler
+                .struct_span_err(span, "empty unicode escape")
+                .span_label(span, "this escape must have at least 1 hex digit")
+                .emit();
+        }
+        EscapeError::ZeroChars => {
+            let msg = "empty character literal";
+            handler.struct_span_err(span, msg).span_label(span, msg).emit();
+        }
+        EscapeError::LoneSlash => {
+            let msg = "invalid trailing slash in literal";
+            handler.struct_span_err(span, msg).span_label(span, msg).emit();
+        }
+        EscapeError::UnskippedWhitespaceWarning => {
+            let (c, char_span) = last_char();
+            let msg =
+                format!("non-ASCII whitespace symbol '{}' is not skipped", c.escape_unicode());
+            handler.struct_span_warn(span, &msg).span_label(char_span, &msg).emit();
+        }
+        EscapeError::MultipleSkippedLinesWarning => {
+            let msg = "multiple lines skipped by escaped newline";
+            let bottom_msg = "skipping everything up to and including this point";
+            handler.struct_span_warn(span, msg).span_label(span, bottom_msg).emit();
+        }
+    }
+}
+
+/// Pushes a character to a message string for error reporting
+pub(crate) fn escaped_char(c: char) -> String {
+    match c {
+        '\u{20}'..='\u{7e}' => {
+            // Don't escape \, ' or " for user-facing messages
+            c.to_string()
+        }
+        _ => c.escape_default().to_string(),
+    }
+}
diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs
new file mode 100644
index 000000000..2c68cc589
--- /dev/null
+++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs
@@ -0,0 +1,386 @@
+// Characters and their corresponding confusables were collected from
+// https://www.unicode.org/Public/security/10.0.0/confusables.txt
+
+use super::StringReader;
+use crate::token::{self, Delimiter};
+use rustc_errors::{Applicability, Diagnostic};
+use rustc_span::{symbol::kw, BytePos, Pos, Span};
+
+#[rustfmt::skip] // for line breaks
+pub(crate) const UNICODE_ARRAY: &[(char, &str, char)] = &[
+    ('
', "Line Separator", ' '),
+    ('
', "Paragraph Separator", ' '),
+    (' ', "Ogham Space mark", ' '),
+    (' ', "En Quad", ' '),
+    (' ', "Em Quad", ' '),
+    (' ', "En Space", ' '),
+    (' ', "Em Space", ' '),
+    (' ', "Three-Per-Em Space", ' '),
+    (' ', "Four-Per-Em Space", ' '),
+    (' ', "Six-Per-Em Space", ' '),
+    (' ', "Punctuation Space", ' '),
+    (' ', "Thin Space", ' '),
+    (' ', "Hair Space", ' '),
+    (' ', "Medium Mathematical Space", ' '),
+    (' ', "No-Break Space", ' '),
+    (' ', "Figure Space", ' '),
+    (' ', "Narrow No-Break Space", ' '),
+    ('　', "Ideographic Space", ' '),
+
+    ('ߺ', "Nko Lajanyalan", '_'),
+    ('﹍', "Dashed Low Line", '_'),
+    ('﹎', "Centreline Low Line", '_'),
+    ('﹏', "Wavy Low Line", '_'),
+    ('＿', "Fullwidth Low Line", '_'),
+
+    ('‐', "Hyphen", '-'),
+    ('‑', "Non-Breaking Hyphen", '-'),
+    ('‒', "Figure Dash", '-'),
+    ('–', "En Dash", '-'),
+    ('—', "Em Dash", '-'),
+    ('﹘', "Small Em Dash", '-'),
+    ('۔', "Arabic Full Stop", '-'),
+    ('⁃', "Hyphen Bullet", '-'),
+    ('˗', "Modifier Letter Minus Sign", '-'),
+    ('−', "Minus Sign", '-'),
+    ('➖', "Heavy Minus Sign", '-'),
+    ('Ⲻ', "Coptic Letter Dialect-P Ni", '-'),
+    ('ー', "Katakana-Hiragana Prolonged Sound Mark", '-'),
+    ('－', "Fullwidth Hyphen-Minus", '-'),
+    ('―', "Horizontal Bar", '-'),
+    ('─', "Box Drawings Light Horizontal", '-'),
+    ('━', "Box Drawings Heavy Horizontal", '-'),
+    ('㇐', "CJK Stroke H", '-'),
+    ('ꟷ', "Latin Epigraphic Letter Sideways I", '-'),
+    ('ᅳ', "Hangul Jungseong Eu", '-'),
+    ('ㅡ', "Hangul Letter Eu", '-'),
+    ('一', "CJK Unified Ideograph-4E00", '-'),
+    ('⼀', "Kangxi Radical One", '-'),
+
+    ('؍', "Arabic Date Separator", ','),
+    ('٫', "Arabic Decimal Separator", ','),
+    ('‚', "Single Low-9 Quotation Mark", ','),
+    ('¸', "Cedilla", ','),
+    ('ꓹ', "Lisu Letter Tone Na Po", ','),
+    ('，', "Fullwidth Comma", ','),
+
+    (';', "Greek Question Mark", ';'),
+    ('；', "Fullwidth Semicolon", ';'),
+    ('︔', "Presentation Form For Vertical Semicolon", ';'),
+
+    ('ः', "Devanagari Sign Visarga", ':'),
+    ('ઃ', "Gujarati Sign Visarga", ':'),
+    ('：', "Fullwidth Colon", ':'),
+    ('։', "Armenian Full Stop", ':'),
+    ('܃', "Syriac Supralinear Colon", ':'),
+    ('܄', "Syriac Sublinear Colon", ':'),
+    ('᛬', "Runic Multiple Punctuation", ':'),
+    ('︰', "Presentation Form For Vertical Two Dot Leader", ':'),
+    ('᠃', "Mongolian Full Stop", ':'),
+    ('᠉', "Mongolian Manchu Full Stop", ':'),
+    ('⁚', "Two Dot Punctuation", ':'),
+    ('׃', "Hebrew Punctuation Sof Pasuq", ':'),
+    ('˸', "Modifier Letter Raised Colon", ':'),
+    ('꞉', "Modifier Letter Colon", ':'),
+    ('∶', "Ratio", ':'),
+    ('ː', "Modifier Letter Triangular Colon", ':'),
+    ('ꓽ', "Lisu Letter Tone Mya Jeu", ':'),
+    ('︓', "Presentation Form For Vertical Colon", ':'),
+
+    ('！', "Fullwidth Exclamation Mark", '!'),
+    ('ǃ', "Latin Letter Retroflex Click", '!'),
+    ('ⵑ', "Tifinagh Letter Tuareg Yang", '!'),
+    ('︕', "Presentation Form For Vertical Exclamation Mark", '!'),
+
+    ('ʔ', "Latin Letter Glottal Stop", '?'),
+    ('Ɂ', "Latin Capital Letter Glottal Stop", '?'),
+    ('ॽ', "Devanagari Letter Glottal Stop", '?'),
+    ('Ꭾ', "Cherokee Letter He", '?'),
+    ('ꛫ', "Bamum Letter Ntuu", '?'),
+    ('？', "Fullwidth Question Mark", '?'),
+    ('︖', "Presentation Form For Vertical Question Mark", '?'),
+
+    ('𝅭', "Musical Symbol Combining Augmentation Dot", '.'),
+    ('․', "One Dot Leader", '.'),
+    ('܁', "Syriac Supralinear Full Stop", '.'),
+    ('܂', "Syriac Sublinear Full Stop", '.'),
+    ('꘎', "Vai Full Stop", '.'),
+    ('𐩐', "Kharoshthi Punctuation Dot", '.'),
+    ('٠', "Arabic-Indic Digit Zero", '.'),
+    ('۰', "Extended Arabic-Indic Digit Zero", '.'),
+    ('ꓸ', "Lisu Letter Tone Mya Ti", '.'),
+    ('·', "Middle Dot", '.'),
+    ('・', "Katakana Middle Dot", '.'),
+    ('･', "Halfwidth Katakana Middle Dot", '.'),
+    ('᛫', "Runic Single Punctuation", '.'),
+    ('·', "Greek Ano Teleia", '.'),
+    ('⸱', "Word Separator Middle Dot", '.'),
+    ('𐄁', "Aegean Word Separator Dot", '.'),
+    ('•', "Bullet", '.'),
+    ('‧', "Hyphenation Point", '.'),
+    ('∙', "Bullet Operator", '.'),
+    ('⋅', "Dot Operator", '.'),
+    ('ꞏ', "Latin Letter Sinological Dot", '.'),
+    ('ᐧ', "Canadian Syllabics Final Middle Dot", '.'),
+    ('ᐧ', "Canadian Syllabics Final Middle Dot", '.'),
+    ('．', "Fullwidth Full Stop", '.'),
+    ('。', "Ideographic Full Stop", '.'),
+    ('︒', "Presentation Form For Vertical Ideographic Full Stop", '.'),
+
+    ('՝', "Armenian Comma", '\''),
+    ('＇', "Fullwidth Apostrophe", '\''),
+    ('‘', "Left Single Quotation Mark", '\''),
+    ('’', "Right Single Quotation Mark", '\''),
+    ('‛', "Single High-Reversed-9 Quotation Mark", '\''),
+    ('′', "Prime", '\''),
+    ('‵', "Reversed Prime", '\''),
+    ('՚', "Armenian Apostrophe", '\''),
+    ('׳', "Hebrew Punctuation Geresh", '\''),
+    ('`', "Grave Accent", '\''),
+    ('`', "Greek Varia", '\''),
+    ('｀', "Fullwidth Grave Accent", '\''),
+    ('´', "Acute Accent", '\''),
+    ('΄', "Greek Tonos", '\''),
+    ('´', "Greek Oxia", '\''),
+    ('᾽', "Greek Koronis", '\''),
+    ('᾿', "Greek Psili", '\''),
+    ('῾', "Greek Dasia", '\''),
+    ('ʹ', "Modifier Letter Prime", '\''),
+    ('ʹ', "Greek Numeral Sign", '\''),
+    ('ˈ', "Modifier Letter Vertical Line", '\''),
+    ('ˊ', "Modifier Letter Acute Accent", '\''),
+    ('ˋ', "Modifier Letter Grave Accent", '\''),
+    ('˴', "Modifier Letter Middle Grave Accent", '\''),
+    ('ʻ', "Modifier Letter Turned Comma", '\''),
+    ('ʽ', "Modifier Letter Reversed Comma", '\''),
+    ('ʼ', "Modifier Letter Apostrophe", '\''),
+    ('ʾ', "Modifier Letter Right Half Ring", '\''),
+    ('ꞌ', "Latin Small Letter Saltillo", '\''),
+    ('י', "Hebrew Letter Yod", '\''),
+    ('ߴ', "Nko High Tone Apostrophe", '\''),
+    ('ߵ', "Nko Low Tone Apostrophe", '\''),
+    ('ᑊ', "Canadian Syllabics West-Cree P", '\''),
+    ('ᛌ', "Runic Letter Short-Twig-Sol S", '\''),
+    ('𖽑', "Miao Sign Aspiration", '\''),
+    ('𖽒', "Miao Sign Reformed Voicing", '\''),
+
+    ('᳓', "Vedic Sign Nihshvasa", '"'),
+    ('＂', "Fullwidth Quotation Mark", '"'),
+    ('“', "Left Double Quotation Mark", '"'),
+    ('”', "Right Double Quotation Mark", '"'),
+    ('‟', "Double High-Reversed-9 Quotation Mark", '"'),
+    ('″', "Double Prime", '"'),
+    ('‶', "Reversed Double Prime", '"'),
+    ('〃', "Ditto Mark", '"'),
+    ('״', "Hebrew Punctuation Gershayim", '"'),
+    ('˝', "Double Acute Accent", '"'),
+    ('ʺ', "Modifier Letter Double Prime", '"'),
+    ('˶', "Modifier Letter Middle Double Acute Accent", '"'),
+    ('˵', "Modifier Letter Middle Double Grave Accent", '"'),
+    ('ˮ', "Modifier Letter Double Apostrophe", '"'),
+    ('ײ', "Hebrew Ligature Yiddish Double Yod", '"'),
+    ('❞', "Heavy Double Comma Quotation Mark Ornament", '"'),
+    ('❝', "Heavy Double Turned Comma Quotation Mark Ornament", '"'),
+
+    ('（', "Fullwidth Left Parenthesis", '('),
+    ('❨', "Medium Left Parenthesis Ornament", '('),
+    ('﴾', "Ornate Left Parenthesis", '('),
+
+    ('）', "Fullwidth Right Parenthesis", ')'),
+    ('❩', "Medium Right Parenthesis Ornament", ')'),
+    ('﴿', "Ornate Right Parenthesis", ')'),
+
+    ('［', "Fullwidth Left Square Bracket", '['),
+    ('❲', "Light Left Tortoise Shell Bracket Ornament", '['),
+    ('「', "Left Corner Bracket", '['),
+    ('『', "Left White Corner Bracket", '['),
+    ('【', "Left Black Lenticular Bracket", '['),
+    ('〔', "Left Tortoise Shell Bracket", '['),
+    ('〖', "Left White Lenticular Bracket", '['),
+    ('〘', "Left White Tortoise Shell Bracket", '['),
+    ('〚', "Left White Square Bracket", '['),
+
+    ('］', "Fullwidth Right Square Bracket", ']'),
+    ('❳', "Light Right Tortoise Shell Bracket Ornament", ']'),
+    ('」', "Right Corner Bracket", ']'),
+    ('』', "Right White Corner Bracket", ']'),
+    ('】', "Right Black Lenticular Bracket", ']'),
+    ('〕', "Right Tortoise Shell Bracket", ']'),
+    ('〗', "Right White Lenticular Bracket", ']'),
+    ('〙', "Right White Tortoise Shell Bracket", ']'),
+    ('〛', "Right White Square Bracket", ']'),
+
+    ('❴', "Medium Left Curly Bracket Ornament", '{'),
+    ('𝄔', "Musical Symbol Brace", '{'),
+    ('｛', "Fullwidth Left Curly Bracket", '{'),
+
+    ('❵', "Medium Right Curly Bracket Ornament", '}'),
+    ('｝', "Fullwidth Right Curly Bracket", '}'),
+
+    ('⁎', "Low Asterisk", '*'),
+    ('٭', "Arabic Five Pointed Star", '*'),
+    ('∗', "Asterisk Operator", '*'),
+    ('𐌟', "Old Italic Letter Ess", '*'),
+    ('＊', "Fullwidth Asterisk", '*'),
+
+    ('᜵', "Philippine Single Punctuation", '/'),
+    ('⁁', "Caret Insertion Point", '/'),
+    ('∕', "Division Slash", '/'),
+    ('⁄', "Fraction Slash", '/'),
+    ('╱', "Box Drawings Light Diagonal Upper Right To Lower Left", '/'),
+    ('⟋', "Mathematical Rising Diagonal", '/'),
+    ('⧸', "Big Solidus", '/'),
+    ('𝈺', "Greek Instrumental Notation Symbol-47", '/'),
+    ('㇓', "CJK Stroke Sp", '/'),
+    ('〳', "Vertical Kana Repeat Mark Upper Half", '/'),
+    ('Ⳇ', "Coptic Capital Letter Old Coptic Esh", '/'),
+    ('ノ', "Katakana Letter No", '/'),
+    ('丿', "CJK Unified Ideograph-4E3F", '/'),
+    ('⼃', "Kangxi Radical Slash", '/'),
+    ('／', "Fullwidth Solidus", '/'),
+
+    ('＼', "Fullwidth Reverse Solidus", '\\'),
+    ('﹨', "Small Reverse Solidus", '\\'),
+    ('∖', "Set Minus", '\\'),
+    ('⟍', "Mathematical Falling Diagonal", '\\'),
+    ('⧵', "Reverse Solidus Operator", '\\'),
+    ('⧹', "Big Reverse Solidus", '\\'),
+    ('⧹', "Greek Vocal Notation Symbol-16", '\\'),
+    ('⧹', "Greek Instrumental Symbol-48", '\\'),
+    ('㇔', "CJK Stroke D", '\\'),
+    ('丶', "CJK Unified Ideograph-4E36", '\\'),
+    ('⼂', "Kangxi Radical Dot", '\\'),
+    ('、', "Ideographic Comma", '\\'),
+    ('ヽ', "Katakana Iteration Mark", '\\'),
+
+    ('ꝸ', "Latin Small Letter Um", '&'),
+    ('＆', "Fullwidth Ampersand", '&'),
+
+    ('᛭', "Runic Cross Punctuation", '+'),
+    ('➕', "Heavy Plus Sign", '+'),
+    ('𐊛', "Lycian Letter H", '+'),
+    ('﬩', "Hebrew Letter Alternative Plus Sign", '+'),
+    ('＋', "Fullwidth Plus Sign", '+'),
+
+    ('‹', "Single Left-Pointing Angle Quotation Mark", '<'),
+    ('❮', "Heavy Left-Pointing Angle Quotation Mark Ornament", '<'),
+    ('˂', "Modifier Letter Left Arrowhead", '<'),
+    ('𝈶', "Greek Instrumental Symbol-40", '<'),
+    ('ᐸ', "Canadian Syllabics Pa", '<'),
+    ('ᚲ', "Runic Letter Kauna", '<'),
+    ('❬', "Medium Left-Pointing Angle Bracket Ornament", '<'),
+    ('⟨', "Mathematical Left Angle Bracket", '<'),
+    ('〈', "Left-Pointing Angle Bracket", '<'),
+    ('〈', "Left Angle Bracket", '<'),
+    ('㇛', "CJK Stroke Pd", '<'),
+    ('く', "Hiragana Letter Ku", '<'),
+    ('𡿨', "CJK Unified Ideograph-21FE8", '<'),
+    ('《', "Left Double Angle Bracket", '<'),
+    ('＜', "Fullwidth Less-Than Sign", '<'),
+
+    ('᐀', "Canadian Syllabics Hyphen", '='),
+    ('⹀', "Double Hyphen", '='),
+    ('゠', "Katakana-Hiragana Double Hyphen", '='),
+    ('꓿', "Lisu Punctuation Full Stop", '='),
+    ('＝', "Fullwidth Equals Sign", '='),
+
+    ('›', "Single Right-Pointing Angle Quotation Mark", '>'),
+    ('❯', "Heavy Right-Pointing Angle Quotation Mark Ornament", '>'),
+    ('˃', "Modifier Letter Right Arrowhead", '>'),
+    ('𝈷', "Greek Instrumental Symbol-42", '>'),
+    ('ᐳ', "Canadian Syllabics Po", '>'),
+    ('𖼿', "Miao Letter Archaic Zza", '>'),
+    ('❭', "Medium Right-Pointing Angle Bracket Ornament", '>'),
+    ('⟩', "Mathematical Right Angle Bracket", '>'),
+    ('〉', "Right-Pointing Angle Bracket", '>'),
+    ('〉', "Right Angle Bracket", '>'),
+    ('》', "Right Double Angle Bracket", '>'),
+    ('＞', "Fullwidth Greater-Than Sign", '>'),
+];
+
+// FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, instead of
+// keeping the substitution token in this table. Ideally, this should be inside `rustc_lexer`.
+// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
+// fancier error recovery to it, as there will be less overall work to do this way.
+const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
+    (' ', "Space", None),
+    ('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
+    ('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
+    (',', "Comma", Some(token::Comma)),
+    (';', "Semicolon", Some(token::Semi)),
+    (':', "Colon", Some(token::Colon)),
+    ('!', "Exclamation Mark", Some(token::Not)),
+    ('?', "Question Mark", Some(token::Question)),
+    ('.', "Period", Some(token::Dot)),
+    ('(', "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
+    (')', "Right Parenthesis", Some(token::CloseDelim(Delimiter::Parenthesis))),
+    ('[', "Left Square Bracket", Some(token::OpenDelim(Delimiter::Bracket))),
+    (']', "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
+    ('{', "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
+    ('}', "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
+    ('*', "Asterisk", Some(token::BinOp(token::Star))),
+    ('/', "Slash", Some(token::BinOp(token::Slash))),
+    ('\\', "Backslash", None),
+    ('&', "Ampersand", Some(token::BinOp(token::And))),
+    ('+', "Plus Sign", Some(token::BinOp(token::Plus))),
+    ('<', "Less-Than Sign", Some(token::Lt)),
+    ('=', "Equals Sign", Some(token::Eq)),
+    ('>', "Greater-Than Sign", Some(token::Gt)),
+    // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
+    // spitting the correct token out.
+    ('\'', "Single Quote", None),
+    ('"', "Quotation Mark", None),
+];
+
+pub(super) fn check_for_substitution<'a>(
+    reader: &StringReader<'a>,
+    pos: BytePos,
+    ch: char,
+    err: &mut Diagnostic,
+) -> Option<token::TokenKind> {
+    let &(_u_char, u_name, ascii_char) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch)?;
+
+    let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8()));
+
+    let Some((_ascii_char, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(c, _, _)| c == ascii_char) else {
+        let msg = format!("substitution character not found for '{}'", ch);
+        reader.sess.span_diagnostic.span_bug_no_panic(span, &msg);
+        return None;
+    };
+
+    // special help suggestion for "directed" double quotes
+    if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') {
+        let msg = format!(
+            "Unicode characters '“' (Left Double Quotation Mark) and \
+             '”' (Right Double Quotation Mark) look like '{}' ({}), but are not",
+            ascii_char, ascii_name
+        );
+        err.span_suggestion(
+            Span::with_root_ctxt(
+                pos,
+                pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()),
+            ),
+            &msg,
+            format!("\"{}\"", s),
+            Applicability::MaybeIncorrect,
+        );
+    } else {
+        let msg = format!(
+            "Unicode character '{}' ({}) looks like '{}' ({}), but it is not",
+            ch, u_name, ascii_char, ascii_name
+        );
+        err.span_suggestion(span, &msg, ascii_char, Applicability::MaybeIncorrect);
+    }
+    token.clone()
+}
+
+/// Extract string if found at current position with given delimiters
+fn peek_delimited(text: &str, from_ch: char, to_ch: char) -> Option<&str> {
+    let mut chars = text.chars();
+    let first_char = chars.next()?;
+    if first_char != from_ch {
+        return None;
+    }
+    let last_char_idx = chars.as_str().find(to_ch)?;
+    Some(&chars.as_str()[..last_char_idx])
+}