diff options
Diffstat (limited to '')
-rw-r--r-- | compiler/rustc_parse/src/lexer/mod.rs | 717 |
1 files changed, 717 insertions, 0 deletions
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs new file mode 100644 index 000000000..848e142e5 --- /dev/null +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -0,0 +1,717 @@ +use crate::lexer::unicode_chars::UNICODE_ARRAY; +use rustc_ast::ast::{self, AttrStyle}; +use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind}; +use rustc_ast::tokenstream::{Spacing, TokenStream}; +use rustc_ast::util::unicode::contains_text_flow_control_chars; +use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult}; +use rustc_lexer::unescape::{self, Mode}; +use rustc_lexer::{Base, DocStyle, RawStrError}; +use rustc_session::lint::builtin::{ + RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT, +}; +use rustc_session::lint::BuiltinLintDiagnostics; +use rustc_session::parse::ParseSess; +use rustc_span::symbol::{sym, Symbol}; +use rustc_span::{edition::Edition, BytePos, Pos, Span}; + +use tracing::debug; + +mod tokentrees; +mod unescape_error_reporting; +mod unicode_chars; + +use unescape_error_reporting::{emit_unescape_error, escaped_char}; + +// This type is used a lot. Make sure it doesn't unintentionally get bigger. +// +// This assertion is in this crate, rather than in `rustc_lexer`, because that +// crate cannot depend on `rustc_data_structures`. +#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] +rustc_data_structures::static_assert_size!(rustc_lexer::Token, 12); + +#[derive(Clone, Debug)] +pub struct UnmatchedBrace { + pub expected_delim: Delimiter, + pub found_delim: Option<Delimiter>, + pub found_span: Span, + pub unclosed_span: Option<Span>, + pub candidate_span: Option<Span>, +} + +pub(crate) fn parse_token_trees<'a>( + sess: &'a ParseSess, + src: &'a str, + start_pos: BytePos, + override_span: Option<Span>, +) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) { + StringReader { sess, start_pos, pos: start_pos, src, override_span }.into_token_trees() +} + +struct StringReader<'a> { + sess: &'a ParseSess, + /// Initial position, read-only. + start_pos: BytePos, + /// The absolute offset within the source_map of the current character. + pos: BytePos, + /// Source text to tokenize. + src: &'a str, + override_span: Option<Span>, +} + +impl<'a> StringReader<'a> { + fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span { + self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi)) + } + + /// Returns the next token, and info about preceding whitespace, if any. + fn next_token(&mut self) -> (Spacing, Token) { + let mut spacing = Spacing::Joint; + + // Skip `#!` at the start of the file + if self.pos == self.start_pos + && let Some(shebang_len) = rustc_lexer::strip_shebang(self.src) + { + self.pos = self.pos + BytePos::from_usize(shebang_len); + spacing = Spacing::Alone; + } + + // Skip trivial (whitespace & comments) tokens + loop { + let start_src_index = self.src_index(self.pos); + let text: &str = &self.src[start_src_index..]; + + if text.is_empty() { + let span = self.mk_sp(self.pos, self.pos); + return (spacing, Token::new(token::Eof, span)); + } + + let token = rustc_lexer::first_token(text); + + let start = self.pos; + self.pos = self.pos + BytePos(token.len); + + debug!("next_token: {:?}({:?})", token.kind, self.str_from(start)); + + match self.cook_lexer_token(token.kind, start) { + Some(kind) => { + let span = self.mk_sp(start, self.pos); + return (spacing, Token::new(kind, span)); + } + None => spacing = Spacing::Alone, + } + } + } + + /// Report a fatal lexical error with a given span. + fn fatal_span(&self, sp: Span, m: &str) -> ! { + self.sess.span_diagnostic.span_fatal(sp, m) + } + + /// Report a lexical error with a given span. + fn err_span(&self, sp: Span, m: &str) { + self.sess.span_diagnostic.struct_span_err(sp, m).emit(); + } + + /// Report a fatal error spanning [`from_pos`, `to_pos`). + fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> ! { + self.fatal_span(self.mk_sp(from_pos, to_pos), m) + } + + /// Report a lexical error spanning [`from_pos`, `to_pos`). + fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) { + self.err_span(self.mk_sp(from_pos, to_pos), m) + } + + fn struct_fatal_span_char( + &self, + from_pos: BytePos, + to_pos: BytePos, + m: &str, + c: char, + ) -> DiagnosticBuilder<'a, !> { + self.sess + .span_diagnostic + .struct_span_fatal(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c))) + } + + fn struct_err_span_char( + &self, + from_pos: BytePos, + to_pos: BytePos, + m: &str, + c: char, + ) -> DiagnosticBuilder<'a, ErrorGuaranteed> { + self.sess + .span_diagnostic + .struct_span_err(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c))) + } + + /// Detect usages of Unicode codepoints changing the direction of the text on screen and loudly + /// complain about it. + fn lint_unicode_text_flow(&self, start: BytePos) { + // Opening delimiter of the length 2 is not included into the comment text. + let content_start = start + BytePos(2); + let content = self.str_from(content_start); + if contains_text_flow_control_chars(content) { + let span = self.mk_sp(start, self.pos); + self.sess.buffer_lint_with_diagnostic( + &TEXT_DIRECTION_CODEPOINT_IN_COMMENT, + span, + ast::CRATE_NODE_ID, + "unicode codepoint changing visible direction of text present in comment", + BuiltinLintDiagnostics::UnicodeTextFlow(span, content.to_string()), + ); + } + } + + /// Turns simple `rustc_lexer::TokenKind` enum into a rich + /// `rustc_ast::TokenKind`. This turns strings into interned + /// symbols and runs additional validation. + fn cook_lexer_token(&self, token: rustc_lexer::TokenKind, start: BytePos) -> Option<TokenKind> { + Some(match token { + rustc_lexer::TokenKind::LineComment { doc_style } => { + // Skip non-doc comments + let Some(doc_style) = doc_style else { + self.lint_unicode_text_flow(start); + return None; + }; + + // Opening delimiter of the length 3 is not included into the symbol. + let content_start = start + BytePos(3); + let content = self.str_from(content_start); + self.cook_doc_comment(content_start, content, CommentKind::Line, doc_style) + } + rustc_lexer::TokenKind::BlockComment { doc_style, terminated } => { + if !terminated { + self.report_unterminated_block_comment(start, doc_style); + } + + // Skip non-doc comments + let Some(doc_style) = doc_style else { + self.lint_unicode_text_flow(start); + return None; + }; + + // Opening delimiter of the length 3 and closing delimiter of the length 2 + // are not included into the symbol. + let content_start = start + BytePos(3); + let content_end = self.pos - BytePos(if terminated { 2 } else { 0 }); + let content = self.str_from_to(content_start, content_end); + self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style) + } + rustc_lexer::TokenKind::Whitespace => return None, + rustc_lexer::TokenKind::Ident + | rustc_lexer::TokenKind::RawIdent + | rustc_lexer::TokenKind::UnknownPrefix => { + let is_raw_ident = token == rustc_lexer::TokenKind::RawIdent; + let is_unknown_prefix = token == rustc_lexer::TokenKind::UnknownPrefix; + let mut ident_start = start; + if is_raw_ident { + ident_start = ident_start + BytePos(2); + } + if is_unknown_prefix { + self.report_unknown_prefix(start); + } + let sym = nfc_normalize(self.str_from(ident_start)); + let span = self.mk_sp(start, self.pos); + self.sess.symbol_gallery.insert(sym, span); + if is_raw_ident { + if !sym.can_be_raw() { + self.err_span(span, &format!("`{}` cannot be a raw identifier", sym)); + } + self.sess.raw_identifier_spans.borrow_mut().push(span); + } + token::Ident(sym, is_raw_ident) + } + rustc_lexer::TokenKind::InvalidIdent + // Do not recover an identifier with emoji if the codepoint is a confusable + // with a recoverable substitution token, like `➖`. + if !UNICODE_ARRAY + .iter() + .any(|&(c, _, _)| { + let sym = self.str_from(start); + sym.chars().count() == 1 && c == sym.chars().next().unwrap() + }) + => + { + let sym = nfc_normalize(self.str_from(start)); + let span = self.mk_sp(start, self.pos); + self.sess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default().push(span); + token::Ident(sym, false) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + let suffix_start = start + BytePos(suffix_start); + let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind); + let suffix = if suffix_start < self.pos { + let string = self.str_from(suffix_start); + if string == "_" { + self.sess + .span_diagnostic + .struct_span_warn( + self.mk_sp(suffix_start, self.pos), + "underscore literal suffix is not allowed", + ) + .warn( + "this was previously accepted by the compiler but is \ + being phased out; it will become a hard error in \ + a future release!", + ) + .note( + "see issue #42326 \ + <https://github.com/rust-lang/rust/issues/42326> \ + for more information", + ) + .emit(); + None + } else { + Some(Symbol::intern(string)) + } + } else { + None + }; + token::Literal(token::Lit { kind, symbol, suffix }) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + // Include the leading `'` in the real identifier, for macro + // expansion purposes. See #12512 for the gory details of why + // this is necessary. + let lifetime_name = self.str_from(start); + if starts_with_number { + self.err_span_(start, self.pos, "lifetimes cannot start with a number"); + } + let ident = Symbol::intern(lifetime_name); + token::Lifetime(ident) + } + rustc_lexer::TokenKind::Semi => token::Semi, + rustc_lexer::TokenKind::Comma => token::Comma, + rustc_lexer::TokenKind::Dot => token::Dot, + rustc_lexer::TokenKind::OpenParen => token::OpenDelim(Delimiter::Parenthesis), + rustc_lexer::TokenKind::CloseParen => token::CloseDelim(Delimiter::Parenthesis), + rustc_lexer::TokenKind::OpenBrace => token::OpenDelim(Delimiter::Brace), + rustc_lexer::TokenKind::CloseBrace => token::CloseDelim(Delimiter::Brace), + rustc_lexer::TokenKind::OpenBracket => token::OpenDelim(Delimiter::Bracket), + rustc_lexer::TokenKind::CloseBracket => token::CloseDelim(Delimiter::Bracket), + rustc_lexer::TokenKind::At => token::At, + rustc_lexer::TokenKind::Pound => token::Pound, + rustc_lexer::TokenKind::Tilde => token::Tilde, + rustc_lexer::TokenKind::Question => token::Question, + rustc_lexer::TokenKind::Colon => token::Colon, + rustc_lexer::TokenKind::Dollar => token::Dollar, + rustc_lexer::TokenKind::Eq => token::Eq, + rustc_lexer::TokenKind::Bang => token::Not, + rustc_lexer::TokenKind::Lt => token::Lt, + rustc_lexer::TokenKind::Gt => token::Gt, + rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus), + rustc_lexer::TokenKind::And => token::BinOp(token::And), + rustc_lexer::TokenKind::Or => token::BinOp(token::Or), + rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus), + rustc_lexer::TokenKind::Star => token::BinOp(token::Star), + rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash), + rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret), + rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent), + + rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => { + let c = self.str_from(start).chars().next().unwrap(); + let mut err = + self.struct_err_span_char(start, self.pos, "unknown start of token", c); + // FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, + // instead of keeping a table in `check_for_substitution`into the token. Ideally, + // this should be inside `rustc_lexer`. However, we should first remove compound + // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, + // as there will be less overall work to do this way. + let token = unicode_chars::check_for_substitution(self, start, c, &mut err); + if c == '\x00' { + err.help("source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used"); + } + err.emit(); + token? + } + }) + } + + fn cook_doc_comment( + &self, + content_start: BytePos, + content: &str, + comment_kind: CommentKind, + doc_style: DocStyle, + ) -> TokenKind { + if content.contains('\r') { + for (idx, _) in content.char_indices().filter(|&(_, c)| c == '\r') { + self.err_span_( + content_start + BytePos(idx as u32), + content_start + BytePos(idx as u32 + 1), + match comment_kind { + CommentKind::Line => "bare CR not allowed in doc-comment", + CommentKind::Block => "bare CR not allowed in block doc-comment", + }, + ); + } + } + + let attr_style = match doc_style { + DocStyle::Outer => AttrStyle::Outer, + DocStyle::Inner => AttrStyle::Inner, + }; + + token::DocComment(comment_kind, attr_style, Symbol::intern(content)) + } + + fn cook_lexer_literal( + &self, + start: BytePos, + suffix_start: BytePos, + kind: rustc_lexer::LiteralKind, + ) -> (token::LitKind, Symbol) { + // prefix means `"` or `br"` or `r###"`, ... + let (lit_kind, mode, prefix_len, postfix_len) = match kind { + rustc_lexer::LiteralKind::Char { terminated } => { + if !terminated { + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start, suffix_start), + "unterminated character literal", + error_code!(E0762), + ) + } + (token::Char, Mode::Char, 1, 1) // ' ' + } + rustc_lexer::LiteralKind::Byte { terminated } => { + if !terminated { + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start + BytePos(1), suffix_start), + "unterminated byte constant", + error_code!(E0763), + ) + } + (token::Byte, Mode::Byte, 2, 1) // b' ' + } + rustc_lexer::LiteralKind::Str { terminated } => { + if !terminated { + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start, suffix_start), + "unterminated double quote string", + error_code!(E0765), + ) + } + (token::Str, Mode::Str, 1, 1) // " " + } + rustc_lexer::LiteralKind::ByteStr { terminated } => { + if !terminated { + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start + BytePos(1), suffix_start), + "unterminated double quote byte string", + error_code!(E0766), + ) + } + (token::ByteStr, Mode::ByteStr, 2, 1) // b" " + } + rustc_lexer::LiteralKind::RawStr { n_hashes } => { + if let Some(n_hashes) = n_hashes { + let n = u32::from(n_hashes); + (token::StrRaw(n_hashes), Mode::RawStr, 2 + n, 1 + n) // r##" "## + } else { + self.report_raw_str_error(start, 1); + } + } + rustc_lexer::LiteralKind::RawByteStr { n_hashes } => { + if let Some(n_hashes) = n_hashes { + let n = u32::from(n_hashes); + (token::ByteStrRaw(n_hashes), Mode::RawByteStr, 3 + n, 1 + n) // br##" "## + } else { + self.report_raw_str_error(start, 2); + } + } + rustc_lexer::LiteralKind::Int { base, empty_int } => { + return if empty_int { + self.sess + .span_diagnostic + .struct_span_err_with_code( + self.mk_sp(start, suffix_start), + "no valid digits found for number", + error_code!(E0768), + ) + .emit(); + (token::Integer, sym::integer(0)) + } else { + self.validate_int_literal(base, start, suffix_start); + (token::Integer, self.symbol_from_to(start, suffix_start)) + }; + } + rustc_lexer::LiteralKind::Float { base, empty_exponent } => { + if empty_exponent { + self.err_span_(start, self.pos, "expected at least one digit in exponent"); + } + + match base { + Base::Hexadecimal => self.err_span_( + start, + suffix_start, + "hexadecimal float literal is not supported", + ), + Base::Octal => { + self.err_span_(start, suffix_start, "octal float literal is not supported") + } + Base::Binary => { + self.err_span_(start, suffix_start, "binary float literal is not supported") + } + _ => (), + } + + let id = self.symbol_from_to(start, suffix_start); + return (token::Float, id); + } + }; + let content_start = start + BytePos(prefix_len); + let content_end = suffix_start - BytePos(postfix_len); + let id = self.symbol_from_to(content_start, content_end); + self.validate_literal_escape(mode, content_start, content_end, prefix_len, postfix_len); + (lit_kind, id) + } + + #[inline] + fn src_index(&self, pos: BytePos) -> usize { + (pos - self.start_pos).to_usize() + } + + /// Slice of the source text from `start` up to but excluding `self.pos`, + /// meaning the slice does not include the character `self.ch`. + fn str_from(&self, start: BytePos) -> &str { + self.str_from_to(start, self.pos) + } + + /// As symbol_from, with an explicit endpoint. + fn symbol_from_to(&self, start: BytePos, end: BytePos) -> Symbol { + debug!("taking an ident from {:?} to {:?}", start, end); + Symbol::intern(self.str_from_to(start, end)) + } + + /// Slice of the source text spanning from `start` up to but excluding `end`. + fn str_from_to(&self, start: BytePos, end: BytePos) -> &str { + &self.src[self.src_index(start)..self.src_index(end)] + } + + fn report_raw_str_error(&self, start: BytePos, prefix_len: u32) -> ! { + match rustc_lexer::validate_raw_str(self.str_from(start), prefix_len) { + Err(RawStrError::InvalidStarter { bad_char }) => { + self.report_non_started_raw_string(start, bad_char) + } + Err(RawStrError::NoTerminator { expected, found, possible_terminator_offset }) => self + .report_unterminated_raw_string(start, expected, possible_terminator_offset, found), + Err(RawStrError::TooManyDelimiters { found }) => { + self.report_too_many_hashes(start, found) + } + Ok(()) => panic!("no error found for supposedly invalid raw string literal"), + } + } + + fn report_non_started_raw_string(&self, start: BytePos, bad_char: char) -> ! { + self.struct_fatal_span_char( + start, + self.pos, + "found invalid character; only `#` is allowed in raw string delimitation", + bad_char, + ) + .emit() + } + + fn report_unterminated_raw_string( + &self, + start: BytePos, + n_hashes: u32, + possible_offset: Option<u32>, + found_terminators: u32, + ) -> ! { + let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code( + self.mk_sp(start, start), + "unterminated raw string", + error_code!(E0748), + ); + + err.span_label(self.mk_sp(start, start), "unterminated raw string"); + + if n_hashes > 0 { + err.note(&format!( + "this raw string should be terminated with `\"{}`", + "#".repeat(n_hashes as usize) + )); + } + + if let Some(possible_offset) = possible_offset { + let lo = start + BytePos(possible_offset as u32); + let hi = lo + BytePos(found_terminators as u32); + let span = self.mk_sp(lo, hi); + err.span_suggestion( + span, + "consider terminating the string here", + "#".repeat(n_hashes as usize), + Applicability::MaybeIncorrect, + ); + } + + err.emit() + } + + fn report_unterminated_block_comment(&self, start: BytePos, doc_style: Option<DocStyle>) { + let msg = match doc_style { + Some(_) => "unterminated block doc-comment", + None => "unterminated block comment", + }; + let last_bpos = self.pos; + let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code( + self.mk_sp(start, last_bpos), + msg, + error_code!(E0758), + ); + let mut nested_block_comment_open_idxs = vec![]; + let mut last_nested_block_comment_idxs = None; + let mut content_chars = self.str_from(start).char_indices().peekable(); + + while let Some((idx, current_char)) = content_chars.next() { + match content_chars.peek() { + Some((_, '*')) if current_char == '/' => { + nested_block_comment_open_idxs.push(idx); + } + Some((_, '/')) if current_char == '*' => { + last_nested_block_comment_idxs = + nested_block_comment_open_idxs.pop().map(|open_idx| (open_idx, idx)); + } + _ => {} + }; + } + + if let Some((nested_open_idx, nested_close_idx)) = last_nested_block_comment_idxs { + err.span_label(self.mk_sp(start, start + BytePos(2)), msg) + .span_label( + self.mk_sp( + start + BytePos(nested_open_idx as u32), + start + BytePos(nested_open_idx as u32 + 2), + ), + "...as last nested comment starts here, maybe you want to close this instead?", + ) + .span_label( + self.mk_sp( + start + BytePos(nested_close_idx as u32), + start + BytePos(nested_close_idx as u32 + 2), + ), + "...and last nested comment terminates here.", + ); + } + + err.emit(); + } + + // RFC 3101 introduced the idea of (reserved) prefixes. As of Rust 2021, + // using a (unknown) prefix is an error. In earlier editions, however, they + // only result in a (allowed by default) lint, and are treated as regular + // identifier tokens. + fn report_unknown_prefix(&self, start: BytePos) { + let prefix_span = self.mk_sp(start, self.pos); + let prefix_str = self.str_from_to(start, self.pos); + let msg = format!("prefix `{}` is unknown", prefix_str); + + let expn_data = prefix_span.ctxt().outer_expn_data(); + + if expn_data.edition >= Edition::Edition2021 { + // In Rust 2021, this is a hard error. + let mut err = self.sess.span_diagnostic.struct_span_err(prefix_span, &msg); + err.span_label(prefix_span, "unknown prefix"); + if prefix_str == "rb" { + err.span_suggestion_verbose( + prefix_span, + "use `br` for a raw byte string", + "br", + Applicability::MaybeIncorrect, + ); + } else if expn_data.is_root() { + err.span_suggestion_verbose( + prefix_span.shrink_to_hi(), + "consider inserting whitespace here", + " ", + Applicability::MaybeIncorrect, + ); + } + err.note("prefixed identifiers and literals are reserved since Rust 2021"); + err.emit(); + } else { + // Before Rust 2021, only emit a lint for migration. + self.sess.buffer_lint_with_diagnostic( + &RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, + prefix_span, + ast::CRATE_NODE_ID, + &msg, + BuiltinLintDiagnostics::ReservedPrefix(prefix_span), + ); + } + } + + fn report_too_many_hashes(&self, start: BytePos, found: u32) -> ! { + self.fatal_span_( + start, + self.pos, + &format!( + "too many `#` symbols: raw strings may be delimited \ + by up to 255 `#` symbols, but found {}", + found + ), + ) + } + + fn validate_literal_escape( + &self, + mode: Mode, + content_start: BytePos, + content_end: BytePos, + prefix_len: u32, + postfix_len: u32, + ) { + let lit_content = self.str_from_to(content_start, content_end); + unescape::unescape_literal(lit_content, mode, &mut |range, result| { + // Here we only check for errors. The actual unescaping is done later. + if let Err(err) = result { + let span_with_quotes = self + .mk_sp(content_start - BytePos(prefix_len), content_end + BytePos(postfix_len)); + let (start, end) = (range.start as u32, range.end as u32); + let lo = content_start + BytePos(start); + let hi = lo + BytePos(end - start); + let span = self.mk_sp(lo, hi); + emit_unescape_error( + &self.sess.span_diagnostic, + lit_content, + span_with_quotes, + span, + mode, + range, + err, + ); + } + }); + } + + fn validate_int_literal(&self, base: Base, content_start: BytePos, content_end: BytePos) { + let base = match base { + Base::Binary => 2, + Base::Octal => 8, + _ => return, + }; + let s = self.str_from_to(content_start + BytePos(2), content_end); + for (idx, c) in s.char_indices() { + let idx = idx as u32; + if c != '_' && c.to_digit(base).is_none() { + let lo = content_start + BytePos(2 + idx); + let hi = content_start + BytePos(2 + idx + c.len_utf8() as u32); + self.err_span_(lo, hi, &format!("invalid digit for a base {} literal", base)); + } + } + } +} + +pub fn nfc_normalize(string: &str) -> Symbol { + use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; + match is_nfc_quick(string.chars()) { + IsNormalized::Yes => Symbol::intern(string), + _ => { + let normalized_str: String = string.chars().nfc().collect(); + Symbol::intern(&normalized_str) + } + } +} |