diff options
Diffstat (limited to 'compiler/rustc_parse/src/lexer/tokentrees.rs')
| -rw-r--r-- | compiler/rustc_parse/src/lexer/tokentrees.rs | 171 |
1 files changed, 118 insertions, 53 deletions
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index 31d91fe80..2bc2789a4 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -3,9 +3,10 @@ use super::diagnostics::same_indentation_level; use super::diagnostics::TokenTreeDiagInfo; use super::{StringReader, UnmatchedDelim}; use rustc_ast::token::{self, Delimiter, Token}; -use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast_pretty::pprust::token_to_string; -use rustc_errors::PErr; +use rustc_errors::{Applicability, PErr}; +use rustc_span::symbol::kw; pub(super) struct TokenTreesReader<'a> { string_reader: StringReader<'a>, @@ -24,54 +25,46 @@ impl<'a> TokenTreesReader<'a> { token: Token::dummy(), diag_info: TokenTreeDiagInfo::default(), }; - let (stream, res) = tt_reader.parse_token_trees(/* is_delimited */ false); + let (_open_spacing, stream, res) = + tt_reader.parse_token_trees(/* is_delimited */ false); (stream, res, tt_reader.diag_info.unmatched_delims) } - // Parse a stream of tokens into a list of `TokenTree`s. + // Parse a stream of tokens into a list of `TokenTree`s. The `Spacing` in + // the result is that of the opening delimiter. fn parse_token_trees( &mut self, is_delimited: bool, - ) -> (TokenStream, Result<(), Vec<PErr<'a>>>) { - self.token = self.string_reader.next_token().0; + ) -> (Spacing, TokenStream, Result<(), Vec<PErr<'a>>>) { + // Move past the opening delimiter. + let (_, open_spacing) = self.bump(false); + let mut buf = Vec::new(); loop { match self.token.kind { token::OpenDelim(delim) => { buf.push(match self.parse_token_tree_open_delim(delim) { Ok(val) => val, - Err(errs) => return (TokenStream::new(buf), Err(errs)), + Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)), }) } token::CloseDelim(delim) => { return ( + open_spacing, TokenStream::new(buf), if is_delimited { Ok(()) } else { Err(vec![self.close_delim_err(delim)]) }, ); } token::Eof => { return ( + open_spacing, TokenStream::new(buf), if is_delimited { Err(vec![self.eof_err()]) } else { Ok(()) }, ); } _ => { - // Get the next normal token. This might require getting multiple adjacent - // single-char tokens and joining them together. - let (this_spacing, next_tok) = loop { - let (next_tok, is_next_tok_preceded_by_whitespace) = - self.string_reader.next_token(); - if is_next_tok_preceded_by_whitespace { - break (Spacing::Alone, next_tok); - } else if let Some(glued) = self.token.glue(&next_tok) { - self.token = glued; - } else { - let this_spacing = - if next_tok.is_punct() { Spacing::Joint } else { Spacing::Alone }; - break (this_spacing, next_tok); - } - }; - let this_tok = std::mem::replace(&mut self.token, next_tok); + // Get the next normal token. + let (this_tok, this_spacing) = self.bump(true); buf.push(TokenTree::Token(this_tok, this_spacing)); } } @@ -80,7 +73,7 @@ impl<'a> TokenTreesReader<'a> { fn eof_err(&mut self) -> PErr<'a> { let msg = "this file contains an unclosed delimiter"; - let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg); + let mut err = self.string_reader.sess.dcx.struct_span_err(self.token.span, msg); for &(_, sp) in &self.diag_info.open_braces { err.span_label(sp, "unclosed delimiter"); self.diag_info.unmatched_delims.push(UnmatchedDelim { @@ -96,7 +89,7 @@ impl<'a> TokenTreesReader<'a> { report_suspicious_mismatch_block( &mut err, &self.diag_info, - &self.string_reader.sess.source_map(), + self.string_reader.sess.source_map(), *delim, ) } @@ -115,32 +108,16 @@ impl<'a> TokenTreesReader<'a> { // Parse the token trees within the delimiters. // We stop at any delimiter so we can try to recover if the user // uses an incorrect delimiter. - let (tts, res) = self.parse_token_trees(/* is_delimited */ true); - if let Err(mut errs) = res { - // If there are unclosed delims, see if there are diff markers and if so, point them - // out instead of complaining about the unclosed delims. - let mut parser = crate::stream_to_parser(self.string_reader.sess, tts, None); - let mut diff_errs = vec![]; - while parser.token != token::Eof { - if let Err(diff_err) = parser.err_diff_marker() { - diff_errs.push(diff_err); - } - parser.bump(); - } - if !diff_errs.is_empty() { - errs.iter_mut().for_each(|err| { - err.delay_as_bug(); - }); - return Err(diff_errs); - } - return Err(errs); + let (open_spacing, tts, res) = self.parse_token_trees(/* is_delimited */ true); + if let Err(errs) = res { + return Err(self.unclosed_delim_err(tts, errs)); } // Expand to cover the entire delimited token tree let delim_span = DelimSpan::from_pair(pre_span, self.token.span); let sm = self.string_reader.sess.source_map(); - match self.token.kind { + let close_spacing = match self.token.kind { // Correct delimiter. token::CloseDelim(close_delim) if close_delim == open_delim => { let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap(); @@ -162,7 +139,7 @@ impl<'a> TokenTreesReader<'a> { } // Move past the closing delimiter. - self.token = self.string_reader.next_token().0; + self.bump(false).1 } // Incorrect delimiter. token::CloseDelim(close_delim) => { @@ -179,7 +156,7 @@ impl<'a> TokenTreesReader<'a> { unclosed_delimiter = Some(sp); }; for (brace, brace_span) in &self.diag_info.open_braces { - if same_indentation_level(&sm, self.token.span, *brace_span) + if same_indentation_level(sm, self.token.span, *brace_span) && brace == &close_delim { // high likelihood of these two corresponding @@ -206,18 +183,106 @@ impl<'a> TokenTreesReader<'a> { // bar(baz( // } // Incorrect delimiter but matches the earlier `{` if !self.diag_info.open_braces.iter().any(|&(b, _)| b == close_delim) { - self.token = self.string_reader.next_token().0; + self.bump(false).1 + } else { + // The choice of value here doesn't matter. + Spacing::Alone } } token::Eof => { // Silently recover, the EOF token will be seen again // and an error emitted then. Thus we don't pop from - // self.open_braces here. + // self.open_braces here. The choice of spacing value here + // doesn't matter. + Spacing::Alone } _ => unreachable!(), - } + }; + + let spacing = DelimSpacing::new(open_spacing, close_spacing); + + Ok(TokenTree::Delimited(delim_span, spacing, open_delim, tts)) + } + + // Move on to the next token, returning the current token and its spacing. + // Will glue adjacent single-char tokens together if `glue` is set. + fn bump(&mut self, glue: bool) -> (Token, Spacing) { + let (this_spacing, next_tok) = loop { + let (next_tok, is_next_tok_preceded_by_whitespace) = self.string_reader.next_token(); - Ok(TokenTree::Delimited(delim_span, open_delim, tts)) + if is_next_tok_preceded_by_whitespace { + break (Spacing::Alone, next_tok); + } else if glue && let Some(glued) = self.token.glue(&next_tok) { + self.token = glued; + } else { + let this_spacing = if next_tok.is_punct() { + Spacing::Joint + } else if next_tok.kind == token::Eof { + Spacing::Alone + } else { + Spacing::JointHidden + }; + break (this_spacing, next_tok); + } + }; + let this_tok = std::mem::replace(&mut self.token, next_tok); + (this_tok, this_spacing) + } + + fn unclosed_delim_err(&mut self, tts: TokenStream, mut errs: Vec<PErr<'a>>) -> Vec<PErr<'a>> { + // If there are unclosed delims, see if there are diff markers and if so, point them + // out instead of complaining about the unclosed delims. + let mut parser = crate::stream_to_parser(self.string_reader.sess, tts, None); + let mut diff_errs = vec![]; + // Suggest removing a `{` we think appears in an `if`/`while` condition + // We want to suggest removing a `{` only if we think we're in an `if`/`while` condition, but + // we have no way of tracking this in the lexer itself, so we piggyback on the parser + let mut in_cond = false; + while parser.token != token::Eof { + if let Err(diff_err) = parser.err_diff_marker() { + diff_errs.push(diff_err); + } else if parser.is_keyword_ahead(0, &[kw::If, kw::While]) { + in_cond = true; + } else if matches!( + parser.token.kind, + token::CloseDelim(Delimiter::Brace) | token::FatArrow + ) { + // end of the `if`/`while` body, or the end of a `match` guard + in_cond = false; + } else if in_cond && parser.token == token::OpenDelim(Delimiter::Brace) { + // Store the `&&` and `let` to use their spans later when creating the diagnostic + let maybe_andand = parser.look_ahead(1, |t| t.clone()); + let maybe_let = parser.look_ahead(2, |t| t.clone()); + if maybe_andand == token::OpenDelim(Delimiter::Brace) { + // This might be the beginning of the `if`/`while` body (i.e., the end of the condition) + in_cond = false; + } else if maybe_andand == token::AndAnd && maybe_let.is_keyword(kw::Let) { + let mut err = parser.struct_span_err( + parser.token.span, + "found a `{` in the middle of a let-chain", + ); + err.span_suggestion( + parser.token.span, + "consider removing this brace to parse the `let` as part of the same chain", + "", + Applicability::MachineApplicable, + ); + err.span_label( + maybe_andand.span.to(maybe_let.span), + "you might have meant to continue the let-chain here", + ); + errs.push(err); + } + } + parser.bump(); + } + if !diff_errs.is_empty() { + errs.iter_mut().for_each(|err| { + err.delay_as_bug(); + }); + return diff_errs; + } + return errs; } fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> { @@ -225,12 +290,12 @@ impl<'a> TokenTreesReader<'a> { // matching opening delimiter). let token_str = token_to_string(&self.token); let msg = format!("unexpected closing delimiter: `{token_str}`"); - let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg); + let mut err = self.string_reader.sess.dcx.struct_span_err(self.token.span, msg); report_suspicious_mismatch_block( &mut err, &self.diag_info, - &self.string_reader.sess.source_map(), + self.string_reader.sess.source_map(), delim, ); err.span_label(self.token.span, "unexpected closing delimiter"); |