diff options
Diffstat (limited to 'compiler/rustc_ast/src/tokenstream.rs')
| -rw-r--r-- | compiler/rustc_ast/src/tokenstream.rs | 141 |
1 files changed, 101 insertions, 40 deletions
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 23b8f9c12..4c0c49658 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -46,7 +46,7 @@ pub enum TokenTree { /// delimiters are implicitly represented by `Delimited`. Token(Token, Spacing), /// A delimited sequence of token trees. - Delimited(DelimSpan, Delimiter, TokenStream), + Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream), } // Ensure all fields of `TokenTree` are `DynSend` and `DynSync`. @@ -62,11 +62,11 @@ where } impl TokenTree { - /// Checks if this `TokenTree` is equal to the other, regardless of span information. + /// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information. pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind, - (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { + (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => { delim == delim2 && tts.eq_unspanned(tts2) } _ => false, @@ -99,6 +99,11 @@ impl TokenTree { TokenTree::Token(Token::new(kind, span), Spacing::Joint) } + /// Create a `TokenTree::Token` with joint-hidden spacing. + pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span), Spacing::JointHidden) + } + pub fn uninterpolate(&self) -> Cow<'_, TokenTree> { match self { TokenTree::Token(token, spacing) => match token.uninterpolate() { @@ -183,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>); #[derive(Clone, Debug, Encodable, Decodable)] pub enum AttrTokenTree { Token(Token, Spacing), - Delimited(DelimSpan, Delimiter, AttrTokenStream), + Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream), /// Stores the attributes for an attribute target, /// along with the tokens for that attribute target. /// See `AttributesData` for more information @@ -208,9 +213,14 @@ impl AttrTokenStream { AttrTokenTree::Token(inner, spacing) => { smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() } - AttrTokenTree::Delimited(span, delim, stream) => { - smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] - .into_iter() + AttrTokenTree::Delimited(span, spacing, delim, stream) => { + smallvec![TokenTree::Delimited( + *span, + *spacing, + *delim, + stream.to_tokenstream() + ),] + .into_iter() } AttrTokenTree::Attributes(data) => { let idx = data @@ -230,7 +240,7 @@ impl AttrTokenStream { let mut found = false; // Check the last two trees (to account for a trailing semi) for tree in target_tokens.iter_mut().rev().take(2) { - if let TokenTree::Delimited(span, delim, delim_tokens) = tree { + if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree { // Inner attributes are only supported on extern blocks, functions, // impls, and modules. All of these have their inner attributes // placed at the beginning of the rightmost outermost braced group: @@ -250,7 +260,7 @@ impl AttrTokenStream { stream.push_stream(inner_attr.tokens()); } stream.push_stream(delim_tokens.clone()); - *tree = TokenTree::Delimited(*span, *delim, stream); + *tree = TokenTree::Delimited(*span, *spacing, *delim, stream); found = true; break; } @@ -303,21 +313,64 @@ pub struct AttributesData { #[derive(Clone, Debug, Default, Encodable, Decodable)] pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>); -/// Similar to `proc_macro::Spacing`, but for tokens. -/// -/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when -/// we convert to `proc_macro::TokenTree` for proc macros only `Punct` -/// `TokenTree`s have a `proc_macro::Spacing`. +/// Indicates whether a token can join with the following token to form a +/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to +/// guide pretty-printing, which is where the `JointHidden` value (which isn't +/// part of `proc_macro::Spacing`) comes in useful. #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] pub enum Spacing { - /// The token is not immediately followed by an operator token (as - /// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`, - /// `+/*foo*/=`, `+ident`, and `+()`. + /// The token cannot join with the following token to form a compound + /// token. + /// + /// In token streams parsed from source code, the compiler will use `Alone` + /// for any token immediately followed by whitespace, a non-doc comment, or + /// EOF. + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed with a space after it, or (b) + /// is the last token in the stream. (In the latter case the choice of + /// spacing doesn't matter because it is never used for the last token. We + /// arbitrarily use `Alone`.) + /// + /// Converts to `proc_macro::Spacing::Alone`, and + /// `proc_macro::Spacing::Alone` converts back to this. Alone, - /// The token is immediately followed by an operator token. E.g. a `+` - /// token is `Joint` in `+=` and `++`. + /// The token can join with the following token to form a compound token. + /// + /// In token streams parsed from source code, the compiler will use `Joint` + /// for any token immediately followed by punctuation (as determined by + /// `Token::is_punct`). + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed without a space after it, and + /// (b) is followed by a punctuation token. + /// + /// Converts to `proc_macro::Spacing::Joint`, and + /// `proc_macro::Spacing::Joint` converts back to this. Joint, + + /// The token can join with the following token to form a compound token, + /// but this will not be visible at the proc macro level. (This is what the + /// `Hidden` means; see below.) + /// + /// In token streams parsed from source code, the compiler will use + /// `JointHidden` for any token immediately followed by anything not + /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, + /// literal, delimiter, doc comment. + /// + /// When constructing token streams, use this for each token that (a) + /// should be pretty-printed without a space after it, and (b) is followed + /// by a non-punctuation token. + /// + /// Converts to `proc_macro::Spacing::Alone`, but + /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. + /// Because of that, pretty-printing of `TokenStream`s produced by proc + /// macros is unavoidably uglier (with more whitespace between tokens) than + /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed + /// source code, internally constructed token streams, and token streams + /// produced by declarative macros). + JointHidden, } impl TokenStream { @@ -421,21 +474,14 @@ impl TokenStream { self } - /// Create a token stream containing a single token with alone spacing. + /// Create a token stream containing a single token with alone spacing. The + /// spacing used for the final token in a constructed stream doesn't matter + /// because it's never used. In practice we arbitrarily use + /// `Spacing::Alone`. pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream { TokenStream::new(vec![TokenTree::token_alone(kind, span)]) } - /// Create a token stream containing a single token with joint spacing. - pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream { - TokenStream::new(vec![TokenTree::token_joint(kind, span)]) - } - - /// Create a token stream containing a single `Delimited`. - pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream { - TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)]) - } - pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { let Some(tokens) = node.tokens() else { panic!("missing tokens for node at {:?}: {:?}", node.span(), node); @@ -477,13 +523,14 @@ impl TokenStream { fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { match &token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => { + token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => { TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } token::Interpolated(nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), + DelimSpacing::new(Spacing::JointHidden, spacing), Delimiter::Invisible, - TokenStream::from_nonterminal_ast(nt).flattened(), + TokenStream::from_nonterminal_ast(&nt.0).flattened(), ), _ => TokenTree::Token(token.clone(), spacing), } @@ -492,8 +539,8 @@ impl TokenStream { fn flatten_token_tree(tree: &TokenTree) -> TokenTree { match tree { TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing), - TokenTree::Delimited(span, delim, tts) => { - TokenTree::Delimited(*span, *delim, tts.flattened()) + TokenTree::Delimited(span, spacing, delim, tts) => { + TokenTree::Delimited(*span, *spacing, *delim, tts.flattened()) } } } @@ -503,7 +550,7 @@ impl TokenStream { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), - TokenTree::Delimited(_, _, inner) => can_skip(inner), + TokenTree::Delimited(.., inner) => can_skip(inner), }) } @@ -517,7 +564,7 @@ impl TokenStream { // If `vec` is not empty, try to glue `tt` onto its last token. The return // value indicates if gluing took place. fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool { - if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last() + if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last() && let TokenTree::Token(tok, spacing) = tt && let Some(glued_tok) = last_tok.glue(tok) { @@ -592,9 +639,10 @@ impl TokenStream { &TokenTree::Token(..) => i += 1, - &TokenTree::Delimited(sp, delim, ref delim_stream) => { + &TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => { if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) { - let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream); + let new_tt = + TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream); Lrc::make_mut(&mut stream.0)[i] = new_tt; modified = true; } @@ -622,10 +670,11 @@ impl TokenStream { num_of_hashes = cmp::max(num_of_hashes, count); } - // `/// foo` becomes `doc = r"foo"`. + // `/// foo` becomes `[doc = r"foo"]`. let delim_span = DelimSpan::from_single(span); let body = TokenTree::Delimited( delim_span, + DelimSpacing::new(Spacing::JointHidden, Spacing::Alone), Delimiter::Bracket, [ TokenTree::token_alone(token::Ident(sym::doc, false), span), @@ -641,7 +690,7 @@ impl TokenStream { if attr_style == AttrStyle::Inner { vec![ - TokenTree::token_alone(token::Pound, span), + TokenTree::token_joint(token::Pound, span), TokenTree::token_alone(token::Not, span), body, ] @@ -738,6 +787,18 @@ impl DelimSpan { } } +#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] +pub struct DelimSpacing { + pub open: Spacing, + pub close: Spacing, +} + +impl DelimSpacing { + pub fn new(open: Spacing, close: Spacing) -> DelimSpacing { + DelimSpacing { open, close } + } +} + // Some types are used a lot. Make sure they don't unintentionally get bigger. #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] mod size_asserts { |