summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_ast/src/tokenstream.rs
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_ast/src/tokenstream.rs')
-rw-r--r--compiler/rustc_ast/src/tokenstream.rs150
1 files changed, 70 insertions, 80 deletions
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 875cd620d..015f5c1ee 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -47,10 +47,6 @@ pub enum TokenTree {
Delimited(DelimSpan, Delimiter, TokenStream),
}
-// This type is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(TokenTree, 32);
-
// Ensure all fields of `TokenTree` is `Send` and `Sync`.
#[cfg(parallel_compiler)]
fn _dummy()
@@ -249,12 +245,12 @@ impl AttrTokenStream {
// properly implemented - we always synthesize fake tokens,
// so we never reach this code.
- let mut builder = TokenStreamBuilder::new();
+ let mut stream = TokenStream::default();
for inner_attr in inner_attrs {
- builder.push(inner_attr.tokens());
+ stream.push_stream(inner_attr.tokens());
}
- builder.push(delim_tokens.clone());
- *tree = TokenTree::Delimited(*span, *delim, builder.build());
+ stream.push_stream(delim_tokens.clone());
+ *tree = TokenTree::Delimited(*span, *delim, stream);
found = true;
break;
}
@@ -308,13 +304,20 @@ pub struct AttributesData {
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
-// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(TokenStream, 8);
-
+/// Similar to `proc_macro::Spacing`, but for tokens.
+///
+/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
+/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
+/// `TokenTree`s have a `proc_macro::Spacing`.
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum Spacing {
+ /// The token is not immediately followed by an operator token (as
+ /// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
+ /// `+/*foo*/=`, `+ident`, and `+()`.
Alone,
+
+ /// The token is immediately followed by an operator token. E.g. a `+`
+ /// token is `Joint` in `+=` and `++`.
Joint,
}
@@ -502,76 +505,49 @@ impl TokenStream {
self.trees().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
}
-}
-// 99.5%+ of the time we have 1 or 2 elements in this vector.
-#[derive(Clone)]
-pub struct TokenStreamBuilder(SmallVec<[TokenStream; 2]>);
-
-impl TokenStreamBuilder {
- pub fn new() -> TokenStreamBuilder {
- TokenStreamBuilder(SmallVec::new())
- }
-
- pub fn push(&mut self, stream: TokenStream) {
- self.0.push(stream);
- }
-
- pub fn build(self) -> TokenStream {
- let mut streams = self.0;
- match streams.len() {
- 0 => TokenStream::default(),
- 1 => streams.pop().unwrap(),
- _ => {
- // We will extend the first stream in `streams` with the
- // elements from the subsequent streams. This requires using
- // `make_mut()` on the first stream, and in practice this
- // doesn't cause cloning 99.9% of the time.
- //
- // One very common use case is when `streams` has two elements,
- // where the first stream has any number of elements within
- // (often 1, but sometimes many more) and the second stream has
- // a single element within.
-
- // Determine how much the first stream will be extended.
- // Needed to avoid quadratic blow up from on-the-fly
- // reallocations (#57735).
- let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum();
-
- // Get the first stream, which will become the result stream.
- // If it's `None`, create an empty stream.
- let mut iter = streams.into_iter();
- let mut res_stream_lrc = iter.next().unwrap().0;
-
- // Append the subsequent elements to the result stream, after
- // reserving space for them.
- let res_vec_mut = Lrc::make_mut(&mut res_stream_lrc);
- res_vec_mut.reserve(num_appends);
- for stream in iter {
- let stream_iter = stream.0.iter().cloned();
-
- // If (a) `res_mut_vec` is not empty and the last tree
- // within it is a token tree marked with `Joint`, and (b)
- // `stream` is not empty and the first tree within it is a
- // token tree, and (c) the two tokens can be glued
- // together...
- if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = res_vec_mut.last()
- && let Some(TokenTree::Token(tok, spacing)) = stream.0.first()
- && let Some(glued_tok) = last_tok.glue(&tok)
- {
- // ...then overwrite the last token tree in
- // `res_vec_mut` with the glued token, and skip the
- // first token tree from `stream`.
- *res_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
- res_vec_mut.extend(stream_iter.skip(1));
- } else {
- // Append all of `stream`.
- res_vec_mut.extend(stream_iter);
- }
- }
+ // If `vec` is not empty, try to glue `tt` onto its last token. The return
+ // value indicates if gluing took place.
+ fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
+ if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
+ && let TokenTree::Token(tok, spacing) = tt
+ && let Some(glued_tok) = last_tok.glue(&tok)
+ {
+ // ...then overwrite the last token tree in `vec` with the
+ // glued token, and skip the first token tree from `stream`.
+ *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
+ true
+ } else {
+ false
+ }
+ }
- TokenStream(res_stream_lrc)
- }
+ // Push `tt` onto the end of the stream, possibly gluing it to the last
+ // token. Uses `make_mut` to maximize efficiency.
+ pub fn push_tree(&mut self, tt: TokenTree) {
+ let vec_mut = Lrc::make_mut(&mut self.0);
+
+ if Self::try_glue_to_last(vec_mut, &tt) {
+ // nothing else to do
+ } else {
+ vec_mut.push(tt);
+ }
+ }
+
+ // Push `stream` onto the end of the stream, possibly gluing the first
+ // token tree to the last token. (No other token trees will be glued.)
+ // Uses `make_mut` to maximize efficiency.
+ pub fn push_stream(&mut self, stream: TokenStream) {
+ let vec_mut = Lrc::make_mut(&mut self.0);
+
+ let stream_iter = stream.0.iter().cloned();
+
+ if let Some(first) = stream.0.first() && Self::try_glue_to_last(vec_mut, first) {
+ // Now skip the first token tree from `stream`.
+ vec_mut.extend(stream_iter.skip(1));
+ } else {
+ // Append all of `stream`.
+ vec_mut.extend(stream_iter);
}
}
}
@@ -664,3 +640,17 @@ impl DelimSpan {
self.open.with_hi(self.close.hi())
}
}
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+mod size_asserts {
+ use super::*;
+ use rustc_data_structures::static_assert_size;
+ // tidy-alphabetical-start
+ static_assert_size!(AttrTokenStream, 8);
+ static_assert_size!(AttrTokenTree, 32);
+ static_assert_size!(LazyAttrTokenStream, 8);
+ static_assert_size!(TokenStream, 8);
+ static_assert_size!(TokenTree, 32);
+ // tidy-alphabetical-end
+}