summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_ast/src/tokenstream.rs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--compiler/rustc_ast/src/tokenstream.rs42
1 files changed, 27 insertions, 15 deletions
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index fabd43a16..f0a6a5e07 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -41,7 +41,8 @@ use std::{fmt, iter};
/// Nothing special happens to misnamed or misplaced `SubstNt`s.
#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum TokenTree {
- /// A single token.
+ /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
+ /// delimiters are implicitly represented by `Delimited`.
Token(Token, Spacing),
/// A delimited sequence of token trees.
Delimited(DelimSpan, Delimiter, TokenStream),
@@ -388,12 +389,12 @@ impl TokenStream {
self.0.len()
}
- pub fn trees(&self) -> CursorRef<'_> {
- CursorRef::new(self)
+ pub fn trees(&self) -> RefTokenTreeCursor<'_> {
+ RefTokenTreeCursor::new(self)
}
- pub fn into_trees(self) -> Cursor {
- Cursor::new(self)
+ pub fn into_trees(self) -> TokenTreeCursor {
+ TokenTreeCursor::new(self)
}
/// Compares two `TokenStream`s, checking equality without regarding span information.
@@ -551,16 +552,17 @@ impl TokenStream {
}
}
-/// By-reference iterator over a [`TokenStream`].
+/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
+/// items.
#[derive(Clone)]
-pub struct CursorRef<'t> {
+pub struct RefTokenTreeCursor<'t> {
stream: &'t TokenStream,
index: usize,
}
-impl<'t> CursorRef<'t> {
+impl<'t> RefTokenTreeCursor<'t> {
fn new(stream: &'t TokenStream) -> Self {
- CursorRef { stream, index: 0 }
+ RefTokenTreeCursor { stream, index: 0 }
}
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
@@ -568,7 +570,7 @@ impl<'t> CursorRef<'t> {
}
}
-impl<'t> Iterator for CursorRef<'t> {
+impl<'t> Iterator for RefTokenTreeCursor<'t> {
type Item = &'t TokenTree;
fn next(&mut self) -> Option<&'t TokenTree> {
@@ -579,15 +581,16 @@ impl<'t> Iterator for CursorRef<'t> {
}
}
-/// Owning by-value iterator over a [`TokenStream`].
+/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
+/// items.
// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
#[derive(Clone)]
-pub struct Cursor {
+pub struct TokenTreeCursor {
pub stream: TokenStream,
index: usize,
}
-impl Iterator for Cursor {
+impl Iterator for TokenTreeCursor {
type Item = TokenTree;
fn next(&mut self) -> Option<TokenTree> {
@@ -598,9 +601,9 @@ impl Iterator for Cursor {
}
}
-impl Cursor {
+impl TokenTreeCursor {
fn new(stream: TokenStream) -> Self {
- Cursor { stream, index: 0 }
+ TokenTreeCursor { stream, index: 0 }
}
#[inline]
@@ -614,6 +617,15 @@ impl Cursor {
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.0.get(self.index + n)
}
+
+ // Replace the previously obtained token tree with `tts`, and rewind to
+ // just before them.
+ pub fn replace_prev_and_rewind(&mut self, tts: Vec<TokenTree>) {
+ assert!(self.index > 0);
+ self.index -= 1;
+ let stream = Lrc::make_mut(&mut self.stream.0);
+ stream.splice(self.index..self.index + 1, tts);
+ }
}
#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]