diff options
Diffstat (limited to 'vendor/syn/src/buffer.rs')
| -rw-r--r-- | vendor/syn/src/buffer.rs | 215 |
1 files changed, 72 insertions, 143 deletions
diff --git a/vendor/syn/src/buffer.rs b/vendor/syn/src/buffer.rs index 2c8e21bdb..161b614c8 100644 --- a/vendor/syn/src/buffer.rs +++ b/vendor/syn/src/buffer.rs @@ -14,23 +14,18 @@ use crate::proc_macro as pm; use crate::Lifetime; use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; -use std::hint; use std::marker::PhantomData; -use std::mem; -use std::ptr; -use std::slice; /// Internal type which is used instead of `TokenTree` to represent a token tree /// within a `TokenBuffer`. enum Entry { // Mimicking types from proc-macro. - Group(Group, TokenBuffer), + // Group entries contain the offset to the matching End entry. + Group(Group, usize), Ident(Ident), Punct(Punct), Literal(Literal), - // End entries contain a raw pointer to the entry from the containing - // token tree, or null if this is the outermost level. - End(*const Entry), + End, } /// A buffer that can be efficiently traversed multiple times, unlike @@ -39,102 +34,29 @@ enum Entry { /// /// *This type is available only if Syn is built with the `"parsing"` feature.* pub struct TokenBuffer { - // NOTE: Do not implement clone on this - there are raw pointers inside - // these entries which will be messed up. Moving the `TokenBuffer` itself is - // safe as the data pointed to won't be moved. - ptr: *const Entry, - len: usize, -} - -impl Drop for TokenBuffer { - fn drop(&mut self) { - unsafe { - let slice = slice::from_raw_parts_mut(self.ptr as *mut Entry, self.len); - let _ = Box::from_raw(slice); - } - } + // NOTE: Do not implement clone on this - while the current design could be + // cloned, other designs which could be desirable may not be cloneable. + entries: Box<[Entry]>, } impl TokenBuffer { - // NOTE: Do not mutate the Vec returned from this function once it returns; - // the address of its backing memory must remain stable. - fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer { - let iterator = stream.into_iter(); - let mut entries = Vec::with_capacity(iterator.size_hint().0 + 1); - let mut next_index_after_last_group = 0; - for tt in iterator { + fn recursive_new(entries: &mut Vec<Entry>, stream: TokenStream) { + for tt in stream { match tt { - TokenTree::Ident(ident) => { - entries.push(Entry::Ident(ident)); - } - TokenTree::Punct(punct) => { - entries.push(Entry::Punct(punct)); - } - TokenTree::Literal(literal) => { - entries.push(Entry::Literal(literal)); - } + TokenTree::Ident(ident) => entries.push(Entry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(Entry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(Entry::Literal(literal)), TokenTree::Group(group) => { - // We cannot fill in a real `End` pointer until `entries` is - // finished growing and getting potentially reallocated. - // Instead, we temporarily coopt the spot where the end - // pointer would go, and use it to string together an - // intrusive linked list of all the Entry::Group entries in - // the vector. Later after `entries` is done growing, we'll - // traverse the linked list and fill in all the end - // pointers with a correct value. - let group_up = - ptr::null::<u8>().wrapping_add(next_index_after_last_group) as *const Entry; - - let inner = Self::inner_new(group.stream(), group_up); - entries.push(Entry::Group(group, inner)); - next_index_after_last_group = entries.len(); + let group_start_index = entries.len(); + entries.push(Entry::End); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + entries.push(Entry::End); + let group_end_offset = group_end_index - group_start_index; + entries[group_start_index] = Entry::Group(group, group_end_offset); } } } - - // Add an `End` entry to the end with a reference to the enclosing token - // stream which was passed in. - entries.push(Entry::End(up)); - - // NOTE: This is done to ensure that we don't accidentally modify the - // length of the backing buffer. The backing buffer must remain at a - // constant address after this point, as we are going to store a raw - // pointer into it. - let entries = entries.into_boxed_slice(); - let len = entries.len(); - - // Convert boxed slice into a pointer to the first element early, to - // avoid invalidating pointers into this slice when we move the Box. - // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326 - let entries = Box::into_raw(entries) as *mut Entry; - - // Traverse intrusive linked list of Entry::Group entries and fill in - // correct End pointers. - while let Some(idx) = next_index_after_last_group.checked_sub(1) { - // We know that idx refers to one of the Entry::Group entries, and - // that the very last entry is an Entry::End, so the next index - // after any group entry is a valid index. - let group_up = unsafe { entries.add(next_index_after_last_group) }; - - // Linked list only takes us to entries which are of type Group. - let token_buffer = match unsafe { &*entries.add(idx) } { - Entry::Group(_group, token_buffer) => token_buffer, - _ => unsafe { hint::unreachable_unchecked() }, - }; - - // Last entry in any TokenBuffer is of type End. - let buffer_ptr = token_buffer.ptr as *mut Entry; - let last_entry = unsafe { &mut *buffer_ptr.add(token_buffer.len - 1) }; - let end_ptr_slot = match last_entry { - Entry::End(end_ptr_slot) => end_ptr_slot, - _ => unsafe { hint::unreachable_unchecked() }, - }; - - // Step to next element in linked list. - next_index_after_last_group = mem::replace(end_ptr_slot, group_up) as usize; - } - - TokenBuffer { ptr: entries, len } } /// Creates a `TokenBuffer` containing all the tokens from the input @@ -153,13 +75,19 @@ impl TokenBuffer { /// Creates a `TokenBuffer` containing all the tokens from the input /// `proc_macro2::TokenStream`. pub fn new2(stream: TokenStream) -> Self { - Self::inner_new(stream, ptr::null()) + let mut entries = Vec::new(); + Self::recursive_new(&mut entries, stream); + entries.push(Entry::End); + Self { + entries: entries.into_boxed_slice(), + } } /// Creates a cursor referencing the first token in the buffer and able to /// traverse until the end of the buffer. pub fn begin(&self) -> Cursor { - unsafe { Cursor::create(self.ptr, self.ptr.add(self.len - 1)) } + let ptr = self.entries.as_ptr(); + unsafe { Cursor::create(ptr, ptr.add(self.entries.len() - 1)) } } } @@ -179,7 +107,7 @@ impl TokenBuffer { pub struct Cursor<'a> { // The current entry which the `Cursor` is pointing at. ptr: *const Entry, - // This is the only `Entry::End(..)` object which this cursor is allowed to + // This is the only `Entry::End` object which this cursor is allowed to // point at. All other `End` objects are skipped over in `Cursor::create`. scope: *const Entry, // Cursor is covariant in 'a. This field ensures that our pointers are still @@ -199,7 +127,7 @@ impl<'a> Cursor<'a> { // object in global storage. struct UnsafeSyncEntry(Entry); unsafe impl Sync for UnsafeSyncEntry {} - static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry)); + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End); Cursor { ptr: &EMPTY_ENTRY.0, @@ -212,15 +140,15 @@ impl<'a> Cursor<'a> { /// `None`-delimited scopes when the cursor reaches the end of them, /// allowing for them to be treated transparently. unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { - // NOTE: If we're looking at a `End(..)`, we want to advance the cursor + // NOTE: If we're looking at a `End`, we want to advance the cursor // past it, unless `ptr == scope`, which means that we're at the edge of // our cursor's scope. We should only have `ptr != scope` at the exit // from None-delimited groups entered with `ignore_none`. - while let Entry::End(exit) = *ptr { + while let Entry::End = *ptr { if ptr == scope { break; } - ptr = exit; + ptr = ptr.add(1); } Cursor { @@ -238,7 +166,10 @@ impl<'a> Cursor<'a> { /// Bump the cursor to point at the next token after the current one. This /// is undefined behavior if the cursor is currently looking at an /// `Entry::End`. - unsafe fn bump(self) -> Cursor<'a> { + /// + /// If the cursor is looking at an `Entry::Group`, the bumped cursor will + /// point at the first token in the group (with the same scope end). + unsafe fn bump_ignore_group(self) -> Cursor<'a> { Cursor::create(self.ptr.offset(1), self.scope) } @@ -248,14 +179,9 @@ impl<'a> Cursor<'a> { /// /// WARNING: This mutates its argument. fn ignore_none(&mut self) { - while let Entry::Group(group, buf) = self.entry() { + while let Entry::Group(group, _) = self.entry() { if group.delimiter() == Delimiter::None { - // NOTE: We call `Cursor::create` here to make sure that - // situations where we should immediately exit the span after - // entering it are handled correctly. - unsafe { - *self = Cursor::create(buf.ptr, self.scope); - } + unsafe { *self = self.bump_ignore_group() }; } else { break; } @@ -279,9 +205,12 @@ impl<'a> Cursor<'a> { self.ignore_none(); } - if let Entry::Group(group, buf) = self.entry() { + if let Entry::Group(group, end_offset) = self.entry() { if group.delimiter() == delim { - return Some((buf.begin(), group.span(), unsafe { self.bump() })); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, group.span(), after_group)); } } @@ -293,7 +222,7 @@ impl<'a> Cursor<'a> { pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { self.ignore_none(); match self.entry() { - Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })), + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump_ignore_group() })), _ => None, } } @@ -304,7 +233,7 @@ impl<'a> Cursor<'a> { self.ignore_none(); match self.entry() { Entry::Punct(punct) if punct.as_char() != '\'' => { - Some((punct.clone(), unsafe { self.bump() })) + Some((punct.clone(), unsafe { self.bump_ignore_group() })) } _ => None, } @@ -315,7 +244,7 @@ impl<'a> Cursor<'a> { pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { self.ignore_none(); match self.entry() { - Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump() })), + Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump_ignore_group() })), _ => None, } } @@ -326,17 +255,13 @@ impl<'a> Cursor<'a> { self.ignore_none(); match self.entry() { Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { - let next = unsafe { self.bump() }; - match next.ident() { - Some((ident, rest)) => { - let lifetime = Lifetime { - apostrophe: punct.span(), - ident, - }; - Some((lifetime, rest)) - } - None => None, - } + let next = unsafe { self.bump_ignore_group() }; + let (ident, rest) = next.ident()?; + let lifetime = Lifetime { + apostrophe: punct.span(), + ident, + }; + Some((lifetime, rest)) } _ => None, } @@ -362,15 +287,16 @@ impl<'a> Cursor<'a> { /// This method does not treat `None`-delimited groups as transparent, and /// will return a `Group(None, ..)` if the cursor is looking at one. pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { - let tree = match self.entry() { - Entry::Group(group, _) => group.clone().into(), - Entry::Literal(literal) => literal.clone().into(), - Entry::Ident(ident) => ident.clone().into(), - Entry::Punct(punct) => punct.clone().into(), - Entry::End(..) => return None, + let (tree, len) = match self.entry() { + Entry::Group(group, end_offset) => (group.clone().into(), *end_offset), + Entry::Literal(literal) => (literal.clone().into(), 1), + Entry::Ident(ident) => (ident.clone().into(), 1), + Entry::Punct(punct) => (punct.clone().into(), 1), + Entry::End => return None, }; - Some((tree, unsafe { self.bump() })) + let rest = unsafe { Cursor::create(self.ptr.add(len), self.scope) }; + Some((tree, rest)) } /// Returns the `Span` of the current token, or `Span::call_site()` if this @@ -381,7 +307,7 @@ impl<'a> Cursor<'a> { Entry::Literal(literal) => literal.span(), Entry::Ident(ident) => ident.span(), Entry::Punct(punct) => punct.span(), - Entry::End(..) => Span::call_site(), + Entry::End => Span::call_site(), } } @@ -390,19 +316,22 @@ impl<'a> Cursor<'a> { /// /// This method treats `'lifetimes` as a single token. pub(crate) fn skip(self) -> Option<Cursor<'a>> { - match self.entry() { - Entry::End(..) => None, + let len = match self.entry() { + Entry::End => return None, // Treat lifetimes as a single tt for the purposes of 'skip'. Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { - let next = unsafe { self.bump() }; - match next.entry() { - Entry::Ident(_) => Some(unsafe { next.bump() }), - _ => Some(next), + match unsafe { &*self.ptr.add(1) } { + Entry::Ident(_) => 2, + _ => 1, } } - _ => Some(unsafe { self.bump() }), - } + + Entry::Group(_, end_offset) => *end_offset, + _ => 1, + }; + + Some(unsafe { Cursor::create(self.ptr.add(len), self.scope) }) } } |