1 files changed, 144 insertions, 0 deletions
diff --git a/vendor/pest/src/iterators/tokens.rs b/vendor/pest/src/iterators/tokens.rs
new file mode 100644
index 000000000..59b75c520
--- /dev/null
+++ b/vendor/pest/src/iterators/tokens.rs
@@ -0,0 +1,144 @@
+// pest. The Elegant Parser
+// Copyright (c) 2018 Dragoș Tiselice
+//
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
+// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. All files in the project carrying such notice may not be copied,
+// modified, or distributed except according to those terms.
+
+use std::fmt;
+use std::rc::Rc;
+use std::str;
+
+use super::queueable_token::QueueableToken;
+use position;
+use token::Token;
+use RuleType;
+
+/// An iterator over [`Token`]s. It is created by [`Pair::tokens`] and [`Pairs::tokens`].
+///
+/// [`Token`]: ../enum.Token.html
+/// [`Pair::tokens`]: struct.Pair.html#method.tokens
+/// [`Pairs::tokens`]: struct.Pairs.html#method.tokens
+#[derive(Clone)]
+pub struct Tokens<'i, R> {
+    /// # Safety:
+    ///
+    /// All `QueueableToken`s' `input_pos` must be valid character boundary indices into `input`.
+    queue: Rc<Vec<QueueableToken<R>>>,
+    input: &'i str,
+    start: usize,
+    end: usize,
+}
+
+// TODO(safety): QueueableTokens must be valid indices into input.
+pub fn new<R: RuleType>(
+    queue: Rc<Vec<QueueableToken<R>>>,
+    input: &str,
+    start: usize,
+    end: usize,
+) -> Tokens<R> {
+    if cfg!(debug_assertions) {
+        for tok in queue.iter() {
+            match *tok {
+                QueueableToken::Start { input_pos, .. } | QueueableToken::End { input_pos, .. } => {
+                    assert!(
+                        input.get(input_pos..).is_some(),
+                        "💥 UNSAFE `Tokens` CREATED 💥"
+                    )
+                }
+            }
+        }
+    }
+
+    Tokens {
+        queue,
+        input,
+        start,
+        end,
+    }
+}
+
+impl<'i, R: RuleType> Tokens<'i, R> {
+    fn create_token(&self, index: usize) -> Token<'i, R> {
+        match self.queue[index] {
+            QueueableToken::Start {
+                end_token_index,
+                input_pos,
+            } => {
+                let rule = match self.queue[end_token_index] {
+                    QueueableToken::End { rule, .. } => rule,
+                    _ => unreachable!(),
+                };
+
+                Token::Start {
+                    rule,
+                    // QueueableTokens are safely created.
+                    pos: unsafe { position::Position::new_unchecked(self.input, input_pos) },
+                }
+            }
+            QueueableToken::End {
+                rule, input_pos, ..
+            } => {
+                Token::End {
+                    rule,
+                    // QueueableTokens are safely created.
+                    pos: unsafe { position::Position::new_unchecked(self.input, input_pos) },
+                }
+            }
+        }
+    }
+}
+
+impl<'i, R: RuleType> Iterator for Tokens<'i, R> {
+    type Item = Token<'i, R>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.start >= self.end {
+            return None;
+        }
+
+        let token = self.create_token(self.start);
+
+        self.start += 1;
+
+        Some(token)
+    }
+}
+
+impl<'i, R: RuleType> DoubleEndedIterator for Tokens<'i, R> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.end <= self.start {
+            return None;
+        }
+
+        let token = self.create_token(self.end - 1);
+
+        self.end -= 1;
+
+        Some(token)
+    }
+}
+
+impl<'i, R: RuleType> fmt::Debug for Tokens<'i, R> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::super::macros::tests::*;
+    use super::super::super::Parser;
+    use super::Token;
+
+    #[test]
+    fn double_ended_iter_for_tokens() {
+        let pairs = AbcParser::parse(Rule::a, "abcde").unwrap();
+        let mut tokens = pairs.clone().tokens().collect::<Vec<Token<Rule>>>();
+        tokens.reverse();
+        let reverse_tokens = pairs.tokens().rev().collect::<Vec<Token<Rule>>>();
+        assert_eq!(tokens, reverse_tokens);
+    }
+}