diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /third_party/rust/toml/src/tokens.rs | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/toml/src/tokens.rs')
-rw-r--r-- | third_party/rust/toml/src/tokens.rs | 742 |
1 files changed, 742 insertions, 0 deletions
diff --git a/third_party/rust/toml/src/tokens.rs b/third_party/rust/toml/src/tokens.rs new file mode 100644 index 0000000000..ea1e7ebd83 --- /dev/null +++ b/third_party/rust/toml/src/tokens.rs @@ -0,0 +1,742 @@ +use std::borrow::Cow; +use std::char; +use std::str; +use std::string; +use std::string::String as StdString; + +use self::Token::*; + +/// A span, designating a range of bytes where a token is located. +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +pub struct Span { + /// The start of the range. + pub start: usize, + /// The end of the range (exclusive). + pub end: usize, +} + +impl From<Span> for (usize, usize) { + fn from(Span { start, end }: Span) -> (usize, usize) { + (start, end) + } +} + +#[derive(Eq, PartialEq, Debug)] +pub enum Token<'a> { + Whitespace(&'a str), + Newline, + Comment(&'a str), + + Equals, + Period, + Comma, + Colon, + Plus, + LeftBrace, + RightBrace, + LeftBracket, + RightBracket, + + Keylike(&'a str), + String { + src: &'a str, + val: Cow<'a, str>, + multiline: bool, + }, +} + +#[derive(Eq, PartialEq, Debug)] +pub enum Error { + InvalidCharInString(usize, char), + InvalidEscape(usize, char), + InvalidHexEscape(usize, char), + InvalidEscapeValue(usize, u32), + NewlineInString(usize), + Unexpected(usize, char), + UnterminatedString(usize), + NewlineInTableKey(usize), + MultilineStringKey(usize), + Wanted { + at: usize, + expected: &'static str, + found: &'static str, + }, +} + +#[derive(Clone)] +pub struct Tokenizer<'a> { + input: &'a str, + chars: CrlfFold<'a>, +} + +#[derive(Clone)] +struct CrlfFold<'a> { + chars: str::CharIndices<'a>, +} + +#[derive(Debug)] +enum MaybeString { + NotEscaped(usize), + Owned(string::String), +} + +impl<'a> Tokenizer<'a> { + pub fn new(input: &'a str) -> Tokenizer<'a> { + let mut t = Tokenizer { + input, + chars: CrlfFold { + chars: input.char_indices(), + }, + }; + // Eat utf-8 BOM + t.eatc('\u{feff}'); + t + } + + pub fn next(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> { + let (start, token) = match self.one() { + Some((start, '\n')) => (start, Newline), + Some((start, ' ')) => (start, self.whitespace_token(start)), + Some((start, '\t')) => (start, self.whitespace_token(start)), + Some((start, '#')) => (start, self.comment_token(start)), + Some((start, '=')) => (start, Equals), + Some((start, '.')) => (start, Period), + Some((start, ',')) => (start, Comma), + Some((start, ':')) => (start, Colon), + Some((start, '+')) => (start, Plus), + Some((start, '{')) => (start, LeftBrace), + Some((start, '}')) => (start, RightBrace), + Some((start, '[')) => (start, LeftBracket), + Some((start, ']')) => (start, RightBracket), + Some((start, '\'')) => { + return self + .literal_string(start) + .map(|t| Some((self.step_span(start), t))) + } + Some((start, '"')) => { + return self + .basic_string(start) + .map(|t| Some((self.step_span(start), t))) + } + Some((start, ch)) if is_keylike(ch) => (start, self.keylike(start)), + + Some((start, ch)) => return Err(Error::Unexpected(start, ch)), + None => return Ok(None), + }; + + let span = self.step_span(start); + Ok(Some((span, token))) + } + + pub fn peek(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> { + self.clone().next() + } + + pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> { + self.eat_spanned(expected).map(|s| s.is_some()) + } + + /// Eat a value, returning it's span if it was consumed. + pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> { + let span = match self.peek()? { + Some((span, ref found)) if expected == *found => span, + Some(_) => return Ok(None), + None => return Ok(None), + }; + + drop(self.next()); + Ok(Some(span)) + } + + pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> { + // ignore span + let _ = self.expect_spanned(expected)?; + Ok(()) + } + + /// Expect the given token returning its span. + pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> { + let current = self.current(); + match self.next()? { + Some((span, found)) => { + if expected == found { + Ok(span) + } else { + Err(Error::Wanted { + at: current, + expected: expected.describe(), + found: found.describe(), + }) + } + } + None => Err(Error::Wanted { + at: self.input.len(), + expected: expected.describe(), + found: "eof", + }), + } + } + + pub fn table_key(&mut self) -> Result<(Span, Cow<'a, str>), Error> { + let current = self.current(); + match self.next()? { + Some((span, Token::Keylike(k))) => Ok((span, k.into())), + Some(( + span, + Token::String { + src, + val, + multiline, + }, + )) => { + let offset = self.substr_offset(src); + if multiline { + return Err(Error::MultilineStringKey(offset)); + } + match src.find('\n') { + None => Ok((span, val)), + Some(i) => Err(Error::NewlineInTableKey(offset + i)), + } + } + Some((_, other)) => Err(Error::Wanted { + at: current, + expected: "a table key", + found: other.describe(), + }), + None => Err(Error::Wanted { + at: self.input.len(), + expected: "a table key", + found: "eof", + }), + } + } + + pub fn eat_whitespace(&mut self) -> Result<(), Error> { + while self.eatc(' ') || self.eatc('\t') { + // ... + } + Ok(()) + } + + pub fn eat_comment(&mut self) -> Result<bool, Error> { + if !self.eatc('#') { + return Ok(false); + } + drop(self.comment_token(0)); + self.eat_newline_or_eof().map(|()| true) + } + + pub fn eat_newline_or_eof(&mut self) -> Result<(), Error> { + let current = self.current(); + match self.next()? { + None | Some((_, Token::Newline)) => Ok(()), + Some((_, other)) => Err(Error::Wanted { + at: current, + expected: "newline", + found: other.describe(), + }), + } + } + + pub fn skip_to_newline(&mut self) { + loop { + match self.one() { + Some((_, '\n')) | None => break, + _ => {} + } + } + } + + fn eatc(&mut self, ch: char) -> bool { + match self.chars.clone().next() { + Some((_, ch2)) if ch == ch2 => { + self.one(); + true + } + _ => false, + } + } + + pub fn current(&mut self) -> usize { + self.chars + .clone() + .next() + .map(|i| i.0) + .unwrap_or_else(|| self.input.len()) + } + + pub fn input(&self) -> &'a str { + self.input + } + + fn whitespace_token(&mut self, start: usize) -> Token<'a> { + while self.eatc(' ') || self.eatc('\t') { + // ... + } + Whitespace(&self.input[start..self.current()]) + } + + fn comment_token(&mut self, start: usize) -> Token<'a> { + while let Some((_, ch)) = self.chars.clone().next() { + if ch != '\t' && !('\u{20}'..='\u{10ffff}').contains(&ch) { + break; + } + self.one(); + } + Comment(&self.input[start..self.current()]) + } + + #[allow(clippy::type_complexity)] + fn read_string( + &mut self, + delim: char, + start: usize, + new_ch: &mut dyn FnMut( + &mut Tokenizer<'_>, + &mut MaybeString, + bool, + usize, + char, + ) -> Result<(), Error>, + ) -> Result<Token<'a>, Error> { + let mut multiline = false; + if self.eatc(delim) { + if self.eatc(delim) { + multiline = true; + } else { + return Ok(String { + src: &self.input[start..start + 2], + val: Cow::Borrowed(""), + multiline: false, + }); + } + } + let mut val = MaybeString::NotEscaped(self.current()); + let mut n = 0; + 'outer: loop { + n += 1; + match self.one() { + Some((i, '\n')) => { + if multiline { + if self.input.as_bytes()[i] == b'\r' { + val.to_owned(&self.input[..i]); + } + if n == 1 { + val = MaybeString::NotEscaped(self.current()); + } else { + val.push('\n'); + } + continue; + } else { + return Err(Error::NewlineInString(i)); + } + } + Some((mut i, ch)) if ch == delim => { + if multiline { + if !self.eatc(delim) { + val.push(delim); + continue 'outer; + } + if !self.eatc(delim) { + val.push(delim); + val.push(delim); + continue 'outer; + } + if self.eatc(delim) { + val.push(delim); + i += 1; + } + if self.eatc(delim) { + val.push(delim); + i += 1; + } + } + return Ok(String { + src: &self.input[start..self.current()], + val: val.into_cow(&self.input[..i]), + multiline, + }); + } + Some((i, c)) => new_ch(self, &mut val, multiline, i, c)?, + None => return Err(Error::UnterminatedString(start)), + } + } + } + + fn literal_string(&mut self, start: usize) -> Result<Token<'a>, Error> { + self.read_string('\'', start, &mut |_me, val, _multi, i, ch| { + if ch == '\u{09}' || (('\u{20}'..='\u{10ffff}').contains(&ch) && ch != '\u{7f}') { + val.push(ch); + Ok(()) + } else { + Err(Error::InvalidCharInString(i, ch)) + } + }) + } + + fn basic_string(&mut self, start: usize) -> Result<Token<'a>, Error> { + self.read_string('"', start, &mut |me, val, multi, i, ch| match ch { + '\\' => { + val.to_owned(&me.input[..i]); + match me.chars.next() { + Some((_, '"')) => val.push('"'), + Some((_, '\\')) => val.push('\\'), + Some((_, 'b')) => val.push('\u{8}'), + Some((_, 'f')) => val.push('\u{c}'), + Some((_, 'n')) => val.push('\n'), + Some((_, 'r')) => val.push('\r'), + Some((_, 't')) => val.push('\t'), + Some((i, c @ 'u')) | Some((i, c @ 'U')) => { + let len = if c == 'u' { 4 } else { 8 }; + val.push(me.hex(start, i, len)?); + } + Some((i, c @ ' ')) | Some((i, c @ '\t')) | Some((i, c @ '\n')) if multi => { + if c != '\n' { + while let Some((_, ch)) = me.chars.clone().next() { + match ch { + ' ' | '\t' => { + me.chars.next(); + continue; + } + '\n' => { + me.chars.next(); + break; + } + _ => return Err(Error::InvalidEscape(i, c)), + } + } + } + while let Some((_, ch)) = me.chars.clone().next() { + match ch { + ' ' | '\t' | '\n' => { + me.chars.next(); + } + _ => break, + } + } + } + Some((i, c)) => return Err(Error::InvalidEscape(i, c)), + None => return Err(Error::UnterminatedString(start)), + } + Ok(()) + } + ch if ch == '\u{09}' || (('\u{20}'..='\u{10ffff}').contains(&ch) && ch != '\u{7f}') => { + val.push(ch); + Ok(()) + } + _ => Err(Error::InvalidCharInString(i, ch)), + }) + } + + fn hex(&mut self, start: usize, i: usize, len: usize) -> Result<char, Error> { + let mut buf = StdString::with_capacity(len); + for _ in 0..len { + match self.one() { + Some((_, ch)) if ch as u32 <= 0x7F && ch.is_ascii_hexdigit() => buf.push(ch), + Some((i, ch)) => return Err(Error::InvalidHexEscape(i, ch)), + None => return Err(Error::UnterminatedString(start)), + } + } + let val = u32::from_str_radix(&buf, 16).unwrap(); + match char::from_u32(val) { + Some(ch) => Ok(ch), + None => Err(Error::InvalidEscapeValue(i, val)), + } + } + + fn keylike(&mut self, start: usize) -> Token<'a> { + while let Some((_, ch)) = self.peek_one() { + if !is_keylike(ch) { + break; + } + self.one(); + } + Keylike(&self.input[start..self.current()]) + } + + pub fn substr_offset(&self, s: &'a str) -> usize { + assert!(s.len() <= self.input.len()); + let a = self.input.as_ptr() as usize; + let b = s.as_ptr() as usize; + assert!(a <= b); + b - a + } + + /// Calculate the span of a single character. + fn step_span(&mut self, start: usize) -> Span { + let end = self + .peek_one() + .map(|t| t.0) + .unwrap_or_else(|| self.input.len()); + Span { start, end } + } + + /// Peek one char without consuming it. + fn peek_one(&mut self) -> Option<(usize, char)> { + self.chars.clone().next() + } + + /// Take one char. + pub fn one(&mut self) -> Option<(usize, char)> { + self.chars.next() + } +} + +impl<'a> Iterator for CrlfFold<'a> { + type Item = (usize, char); + + fn next(&mut self) -> Option<(usize, char)> { + self.chars.next().map(|(i, c)| { + if c == '\r' { + let mut attempt = self.chars.clone(); + if let Some((_, '\n')) = attempt.next() { + self.chars = attempt; + return (i, '\n'); + } + } + (i, c) + }) + } +} + +impl MaybeString { + fn push(&mut self, ch: char) { + match *self { + MaybeString::NotEscaped(..) => {} + MaybeString::Owned(ref mut s) => s.push(ch), + } + } + + #[allow(clippy::wrong_self_convention)] + fn to_owned(&mut self, input: &str) { + match *self { + MaybeString::NotEscaped(start) => { + *self = MaybeString::Owned(input[start..].to_owned()); + } + MaybeString::Owned(..) => {} + } + } + + fn into_cow(self, input: &str) -> Cow<'_, str> { + match self { + MaybeString::NotEscaped(start) => Cow::Borrowed(&input[start..]), + MaybeString::Owned(s) => Cow::Owned(s), + } + } +} + +fn is_keylike(ch: char) -> bool { + ('A'..='Z').contains(&ch) + || ('a'..='z').contains(&ch) + || ('0'..='9').contains(&ch) + || ch == '-' + || ch == '_' +} + +impl<'a> Token<'a> { + pub fn describe(&self) -> &'static str { + match *self { + Token::Keylike(_) => "an identifier", + Token::Equals => "an equals", + Token::Period => "a period", + Token::Comment(_) => "a comment", + Token::Newline => "a newline", + Token::Whitespace(_) => "whitespace", + Token::Comma => "a comma", + Token::RightBrace => "a right brace", + Token::LeftBrace => "a left brace", + Token::RightBracket => "a right bracket", + Token::LeftBracket => "a left bracket", + Token::String { multiline, .. } => { + if multiline { + "a multiline string" + } else { + "a string" + } + } + Token::Colon => "a colon", + Token::Plus => "a plus", + } + } +} + +#[cfg(test)] +mod tests { + use super::{Error, Token, Tokenizer}; + use std::borrow::Cow; + + fn err(input: &str, err: Error) { + let mut t = Tokenizer::new(input); + let token = t.next().unwrap_err(); + assert_eq!(token, err); + assert!(t.next().unwrap().is_none()); + } + + #[test] + fn literal_strings() { + fn t(input: &str, val: &str, multiline: bool) { + let mut t = Tokenizer::new(input); + let (_, token) = t.next().unwrap().unwrap(); + assert_eq!( + token, + Token::String { + src: input, + val: Cow::Borrowed(val), + multiline, + } + ); + assert!(t.next().unwrap().is_none()); + } + + t("''", "", false); + t("''''''", "", true); + t("'''\n'''", "", true); + t("'a'", "a", false); + t("'\"a'", "\"a", false); + t("''''a'''", "'a", true); + t("'''\n'a\n'''", "'a\n", true); + t("'''a\n'a\r\n'''", "a\n'a\n", true); + } + + #[test] + fn basic_strings() { + fn t(input: &str, val: &str, multiline: bool) { + let mut t = Tokenizer::new(input); + let (_, token) = t.next().unwrap().unwrap(); + assert_eq!( + token, + Token::String { + src: input, + val: Cow::Borrowed(val), + multiline, + } + ); + assert!(t.next().unwrap().is_none()); + } + + t(r#""""#, "", false); + t(r#""""""""#, "", true); + t(r#""a""#, "a", false); + t(r#""""a""""#, "a", true); + t(r#""\t""#, "\t", false); + t(r#""\u0000""#, "\0", false); + t(r#""\U00000000""#, "\0", false); + t(r#""\U000A0000""#, "\u{A0000}", false); + t(r#""\\t""#, "\\t", false); + t("\"\t\"", "\t", false); + t("\"\"\"\n\t\"\"\"", "\t", true); + t("\"\"\"\\\n\"\"\"", "", true); + t( + "\"\"\"\\\n \t \t \\\r\n \t \n \t \r\n\"\"\"", + "", + true, + ); + t(r#""\r""#, "\r", false); + t(r#""\n""#, "\n", false); + t(r#""\b""#, "\u{8}", false); + t(r#""a\fa""#, "a\u{c}a", false); + t(r#""\"a""#, "\"a", false); + t("\"\"\"\na\"\"\"", "a", true); + t("\"\"\"\n\"\"\"", "", true); + t(r#""""a\"""b""""#, "a\"\"\"b", true); + err(r#""\a"#, Error::InvalidEscape(2, 'a')); + err("\"\\\n", Error::InvalidEscape(2, '\n')); + err("\"\\\r\n", Error::InvalidEscape(2, '\n')); + err("\"\\", Error::UnterminatedString(0)); + err("\"\u{0}", Error::InvalidCharInString(1, '\u{0}')); + err(r#""\U00""#, Error::InvalidHexEscape(5, '"')); + err(r#""\U00"#, Error::UnterminatedString(0)); + err(r#""\uD800"#, Error::InvalidEscapeValue(2, 0xd800)); + err(r#""\UFFFFFFFF"#, Error::InvalidEscapeValue(2, 0xffff_ffff)); + } + + #[test] + fn keylike() { + fn t(input: &str) { + let mut t = Tokenizer::new(input); + let (_, token) = t.next().unwrap().unwrap(); + assert_eq!(token, Token::Keylike(input)); + assert!(t.next().unwrap().is_none()); + } + t("foo"); + t("0bar"); + t("bar0"); + t("1234"); + t("a-b"); + t("a_B"); + t("-_-"); + t("___"); + } + + #[test] + fn all() { + fn t(input: &str, expected: &[((usize, usize), Token<'_>, &str)]) { + let mut tokens = Tokenizer::new(input); + let mut actual: Vec<((usize, usize), Token<'_>, &str)> = Vec::new(); + while let Some((span, token)) = tokens.next().unwrap() { + actual.push((span.into(), token, &input[span.start..span.end])); + } + for (a, b) in actual.iter().zip(expected) { + assert_eq!(a, b); + } + assert_eq!(actual.len(), expected.len()); + } + + t( + " a ", + &[ + ((0, 1), Token::Whitespace(" "), " "), + ((1, 2), Token::Keylike("a"), "a"), + ((2, 3), Token::Whitespace(" "), " "), + ], + ); + + t( + " a\t [[]] \t [] {} , . =\n# foo \r\n#foo \n ", + &[ + ((0, 1), Token::Whitespace(" "), " "), + ((1, 2), Token::Keylike("a"), "a"), + ((2, 4), Token::Whitespace("\t "), "\t "), + ((4, 5), Token::LeftBracket, "["), + ((5, 6), Token::LeftBracket, "["), + ((6, 7), Token::RightBracket, "]"), + ((7, 8), Token::RightBracket, "]"), + ((8, 11), Token::Whitespace(" \t "), " \t "), + ((11, 12), Token::LeftBracket, "["), + ((12, 13), Token::RightBracket, "]"), + ((13, 14), Token::Whitespace(" "), " "), + ((14, 15), Token::LeftBrace, "{"), + ((15, 16), Token::RightBrace, "}"), + ((16, 17), Token::Whitespace(" "), " "), + ((17, 18), Token::Comma, ","), + ((18, 19), Token::Whitespace(" "), " "), + ((19, 20), Token::Period, "."), + ((20, 21), Token::Whitespace(" "), " "), + ((21, 22), Token::Equals, "="), + ((22, 23), Token::Newline, "\n"), + ((23, 29), Token::Comment("# foo "), "# foo "), + ((29, 31), Token::Newline, "\r\n"), + ((31, 36), Token::Comment("#foo "), "#foo "), + ((36, 37), Token::Newline, "\n"), + ((37, 38), Token::Whitespace(" "), " "), + ], + ); + } + + #[test] + fn bare_cr_bad() { + err("\r", Error::Unexpected(0, '\r')); + err("'\n", Error::NewlineInString(1)); + err("'\u{0}", Error::InvalidCharInString(1, '\u{0}')); + err("'", Error::UnterminatedString(0)); + err("\u{0}", Error::Unexpected(0, '\u{0}')); + } + + #[test] + fn bad_comment() { + let mut t = Tokenizer::new("#\u{0}"); + t.next().unwrap().unwrap(); + assert_eq!(t.next(), Err(Error::Unexpected(1, '\u{0}'))); + assert!(t.next().unwrap().is_none()); + } +} |