diff options
Diffstat (limited to 'third_party/rust/litrs/src/bytestr')
| -rw-r--r-- | third_party/rust/litrs/src/bytestr/mod.rs | 126 | ||||
| -rw-r--r-- | third_party/rust/litrs/src/bytestr/tests.rs | 224 |
2 files changed, 350 insertions, 0 deletions
diff --git a/third_party/rust/litrs/src/bytestr/mod.rs b/third_party/rust/litrs/src/bytestr/mod.rs new file mode 100644 index 0000000000..a0e09727f4 --- /dev/null +++ b/third_party/rust/litrs/src/bytestr/mod.rs @@ -0,0 +1,126 @@ +use std::{fmt, ops::Range}; + +use crate::{ + Buffer, ParseError, + err::{perr, ParseErrorKind::*}, + escape::{scan_raw_string, unescape_string}, +}; + + +/// A byte string or raw byte string literal, e.g. `b"hello"` or `br#"abc"def"#`. +/// +/// See [the reference][ref] for more information. +/// +/// [ref]: https://doc.rust-lang.org/reference/tokens.html#byte-string-literals +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ByteStringLit<B: Buffer> { + /// The raw input. + raw: B, + + /// The string value (with all escaped unescaped), or `None` if there were + /// no escapes. In the latter case, `input` is the string value. + value: Option<Vec<u8>>, + + /// The number of hash signs in case of a raw string literal, or `None` if + /// it's not a raw string literal. + num_hashes: Option<u32>, + + /// Start index of the suffix or `raw.len()` if there is no suffix. + start_suffix: usize, +} + +impl<B: Buffer> ByteStringLit<B> { + /// Parses the input as a (raw) byte string literal. Returns an error if the + /// input is invalid or represents a different kind of literal. + pub fn parse(input: B) -> Result<Self, ParseError> { + if input.is_empty() { + return Err(perr(None, Empty)); + } + if !input.starts_with(r#"b""#) && !input.starts_with("br") { + return Err(perr(None, InvalidByteStringLiteralStart)); + } + + let (value, num_hashes, start_suffix) = parse_impl(&input)?; + Ok(Self { raw: input, value, num_hashes, start_suffix }) + } + + /// Returns the string value this literal represents (where all escapes have + /// been turned into their respective values). + pub fn value(&self) -> &[u8] { + self.value.as_deref().unwrap_or(&self.raw.as_bytes()[self.inner_range()]) + } + + /// Like `value` but returns a potentially owned version of the value. + /// + /// The return value is either `Cow<'static, [u8]>` if `B = String`, or + /// `Cow<'a, [u8]>` if `B = &'a str`. + pub fn into_value(self) -> B::ByteCow { + let inner_range = self.inner_range(); + let Self { raw, value, .. } = self; + value.map(B::ByteCow::from).unwrap_or_else(|| raw.cut(inner_range).into_byte_cow()) + } + + /// The optional suffix. Returns `""` if the suffix is empty/does not exist. + pub fn suffix(&self) -> &str { + &(*self.raw)[self.start_suffix..] + } + + /// Returns whether this literal is a raw string literal (starting with + /// `r`). + pub fn is_raw_byte_string(&self) -> bool { + self.num_hashes.is_some() + } + + /// Returns the raw input that was passed to `parse`. + pub fn raw_input(&self) -> &str { + &self.raw + } + + /// Returns the raw input that was passed to `parse`, potentially owned. + pub fn into_raw_input(self) -> B { + self.raw + } + + /// The range within `self.raw` that excludes the quotes and potential `r#`. + fn inner_range(&self) -> Range<usize> { + match self.num_hashes { + None => 2..self.start_suffix - 1, + Some(n) => 2 + n as usize + 1..self.start_suffix - n as usize - 1, + } + } +} + +impl ByteStringLit<&str> { + /// Makes a copy of the underlying buffer and returns the owned version of + /// `Self`. + pub fn into_owned(self) -> ByteStringLit<String> { + ByteStringLit { + raw: self.raw.to_owned(), + value: self.value, + num_hashes: self.num_hashes, + start_suffix: self.start_suffix, + } + } +} + +impl<B: Buffer> fmt::Display for ByteStringLit<B> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(&self.raw) + } +} + + +/// Precondition: input has to start with either `b"` or `br`. +#[inline(never)] +fn parse_impl(input: &str) -> Result<(Option<Vec<u8>>, Option<u32>, usize), ParseError> { + if input.starts_with("br") { + scan_raw_string::<u8>(&input, 2) + .map(|(v, num, start_suffix)| (v.map(String::into_bytes), Some(num), start_suffix)) + } else { + unescape_string::<u8>(&input, 2) + .map(|(v, start_suffix)| (v.map(String::into_bytes), None, start_suffix)) + } +} + +#[cfg(test)] +mod tests; diff --git a/third_party/rust/litrs/src/bytestr/tests.rs b/third_party/rust/litrs/src/bytestr/tests.rs new file mode 100644 index 0000000000..2afef5a99c --- /dev/null +++ b/third_party/rust/litrs/src/bytestr/tests.rs @@ -0,0 +1,224 @@ +use crate::{Literal, ByteStringLit, test_util::{assert_parse_ok_eq, assert_roundtrip}}; + +// ===== Utility functions ======================================================================= + +macro_rules! check { + ($lit:literal, $has_escapes:expr, $num_hashes:expr) => { + check!($lit, stringify!($lit), $has_escapes, $num_hashes, "") + }; + ($lit:literal, $input:expr, $has_escapes:expr, $num_hashes:expr, $suffix:literal) => { + let input = $input; + let expected = ByteStringLit { + raw: input, + value: if $has_escapes { Some($lit.to_vec()) } else { None }, + num_hashes: $num_hashes, + start_suffix: input.len() - $suffix.len(), + }; + + assert_parse_ok_eq( + input, ByteStringLit::parse(input), expected.clone(), "ByteStringLit::parse"); + assert_parse_ok_eq( + input, Literal::parse(input), Literal::ByteString(expected.clone()), "Literal::parse"); + let lit = ByteStringLit::parse(input).unwrap(); + assert_eq!(lit.value(), $lit); + assert_eq!(lit.suffix(), $suffix); + assert_eq!(lit.into_value().as_ref(), $lit); + assert_roundtrip(expected.into_owned(), input); + }; +} + + +// ===== Actual tests ============================================================================ + +#[test] +fn simple() { + check!(b"", false, None); + check!(b"a", false, None); + check!(b"peter", false, None); +} + +#[test] +fn special_whitespace() { + let strings = ["\n", "\t", "foo\tbar", "baz\n"]; + + for &s in &strings { + let input = format!(r#"b"{}""#, s); + let input_raw = format!(r#"br"{}""#, s); + for (input, num_hashes) in vec![(input, None), (input_raw, Some(0))] { + let expected = ByteStringLit { + raw: &*input, + value: None, + num_hashes, + start_suffix: input.len(), + }; + assert_parse_ok_eq( + &input, ByteStringLit::parse(&*input), expected.clone(), "ByteStringLit::parse"); + assert_parse_ok_eq( + &input, Literal::parse(&*input), Literal::ByteString(expected), "Literal::parse"); + assert_eq!(ByteStringLit::parse(&*input).unwrap().value(), s.as_bytes()); + assert_eq!(ByteStringLit::parse(&*input).unwrap().into_value(), s.as_bytes()); + } + } + + let res = ByteStringLit::parse("br\"\r\"").expect("failed to parse"); + assert_eq!(res.value(), b"\r"); +} + +#[test] +fn simple_escapes() { + check!(b"a\nb", true, None); + check!(b"\nb", true, None); + check!(b"a\n", true, None); + check!(b"\n", true, None); + + check!(b"\x60foo \t bar\rbaz\n banana \0kiwi", true, None); + check!(b"foo \\ferris", true, None); + check!(b"baz \\ferris\"box", true, None); + check!(b"\\foo\\ banana\" baz\"", true, None); + check!(b"\"foo \\ferris \" baz\\", true, None); + + check!(b"\x00", true, None); + check!(b" \x01", true, None); + check!(b"\x0c foo", true, None); + check!(b" foo\x0D ", true, None); + check!(b"\\x13", true, None); + check!(b"\"x30", true, None); +} + +#[test] +fn string_continue() { + check!(b"foo\ + bar", true, None); + check!(b"foo\ +bar", true, None); + + check!(b"foo\ + + banana", true, None); + + // Weird whitespace characters + let lit = ByteStringLit::parse("b\"foo\\\n\r\t\n \n\tbar\"").expect("failed to parse"); + assert_eq!(lit.value(), b"foobar"); + + // Raw strings do not handle "string continues" + check!(br"foo\ + bar", false, Some(0)); +} + +#[test] +fn crlf_newlines() { + let lit = ByteStringLit::parse("b\"foo\r\nbar\"").expect("failed to parse"); + assert_eq!(lit.value(), b"foo\nbar"); + + let lit = ByteStringLit::parse("b\"\r\nbar\"").expect("failed to parse"); + assert_eq!(lit.value(), b"\nbar"); + + let lit = ByteStringLit::parse("b\"foo\r\n\"").expect("failed to parse"); + assert_eq!(lit.value(), b"foo\n"); + + let lit = ByteStringLit::parse("br\"foo\r\nbar\"").expect("failed to parse"); + assert_eq!(lit.value(), b"foo\nbar"); + + let lit = ByteStringLit::parse("br#\"\r\nbar\"#").expect("failed to parse"); + assert_eq!(lit.value(), b"\nbar"); + + let lit = ByteStringLit::parse("br##\"foo\r\n\"##").expect("failed to parse"); + assert_eq!(lit.value(), b"foo\n"); +} + +#[test] +fn raw_byte_string() { + check!(br"", false, Some(0)); + check!(br"a", false, Some(0)); + check!(br"peter", false, Some(0)); + check!(br"Greetings jason!", false, Some(0)); + + check!(br#""#, false, Some(1)); + check!(br#"a"#, false, Some(1)); + check!(br##"peter"##, false, Some(2)); + check!(br###"Greetings # Jason!"###, false, Some(3)); + check!(br########"we ## need #### more ####### hashtags"########, false, Some(8)); + + check!(br#"foo " bar"#, false, Some(1)); + check!(br##"foo " bar"##, false, Some(2)); + check!(br#"foo """" '"'" bar"#, false, Some(1)); + check!(br#""foo""#, false, Some(1)); + check!(br###""foo'"###, false, Some(3)); + check!(br#""x'#_#s'"#, false, Some(1)); + check!(br"#", false, Some(0)); + check!(br"foo#", false, Some(0)); + check!(br"##bar", false, Some(0)); + check!(br###""##foo"##bar'"###, false, Some(3)); + + check!(br"foo\n\t\r\0\\x60\u{123}doggo", false, Some(0)); + check!(br#"cat\n\t\r\0\\x60\u{123}doggo"#, false, Some(1)); +} + +#[test] +fn suffixes() { + check!(b"hello", r###"b"hello"suffix"###, false, None, "suffix"); + check!(b"fox", r#"b"fox"peter"#, false, None, "peter"); + check!(b"a\x0cb\\", r#"b"a\x0cb\\"_jürgen"#, true, None, "_jürgen"); + check!(br"a\x0cb\\", r###"br#"a\x0cb\\"#_jürgen"###, false, Some(1), "_jürgen"); +} + +#[test] +fn parse_err() { + assert_err!(ByteStringLit, r#"b""#, UnterminatedString, None); + assert_err!(ByteStringLit, r#"b"cat"#, UnterminatedString, None); + assert_err!(ByteStringLit, r#"b"Jurgen"#, UnterminatedString, None); + assert_err!(ByteStringLit, r#"b"foo bar baz"#, UnterminatedString, None); + + assert_err!(ByteStringLit, r#"b"fox"peter""#, InvalidSuffix, 6); + assert_err!(ByteStringLit, r###"br#"foo "# bar"#"###, UnexpectedChar, 10); + + assert_err!(ByteStringLit, "b\"\r\"", IsolatedCr, 2); + assert_err!(ByteStringLit, "b\"fo\rx\"", IsolatedCr, 4); + + assert_err!(ByteStringLit, r##"br####""##, UnterminatedRawString, None); + assert_err!(ByteStringLit, r#####"br##"foo"#bar"#####, UnterminatedRawString, None); + assert_err!(ByteStringLit, r##"br####"##, InvalidLiteral, None); + assert_err!(ByteStringLit, r##"br####x"##, InvalidLiteral, None); +} + +#[test] +fn non_ascii() { + assert_err!(ByteStringLit, r#"b"న""#, NonAsciiInByteLiteral, 2); + assert_err!(ByteStringLit, r#"b"foo犬""#, NonAsciiInByteLiteral, 5); + assert_err!(ByteStringLit, r#"b"x🦊baz""#, NonAsciiInByteLiteral, 3); + assert_err!(ByteStringLit, r#"br"న""#, NonAsciiInByteLiteral, 3); + assert_err!(ByteStringLit, r#"br"foo犬""#, NonAsciiInByteLiteral, 6); + assert_err!(ByteStringLit, r#"br"x🦊baz""#, NonAsciiInByteLiteral, 4); +} + +#[test] +fn invalid_escapes() { + assert_err!(ByteStringLit, r#"b"\a""#, UnknownEscape, 2..4); + assert_err!(ByteStringLit, r#"b"foo\y""#, UnknownEscape, 5..7); + assert_err!(ByteStringLit, r#"b"\"#, UnterminatedEscape, 2); + assert_err!(ByteStringLit, r#"b"\x""#, UnterminatedEscape, 2..4); + assert_err!(ByteStringLit, r#"b"foo\x1""#, UnterminatedEscape, 5..8); + assert_err!(ByteStringLit, r#"b" \xaj""#, InvalidXEscape, 3..7); + assert_err!(ByteStringLit, r#"b"\xjbbaz""#, InvalidXEscape, 2..6); +} + +#[test] +fn unicode_escape_not_allowed() { + assert_err!(ByteStringLit, r#"b"\u{0}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{00}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{b}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{B}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{7e}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{E4}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{e4}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{fc}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{Fc}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{fC}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{FC}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{b10}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{B10}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{0b10}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{2764}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{1f602}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!(ByteStringLit, r#"b"\u{1F602}""#, UnicodeEscapeInByteLiteral, 2..4); +} |