diff options
Diffstat (limited to 'vendor/bstr/src/utf8.rs')
| -rw-r--r-- | vendor/bstr/src/utf8.rs | 1370 |
1 files changed, 0 insertions, 1370 deletions
diff --git a/vendor/bstr/src/utf8.rs b/vendor/bstr/src/utf8.rs deleted file mode 100644 index 5c7de363d..000000000 --- a/vendor/bstr/src/utf8.rs +++ /dev/null @@ -1,1370 +0,0 @@ -use core::char; -use core::cmp; -use core::fmt; -use core::str; -#[cfg(feature = "std")] -use std::error; - -use crate::ascii; -use crate::bstr::BStr; -use crate::ext_slice::ByteSlice; - -// The UTF-8 decoder provided here is based on the one presented here: -// https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ -// -// We *could* have done UTF-8 decoding by using a DFA generated by `\p{any}` -// using regex-automata that is roughly the same size. The real benefit of -// Hoehrmann's formulation is that the byte class mapping below is manually -// tailored such that each byte's class doubles as a shift to mask out the -// bits necessary for constructing the leading bits of each codepoint value -// from the initial byte. -// -// There are some minor differences between this implementation and Hoehrmann's -// formulation. -// -// Firstly, we make REJECT have state ID 0, since it makes the state table -// itself a little easier to read and is consistent with the notion that 0 -// means "false" or "bad." -// -// Secondly, when doing bulk decoding, we add a SIMD accelerated ASCII fast -// path. -// -// Thirdly, we pre-multiply the state IDs to avoid a multiplication instruction -// in the core decoding loop. (Which is what regex-automata would do by -// default.) -// -// Fourthly, we split the byte class mapping and transition table into two -// arrays because it's clearer. -// -// It is unlikely that this is the fastest way to do UTF-8 decoding, however, -// it is fairly simple. - -const ACCEPT: usize = 12; -const REJECT: usize = 0; - -/// SAFETY: The decode below function relies on the correctness of these -/// equivalence classes. -#[cfg_attr(rustfmt, rustfmt::skip)] -const CLASSES: [u8; 256] = [ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, -]; - -/// SAFETY: The decode below function relies on the correctness of this state -/// machine. -#[cfg_attr(rustfmt, rustfmt::skip)] -const STATES_FORWARD: &'static [u8] = &[ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 24, 36, 60, 96, 84, 0, 0, 0, 48, 72, - 0, 12, 0, 0, 0, 0, 0, 12, 0, 12, 0, 0, - 0, 24, 0, 0, 0, 0, 0, 24, 0, 24, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, - 0, 24, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 36, 0, 36, 0, 0, - 0, 36, 0, 0, 0, 0, 0, 36, 0, 36, 0, 0, - 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -]; - -/// An iterator over Unicode scalar values in a byte string. -/// -/// When invalid UTF-8 byte sequences are found, they are substituted with the -/// Unicode replacement codepoint (`U+FFFD`) using the -/// ["maximal subpart" strategy](http://www.unicode.org/review/pr-121.html). -/// -/// This iterator is created by the -/// [`chars`](trait.ByteSlice.html#method.chars) method provided by the -/// [`ByteSlice`](trait.ByteSlice.html) extension trait for `&[u8]`. -#[derive(Clone, Debug)] -pub struct Chars<'a> { - bs: &'a [u8], -} - -impl<'a> Chars<'a> { - pub(crate) fn new(bs: &'a [u8]) -> Chars<'a> { - Chars { bs } - } - - /// View the underlying data as a subslice of the original data. - /// - /// The slice returned has the same lifetime as the original slice, and so - /// the iterator can continue to be used while this exists. - /// - /// # Examples - /// - /// ``` - /// use bstr::ByteSlice; - /// - /// let mut chars = b"abc".chars(); - /// - /// assert_eq!(b"abc", chars.as_bytes()); - /// chars.next(); - /// assert_eq!(b"bc", chars.as_bytes()); - /// chars.next(); - /// chars.next(); - /// assert_eq!(b"", chars.as_bytes()); - /// ``` - #[inline] - pub fn as_bytes(&self) -> &'a [u8] { - self.bs - } -} - -impl<'a> Iterator for Chars<'a> { - type Item = char; - - #[inline] - fn next(&mut self) -> Option<char> { - let (ch, size) = decode_lossy(self.bs); - if size == 0 { - return None; - } - self.bs = &self.bs[size..]; - Some(ch) - } -} - -impl<'a> DoubleEndedIterator for Chars<'a> { - #[inline] - fn next_back(&mut self) -> Option<char> { - let (ch, size) = decode_last_lossy(self.bs); - if size == 0 { - return None; - } - self.bs = &self.bs[..self.bs.len() - size]; - Some(ch) - } -} - -/// An iterator over Unicode scalar values in a byte string and their -/// byte index positions. -/// -/// When invalid UTF-8 byte sequences are found, they are substituted with the -/// Unicode replacement codepoint (`U+FFFD`) using the -/// ["maximal subpart" strategy](http://www.unicode.org/review/pr-121.html). -/// -/// Note that this is slightly different from the `CharIndices` iterator -/// provided by the standard library. Aside from working on possibly invalid -/// UTF-8, this iterator provides both the corresponding starting and ending -/// byte indices of each codepoint yielded. The ending position is necessary to -/// slice the original byte string when invalid UTF-8 bytes are converted into -/// a Unicode replacement codepoint, since a single replacement codepoint can -/// substitute anywhere from 1 to 3 invalid bytes (inclusive). -/// -/// This iterator is created by the -/// [`char_indices`](trait.ByteSlice.html#method.char_indices) method provided -/// by the [`ByteSlice`](trait.ByteSlice.html) extension trait for `&[u8]`. -#[derive(Clone, Debug)] -pub struct CharIndices<'a> { - bs: &'a [u8], - forward_index: usize, - reverse_index: usize, -} - -impl<'a> CharIndices<'a> { - pub(crate) fn new(bs: &'a [u8]) -> CharIndices<'a> { - CharIndices { bs: bs, forward_index: 0, reverse_index: bs.len() } - } - - /// View the underlying data as a subslice of the original data. - /// - /// The slice returned has the same lifetime as the original slice, and so - /// the iterator can continue to be used while this exists. - /// - /// # Examples - /// - /// ``` - /// use bstr::ByteSlice; - /// - /// let mut it = b"abc".char_indices(); - /// - /// assert_eq!(b"abc", it.as_bytes()); - /// it.next(); - /// assert_eq!(b"bc", it.as_bytes()); - /// it.next(); - /// it.next(); - /// assert_eq!(b"", it.as_bytes()); - /// ``` - #[inline] - pub fn as_bytes(&self) -> &'a [u8] { - self.bs - } -} - -impl<'a> Iterator for CharIndices<'a> { - type Item = (usize, usize, char); - - #[inline] - fn next(&mut self) -> Option<(usize, usize, char)> { - let index = self.forward_index; - let (ch, size) = decode_lossy(self.bs); - if size == 0 { - return None; - } - self.bs = &self.bs[size..]; - self.forward_index += size; - Some((index, index + size, ch)) - } -} - -impl<'a> DoubleEndedIterator for CharIndices<'a> { - #[inline] - fn next_back(&mut self) -> Option<(usize, usize, char)> { - let (ch, size) = decode_last_lossy(self.bs); - if size == 0 { - return None; - } - self.bs = &self.bs[..self.bs.len() - size]; - self.reverse_index -= size; - Some((self.reverse_index, self.reverse_index + size, ch)) - } -} - -impl<'a> ::core::iter::FusedIterator for CharIndices<'a> {} - -/// An iterator over chunks of valid UTF-8 in a byte slice. -/// -/// See [`utf8_chunks`](trait.ByteSlice.html#method.utf8_chunks). -#[derive(Clone, Debug)] -pub struct Utf8Chunks<'a> { - pub(super) bytes: &'a [u8], -} - -/// A chunk of valid UTF-8, possibly followed by invalid UTF-8 bytes. -/// -/// This is yielded by the -/// [`Utf8Chunks`](struct.Utf8Chunks.html) -/// iterator, which can be created via the -/// [`ByteSlice::utf8_chunks`](trait.ByteSlice.html#method.utf8_chunks) -/// method. -/// -/// The `'a` lifetime parameter corresponds to the lifetime of the bytes that -/// are being iterated over. -#[cfg_attr(test, derive(Debug, PartialEq))] -pub struct Utf8Chunk<'a> { - /// A valid UTF-8 piece, at the start, end, or between invalid UTF-8 bytes. - /// - /// This is empty between adjacent invalid UTF-8 byte sequences. - valid: &'a str, - /// A sequence of invalid UTF-8 bytes. - /// - /// Can only be empty in the last chunk. - /// - /// Should be replaced by a single unicode replacement character, if not - /// empty. - invalid: &'a BStr, - /// Indicates whether the invalid sequence could've been valid if there - /// were more bytes. - /// - /// Can only be true in the last chunk. - incomplete: bool, -} - -impl<'a> Utf8Chunk<'a> { - /// Returns the (possibly empty) valid UTF-8 bytes in this chunk. - /// - /// This may be empty if there are consecutive sequences of invalid UTF-8 - /// bytes. - #[inline] - pub fn valid(&self) -> &'a str { - self.valid - } - - /// Returns the (possibly empty) invalid UTF-8 bytes in this chunk that - /// immediately follow the valid UTF-8 bytes in this chunk. - /// - /// This is only empty when this chunk corresponds to the last chunk in - /// the original bytes. - /// - /// The maximum length of this slice is 3. That is, invalid UTF-8 byte - /// sequences greater than 1 always correspond to a valid _prefix_ of - /// a valid UTF-8 encoded codepoint. This corresponds to the "substitution - /// of maximal subparts" strategy that is described in more detail in the - /// docs for the - /// [`ByteSlice::to_str_lossy`](trait.ByteSlice.html#method.to_str_lossy) - /// method. - #[inline] - pub fn invalid(&self) -> &'a [u8] { - self.invalid.as_bytes() - } - - /// Returns whether the invalid sequence might still become valid if more - /// bytes are added. - /// - /// Returns true if the end of the input was reached unexpectedly, - /// without encountering an unexpected byte. - /// - /// This can only be the case for the last chunk. - #[inline] - pub fn incomplete(&self) -> bool { - self.incomplete - } -} - -impl<'a> Iterator for Utf8Chunks<'a> { - type Item = Utf8Chunk<'a>; - - #[inline] - fn next(&mut self) -> Option<Utf8Chunk<'a>> { - if self.bytes.is_empty() { - return None; - } - match validate(self.bytes) { - Ok(()) => { - let valid = self.bytes; - self.bytes = &[]; - Some(Utf8Chunk { - // SAFETY: This is safe because of the guarantees provided - // by utf8::validate. - valid: unsafe { str::from_utf8_unchecked(valid) }, - invalid: [].as_bstr(), - incomplete: false, - }) - } - Err(e) => { - let (valid, rest) = self.bytes.split_at(e.valid_up_to()); - // SAFETY: This is safe because of the guarantees provided by - // utf8::validate. - let valid = unsafe { str::from_utf8_unchecked(valid) }; - let (invalid_len, incomplete) = match e.error_len() { - Some(n) => (n, false), - None => (rest.len(), true), - }; - let (invalid, rest) = rest.split_at(invalid_len); - self.bytes = rest; - Some(Utf8Chunk { - valid, - invalid: invalid.as_bstr(), - incomplete, - }) - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - if self.bytes.is_empty() { - (0, Some(0)) - } else { - (1, Some(self.bytes.len())) - } - } -} - -impl<'a> ::core::iter::FusedIterator for Utf8Chunks<'a> {} - -/// An error that occurs when UTF-8 decoding fails. -/// -/// This error occurs when attempting to convert a non-UTF-8 byte -/// string to a Rust string that must be valid UTF-8. For example, -/// [`to_str`](trait.ByteSlice.html#method.to_str) is one such method. -/// -/// # Example -/// -/// This example shows what happens when a given byte sequence is invalid, -/// but ends with a sequence that is a possible prefix of valid UTF-8. -/// -/// ``` -/// use bstr::{B, ByteSlice}; -/// -/// let s = B(b"foobar\xF1\x80\x80"); -/// let err = s.to_str().unwrap_err(); -/// assert_eq!(err.valid_up_to(), 6); -/// assert_eq!(err.error_len(), None); -/// ``` -/// -/// This example shows what happens when a given byte sequence contains -/// invalid UTF-8. -/// -/// ``` -/// use bstr::ByteSlice; -/// -/// let s = b"foobar\xF1\x80\x80quux"; -/// let err = s.to_str().unwrap_err(); -/// assert_eq!(err.valid_up_to(), 6); -/// // The error length reports the maximum number of bytes that correspond to -/// // a valid prefix of a UTF-8 encoded codepoint. -/// assert_eq!(err.error_len(), Some(3)); -/// -/// // In contrast to the above which contains a single invalid prefix, -/// // consider the case of multiple individal bytes that are never valid -/// // prefixes. Note how the value of error_len changes! -/// let s = b"foobar\xFF\xFFquux"; -/// let err = s.to_str().unwrap_err(); -/// assert_eq!(err.valid_up_to(), 6); -/// assert_eq!(err.error_len(), Some(1)); -/// -/// // The fact that it's an invalid prefix does not change error_len even -/// // when it immediately precedes the end of the string. -/// let s = b"foobar\xFF"; -/// let err = s.to_str().unwrap_err(); -/// assert_eq!(err.valid_up_to(), 6); -/// assert_eq!(err.error_len(), Some(1)); -/// ``` -#[derive(Debug, Eq, PartialEq)] -pub struct Utf8Error { - valid_up_to: usize, - error_len: Option<usize>, -} - -impl Utf8Error { - /// Returns the byte index of the position immediately following the last - /// valid UTF-8 byte. - /// - /// # Example - /// - /// This examples shows how `valid_up_to` can be used to retrieve a - /// possibly empty prefix that is guaranteed to be valid UTF-8: - /// - /// ``` - /// use bstr::ByteSlice; - /// - /// let s = b"foobar\xF1\x80\x80quux"; - /// let err = s.to_str().unwrap_err(); - /// - /// // This is guaranteed to never panic. - /// let string = s[..err.valid_up_to()].to_str().unwrap(); - /// assert_eq!(string, "foobar"); - /// ``` - #[inline] - pub fn valid_up_to(&self) -> usize { - self.valid_up_to - } - - /// Returns the total number of invalid UTF-8 bytes immediately following - /// the position returned by `valid_up_to`. This value is always at least - /// `1`, but can be up to `3` if bytes form a valid prefix of some UTF-8 - /// encoded codepoint. - /// - /// If the end of the original input was found before a valid UTF-8 encoded - /// codepoint could be completed, then this returns `None`. This is useful - /// when processing streams, where a `None` value signals that more input - /// might be needed. - #[inline] - pub fn error_len(&self) -> Option<usize> { - self.error_len - } -} - -#[cfg(feature = "std")] -impl error::Error for Utf8Error { - fn description(&self) -> &str { - "invalid UTF-8" - } -} - -impl fmt::Display for Utf8Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "invalid UTF-8 found at byte offset {}", self.valid_up_to) - } -} - -/// Returns OK if and only if the given slice is completely valid UTF-8. -/// -/// If the slice isn't valid UTF-8, then an error is returned that explains -/// the first location at which invalid UTF-8 was detected. -pub fn validate(slice: &[u8]) -> Result<(), Utf8Error> { - // The fast path for validating UTF-8. It steps through a UTF-8 automaton - // and uses a SIMD accelerated ASCII fast path on x86_64. If an error is - // detected, it backs up and runs the slower version of the UTF-8 automaton - // to determine correct error information. - fn fast(slice: &[u8]) -> Result<(), Utf8Error> { - let mut state = ACCEPT; - let mut i = 0; - - while i < slice.len() { - let b = slice[i]; - - // ASCII fast path. If we see two consecutive ASCII bytes, then try - // to validate as much ASCII as possible very quickly. - if state == ACCEPT - && b <= 0x7F - && slice.get(i + 1).map_or(false, |&b| b <= 0x7F) - { - i += ascii::first_non_ascii_byte(&slice[i..]); - continue; - } - - state = step(state, b); - if state == REJECT { - return Err(find_valid_up_to(slice, i)); - } - i += 1; - } - if state != ACCEPT { - Err(find_valid_up_to(slice, slice.len())) - } else { - Ok(()) - } - } - - // Given the first position at which a UTF-8 sequence was determined to be - // invalid, return an error that correctly reports the position at which - // the last complete UTF-8 sequence ends. - #[inline(never)] - fn find_valid_up_to(slice: &[u8], rejected_at: usize) -> Utf8Error { - // In order to find the last valid byte, we need to back up an amount - // that guarantees every preceding byte is part of a valid UTF-8 - // code unit sequence. To do this, we simply locate the last leading - // byte that occurs before rejected_at. - let mut backup = rejected_at.saturating_sub(1); - while backup > 0 && !is_leading_or_invalid_utf8_byte(slice[backup]) { - backup -= 1; - } - let upto = cmp::min(slice.len(), rejected_at.saturating_add(1)); - let mut err = slow(&slice[backup..upto]).unwrap_err(); - err.valid_up_to += backup; - err - } - - // Like top-level UTF-8 decoding, except it correctly reports a UTF-8 error - // when an invalid sequence is found. This is split out from validate so - // that the fast path doesn't need to keep track of the position of the - // last valid UTF-8 byte. In particular, tracking this requires checking - // for an ACCEPT state on each byte, which degrades throughput pretty - // badly. - fn slow(slice: &[u8]) -> Result<(), Utf8Error> { - let mut state = ACCEPT; - let mut valid_up_to = 0; - for (i, &b) in slice.iter().enumerate() { - state = step(state, b); - if state == ACCEPT { - valid_up_to = i + 1; - } else if state == REJECT { - // Our error length must always be at least 1. - let error_len = Some(cmp::max(1, i - valid_up_to)); - return Err(Utf8Error { valid_up_to, error_len }); - } - } - if state != ACCEPT { - Err(Utf8Error { valid_up_to, error_len: None }) - } else { - Ok(()) - } - } - - // Advance to the next state given the current state and current byte. - fn step(state: usize, b: u8) -> usize { - let class = CLASSES[b as usize]; - // SAFETY: This is safe because 'class' is always <=11 and 'state' is - // always <=96. Therefore, the maximal index is 96+11 = 107, where - // STATES_FORWARD.len() = 108 such that every index is guaranteed to be - // valid by construction of the state machine and the byte equivalence - // classes. - unsafe { - *STATES_FORWARD.get_unchecked(state + class as usize) as usize - } - } - - fast(slice) -} - -/// UTF-8 decode a single Unicode scalar value from the beginning of a slice. -/// -/// When successful, the corresponding Unicode scalar value is returned along -/// with the number of bytes it was encoded with. The number of bytes consumed -/// for a successful decode is always between 1 and 4, inclusive. -/// -/// When unsuccessful, `None` is returned along with the number of bytes that -/// make up a maximal prefix of a valid UTF-8 code unit sequence. In this case, -/// the number of bytes consumed is always between 0 and 3, inclusive, where -/// 0 is only returned when `slice` is empty. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use bstr::decode_utf8; -/// -/// // Decoding a valid codepoint. -/// let (ch, size) = decode_utf8(b"\xE2\x98\x83"); -/// assert_eq!(Some('☃'), ch); -/// assert_eq!(3, size); -/// -/// // Decoding an incomplete codepoint. -/// let (ch, size) = decode_utf8(b"\xE2\x98"); -/// assert_eq!(None, ch); -/// assert_eq!(2, size); -/// ``` -/// -/// This example shows how to iterate over all codepoints in UTF-8 encoded -/// bytes, while replacing invalid UTF-8 sequences with the replacement -/// codepoint: -/// -/// ``` -/// use bstr::{B, decode_utf8}; -/// -/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61"); -/// let mut chars = vec![]; -/// while !bytes.is_empty() { -/// let (ch, size) = decode_utf8(bytes); -/// bytes = &bytes[size..]; -/// chars.push(ch.unwrap_or('\u{FFFD}')); -/// } -/// assert_eq!(vec!['☃', '\u{FFFD}', '𝞃', '\u{FFFD}', 'a'], chars); -/// ``` -#[inline] -pub fn decode<B: AsRef<[u8]>>(slice: B) -> (Option<char>, usize) { - let slice = slice.as_ref(); - match slice.get(0) { - None => return (None, 0), - Some(&b) if b <= 0x7F => return (Some(b as char), 1), - _ => {} - } - - let (mut state, mut cp, mut i) = (ACCEPT, 0, 0); - while i < slice.len() { - decode_step(&mut state, &mut cp, slice[i]); - i += 1; - - if state == ACCEPT { - // SAFETY: This is safe because `decode_step` guarantees that - // `cp` is a valid Unicode scalar value in an ACCEPT state. - let ch = unsafe { char::from_u32_unchecked(cp) }; - return (Some(ch), i); - } else if state == REJECT { - // At this point, we always want to advance at least one byte. - return (None, cmp::max(1, i.saturating_sub(1))); - } - } - (None, i) -} - -/// Lossily UTF-8 decode a single Unicode scalar value from the beginning of a -/// slice. -/// -/// When successful, the corresponding Unicode scalar value is returned along -/// with the number of bytes it was encoded with. The number of bytes consumed -/// for a successful decode is always between 1 and 4, inclusive. -/// -/// When unsuccessful, the Unicode replacement codepoint (`U+FFFD`) is returned -/// along with the number of bytes that make up a maximal prefix of a valid -/// UTF-8 code unit sequence. In this case, the number of bytes consumed is -/// always between 0 and 3, inclusive, where 0 is only returned when `slice` is -/// empty. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ```ignore -/// use bstr::decode_utf8_lossy; -/// -/// // Decoding a valid codepoint. -/// let (ch, size) = decode_utf8_lossy(b"\xE2\x98\x83"); -/// assert_eq!('☃', ch); -/// assert_eq!(3, size); -/// -/// // Decoding an incomplete codepoint. -/// let (ch, size) = decode_utf8_lossy(b"\xE2\x98"); -/// assert_eq!('\u{FFFD}', ch); -/// assert_eq!(2, size); -/// ``` -/// -/// This example shows how to iterate over all codepoints in UTF-8 encoded -/// bytes, while replacing invalid UTF-8 sequences with the replacement -/// codepoint: -/// -/// ```ignore -/// use bstr::{B, decode_utf8_lossy}; -/// -/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61"); -/// let mut chars = vec![]; -/// while !bytes.is_empty() { -/// let (ch, size) = decode_utf8_lossy(bytes); -/// bytes = &bytes[size..]; -/// chars.push(ch); -/// } -/// assert_eq!(vec!['☃', '\u{FFFD}', '𝞃', '\u{FFFD}', 'a'], chars); -/// ``` -#[inline] -pub fn decode_lossy<B: AsRef<[u8]>>(slice: B) -> (char, usize) { - match decode(slice) { - (Some(ch), size) => (ch, size), - (None, size) => ('\u{FFFD}', size), - } -} - -/// UTF-8 decode a single Unicode scalar value from the end of a slice. -/// -/// When successful, the corresponding Unicode scalar value is returned along -/// with the number of bytes it was encoded with. The number of bytes consumed -/// for a successful decode is always between 1 and 4, inclusive. -/// -/// When unsuccessful, `None` is returned along with the number of bytes that -/// make up a maximal prefix of a valid UTF-8 code unit sequence. In this case, -/// the number of bytes consumed is always between 0 and 3, inclusive, where -/// 0 is only returned when `slice` is empty. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use bstr::decode_last_utf8; -/// -/// // Decoding a valid codepoint. -/// let (ch, size) = decode_last_utf8(b"\xE2\x98\x83"); -/// assert_eq!(Some('☃'), ch); -/// assert_eq!(3, size); -/// -/// // Decoding an incomplete codepoint. -/// let (ch, size) = decode_last_utf8(b"\xE2\x98"); -/// assert_eq!(None, ch); -/// assert_eq!(2, size); -/// ``` -/// -/// This example shows how to iterate over all codepoints in UTF-8 encoded -/// bytes in reverse, while replacing invalid UTF-8 sequences with the -/// replacement codepoint: -/// -/// ``` -/// use bstr::{B, decode_last_utf8}; -/// -/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61"); -/// let mut chars = vec![]; -/// while !bytes.is_empty() { -/// let (ch, size) = decode_last_utf8(bytes); -/// bytes = &bytes[..bytes.len()-size]; -/// chars.push(ch.unwrap_or('\u{FFFD}')); -/// } -/// assert_eq!(vec!['a', '\u{FFFD}', '𝞃', '\u{FFFD}', '☃'], chars); -/// ``` -#[inline] -pub fn decode_last<B: AsRef<[u8]>>(slice: B) -> (Option<char>, usize) { - // TODO: We could implement this by reversing the UTF-8 automaton, but for - // now, we do it the slow way by using the forward automaton. - - let slice = slice.as_ref(); - if slice.is_empty() { - return (None, 0); - } - let mut start = slice.len() - 1; - let limit = slice.len().saturating_sub(4); - while start > limit && !is_leading_or_invalid_utf8_byte(slice[start]) { - start -= 1; - } - let (ch, size) = decode(&slice[start..]); - // If we didn't consume all of the bytes, then that means there's at least - // one stray byte that never occurs in a valid code unit prefix, so we can - // advance by one byte. - if start + size != slice.len() { - (None, 1) - } else { - (ch, size) - } -} - -/// Lossily UTF-8 decode a single Unicode scalar value from the end of a slice. -/// -/// When successful, the corresponding Unicode scalar value is returned along -/// with the number of bytes it was encoded with. The number of bytes consumed -/// for a successful decode is always between 1 and 4, inclusive. -/// -/// When unsuccessful, the Unicode replacement codepoint (`U+FFFD`) is returned -/// along with the number of bytes that make up a maximal prefix of a valid -/// UTF-8 code unit sequence. In this case, the number of bytes consumed is -/// always between 0 and 3, inclusive, where 0 is only returned when `slice` is -/// empty. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ```ignore -/// use bstr::decode_last_utf8_lossy; -/// -/// // Decoding a valid codepoint. -/// let (ch, size) = decode_last_utf8_lossy(b"\xE2\x98\x83"); -/// assert_eq!('☃', ch); -/// assert_eq!(3, size); -/// -/// // Decoding an incomplete codepoint. -/// let (ch, size) = decode_last_utf8_lossy(b"\xE2\x98"); -/// assert_eq!('\u{FFFD}', ch); -/// assert_eq!(2, size); -/// ``` -/// -/// This example shows how to iterate over all codepoints in UTF-8 encoded -/// bytes in reverse, while replacing invalid UTF-8 sequences with the -/// replacement codepoint: -/// -/// ```ignore -/// use bstr::decode_last_utf8_lossy; -/// -/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61"); -/// let mut chars = vec![]; -/// while !bytes.is_empty() { -/// let (ch, size) = decode_last_utf8_lossy(bytes); -/// bytes = &bytes[..bytes.len()-size]; -/// chars.push(ch); -/// } -/// assert_eq!(vec!['a', '\u{FFFD}', '𝞃', '\u{FFFD}', '☃'], chars); -/// ``` -#[inline] -pub fn decode_last_lossy<B: AsRef<[u8]>>(slice: B) -> (char, usize) { - match decode_last(slice) { - (Some(ch), size) => (ch, size), - (None, size) => ('\u{FFFD}', size), - } -} - -/// SAFETY: The decode function relies on state being equal to ACCEPT only if -/// cp is a valid Unicode scalar value. -#[inline] -pub fn decode_step(state: &mut usize, cp: &mut u32, b: u8) { - let class = CLASSES[b as usize]; - if *state == ACCEPT { - *cp = (0xFF >> class) & (b as u32); - } else { - *cp = (b as u32 & 0b111111) | (*cp << 6); - } - *state = STATES_FORWARD[*state + class as usize] as usize; -} - -/// Returns true if and only if the given byte is either a valid leading UTF-8 -/// byte, or is otherwise an invalid byte that can never appear anywhere in a -/// valid UTF-8 sequence. -fn is_leading_or_invalid_utf8_byte(b: u8) -> bool { - // In the ASCII case, the most significant bit is never set. The leading - // byte of a 2/3/4-byte sequence always has the top two most significant - // bits set. For bytes that can never appear anywhere in valid UTF-8, this - // also returns true, since every such byte has its two most significant - // bits set: - // - // \xC0 :: 11000000 - // \xC1 :: 11000001 - // \xF5 :: 11110101 - // \xF6 :: 11110110 - // \xF7 :: 11110111 - // \xF8 :: 11111000 - // \xF9 :: 11111001 - // \xFA :: 11111010 - // \xFB :: 11111011 - // \xFC :: 11111100 - // \xFD :: 11111101 - // \xFE :: 11111110 - // \xFF :: 11111111 - (b & 0b1100_0000) != 0b1000_0000 -} - -#[cfg(test)] -mod tests { - use std::char; - - use crate::ext_slice::{ByteSlice, B}; - use crate::tests::LOSSY_TESTS; - use crate::utf8::{self, Utf8Error}; - - fn utf8e(valid_up_to: usize) -> Utf8Error { - Utf8Error { valid_up_to, error_len: None } - } - - fn utf8e2(valid_up_to: usize, error_len: usize) -> Utf8Error { - Utf8Error { valid_up_to, error_len: Some(error_len) } - } - - #[test] - fn validate_all_codepoints() { - for i in 0..(0x10FFFF + 1) { - let cp = match char::from_u32(i) { - None => continue, - Some(cp) => cp, - }; - let mut buf = [0; 4]; - let s = cp.encode_utf8(&mut buf); - assert_eq!(Ok(()), utf8::validate(s.as_bytes())); - } - } - - #[test] - fn validate_multiple_codepoints() { - assert_eq!(Ok(()), utf8::validate(b"abc")); - assert_eq!(Ok(()), utf8::validate(b"a\xE2\x98\x83a")); - assert_eq!(Ok(()), utf8::validate(b"a\xF0\x9D\x9C\xB7a")); - assert_eq!(Ok(()), utf8::validate(b"\xE2\x98\x83\xF0\x9D\x9C\xB7",)); - assert_eq!( - Ok(()), - utf8::validate(b"a\xE2\x98\x83a\xF0\x9D\x9C\xB7a",) - ); - assert_eq!( - Ok(()), - utf8::validate(b"\xEF\xBF\xBD\xE2\x98\x83\xEF\xBF\xBD",) - ); - } - - #[test] - fn validate_errors() { - // single invalid byte - assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xFF")); - // single invalid byte after ASCII - assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xFF")); - // single invalid byte after 2 byte sequence - assert_eq!(Err(utf8e2(2, 1)), utf8::validate(b"\xCE\xB2\xFF")); - // single invalid byte after 3 byte sequence - assert_eq!(Err(utf8e2(3, 1)), utf8::validate(b"\xE2\x98\x83\xFF")); - // single invalid byte after 4 byte sequence - assert_eq!(Err(utf8e2(4, 1)), utf8::validate(b"\xF0\x9D\x9D\xB1\xFF")); - - // An invalid 2-byte sequence with a valid 1-byte prefix. - assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xCE\xF0")); - // An invalid 3-byte sequence with a valid 2-byte prefix. - assert_eq!(Err(utf8e2(0, 2)), utf8::validate(b"\xE2\x98\xF0")); - // An invalid 4-byte sequence with a valid 3-byte prefix. - assert_eq!(Err(utf8e2(0, 3)), utf8::validate(b"\xF0\x9D\x9D\xF0")); - - // An overlong sequence. Should be \xE2\x82\xAC, but we encode the - // same codepoint value in 4 bytes. This not only tests that we reject - // overlong sequences, but that we get valid_up_to correct. - assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xF0\x82\x82\xAC")); - assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xF0\x82\x82\xAC")); - assert_eq!( - Err(utf8e2(3, 1)), - utf8::validate(b"\xE2\x98\x83\xF0\x82\x82\xAC",) - ); - - // Check that encoding a surrogate codepoint using the UTF-8 scheme - // fails validation. - assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xED\xA0\x80")); - assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xED\xA0\x80")); - assert_eq!( - Err(utf8e2(3, 1)), - utf8::validate(b"\xE2\x98\x83\xED\xA0\x80",) - ); - - // Check that an incomplete 2-byte sequence fails. - assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xCEa")); - assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xCEa")); - assert_eq!( - Err(utf8e2(3, 1)), - utf8::validate(b"\xE2\x98\x83\xCE\xE2\x98\x83",) - ); - // Check that an incomplete 3-byte sequence fails. - assert_eq!(Err(utf8e2(0, 2)), utf8::validate(b"\xE2\x98a")); - assert_eq!(Err(utf8e2(1, 2)), utf8::validate(b"a\xE2\x98a")); - assert_eq!( - Err(utf8e2(3, 2)), - utf8::validate(b"\xE2\x98\x83\xE2\x98\xE2\x98\x83",) - ); - // Check that an incomplete 4-byte sequence fails. - assert_eq!(Err(utf8e2(0, 3)), utf8::validate(b"\xF0\x9D\x9Ca")); - assert_eq!(Err(utf8e2(1, 3)), utf8::validate(b"a\xF0\x9D\x9Ca")); - assert_eq!( - Err(utf8e2(4, 3)), - utf8::validate(b"\xF0\x9D\x9C\xB1\xF0\x9D\x9C\xE2\x98\x83",) - ); - assert_eq!( - Err(utf8e2(6, 3)), - utf8::validate(b"foobar\xF1\x80\x80quux",) - ); - - // Check that an incomplete (EOF) 2-byte sequence fails. - assert_eq!(Err(utf8e(0)), utf8::validate(b"\xCE")); - assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xCE")); - assert_eq!(Err(utf8e(3)), utf8::validate(b"\xE2\x98\x83\xCE")); - // Check that an incomplete (EOF) 3-byte sequence fails. - assert_eq!(Err(utf8e(0)), utf8::validate(b"\xE2\x98")); - assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xE2\x98")); - assert_eq!(Err(utf8e(3)), utf8::validate(b"\xE2\x98\x83\xE2\x98")); - // Check that an incomplete (EOF) 4-byte sequence fails. - assert_eq!(Err(utf8e(0)), utf8::validate(b"\xF0\x9D\x9C")); - assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xF0\x9D\x9C")); - assert_eq!( - Err(utf8e(4)), - utf8::validate(b"\xF0\x9D\x9C\xB1\xF0\x9D\x9C",) - ); - - // Test that we errors correct even after long valid sequences. This - // checks that our "backup" logic for detecting errors is correct. - assert_eq!( - Err(utf8e2(8, 1)), - utf8::validate(b"\xe2\x98\x83\xce\xb2\xe3\x83\x84\xFF",) - ); - } - - #[test] - fn decode_valid() { - fn d(mut s: &str) -> Vec<char> { - let mut chars = vec![]; - while !s.is_empty() { - let (ch, size) = utf8::decode(s.as_bytes()); - s = &s[size..]; - chars.push(ch.unwrap()); - } - chars - } - - assert_eq!(vec!['☃'], d("☃")); - assert_eq!(vec!['☃', '☃'], d("☃☃")); - assert_eq!(vec!['α', 'β', 'γ', 'δ', 'ε'], d("αβγδε")); - assert_eq!(vec!['☃', '⛄', '⛇'], d("☃⛄⛇")); - assert_eq!(vec!['𝗮', '𝗯', '𝗰', '𝗱', '𝗲'], d("𝗮𝗯𝗰𝗱𝗲")); - } - - #[test] - fn decode_invalid() { - let (ch, size) = utf8::decode(b""); - assert_eq!(None, ch); - assert_eq!(0, size); - - let (ch, size) = utf8::decode(b"\xFF"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode(b"\xCE\xF0"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode(b"\xE2\x98\xF0"); - assert_eq!(None, ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode(b"\xF0\x9D\x9D"); - assert_eq!(None, ch); - assert_eq!(3, size); - - let (ch, size) = utf8::decode(b"\xF0\x9D\x9D\xF0"); - assert_eq!(None, ch); - assert_eq!(3, size); - - let (ch, size) = utf8::decode(b"\xF0\x82\x82\xAC"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode(b"\xED\xA0\x80"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode(b"\xCEa"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode(b"\xE2\x98a"); - assert_eq!(None, ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode(b"\xF0\x9D\x9Ca"); - assert_eq!(None, ch); - assert_eq!(3, size); - } - - #[test] - fn decode_lossy() { - let (ch, size) = utf8::decode_lossy(b""); - assert_eq!('\u{FFFD}', ch); - assert_eq!(0, size); - - let (ch, size) = utf8::decode_lossy(b"\xFF"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_lossy(b"\xCE\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_lossy(b"\xE2\x98\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_lossy(b"\xF0\x9D\x9D\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(3, size); - - let (ch, size) = utf8::decode_lossy(b"\xF0\x82\x82\xAC"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_lossy(b"\xED\xA0\x80"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_lossy(b"\xCEa"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_lossy(b"\xE2\x98a"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_lossy(b"\xF0\x9D\x9Ca"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(3, size); - } - - #[test] - fn decode_last_valid() { - fn d(mut s: &str) -> Vec<char> { - let mut chars = vec![]; - while !s.is_empty() { - let (ch, size) = utf8::decode_last(s.as_bytes()); - s = &s[..s.len() - size]; - chars.push(ch.unwrap()); - } - chars - } - - assert_eq!(vec!['☃'], d("☃")); - assert_eq!(vec!['☃', '☃'], d("☃☃")); - assert_eq!(vec!['ε', 'δ', 'γ', 'β', 'α'], d("αβγδε")); - assert_eq!(vec!['⛇', '⛄', '☃'], d("☃⛄⛇")); - assert_eq!(vec!['𝗲', '𝗱', '𝗰', '𝗯', '𝗮'], d("𝗮𝗯𝗰𝗱𝗲")); - } - - #[test] - fn decode_last_invalid() { - let (ch, size) = utf8::decode_last(b""); - assert_eq!(None, ch); - assert_eq!(0, size); - - let (ch, size) = utf8::decode_last(b"\xFF"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xCE\xF0"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xCE"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xE2\x98\xF0"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xE2\x98"); - assert_eq!(None, ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_last(b"\xF0\x9D\x9D\xF0"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xF0\x9D\x9D"); - assert_eq!(None, ch); - assert_eq!(3, size); - - let (ch, size) = utf8::decode_last(b"\xF0\x82\x82\xAC"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xED\xA0\x80"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xED\xA0"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"\xED"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"a\xCE"); - assert_eq!(None, ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last(b"a\xE2\x98"); - assert_eq!(None, ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_last(b"a\xF0\x9D\x9C"); - assert_eq!(None, ch); - assert_eq!(3, size); - } - - #[test] - fn decode_last_lossy() { - let (ch, size) = utf8::decode_last_lossy(b""); - assert_eq!('\u{FFFD}', ch); - assert_eq!(0, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xFF"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xCE\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xCE"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xE2\x98\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xE2\x98"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xF0\x9D\x9D\xF0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xF0\x9D\x9D"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(3, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xF0\x82\x82\xAC"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xED\xA0\x80"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xED\xA0"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"\xED"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"a\xCE"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(1, size); - - let (ch, size) = utf8::decode_last_lossy(b"a\xE2\x98"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(2, size); - - let (ch, size) = utf8::decode_last_lossy(b"a\xF0\x9D\x9C"); - assert_eq!('\u{FFFD}', ch); - assert_eq!(3, size); - } - - #[test] - fn chars() { - for (i, &(expected, input)) in LOSSY_TESTS.iter().enumerate() { - let got: String = B(input).chars().collect(); - assert_eq!( - expected, got, - "chars(ith: {:?}, given: {:?})", - i, input, - ); - let got: String = - B(input).char_indices().map(|(_, _, ch)| ch).collect(); - assert_eq!( - expected, got, - "char_indices(ith: {:?}, given: {:?})", - i, input, - ); - - let expected: String = expected.chars().rev().collect(); - - let got: String = B(input).chars().rev().collect(); - assert_eq!( - expected, got, - "chars.rev(ith: {:?}, given: {:?})", - i, input, - ); - let got: String = - B(input).char_indices().rev().map(|(_, _, ch)| ch).collect(); - assert_eq!( - expected, got, - "char_indices.rev(ith: {:?}, given: {:?})", - i, input, - ); - } - } - - #[test] - fn utf8_chunks() { - let mut c = utf8::Utf8Chunks { bytes: b"123\xC0" }; - assert_eq!( - (c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xC0".as_bstr(), - incomplete: false, - }), - None, - ) - ); - - let mut c = utf8::Utf8Chunks { bytes: b"123\xFF\xFF" }; - assert_eq!( - (c.next(), c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xFF".as_bstr(), - incomplete: false, - }), - Some(utf8::Utf8Chunk { - valid: "", - invalid: b"\xFF".as_bstr(), - incomplete: false, - }), - None, - ) - ); - - let mut c = utf8::Utf8Chunks { bytes: b"123\xD0" }; - assert_eq!( - (c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xD0".as_bstr(), - incomplete: true, - }), - None, - ) - ); - - let mut c = utf8::Utf8Chunks { bytes: b"123\xD0456" }; - assert_eq!( - (c.next(), c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xD0".as_bstr(), - incomplete: false, - }), - Some(utf8::Utf8Chunk { - valid: "456", - invalid: b"".as_bstr(), - incomplete: false, - }), - None, - ) - ); - - let mut c = utf8::Utf8Chunks { bytes: b"123\xE2\x98" }; - assert_eq!( - (c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xE2\x98".as_bstr(), - incomplete: true, - }), - None, - ) - ); - - let mut c = utf8::Utf8Chunks { bytes: b"123\xF4\x8F\xBF" }; - assert_eq!( - (c.next(), c.next()), - ( - Some(utf8::Utf8Chunk { - valid: "123", - invalid: b"\xF4\x8F\xBF".as_bstr(), - incomplete: true, - }), - None, - ) - ); - } -} |