/*! Types and routines for working with look-around assertions. This module principally defines two types: * [`Look`] enumerates all of the assertions supported by this crate. * [`LookSet`] provides a way to efficiently store a set of [`Look`] values. * [`LookMatcher`] provides routines for checking whether a `Look` or a `LookSet` matches at a particular position in a haystack. */ // LAMENTATION: Sadly, a lot of the API of `Look` and `LookSet` were basically // copied verbatim from the regex-syntax crate. I would have no problems using // the regex-syntax types and defining the matching routines (only found // in this crate) as free functions, except the `Look` and `LookSet` types // are used in lots of places. Including in places we expect to work when // regex-syntax is *not* enabled, such as in the definition of the NFA itself. // // Thankfully the code we copy is pretty simple and there isn't much of it. // Otherwise, the rest of this module deals with *matching* the assertions, // which is not something that regex-syntax handles. use crate::util::{escape::DebugByte, utf8}; /// A look-around assertion. /// /// An assertion matches at a position between characters in a haystack. /// Namely, it does not actually "consume" any input as most parts of a regular /// expression do. Assertions are a way of stating that some property must be /// true at a particular point during matching. /// /// For example, `(?m)^[a-z]+$` is a pattern that: /// /// * Scans the haystack for a position at which `(?m:^)` is satisfied. That /// occurs at either the beginning of the haystack, or immediately following /// a `\n` character. /// * Looks for one or more occurrences of `[a-z]`. /// * Once `[a-z]+` has matched as much as it can, an overall match is only /// reported when `[a-z]+` stops just before a `\n`. /// /// So in this case, `abc` and `\nabc\n` match, but `\nabc1\n` does not. /// /// Assertions are also called "look-around," "look-behind" and "look-ahead." /// Specifically, some assertions are look-behind (like `^`), other assertions /// are look-ahead (like `$`) and yet other assertions are both look-ahead and /// look-behind (like `\b`). /// /// # Assertions in an NFA /// /// An assertion in a [`thompson::NFA`](crate::nfa::thompson::NFA) can be /// thought of as a conditional epsilon transition. That is, a matching engine /// like the [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) only permits /// moving through conditional epsilon transitions when their condition /// is satisfied at whatever position the `PikeVM` is currently at in the /// haystack. /// /// How assertions are handled in a `DFA` is trickier, since a DFA does not /// have epsilon transitions at all. In this case, they are compiled into the /// automaton itself, at the expense of more states than what would be required /// without an assertion. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Look { /// Match the beginning of text. Specifically, this matches at the starting /// position of the input. Start = 1 << 0, /// Match the end of text. Specifically, this matches at the ending /// position of the input. End = 1 << 1, /// Match the beginning of a line or the beginning of text. Specifically, /// this matches at the starting position of the input, or at the position /// immediately following a `\n` character. StartLF = 1 << 2, /// Match the end of a line or the end of text. Specifically, this matches /// at the end position of the input, or at the position immediately /// preceding a `\n` character. EndLF = 1 << 3, /// Match the beginning of a line or the beginning of text. Specifically, /// this matches at the starting position of the input, or at the position /// immediately following either a `\r` or `\n` character, but never after /// a `\r` when a `\n` follows. StartCRLF = 1 << 4, /// Match the end of a line or the end of text. Specifically, this matches /// at the end position of the input, or at the position immediately /// preceding a `\r` or `\n` character, but never before a `\n` when a `\r` /// precedes it. EndCRLF = 1 << 5, /// Match an ASCII-only word boundary. That is, this matches a position /// where the left adjacent character and right adjacent character /// correspond to a word and non-word or a non-word and word character. WordAscii = 1 << 6, /// Match an ASCII-only negation of a word boundary. WordAsciiNegate = 1 << 7, /// Match a Unicode-aware word boundary. That is, this matches a position /// where the left adjacent character and right adjacent character /// correspond to a word and non-word or a non-word and word character. WordUnicode = 1 << 8, /// Match a Unicode-aware negation of a word boundary. WordUnicodeNegate = 1 << 9, } impl Look { /// Flip the look-around assertion to its equivalent for reverse searches. /// For example, `StartLF` gets translated to `EndLF`. /// /// Some assertions, such as `WordUnicode`, remain the same since they /// match the same positions regardless of the direction of the search. #[inline] pub const fn reversed(self) -> Look { match self { Look::Start => Look::End, Look::End => Look::Start, Look::StartLF => Look::EndLF, Look::EndLF => Look::StartLF, Look::StartCRLF => Look::EndCRLF, Look::EndCRLF => Look::StartCRLF, Look::WordAscii => Look::WordAscii, Look::WordAsciiNegate => Look::WordAsciiNegate, Look::WordUnicode => Look::WordUnicode, Look::WordUnicodeNegate => Look::WordUnicodeNegate, } } /// Return the underlying representation of this look-around enumeration /// as an integer. Giving the return value to the [`Look::from_repr`] /// constructor is guaranteed to return the same look-around variant that /// one started with within a semver compatible release of this crate. #[inline] pub const fn as_repr(self) -> u16 { // AFAIK, 'as' is the only way to zero-cost convert an int enum to an // actual int. self as u16 } /// Given the underlying representation of a `Look` value, return the /// corresponding `Look` value if the representation is valid. Otherwise /// `None` is returned. #[inline] pub const fn from_repr(repr: u16) -> Option { match repr { 0b00_0000_0001 => Some(Look::Start), 0b00_0000_0010 => Some(Look::End), 0b00_0000_0100 => Some(Look::StartLF), 0b00_0000_1000 => Some(Look::EndLF), 0b00_0001_0000 => Some(Look::StartCRLF), 0b00_0010_0000 => Some(Look::EndCRLF), 0b00_0100_0000 => Some(Look::WordAscii), 0b00_1000_0000 => Some(Look::WordAsciiNegate), 0b01_0000_0000 => Some(Look::WordUnicode), 0b10_0000_0000 => Some(Look::WordUnicodeNegate), _ => None, } } /// Returns a convenient single codepoint representation of this /// look-around assertion. Each assertion is guaranteed to be represented /// by a distinct character. /// /// This is useful for succinctly representing a look-around assertion in /// human friendly but succinct output intended for a programmer working on /// regex internals. #[inline] pub const fn as_char(self) -> char { match self { Look::Start => 'A', Look::End => 'z', Look::StartLF => '^', Look::EndLF => '$', Look::StartCRLF => 'r', Look::EndCRLF => 'R', Look::WordAscii => 'b', Look::WordAsciiNegate => 'B', Look::WordUnicode => '𝛃', Look::WordUnicodeNegate => '𝚩', } } } /// LookSet is a memory-efficient set of look-around assertions. /// /// This is useful for efficiently tracking look-around assertions. For /// example, a [`thompson::NFA`](crate::nfa::thompson::NFA) provides properties /// that return `LookSet`s. #[derive(Clone, Copy, Default, Eq, PartialEq)] pub struct LookSet { /// The underlying representation this set is exposed to make it possible /// to store it somewhere efficiently. The representation is that /// of a bitset, where each assertion occupies bit `i` where `i = /// Look::as_repr()`. /// /// Note that users of this internal representation must permit the full /// range of `u16` values to be represented. For example, even if the /// current implementation only makes use of the 10 least significant bits, /// it may use more bits in a future semver compatible release. pub bits: u16, } impl LookSet { /// Create an empty set of look-around assertions. #[inline] pub fn empty() -> LookSet { LookSet { bits: 0 } } /// Create a full set of look-around assertions. /// /// This set contains all possible look-around assertions. #[inline] pub fn full() -> LookSet { LookSet { bits: !0 } } /// Create a look-around set containing the look-around assertion given. /// /// This is a convenience routine for creating an empty set and inserting /// one look-around assertions. #[inline] pub fn singleton(look: Look) -> LookSet { LookSet::empty().insert(look) } /// Returns the total number of look-around assertions in this set. #[inline] pub fn len(self) -> usize { // OK because max value always fits in a u8, which in turn always // fits in a usize, regardless of target. usize::try_from(self.bits.count_ones()).unwrap() } /// Returns true if and only if this set is empty. #[inline] pub fn is_empty(self) -> bool { self.len() == 0 } /// Returns true if and only if the given look-around assertion is in this /// set. #[inline] pub fn contains(self, look: Look) -> bool { self.bits & look.as_repr() != 0 } /// Returns true if and only if this set contains any anchor assertions. /// This includes both "start/end of haystack" and "start/end of line." #[inline] pub fn contains_anchor(&self) -> bool { self.contains_anchor_haystack() || self.contains_anchor_line() } /// Returns true if and only if this set contains any "start/end of /// haystack" anchors. This doesn't include "start/end of line" anchors. #[inline] pub fn contains_anchor_haystack(&self) -> bool { self.contains(Look::Start) || self.contains(Look::End) } /// Returns true if and only if this set contains any "start/end of line" /// anchors. This doesn't include "start/end of haystack" anchors. This /// includes both `\n` line anchors and CRLF (`\r\n`) aware line anchors. #[inline] pub fn contains_anchor_line(&self) -> bool { self.contains(Look::StartLF) || self.contains(Look::EndLF) || self.contains(Look::StartCRLF) || self.contains(Look::EndCRLF) } /// Returns true if and only if this set contains any "start/end of line" /// anchors that only treat `\n` as line terminators. This does not include /// haystack anchors or CRLF aware line anchors. #[inline] pub fn contains_anchor_lf(&self) -> bool { self.contains(Look::StartLF) || self.contains(Look::EndLF) } /// Returns true if and only if this set contains any "start/end of line" /// anchors that are CRLF-aware. This doesn't include "start/end of /// haystack" or "start/end of line-feed" anchors. #[inline] pub fn contains_anchor_crlf(&self) -> bool { self.contains(Look::StartCRLF) || self.contains(Look::EndCRLF) } /// Returns true if and only if this set contains any word boundary or /// negated word boundary assertions. This include both Unicode and ASCII /// word boundaries. #[inline] pub fn contains_word(self) -> bool { self.contains_word_unicode() || self.contains_word_ascii() } /// Returns true if and only if this set contains any Unicode word boundary /// or negated Unicode word boundary assertions. #[inline] pub fn contains_word_unicode(self) -> bool { self.contains(Look::WordUnicode) || self.contains(Look::WordUnicodeNegate) } /// Returns true if and only if this set contains any ASCII word boundary /// or negated ASCII word boundary assertions. #[inline] pub fn contains_word_ascii(self) -> bool { self.contains(Look::WordAscii) || self.contains(Look::WordAsciiNegate) } /// Returns an iterator over all of the look-around assertions in this set. #[inline] pub fn iter(self) -> LookSetIter { LookSetIter { set: self } } /// Return a new set that is equivalent to the original, but with the given /// assertion added to it. If the assertion is already in the set, then the /// returned set is equivalent to the original. #[inline] pub fn insert(self, look: Look) -> LookSet { LookSet { bits: self.bits | look.as_repr() } } /// Updates this set in place with the result of inserting the given /// assertion into this set. #[inline] pub fn set_insert(&mut self, look: Look) { *self = self.insert(look); } /// Return a new set that is equivalent to the original, but with the given /// assertion removed from it. If the assertion is not in the set, then the /// returned set is equivalent to the original. #[inline] pub fn remove(self, look: Look) -> LookSet { LookSet { bits: self.bits & !look.as_repr() } } /// Updates this set in place with the result of removing the given /// assertion from this set. #[inline] pub fn set_remove(&mut self, look: Look) { *self = self.remove(look); } /// Returns a new set that is the result of subtracting the given set from /// this set. #[inline] pub fn subtract(self, other: LookSet) -> LookSet { LookSet { bits: self.bits & !other.bits } } /// Updates this set in place with the result of subtracting the given set /// from this set. #[inline] pub fn set_subtract(&mut self, other: LookSet) { *self = self.subtract(other); } /// Returns a new set that is the union of this and the one given. #[inline] pub fn union(self, other: LookSet) -> LookSet { LookSet { bits: self.bits | other.bits } } /// Updates this set in place with the result of unioning it with the one /// given. #[inline] pub fn set_union(&mut self, other: LookSet) { *self = self.union(other); } /// Returns a new set that is the intersection of this and the one given. #[inline] pub fn intersect(self, other: LookSet) -> LookSet { LookSet { bits: self.bits & other.bits } } /// Updates this set in place with the result of intersecting it with the /// one given. #[inline] pub fn set_intersect(&mut self, other: LookSet) { *self = self.intersect(other); } /// Return a `LookSet` from the slice given as a native endian 16-bit /// integer. /// /// # Panics /// /// This panics if `slice.len() < 2`. #[inline] pub fn read_repr(slice: &[u8]) -> LookSet { let bits = u16::from_ne_bytes(slice[..2].try_into().unwrap()); LookSet { bits } } /// Write a `LookSet` as a native endian 16-bit integer to the beginning /// of the slice given. /// /// # Panics /// /// This panics if `slice.len() < 2`. #[inline] pub fn write_repr(self, slice: &mut [u8]) { let raw = self.bits.to_ne_bytes(); slice[0] = raw[0]; slice[1] = raw[1]; } /// Checks that all assertions in this set can be matched. /// /// Some assertions, such as Unicode word boundaries, require optional (but /// enabled by default) tables that may not be available. If there are /// assertions in this set that require tables that are not available, then /// this will return an error. /// /// Specifically, this returns an error when the the /// `unicode-word-boundary` feature is _not_ enabled _and_ this set /// contains a Unicode word boundary assertion. /// /// It can be useful to use this on the result of /// [`NFA::look_set_any`](crate::nfa::thompson::NFA::look_set_any) /// when building a matcher engine to ensure methods like /// [`LookMatcher::matches_set`] do not panic at search time. pub fn available(self) -> Result<(), UnicodeWordBoundaryError> { if self.contains_word_unicode() { UnicodeWordBoundaryError::check()?; } Ok(()) } } impl core::fmt::Debug for LookSet { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { if self.is_empty() { return write!(f, "∅"); } for look in self.iter() { write!(f, "{}", look.as_char())?; } Ok(()) } } /// An iterator over all look-around assertions in a [`LookSet`]. /// /// This iterator is created by [`LookSet::iter`]. #[derive(Clone, Debug)] pub struct LookSetIter { set: LookSet, } impl Iterator for LookSetIter { type Item = Look; #[inline] fn next(&mut self) -> Option { if self.set.is_empty() { return None; } // We'll never have more than u8::MAX distinct look-around assertions, // so 'repr' will always fit into a u16. let repr = u16::try_from(self.set.bits.trailing_zeros()).unwrap(); let look = Look::from_repr(1 << repr)?; self.set = self.set.remove(look); Some(look) } } /// A matcher for look-around assertions. /// /// This matcher permits configuring aspects of how look-around assertions are /// matched. /// /// # Example /// /// A `LookMatcher` can change the line terminator used for matching multi-line /// anchors such as `(?m:^)` and `(?m:$)`. /// /// ``` /// use regex_automata::{ /// nfa::thompson::{self, pikevm::PikeVM}, /// util::look::LookMatcher, /// Match, Input, /// }; /// /// let mut lookm = LookMatcher::new(); /// lookm.set_line_terminator(b'\x00'); /// /// let re = PikeVM::builder() /// .thompson(thompson::Config::new().look_matcher(lookm)) /// .build(r"(?m)^[a-z]+$")?; /// let mut cache = re.create_cache(); /// /// // Multi-line assertions now use NUL as a terminator. /// assert_eq!( /// Some(Match::must(0, 1..4)), /// re.find(&mut cache, b"\x00abc\x00"), /// ); /// // ... and \n is no longer recognized as a terminator. /// assert_eq!( /// None, /// re.find(&mut cache, b"\nabc\n"), /// ); /// /// # Ok::<(), Box>(()) /// ``` #[derive(Clone, Debug)] pub struct LookMatcher { lineterm: DebugByte, } impl LookMatcher { /// Creates a new default matcher for look-around assertions. pub fn new() -> LookMatcher { LookMatcher { lineterm: DebugByte(b'\n') } } /// Sets the line terminator for use with `(?m:^)` and `(?m:$)`. /// /// Namely, instead of `^` matching after `\n` and `$` matching immediately /// before a `\n`, this will cause it to match after and before the byte /// given. /// /// It can occasionally be useful to use this to configure the line /// terminator to the NUL byte when searching binary data. /// /// Note that this does not apply to CRLF-aware line anchors such as /// `(?Rm:^)` and `(?Rm:$)`. CRLF-aware line anchors are hard-coded to /// use `\r` and `\n`. pub fn set_line_terminator(&mut self, byte: u8) -> &mut LookMatcher { self.lineterm.0 = byte; self } /// Returns the line terminator that was configured for this matcher. /// /// If no line terminator was configured, then this returns `\n`. /// /// Note that the line terminator should only be used for matching `(?m:^)` /// and `(?m:$)` assertions. It specifically should _not_ be used for /// matching the CRLF aware assertions `(?Rm:^)` and `(?Rm:$)`. pub fn get_line_terminator(&self) -> u8 { self.lineterm.0 } /// Returns true when the position `at` in `haystack` satisfies the given /// look-around assertion. /// /// # Panics /// /// This panics when testing any Unicode word boundary assertion in this /// set and when the Unicode word data is not available. Specifically, this /// only occurs when the `unicode-word-boundary` feature is not enabled. /// /// Since it's generally expected that this routine is called inside of /// a matching engine, callers should check the error condition when /// building the matching engine. If there is a Unicode word boundary /// in the matcher and the data isn't available, then the matcher should /// fail to build. /// /// Callers can check the error condition with [`LookSet::available`]. /// /// This also may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn matches(&self, look: Look, haystack: &[u8], at: usize) -> bool { self.matches_inline(look, haystack, at) } /// Like `matches`, but forcefully inlined. #[cfg_attr(feature = "perf-inline", inline(always))] pub(crate) fn matches_inline( &self, look: Look, haystack: &[u8], at: usize, ) -> bool { match look { Look::Start => self.is_start(haystack, at), Look::End => self.is_end(haystack, at), Look::StartLF => self.is_start_lf(haystack, at), Look::EndLF => self.is_end_lf(haystack, at), Look::StartCRLF => self.is_start_crlf(haystack, at), Look::EndCRLF => self.is_end_crlf(haystack, at), Look::WordAscii => self.is_word_ascii(haystack, at), Look::WordAsciiNegate => self.is_word_ascii_negate(haystack, at), Look::WordUnicode => self.is_word_unicode(haystack, at).unwrap(), Look::WordUnicodeNegate => { self.is_word_unicode_negate(haystack, at).unwrap() } } } /// Returns true when _all_ of the assertions in the given set match at the /// given position in the haystack. /// /// # Panics /// /// This panics when testing any Unicode word boundary assertion in this /// set and when the Unicode word data is not available. Specifically, this /// only occurs when the `unicode-word-boundary` feature is not enabled. /// /// Since it's generally expected that this routine is called inside of /// a matching engine, callers should check the error condition when /// building the matching engine. If there is a Unicode word boundary /// in the matcher and the data isn't available, then the matcher should /// fail to build. /// /// Callers can check the error condition with [`LookSet::available`]. /// /// This also may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn matches_set( &self, set: LookSet, haystack: &[u8], at: usize, ) -> bool { self.matches_set_inline(set, haystack, at) } /// Like `LookSet::matches`, but forcefully inlined for perf. #[cfg_attr(feature = "perf-inline", inline(always))] pub(crate) fn matches_set_inline( &self, set: LookSet, haystack: &[u8], at: usize, ) -> bool { // This used to luse LookSet::iter with Look::matches on each element, // but that proved to be quite diastrous for perf. The manual "if // the set has this assertion, check it" turns out to be quite a bit // faster. if set.contains(Look::Start) { if !self.is_start(haystack, at) { return false; } } if set.contains(Look::End) { if !self.is_end(haystack, at) { return false; } } if set.contains(Look::StartLF) { if !self.is_start_lf(haystack, at) { return false; } } if set.contains(Look::EndLF) { if !self.is_end_lf(haystack, at) { return false; } } if set.contains(Look::StartCRLF) { if !self.is_start_crlf(haystack, at) { return false; } } if set.contains(Look::EndCRLF) { if !self.is_end_crlf(haystack, at) { return false; } } if set.contains(Look::WordAscii) { if !self.is_word_ascii(haystack, at) { return false; } } if set.contains(Look::WordAsciiNegate) { if !self.is_word_ascii_negate(haystack, at) { return false; } } if set.contains(Look::WordUnicode) { if !self.is_word_unicode(haystack, at).unwrap() { return false; } } if set.contains(Look::WordUnicodeNegate) { if !self.is_word_unicode_negate(haystack, at).unwrap() { return false; } } true } /// Split up the given byte classes into equivalence classes in a way that /// is consistent with this look-around assertion. #[cfg(feature = "alloc")] pub(crate) fn add_to_byteset( &self, look: Look, set: &mut crate::util::alphabet::ByteClassSet, ) { match look { Look::Start | Look::End => {} Look::StartLF | Look::EndLF => { set.set_range(self.lineterm.0, self.lineterm.0); } Look::StartCRLF | Look::EndCRLF => { set.set_range(b'\r', b'\r'); set.set_range(b'\n', b'\n'); } Look::WordAscii | Look::WordAsciiNegate | Look::WordUnicode | Look::WordUnicodeNegate => { // We need to mark all ranges of bytes whose pairs result in // evaluating \b differently. This isn't technically correct // for Unicode word boundaries, but DFAs can't handle those // anyway, and thus, the byte classes don't need to either // since they are themselves only used in DFAs. // // FIXME: It seems like the calls to 'set_range' here are // completely invariant, which means we could just hard-code // them here without needing to write a loop. And we only need // to do this dance at most once per regex. // // FIXME: Is this correct for \B? let iswb = utf8::is_word_byte; // This unwrap is OK because we guard every use of 'asu8' with // a check that the input is <= 255. let asu8 = |b: u16| u8::try_from(b).unwrap(); let mut b1: u16 = 0; let mut b2: u16; while b1 <= 255 { b2 = b1 + 1; while b2 <= 255 && iswb(asu8(b1)) == iswb(asu8(b2)) { b2 += 1; } // The guards above guarantee that b2 can never get any // bigger. assert!(b2 <= 256); // Subtracting 1 from b2 is always OK because it is always // at least 1 greater than b1, and the assert above // guarantees that the asu8 conversion will succeed. set.set_range(asu8(b1), asu8(b2.checked_sub(1).unwrap())); b1 = b2; } } } } /// Returns true when [`Look::Start`] is satisfied `at` the given position /// in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_start(&self, _haystack: &[u8], at: usize) -> bool { at == 0 } /// Returns true when [`Look::End`] is satisfied `at` the given position in /// `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_end(&self, haystack: &[u8], at: usize) -> bool { at == haystack.len() } /// Returns true when [`Look::StartLF`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_start_lf(&self, haystack: &[u8], at: usize) -> bool { self.is_start(haystack, at) || haystack[at - 1] == self.lineterm.0 } /// Returns true when [`Look::EndLF`] is satisfied `at` the given position /// in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_end_lf(&self, haystack: &[u8], at: usize) -> bool { self.is_end(haystack, at) || haystack[at] == self.lineterm.0 } /// Returns true when [`Look::StartCRLF`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_start_crlf(&self, haystack: &[u8], at: usize) -> bool { self.is_start(haystack, at) || haystack[at - 1] == b'\n' || (haystack[at - 1] == b'\r' && (at >= haystack.len() || haystack[at] != b'\n')) } /// Returns true when [`Look::EndCRLF`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_end_crlf(&self, haystack: &[u8], at: usize) -> bool { self.is_end(haystack, at) || haystack[at] == b'\r' || (haystack[at] == b'\n' && (at == 0 || haystack[at - 1] != b'\r')) } /// Returns true when [`Look::WordAscii`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_word_ascii(&self, haystack: &[u8], at: usize) -> bool { let word_before = at > 0 && utf8::is_word_byte(haystack[at - 1]); let word_after = at < haystack.len() && utf8::is_word_byte(haystack[at]); word_before != word_after } /// Returns true when [`Look::WordAsciiNegate`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. #[inline] pub fn is_word_ascii_negate(&self, haystack: &[u8], at: usize) -> bool { !self.is_word_ascii(haystack, at) } /// Returns true when [`Look::WordUnicode`] is satisfied `at` the given /// position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. /// /// # Errors /// /// This returns an error when Unicode word boundary tables /// are not available. Specifically, this only occurs when the /// `unicode-word-boundary` feature is not enabled. #[inline] pub fn is_word_unicode( &self, haystack: &[u8], at: usize, ) -> Result { let word_before = is_word_char::rev(haystack, at)?; let word_after = is_word_char::fwd(haystack, at)?; Ok(word_before != word_after) } /// Returns true when [`Look::WordUnicodeNegate`] is satisfied `at` the /// given position in `haystack`. /// /// # Panics /// /// This may panic when `at > haystack.len()`. Note that `at == /// haystack.len()` is legal and guaranteed not to panic. /// /// # Errors /// /// This returns an error when Unicode word boundary tables /// are not available. Specifically, this only occurs when the /// `unicode-word-boundary` feature is not enabled. #[inline] pub fn is_word_unicode_negate( &self, haystack: &[u8], at: usize, ) -> Result { // This is pretty subtle. Why do we need to do UTF-8 decoding here? // Well... at time of writing, the is_word_char_{fwd,rev} routines will // only return true if there is a valid UTF-8 encoding of a "word" // codepoint, and false in every other case (including invalid UTF-8). // This means that in regions of invalid UTF-8 (which might be a // subset of valid UTF-8!), it would result in \B matching. While this // would be questionable in the context of truly invalid UTF-8, it is // *certainly* wrong to report match boundaries that split the encoding // of a codepoint. So to work around this, we ensure that we can decode // a codepoint on either side of `at`. If either direction fails, then // we don't permit \B to match at all. // // Now, this isn't exactly optimal from a perf perspective. We could // try and detect this in is_word_char::{fwd,rev}, but it's not clear // if it's worth it. \B is, after all, rarely used. Even worse, // is_word_char::{fwd,rev} could do its own UTF-8 decoding, and so this // will wind up doing UTF-8 decoding twice. Owch. We could fix this // with more code complexity, but it just doesn't feel worth it for \B. // // And in particular, we do *not* have to do this with \b, because \b // *requires* that at least one side of `at` be a "word" codepoint, // which in turn implies one side of `at` must be valid UTF-8. This in // turn implies that \b can never split a valid UTF-8 encoding of a // codepoint. In the case where one side of `at` is truly invalid UTF-8 // and the other side IS a word codepoint, then we want \b to match // since it represents a valid UTF-8 boundary. It also makes sense. For // example, you'd want \b\w+\b to match 'abc' in '\xFFabc\xFF'. // // Note also that this is not just '!is_word_unicode(..)' like it is // for the ASCII case. For example, neither \b nor \B is satisfied // within invalid UTF-8 sequences. let word_before = at > 0 && match utf8::decode_last(&haystack[..at]) { None | Some(Err(_)) => return Ok(false), Some(Ok(_)) => is_word_char::rev(haystack, at)?, }; let word_after = at < haystack.len() && match utf8::decode(&haystack[at..]) { None | Some(Err(_)) => return Ok(false), Some(Ok(_)) => is_word_char::fwd(haystack, at)?, }; Ok(word_before == word_after) } } impl Default for LookMatcher { fn default() -> LookMatcher { LookMatcher::new() } } /// An error that occurs when the Unicode-aware `\w` class is unavailable. /// /// This error can occur when the data tables necessary for the Unicode aware /// Perl character class `\w` are unavailable. The `\w` class is used to /// determine whether a codepoint is considered a word character or not when /// determining whether a Unicode aware `\b` (or `\B`) matches at a particular /// position. /// /// This error can only occur when the `unicode-word-boundary` feature is /// disabled. #[derive(Clone, Debug)] pub struct UnicodeWordBoundaryError(()); impl UnicodeWordBoundaryError { #[cfg(not(feature = "unicode-word-boundary"))] pub(crate) fn new() -> UnicodeWordBoundaryError { UnicodeWordBoundaryError(()) } /// Returns an error if and only if Unicode word boundary data is /// unavailable. pub fn check() -> Result<(), UnicodeWordBoundaryError> { is_word_char::check() } } #[cfg(feature = "std")] impl std::error::Error for UnicodeWordBoundaryError {} impl core::fmt::Display for UnicodeWordBoundaryError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, "Unicode-aware \\b and \\B are unavailable because the \ requisite data tables are missing, please enable the \ unicode-word-boundary feature" ) } } // Below are FOUR different ways for checking whether whether a "word" // codepoint exists at a particular position in the haystack. The four // different approaches are, in order of preference: // // 1. Parse '\w', convert to an NFA, convert to a fully compiled DFA on the // first call, and then use that DFA for all subsequent calls. // 2. Do UTF-8 decoding and use regex_syntax::is_word_character if available. // 3. Do UTF-8 decoding and use our own 'perl_word' table. // 4. Return an error. // // The reason for all of these approaches is a combination of perf and // permitting one to build regex-automata without the Unicode data necessary // for handling Unicode-aware word boundaries. (In which case, '(?-u:\b)' would // still work.) // // The DFA approach is the fastest, but it requires the regex parser, the // NFA compiler, the DFA builder and the DFA search runtime. That's a lot to // bring in, but if it's available, it's (probably) the best we can do. // // Approaches (2) and (3) are effectively equivalent, but (2) reuses the // data in regex-syntax and avoids duplicating it in regex-automata. // // Finally, (4) unconditionally returns an error since the requisite data isn't // available anywhere. // // There are actually more approaches possible that we didn't implement. For // example, if the DFA builder is available but the syntax parser is not, we // could technically hand construct our own NFA from the 'perl_word' data // table. But to avoid some pretty hairy code duplication, we would in turn // need to pull the UTF-8 compiler out of the NFA compiler. Yikes. // // A possibly more sensible alternative is to use a lazy DFA when the full // DFA builder isn't available... // // Yet another choice would be to build the full DFA and then embed it into the // source. Then we'd only need to bring in the DFA search runtime, which is // considerably smaller than the DFA builder code. The problem here is that the // Debian people have spooked me[1] into avoiding cyclic dependencies. Namely, // we'd need to build regex-cli, which depends on regex-automata in order to // build some part of regex-automata. But to be honest, something like this has // to be allowed somehow? I just don't know what the right process is. // // There are perhaps other choices as well. Why did I stop at these 4? Because // I wanted to preserve my sanity. I suspect I'll wind up adding the lazy DFA // approach eventually, as the benefits of the DFA approach are somewhat // compelling. The 'boundary-words-holmes' benchmark tests this: // // $ regex-cli bench measure -f boundary-words-holmes -e pikevm > dfa.csv // // Then I changed the code below so that the util/unicode_data/perl_word table // was used and re-ran the benchmark: // // $ regex-cli bench measure -f boundary-words-holmes -e pikevm > table.csv // // And compared them: // // $ regex-cli bench diff dfa.csv table.csv // benchmark engine dfa table // --------- ------ --- ----- // internal/count/boundary-words-holmes regex/automata/pikevm 18.6 MB/s 12.9 MB/s // // Which is a nice improvement. // // UPDATE: It turns out that it takes approximately 22ms to build the reverse // DFA for \w. (And about 3ms for the forward DFA.) It's probably not much in // the grand scheme things, but that is a significant latency cost. So I'm not // sure that's a good idea. I then tried using a lazy DFA instead, and that // eliminated the overhead, but since the lazy DFA requires mutable working // memory, that requires introducing a 'Cache' for every simultaneous call. // // I ended up deciding for now to just keep the "UTF-8 decode and check the // table." The DFA and lazy DFA approaches are still below, but commented out. // // [1]: https://github.com/BurntSushi/ucd-generate/issues/11 /* /// A module that looks for word codepoints using lazy DFAs. #[cfg(all( feature = "unicode-word-boundary", feature = "syntax", feature = "unicode-perl", feature = "hybrid" ))] mod is_word_char { use alloc::vec::Vec; use crate::{ hybrid::dfa::{Cache, DFA}, nfa::thompson::NFA, util::{lazy::Lazy, pool::Pool, primitives::StateID}, Anchored, Input, }; pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { Ok(()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn fwd( haystack: &[u8], mut at: usize, ) -> Result { static WORD: Lazy = Lazy::new(|| DFA::new(r"\w").unwrap()); static CACHE: Lazy> = Lazy::new(|| Pool::new(|| WORD.create_cache())); let dfa = Lazy::get(&WORD); let mut cache = Lazy::get(&CACHE).get(); let mut sid = dfa .start_state_forward( &mut cache, &Input::new("").anchored(Anchored::Yes), ) .unwrap(); while at < haystack.len() { let byte = haystack[at]; sid = dfa.next_state(&mut cache, sid, byte).unwrap(); at += 1; if sid.is_tagged() { if sid.is_match() { return Ok(true); } else if sid.is_dead() { return Ok(false); } } } Ok(dfa.next_eoi_state(&mut cache, sid).unwrap().is_match()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn rev( haystack: &[u8], mut at: usize, ) -> Result { static WORD: Lazy = Lazy::new(|| { DFA::builder() .thompson(NFA::config().reverse(true)) .build(r"\w") .unwrap() }); static CACHE: Lazy> = Lazy::new(|| Pool::new(|| WORD.create_cache())); let dfa = Lazy::get(&WORD); let mut cache = Lazy::get(&CACHE).get(); let mut sid = dfa .start_state_reverse( &mut cache, &Input::new("").anchored(Anchored::Yes), ) .unwrap(); while at > 0 { at -= 1; let byte = haystack[at]; sid = dfa.next_state(&mut cache, sid, byte).unwrap(); if sid.is_tagged() { if sid.is_match() { return Ok(true); } else if sid.is_dead() { return Ok(false); } } } Ok(dfa.next_eoi_state(&mut cache, sid).unwrap().is_match()) } } */ /* /// A module that looks for word codepoints using fully compiled DFAs. #[cfg(all( feature = "unicode-word-boundary", feature = "syntax", feature = "unicode-perl", feature = "dfa-build" ))] mod is_word_char { use alloc::vec::Vec; use crate::{ dfa::{dense::DFA, Automaton, StartKind}, nfa::thompson::NFA, util::{lazy::Lazy, primitives::StateID}, Anchored, Input, }; pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { Ok(()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn fwd( haystack: &[u8], mut at: usize, ) -> Result { static WORD: Lazy<(DFA>, StateID)> = Lazy::new(|| { let dfa = DFA::builder() .configure(DFA::config().start_kind(StartKind::Anchored)) .build(r"\w") .unwrap(); // OK because our regex has no look-around. let start_id = dfa.universal_start_state(Anchored::Yes).unwrap(); (dfa, start_id) }); let &(ref dfa, mut sid) = Lazy::get(&WORD); while at < haystack.len() { let byte = haystack[at]; sid = dfa.next_state(sid, byte); at += 1; if dfa.is_special_state(sid) { if dfa.is_match_state(sid) { return Ok(true); } else if dfa.is_dead_state(sid) { return Ok(false); } } } Ok(dfa.is_match_state(dfa.next_eoi_state(sid))) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn rev( haystack: &[u8], mut at: usize, ) -> Result { static WORD: Lazy<(DFA>, StateID)> = Lazy::new(|| { let dfa = DFA::builder() .configure(DFA::config().start_kind(StartKind::Anchored)) // From ad hoc measurements, it looks like setting // shrink==false is slightly faster than shrink==true. I kind // of feel like this indicates that shrinking is probably a // failure, although it can help in some cases. Sigh. .thompson(NFA::config().reverse(true).shrink(false)) .build(r"\w") .unwrap(); // OK because our regex has no look-around. let start_id = dfa.universal_start_state(Anchored::Yes).unwrap(); (dfa, start_id) }); let &(ref dfa, mut sid) = Lazy::get(&WORD); while at > 0 { at -= 1; let byte = haystack[at]; sid = dfa.next_state(sid, byte); if dfa.is_special_state(sid) { if dfa.is_match_state(sid) { return Ok(true); } else if dfa.is_dead_state(sid) { return Ok(false); } } } Ok(dfa.is_match_state(dfa.next_eoi_state(sid))) } } */ /// A module that looks for word codepoints using regex-syntax's data tables. #[cfg(all( feature = "unicode-word-boundary", feature = "syntax", feature = "unicode-perl", ))] mod is_word_char { use regex_syntax::try_is_word_character; use crate::util::utf8; pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { Ok(()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn fwd( haystack: &[u8], at: usize, ) -> Result { Ok(match utf8::decode(&haystack[at..]) { None | Some(Err(_)) => false, Some(Ok(ch)) => try_is_word_character(ch).expect( "since unicode-word-boundary, syntax and unicode-perl \ are all enabled, it is expected that \ try_is_word_character succeeds", ), }) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn rev( haystack: &[u8], at: usize, ) -> Result { Ok(match utf8::decode_last(&haystack[..at]) { None | Some(Err(_)) => false, Some(Ok(ch)) => try_is_word_character(ch).expect( "since unicode-word-boundary, syntax and unicode-perl \ are all enabled, it is expected that \ try_is_word_character succeeds", ), }) } } /// A module that looks for word codepoints using regex-automata's data tables /// (which are only compiled when regex-syntax's tables aren't available). /// /// Note that the cfg should match the one in src/util/unicode_data/mod.rs for /// perl_word. #[cfg(all( feature = "unicode-word-boundary", not(all(feature = "syntax", feature = "unicode-perl")), ))] mod is_word_char { use crate::util::utf8; pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { Ok(()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn fwd( haystack: &[u8], at: usize, ) -> Result { Ok(match utf8::decode(&haystack[at..]) { None | Some(Err(_)) => false, Some(Ok(ch)) => is_word_character(ch), }) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn rev( haystack: &[u8], at: usize, ) -> Result { Ok(match utf8::decode_last(&haystack[..at]) { None | Some(Err(_)) => false, Some(Ok(ch)) => is_word_character(ch), }) } #[cfg_attr(feature = "perf-inline", inline(always))] fn is_word_character(c: char) -> bool { use crate::util::{unicode_data::perl_word::PERL_WORD, utf8}; // MSRV(1.59): Use 'u8::try_from(c)' instead. if u8::try_from(u32::from(c)).map_or(false, utf8::is_word_byte) { return true; } PERL_WORD .binary_search_by(|&(start, end)| { use core::cmp::Ordering; if start <= c && c <= end { Ordering::Equal } else if start > c { Ordering::Greater } else { Ordering::Less } }) .is_ok() } } /// A module that always returns an error if Unicode word boundaries are /// disabled. When this feature is disabled, then regex-automata will not /// include its own data tables even if regex-syntax is disabled. #[cfg(not(feature = "unicode-word-boundary"))] mod is_word_char { pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { Err(super::UnicodeWordBoundaryError::new()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn fwd( _bytes: &[u8], _at: usize, ) -> Result { Err(super::UnicodeWordBoundaryError::new()) } #[cfg_attr(feature = "perf-inline", inline(always))] pub(super) fn rev( _bytes: &[u8], _at: usize, ) -> Result { Err(super::UnicodeWordBoundaryError::new()) } } #[cfg(test)] mod tests { use super::*; macro_rules! testlook { ($look:expr, $haystack:expr, $at:expr) => { LookMatcher::default().matches($look, $haystack.as_bytes(), $at) }; } #[test] fn look_matches_start_line() { let look = Look::StartLF; assert!(testlook!(look, "", 0)); assert!(testlook!(look, "\n", 0)); assert!(testlook!(look, "\n", 1)); assert!(testlook!(look, "a", 0)); assert!(testlook!(look, "\na", 1)); assert!(!testlook!(look, "a", 1)); assert!(!testlook!(look, "a\na", 1)); } #[test] fn look_matches_end_line() { let look = Look::EndLF; assert!(testlook!(look, "", 0)); assert!(testlook!(look, "\n", 1)); assert!(testlook!(look, "\na", 0)); assert!(testlook!(look, "\na", 2)); assert!(testlook!(look, "a\na", 1)); assert!(!testlook!(look, "a", 0)); assert!(!testlook!(look, "\na", 1)); assert!(!testlook!(look, "a\na", 0)); assert!(!testlook!(look, "a\na", 2)); } #[test] fn look_matches_start_text() { let look = Look::Start; assert!(testlook!(look, "", 0)); assert!(testlook!(look, "\n", 0)); assert!(testlook!(look, "a", 0)); assert!(!testlook!(look, "\n", 1)); assert!(!testlook!(look, "\na", 1)); assert!(!testlook!(look, "a", 1)); assert!(!testlook!(look, "a\na", 1)); } #[test] fn look_matches_end_text() { let look = Look::End; assert!(testlook!(look, "", 0)); assert!(testlook!(look, "\n", 1)); assert!(testlook!(look, "\na", 2)); assert!(!testlook!(look, "\na", 0)); assert!(!testlook!(look, "a\na", 1)); assert!(!testlook!(look, "a", 0)); assert!(!testlook!(look, "\na", 1)); assert!(!testlook!(look, "a\na", 0)); assert!(!testlook!(look, "a\na", 2)); } #[test] #[cfg(all(not(miri), feature = "unicode-word-boundary"))] fn look_matches_word_unicode() { let look = Look::WordUnicode; // \xF0\x9D\x9B\x83 = 𝛃 (in \w) // \xF0\x90\x86\x80 = 𐆀 (not in \w) // Simple ASCII word boundaries. assert!(testlook!(look, "a", 0)); assert!(testlook!(look, "a", 1)); assert!(testlook!(look, "a ", 1)); assert!(testlook!(look, " a ", 1)); assert!(testlook!(look, " a ", 2)); // Unicode word boundaries with a non-ASCII codepoint. assert!(testlook!(look, "𝛃", 0)); assert!(testlook!(look, "𝛃", 4)); assert!(testlook!(look, "𝛃 ", 4)); assert!(testlook!(look, " 𝛃 ", 1)); assert!(testlook!(look, " 𝛃 ", 5)); // Unicode word boundaries between non-ASCII codepoints. assert!(testlook!(look, "𝛃𐆀", 0)); assert!(testlook!(look, "𝛃𐆀", 4)); // Non word boundaries for ASCII. assert!(!testlook!(look, "", 0)); assert!(!testlook!(look, "ab", 1)); assert!(!testlook!(look, "a ", 2)); assert!(!testlook!(look, " a ", 0)); assert!(!testlook!(look, " a ", 3)); // Non word boundaries with a non-ASCII codepoint. assert!(!testlook!(look, "𝛃b", 4)); assert!(!testlook!(look, "𝛃 ", 5)); assert!(!testlook!(look, " 𝛃 ", 0)); assert!(!testlook!(look, " 𝛃 ", 6)); assert!(!testlook!(look, "𝛃", 1)); assert!(!testlook!(look, "𝛃", 2)); assert!(!testlook!(look, "𝛃", 3)); // Non word boundaries with non-ASCII codepoints. assert!(!testlook!(look, "𝛃𐆀", 1)); assert!(!testlook!(look, "𝛃𐆀", 2)); assert!(!testlook!(look, "𝛃𐆀", 3)); assert!(!testlook!(look, "𝛃𐆀", 5)); assert!(!testlook!(look, "𝛃𐆀", 6)); assert!(!testlook!(look, "𝛃𐆀", 7)); assert!(!testlook!(look, "𝛃𐆀", 8)); } #[test] fn look_matches_word_ascii() { let look = Look::WordAscii; // \xF0\x9D\x9B\x83 = 𝛃 (in \w) // \xF0\x90\x86\x80 = 𐆀 (not in \w) // Simple ASCII word boundaries. assert!(testlook!(look, "a", 0)); assert!(testlook!(look, "a", 1)); assert!(testlook!(look, "a ", 1)); assert!(testlook!(look, " a ", 1)); assert!(testlook!(look, " a ", 2)); // Unicode word boundaries with a non-ASCII codepoint. Since this is // an ASCII word boundary, none of these match. assert!(!testlook!(look, "𝛃", 0)); assert!(!testlook!(look, "𝛃", 4)); assert!(!testlook!(look, "𝛃 ", 4)); assert!(!testlook!(look, " 𝛃 ", 1)); assert!(!testlook!(look, " 𝛃 ", 5)); // Unicode word boundaries between non-ASCII codepoints. Again, since // this is an ASCII word boundary, none of these match. assert!(!testlook!(look, "𝛃𐆀", 0)); assert!(!testlook!(look, "𝛃𐆀", 4)); // Non word boundaries for ASCII. assert!(!testlook!(look, "", 0)); assert!(!testlook!(look, "ab", 1)); assert!(!testlook!(look, "a ", 2)); assert!(!testlook!(look, " a ", 0)); assert!(!testlook!(look, " a ", 3)); // Non word boundaries with a non-ASCII codepoint. assert!(testlook!(look, "𝛃b", 4)); assert!(!testlook!(look, "𝛃 ", 5)); assert!(!testlook!(look, " 𝛃 ", 0)); assert!(!testlook!(look, " 𝛃 ", 6)); assert!(!testlook!(look, "𝛃", 1)); assert!(!testlook!(look, "𝛃", 2)); assert!(!testlook!(look, "𝛃", 3)); // Non word boundaries with non-ASCII codepoints. assert!(!testlook!(look, "𝛃𐆀", 1)); assert!(!testlook!(look, "𝛃𐆀", 2)); assert!(!testlook!(look, "𝛃𐆀", 3)); assert!(!testlook!(look, "𝛃𐆀", 5)); assert!(!testlook!(look, "𝛃𐆀", 6)); assert!(!testlook!(look, "𝛃𐆀", 7)); assert!(!testlook!(look, "𝛃𐆀", 8)); } #[test] #[cfg(all(not(miri), feature = "unicode-word-boundary"))] fn look_matches_word_unicode_negate() { let look = Look::WordUnicodeNegate; // \xF0\x9D\x9B\x83 = 𝛃 (in \w) // \xF0\x90\x86\x80 = 𐆀 (not in \w) // Simple ASCII word boundaries. assert!(!testlook!(look, "a", 0)); assert!(!testlook!(look, "a", 1)); assert!(!testlook!(look, "a ", 1)); assert!(!testlook!(look, " a ", 1)); assert!(!testlook!(look, " a ", 2)); // Unicode word boundaries with a non-ASCII codepoint. assert!(!testlook!(look, "𝛃", 0)); assert!(!testlook!(look, "𝛃", 4)); assert!(!testlook!(look, "𝛃 ", 4)); assert!(!testlook!(look, " 𝛃 ", 1)); assert!(!testlook!(look, " 𝛃 ", 5)); // Unicode word boundaries between non-ASCII codepoints. assert!(!testlook!(look, "𝛃𐆀", 0)); assert!(!testlook!(look, "𝛃𐆀", 4)); // Non word boundaries for ASCII. assert!(testlook!(look, "", 0)); assert!(testlook!(look, "ab", 1)); assert!(testlook!(look, "a ", 2)); assert!(testlook!(look, " a ", 0)); assert!(testlook!(look, " a ", 3)); // Non word boundaries with a non-ASCII codepoint. assert!(testlook!(look, "𝛃b", 4)); assert!(testlook!(look, "𝛃 ", 5)); assert!(testlook!(look, " 𝛃 ", 0)); assert!(testlook!(look, " 𝛃 ", 6)); // These don't match because they could otherwise return an offset that // splits the UTF-8 encoding of a codepoint. assert!(!testlook!(look, "𝛃", 1)); assert!(!testlook!(look, "𝛃", 2)); assert!(!testlook!(look, "𝛃", 3)); // Non word boundaries with non-ASCII codepoints. These also don't // match because they could otherwise return an offset that splits the // UTF-8 encoding of a codepoint. assert!(!testlook!(look, "𝛃𐆀", 1)); assert!(!testlook!(look, "𝛃𐆀", 2)); assert!(!testlook!(look, "𝛃𐆀", 3)); assert!(!testlook!(look, "𝛃𐆀", 5)); assert!(!testlook!(look, "𝛃𐆀", 6)); assert!(!testlook!(look, "𝛃𐆀", 7)); // But this one does, since 𐆀 isn't a word codepoint, and 8 is the end // of the haystack. So the "end" of the haystack isn't a word and 𐆀 // isn't a word, thus, \B matches. assert!(testlook!(look, "𝛃𐆀", 8)); } #[test] fn look_matches_word_ascii_negate() { let look = Look::WordAsciiNegate; // \xF0\x9D\x9B\x83 = 𝛃 (in \w) // \xF0\x90\x86\x80 = 𐆀 (not in \w) // Simple ASCII word boundaries. assert!(!testlook!(look, "a", 0)); assert!(!testlook!(look, "a", 1)); assert!(!testlook!(look, "a ", 1)); assert!(!testlook!(look, " a ", 1)); assert!(!testlook!(look, " a ", 2)); // Unicode word boundaries with a non-ASCII codepoint. Since this is // an ASCII word boundary, none of these match. assert!(testlook!(look, "𝛃", 0)); assert!(testlook!(look, "𝛃", 4)); assert!(testlook!(look, "𝛃 ", 4)); assert!(testlook!(look, " 𝛃 ", 1)); assert!(testlook!(look, " 𝛃 ", 5)); // Unicode word boundaries between non-ASCII codepoints. Again, since // this is an ASCII word boundary, none of these match. assert!(testlook!(look, "𝛃𐆀", 0)); assert!(testlook!(look, "𝛃𐆀", 4)); // Non word boundaries for ASCII. assert!(testlook!(look, "", 0)); assert!(testlook!(look, "ab", 1)); assert!(testlook!(look, "a ", 2)); assert!(testlook!(look, " a ", 0)); assert!(testlook!(look, " a ", 3)); // Non word boundaries with a non-ASCII codepoint. assert!(!testlook!(look, "𝛃b", 4)); assert!(testlook!(look, "𝛃 ", 5)); assert!(testlook!(look, " 𝛃 ", 0)); assert!(testlook!(look, " 𝛃 ", 6)); assert!(testlook!(look, "𝛃", 1)); assert!(testlook!(look, "𝛃", 2)); assert!(testlook!(look, "𝛃", 3)); // Non word boundaries with non-ASCII codepoints. assert!(testlook!(look, "𝛃𐆀", 1)); assert!(testlook!(look, "𝛃𐆀", 2)); assert!(testlook!(look, "𝛃𐆀", 3)); assert!(testlook!(look, "𝛃𐆀", 5)); assert!(testlook!(look, "𝛃𐆀", 6)); assert!(testlook!(look, "𝛃𐆀", 7)); assert!(testlook!(look, "𝛃𐆀", 8)); } #[test] fn look_set() { let mut f = LookSet::default(); assert!(!f.contains(Look::Start)); assert!(!f.contains(Look::End)); assert!(!f.contains(Look::StartLF)); assert!(!f.contains(Look::EndLF)); assert!(!f.contains(Look::WordUnicode)); assert!(!f.contains(Look::WordUnicodeNegate)); assert!(!f.contains(Look::WordAscii)); assert!(!f.contains(Look::WordAsciiNegate)); f = f.insert(Look::Start); assert!(f.contains(Look::Start)); f = f.remove(Look::Start); assert!(!f.contains(Look::Start)); f = f.insert(Look::End); assert!(f.contains(Look::End)); f = f.remove(Look::End); assert!(!f.contains(Look::End)); f = f.insert(Look::StartLF); assert!(f.contains(Look::StartLF)); f = f.remove(Look::StartLF); assert!(!f.contains(Look::StartLF)); f = f.insert(Look::EndLF); assert!(f.contains(Look::EndLF)); f = f.remove(Look::EndLF); assert!(!f.contains(Look::EndLF)); f = f.insert(Look::StartCRLF); assert!(f.contains(Look::StartCRLF)); f = f.remove(Look::StartCRLF); assert!(!f.contains(Look::StartCRLF)); f = f.insert(Look::EndCRLF); assert!(f.contains(Look::EndCRLF)); f = f.remove(Look::EndCRLF); assert!(!f.contains(Look::EndCRLF)); f = f.insert(Look::WordUnicode); assert!(f.contains(Look::WordUnicode)); f = f.remove(Look::WordUnicode); assert!(!f.contains(Look::WordUnicode)); f = f.insert(Look::WordUnicodeNegate); assert!(f.contains(Look::WordUnicodeNegate)); f = f.remove(Look::WordUnicodeNegate); assert!(!f.contains(Look::WordUnicodeNegate)); f = f.insert(Look::WordAscii); assert!(f.contains(Look::WordAscii)); f = f.remove(Look::WordAscii); assert!(!f.contains(Look::WordAscii)); f = f.insert(Look::WordAsciiNegate); assert!(f.contains(Look::WordAsciiNegate)); f = f.remove(Look::WordAsciiNegate); assert!(!f.contains(Look::WordAsciiNegate)); } #[test] fn look_set_iter() { let set = LookSet::empty(); assert_eq!(0, set.iter().count()); let set = LookSet::full(); assert_eq!(10, set.iter().count()); let set = LookSet::empty().insert(Look::StartLF).insert(Look::WordUnicode); assert_eq!(2, set.iter().count()); let set = LookSet::empty().insert(Look::StartLF); assert_eq!(1, set.iter().count()); let set = LookSet::empty().insert(Look::WordAsciiNegate); assert_eq!(1, set.iter().count()); } #[test] #[cfg(feature = "alloc")] fn look_set_debug() { let res = alloc::format!("{:?}", LookSet::empty()); assert_eq!("∅", res); let res = alloc::format!("{:?}", LookSet::full()); assert_eq!("Az^$rRbB𝛃𝚩", res); } }