1 files changed, 55 insertions, 44 deletions

diff --git a/third_party/rust/hashbrown/src/raw/bitmask.rs b/third_party/rust/hashbrown/src/raw/bitmask.rs
index 7d4f9fc387..6576b3c5c0 100644
--- a/third_party/rust/hashbrown/src/raw/bitmask.rs
+++ b/third_party/rust/hashbrown/src/raw/bitmask.rs
@@ -1,6 +1,6 @@
-use super::imp::{BitMaskWord, BITMASK_MASK, BITMASK_STRIDE};
-#[cfg(feature = "nightly")]
-use core::intrinsics;
+use super::imp::{
+    BitMaskWord, NonZeroBitMaskWord, BITMASK_ITER_MASK, BITMASK_MASK, BITMASK_STRIDE,
+};
 
 /// A bit mask which contains the result of a `Match` operation on a `Group` and
 /// allows iterating through them.
@@ -8,75 +8,55 @@ use core::intrinsics;
 /// The bit mask is arranged so that low-order bits represent lower memory
 /// addresses for group match results.
 ///
-/// For implementation reasons, the bits in the set may be sparsely packed, so
-/// that there is only one bit-per-byte used (the high bit, 7). If this is the
+/// For implementation reasons, the bits in the set may be sparsely packed with
+/// groups of 8 bits representing one element. If any of these bits are non-zero
+/// then this element is considered to true in the mask. If this is the
 /// case, `BITMASK_STRIDE` will be 8 to indicate a divide-by-8 should be
 /// performed on counts/indices to normalize this difference. `BITMASK_MASK` is
 /// similarly a mask of all the actually-used bits.
+///
+/// To iterate over a bit mask, it must be converted to a form where only 1 bit
+/// is set per element. This is done by applying `BITMASK_ITER_MASK` on the
+/// mask bits.
 #[derive(Copy, Clone)]
-pub struct BitMask(pub BitMaskWord);
+pub(crate) struct BitMask(pub(crate) BitMaskWord);
 
 #[allow(clippy::use_self)]
 impl BitMask {
     /// Returns a new `BitMask` with all bits inverted.
     #[inline]
     #[must_use]
-    pub fn invert(self) -> Self {
+    #[allow(dead_code)]
+    pub(crate) fn invert(self) -> Self {
         BitMask(self.0 ^ BITMASK_MASK)
     }
 
-    /// Flip the bit in the mask for the entry at the given index.
-    ///
-    /// Returns the bit's previous state.
-    #[inline]
-    #[allow(clippy::cast_ptr_alignment)]
-    #[cfg(feature = "raw")]
-    pub unsafe fn flip(&mut self, index: usize) -> bool {
-        // NOTE: The + BITMASK_STRIDE - 1 is to set the high bit.
-        let mask = 1 << (index * BITMASK_STRIDE + BITMASK_STRIDE - 1);
-        self.0 ^= mask;
-        // The bit was set if the bit is now 0.
-        self.0 & mask == 0
-    }
-
     /// Returns a new `BitMask` with the lowest bit removed.
     #[inline]
     #[must_use]
-    pub fn remove_lowest_bit(self) -> Self {
+    fn remove_lowest_bit(self) -> Self {
         BitMask(self.0 & (self.0 - 1))
     }
+
     /// Returns whether the `BitMask` has at least one set bit.
     #[inline]
-    pub fn any_bit_set(self) -> bool {
+    pub(crate) fn any_bit_set(self) -> bool {
         self.0 != 0
     }
 
     /// Returns the first set bit in the `BitMask`, if there is one.
     #[inline]
-    pub fn lowest_set_bit(self) -> Option<usize> {
-        if self.0 == 0 {
-            None
+    pub(crate) fn lowest_set_bit(self) -> Option<usize> {
+        if let Some(nonzero) = NonZeroBitMaskWord::new(self.0) {
+            Some(Self::nonzero_trailing_zeros(nonzero))
         } else {
-            Some(unsafe { self.lowest_set_bit_nonzero() })
+            None
         }
     }
 
-    /// Returns the first set bit in the `BitMask`, if there is one. The
-    /// bitmask must not be empty.
-    #[inline]
-    #[cfg(feature = "nightly")]
-    pub unsafe fn lowest_set_bit_nonzero(self) -> usize {
-        intrinsics::cttz_nonzero(self.0) as usize / BITMASK_STRIDE
-    }
-    #[inline]
-    #[cfg(not(feature = "nightly"))]
-    pub unsafe fn lowest_set_bit_nonzero(self) -> usize {
-        self.trailing_zeros()
-    }
-
     /// Returns the number of trailing zeroes in the `BitMask`.
     #[inline]
-    pub fn trailing_zeros(self) -> usize {
+    pub(crate) fn trailing_zeros(self) -> usize {
         // ARM doesn't have a trailing_zeroes instruction, and instead uses
         // reverse_bits (RBIT) + leading_zeroes (CLZ). However older ARM
         // versions (pre-ARMv7) don't have RBIT and need to emulate it
@@ -89,9 +69,21 @@ impl BitMask {
         }
     }
 
+    /// Same as above but takes a `NonZeroBitMaskWord`.
+    #[inline]
+    fn nonzero_trailing_zeros(nonzero: NonZeroBitMaskWord) -> usize {
+        if cfg!(target_arch = "arm") && BITMASK_STRIDE % 8 == 0 {
+            // SAFETY: A byte-swapped non-zero value is still non-zero.
+            let swapped = unsafe { NonZeroBitMaskWord::new_unchecked(nonzero.get().swap_bytes()) };
+            swapped.leading_zeros() as usize / BITMASK_STRIDE
+        } else {
+            nonzero.trailing_zeros() as usize / BITMASK_STRIDE
+        }
+    }
+
     /// Returns the number of leading zeroes in the `BitMask`.
     #[inline]
-    pub fn leading_zeros(self) -> usize {
+    pub(crate) fn leading_zeros(self) -> usize {
         self.0.leading_zeros() as usize / BITMASK_STRIDE
     }
 }
@@ -102,13 +94,32 @@ impl IntoIterator for BitMask {
 
     #[inline]
     fn into_iter(self) -> BitMaskIter {
-        BitMaskIter(self)
+        // A BitMask only requires each element (group of bits) to be non-zero.
+        // However for iteration we need each element to only contain 1 bit.
+        BitMaskIter(BitMask(self.0 & BITMASK_ITER_MASK))
     }
 }
 
 /// Iterator over the contents of a `BitMask`, returning the indices of set
 /// bits.
-pub struct BitMaskIter(BitMask);
+#[derive(Copy, Clone)]
+pub(crate) struct BitMaskIter(pub(crate) BitMask);
+
+impl BitMaskIter {
+    /// Flip the bit in the mask for the entry at the given index.
+    ///
+    /// Returns the bit's previous state.
+    #[inline]
+    #[allow(clippy::cast_ptr_alignment)]
+    #[cfg(feature = "raw")]
+    pub(crate) unsafe fn flip(&mut self, index: usize) -> bool {
+        // NOTE: The + BITMASK_STRIDE - 1 is to set the high bit.
+        let mask = 1 << (index * BITMASK_STRIDE + BITMASK_STRIDE - 1);
+        self.0 .0 ^= mask;
+        // The bit was set if the bit is now 0.
+        self.0 .0 & mask == 0
+    }
+}
 
 impl Iterator for BitMaskIter {
     type Item = usize;