use crate::fx::{FxHashMap, FxHasher}; #[cfg(parallel_compiler)] use crate::sync::is_dyn_thread_safe; use crate::sync::{CacheAligned, Lock, LockGuard}; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; use std::mem; #[cfg(parallel_compiler)] // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700, // but this should be tested on higher core count CPUs. How the `Sharded` type gets used // may also affect the ideal number of shards. const SHARD_BITS: usize = 5; #[cfg(not(parallel_compiler))] const SHARD_BITS: usize = 0; pub const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. pub struct Sharded { /// This mask is used to ensure that accesses are inbounds of `shards`. /// When dynamic thread safety is off, this field is set to 0 causing only /// a single shard to be used for greater cache efficiency. #[cfg(parallel_compiler)] mask: usize, shards: [CacheAligned>; SHARDS], } impl Default for Sharded { #[inline] fn default() -> Self { Self::new(T::default) } } impl Sharded { #[inline] pub fn new(mut value: impl FnMut() -> T) -> Self { Sharded { #[cfg(parallel_compiler)] mask: if is_dyn_thread_safe() { SHARDS - 1 } else { 0 }, shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))), } } #[inline(always)] fn mask(&self) -> usize { #[cfg(parallel_compiler)] { if SHARDS == 1 { 0 } else { self.mask } } #[cfg(not(parallel_compiler))] { 0 } } #[inline(always)] fn count(&self) -> usize { // `self.mask` is always one below the used shard count self.mask() + 1 } /// The shard is selected by hashing `val` with `FxHasher`. #[inline] pub fn get_shard_by_value(&self, val: &K) -> &Lock { self.get_shard_by_hash(if SHARDS == 1 { 0 } else { make_hash(val) }) } #[inline] pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { self.get_shard_by_index(get_shard_hash(hash)) } #[inline] pub fn get_shard_by_index(&self, i: usize) -> &Lock { // SAFETY: The index get ANDed with the mask, ensuring it is always inbounds. unsafe { &self.shards.get_unchecked(i & self.mask()).0 } } pub fn lock_shards(&self) -> Vec> { (0..self.count()).map(|i| self.get_shard_by_index(i).lock()).collect() } pub fn try_lock_shards(&self) -> Option>> { (0..self.count()).map(|i| self.get_shard_by_index(i).try_lock()).collect() } } pub type ShardedHashMap = Sharded>; impl ShardedHashMap { pub fn len(&self) -> usize { self.lock_shards().iter().map(|shard| shard.len()).sum() } } impl ShardedHashMap { #[inline] pub fn intern_ref(&self, value: &Q, make: impl FnOnce() -> K) -> K where K: Borrow, Q: Hash + Eq, { let hash = make_hash(value); let mut shard = self.get_shard_by_hash(hash).lock(); let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); match entry { RawEntryMut::Occupied(e) => *e.key(), RawEntryMut::Vacant(e) => { let v = make(); e.insert_hashed_nocheck(hash, v, ()); v } } } #[inline] pub fn intern(&self, value: Q, make: impl FnOnce(Q) -> K) -> K where K: Borrow, Q: Hash + Eq, { let hash = make_hash(&value); let mut shard = self.get_shard_by_hash(hash).lock(); let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); match entry { RawEntryMut::Occupied(e) => *e.key(), RawEntryMut::Vacant(e) => { let v = make(value); e.insert_hashed_nocheck(hash, v, ()); v } } } } pub trait IntoPointer { /// Returns a pointer which outlives `self`. fn into_pointer(&self) -> *const (); } impl ShardedHashMap { pub fn contains_pointer_to(&self, value: &T) -> bool { let hash = make_hash(&value); let shard = self.get_shard_by_hash(hash).lock(); let value = value.into_pointer(); shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() } } #[inline] pub fn make_hash(val: &K) -> u64 { let mut state = FxHasher::default(); val.hash(&mut state); state.finish() } /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is /// ever used in combination with `get_shard_by_hash` on a single `Sharded` /// instance, then `hash` must be computed with `FxHasher`. Otherwise, /// `hash` can be computed with any hasher, so long as that hasher is used /// consistently for each `Sharded` instance. #[inline] fn get_shard_hash(hash: u64) -> usize { let hash_len = mem::size_of::(); // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. // hashbrown also uses the lowest bits, so we can't use those (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize }