From c23a457e72abe608715ac76f076f47dc42af07a5 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 30 May 2024 20:31:44 +0200 Subject: Merging upstream version 1.74.1+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_data_structures/src/flock/unix.rs | 24 +- .../src/graph/dominators/mod.rs | 6 +- .../src/graph/implementation/mod.rs | 23 +- compiler/rustc_data_structures/src/lib.rs | 24 +- compiler/rustc_data_structures/src/marker.rs | 2 - compiler/rustc_data_structures/src/memmap.rs | 7 +- compiler/rustc_data_structures/src/profiling.rs | 4 +- compiler/rustc_data_structures/src/sharded.rs | 159 +++++++--- compiler/rustc_data_structures/src/small_c_str.rs | 6 + compiler/rustc_data_structures/src/sync.rs | 338 ++------------------- compiler/rustc_data_structures/src/sync/freeze.rs | 200 ++++++++++++ compiler/rustc_data_structures/src/sync/lock.rs | 275 +++++++++++++++++ .../rustc_data_structures/src/sync/parallel.rs | 188 ++++++++++++ .../rustc_data_structures/src/sync/worker_local.rs | 20 +- 14 files changed, 875 insertions(+), 401 deletions(-) create mode 100644 compiler/rustc_data_structures/src/sync/freeze.rs create mode 100644 compiler/rustc_data_structures/src/sync/lock.rs create mode 100644 compiler/rustc_data_structures/src/sync/parallel.rs (limited to 'compiler/rustc_data_structures') diff --git a/compiler/rustc_data_structures/src/flock/unix.rs b/compiler/rustc_data_structures/src/flock/unix.rs index 4e5297d58..eff9e8f83 100644 --- a/compiler/rustc_data_structures/src/flock/unix.rs +++ b/compiler/rustc_data_structures/src/flock/unix.rs @@ -21,8 +21,16 @@ impl Lock { let lock_type = if exclusive { libc::F_WRLCK } else { libc::F_RDLCK }; let mut flock: libc::flock = unsafe { mem::zeroed() }; - flock.l_type = lock_type as libc::c_short; - flock.l_whence = libc::SEEK_SET as libc::c_short; + #[cfg(not(all(target_os = "hurd", target_arch = "x86")))] + { + flock.l_type = lock_type as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + } + #[cfg(all(target_os = "hurd", target_arch = "x86"))] + { + flock.l_type = lock_type as libc::c_int; + flock.l_whence = libc::SEEK_SET as libc::c_int; + } flock.l_start = 0; flock.l_len = 0; @@ -39,8 +47,16 @@ impl Lock { impl Drop for Lock { fn drop(&mut self) { let mut flock: libc::flock = unsafe { mem::zeroed() }; - flock.l_type = libc::F_UNLCK as libc::c_short; - flock.l_whence = libc::SEEK_SET as libc::c_short; + #[cfg(not(all(target_os = "hurd", target_arch = "x86")))] + { + flock.l_type = libc::F_UNLCK as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + } + #[cfg(all(target_os = "hurd", target_arch = "x86"))] + { + flock.l_type = libc::F_UNLCK as libc::c_int; + flock.l_whence = libc::SEEK_SET as libc::c_int; + } flock.l_start = 0; flock.l_len = 0; diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 85ef2de9b..4075481e5 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -51,7 +51,7 @@ pub fn dominators(graph: &G) -> Dominators { // Traverse the graph, collecting a number of things: // // * Preorder mapping (to it, and back to the actual ordering) - // * Postorder mapping (used exclusively for rank_partial_cmp on the final product) + // * Postorder mapping (used exclusively for `cmp_in_dominator_order` on the final product) // * Parents for each vertex in the preorder tree // // These are all done here rather than through one of the 'standard' @@ -342,8 +342,8 @@ impl Dominators { /// relationship, the dominator will always precede the dominated. (The relative ordering /// of two unrelated nodes will also be consistent, but otherwise the order has no /// meaning.) This method cannot be used to determine if either Node dominates the other. - pub fn rank_partial_cmp(&self, lhs: Node, rhs: Node) -> Option { - self.post_order_rank[rhs].partial_cmp(&self.post_order_rank[lhs]) + pub fn cmp_in_dominator_order(&self, lhs: Node, rhs: Node) -> Ordering { + self.post_order_rank[rhs].cmp(&self.post_order_rank[lhs]) } /// Returns true if `a` dominates `b`. diff --git a/compiler/rustc_data_structures/src/graph/implementation/mod.rs b/compiler/rustc_data_structures/src/graph/implementation/mod.rs index 9ff401c3c..3910c6fa4 100644 --- a/compiler/rustc_data_structures/src/graph/implementation/mod.rs +++ b/compiler/rustc_data_structures/src/graph/implementation/mod.rs @@ -20,7 +20,6 @@ //! the field `next_edge`). Each of those fields is an array that should //! be indexed by the direction (see the type `Direction`). -use crate::snapshot_vec::{SnapshotVec, SnapshotVecDelegate}; use rustc_index::bit_set::BitSet; use std::fmt::Debug; @@ -28,8 +27,8 @@ use std::fmt::Debug; mod tests; pub struct Graph { - nodes: SnapshotVec>, - edges: SnapshotVec>, + nodes: Vec>, + edges: Vec>, } pub struct Node { @@ -45,20 +44,6 @@ pub struct Edge { pub data: E, } -impl SnapshotVecDelegate for Node { - type Value = Node; - type Undo = (); - - fn reverse(_: &mut Vec>, _: ()) {} -} - -impl SnapshotVecDelegate for Edge { - type Value = Edge; - type Undo = (); - - fn reverse(_: &mut Vec>, _: ()) {} -} - #[derive(Copy, Clone, PartialEq, Debug)] pub struct NodeIndex(pub usize); @@ -86,11 +71,11 @@ impl NodeIndex { impl Graph { pub fn new() -> Graph { - Graph { nodes: SnapshotVec::new(), edges: SnapshotVec::new() } + Graph { nodes: Vec::new(), edges: Vec::new() } } pub fn with_capacity(nodes: usize, edges: usize) -> Graph { - Graph { nodes: SnapshotVec::with_capacity(nodes), edges: SnapshotVec::with_capacity(edges) } + Graph { nodes: Vec::with_capacity(nodes), edges: Vec::with_capacity(edges) } } // # Simple accessors diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 337720897..461ec3a90 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -37,7 +37,7 @@ #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] -#![cfg_attr(not(bootstrap), allow(internal_features))] +#![allow(internal_features)] #![deny(unsafe_op_in_unsafe_fn)] #[macro_use] @@ -47,11 +47,14 @@ extern crate cfg_if; #[macro_use] extern crate rustc_macros; +use std::fmt; + pub use rustc_index::static_assert_size; +/// This calls the passed function while ensuring it won't be inlined into the caller. #[inline(never)] #[cold] -pub fn cold_path R, R>(f: F) -> R { +pub fn outline R, R>(f: F) -> R { f() } @@ -126,6 +129,23 @@ impl Drop for OnDrop { } } +/// Turns a closure that takes an `&mut Formatter` into something that can be display-formatted. +pub fn make_display(f: impl Fn(&mut fmt::Formatter<'_>) -> fmt::Result) -> impl fmt::Display { + struct Printer { + f: F, + } + impl fmt::Display for Printer + where + F: Fn(&mut fmt::Formatter<'_>) -> fmt::Result, + { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + (self.f)(fmt) + } + } + + Printer { f } +} + // See comments in src/librustc_middle/lib.rs #[doc(hidden)] pub fn __noop_fix_for_27438() {} diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs index f8c06f9a8..b067f9d45 100644 --- a/compiler/rustc_data_structures/src/marker.rs +++ b/compiler/rustc_data_structures/src/marker.rs @@ -92,7 +92,6 @@ cfg_if!( [std::collections::BTreeMap where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend] [Vec where T: DynSend, A: std::alloc::Allocator + DynSend] [Box where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend] - [crate::sync::Lock where T: DynSend] [crate::sync::RwLock where T: DynSend] [crate::tagged_ptr::CopyTaggedPtr where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool] [rustc_arena::TypedArena where T: DynSend] @@ -171,7 +170,6 @@ cfg_if!( [std::collections::BTreeMap where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync] [Vec where T: DynSync, A: std::alloc::Allocator + DynSync] [Box where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync] - [crate::sync::Lock where T: DynSend] [crate::sync::RwLock where T: DynSend + DynSync] [crate::sync::OneThread where T] [crate::sync::WorkerLocal where T: DynSend] diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs index ca908671a..30403a614 100644 --- a/compiler/rustc_data_structures/src/memmap.rs +++ b/compiler/rustc_data_structures/src/memmap.rs @@ -11,9 +11,14 @@ pub struct Mmap(Vec); #[cfg(not(target_arch = "wasm32"))] impl Mmap { + /// # Safety + /// + /// The given file must not be mutated (i.e., not written, not truncated, ...) until the mapping is closed. + /// + /// However in practice most callers do not ensure this, so uses of this function are likely unsound. #[inline] pub unsafe fn map(file: File) -> io::Result { - // Safety: this is in fact not safe. + // Safety: the caller must ensure that this is safe. unsafe { memmap2::Mmap::map(&file).map(Mmap) } } } diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs index 3c76c2b79..e688feb5f 100644 --- a/compiler/rustc_data_structures/src/profiling.rs +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -81,8 +81,8 @@ //! //! [mm]: https://github.com/rust-lang/measureme/ -use crate::cold_path; use crate::fx::FxHashMap; +use crate::outline; use std::borrow::Borrow; use std::collections::hash_map::Entry; @@ -697,7 +697,7 @@ impl<'a> TimingGuard<'a> { #[inline] pub fn finish_with_query_invocation_id(self, query_invocation_id: QueryInvocationId) { if let Some(guard) = self.0 { - cold_path(|| { + outline(|| { let event_id = StringId::new_virtual(query_invocation_id.0); let event_id = EventId::from_virtual(event_id); guard.finish_with_override_event_id(event_id); diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 40cbf1495..29516fffd 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,31 +1,29 @@ use crate::fx::{FxHashMap, FxHasher}; #[cfg(parallel_compiler)] -use crate::sync::is_dyn_thread_safe; -use crate::sync::{CacheAligned, Lock, LockGuard}; +use crate::sync::{is_dyn_thread_safe, CacheAligned}; +use crate::sync::{Lock, LockGuard, Mode}; +#[cfg(parallel_compiler)] +use itertools::Either; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; +use std::iter; use std::mem; -#[cfg(parallel_compiler)] // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700, // but this should be tested on higher core count CPUs. How the `Sharded` type gets used // may also affect the ideal number of shards. const SHARD_BITS: usize = 5; -#[cfg(not(parallel_compiler))] -const SHARD_BITS: usize = 0; - -pub const SHARDS: usize = 1 << SHARD_BITS; +#[cfg(parallel_compiler)] +const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. -pub struct Sharded { - /// This mask is used to ensure that accesses are inbounds of `shards`. - /// When dynamic thread safety is off, this field is set to 0 causing only - /// a single shard to be used for greater cache efficiency. +/// A single field is used when the compiler uses only one thread. +pub enum Sharded { + Single(Lock), #[cfg(parallel_compiler)] - mask: usize, - shards: [CacheAligned>; SHARDS], + Shards(Box<[CacheAligned>; SHARDS]>), } impl Default for Sharded { @@ -38,62 +36,133 @@ impl Default for Sharded { impl Sharded { #[inline] pub fn new(mut value: impl FnMut() -> T) -> Self { - Sharded { - #[cfg(parallel_compiler)] - mask: if is_dyn_thread_safe() { SHARDS - 1 } else { 0 }, - shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))), + #[cfg(parallel_compiler)] + if is_dyn_thread_safe() { + return Sharded::Shards(Box::new( + [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))), + )); } + + Sharded::Single(Lock::new(value())) } - #[inline(always)] - fn mask(&self) -> usize { - #[cfg(parallel_compiler)] - { - if SHARDS == 1 { 0 } else { self.mask } - } - #[cfg(not(parallel_compiler))] - { - 0 + /// The shard is selected by hashing `val` with `FxHasher`. + #[inline] + pub fn get_shard_by_value(&self, _val: &K) -> &Lock { + match self { + Self::Single(single) => &single, + #[cfg(parallel_compiler)] + Self::Shards(..) => self.get_shard_by_hash(make_hash(_val)), } } - #[inline(always)] - fn count(&self) -> usize { - // `self.mask` is always one below the used shard count - self.mask() + 1 + #[inline] + pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { + self.get_shard_by_index(get_shard_hash(hash)) + } + + #[inline] + pub fn get_shard_by_index(&self, _i: usize) -> &Lock { + match self { + Self::Single(single) => &single, + #[cfg(parallel_compiler)] + Self::Shards(shards) => { + // SAFETY: The index gets ANDed with the shard mask, ensuring it is always inbounds. + unsafe { &shards.get_unchecked(_i & (SHARDS - 1)).0 } + } + } } /// The shard is selected by hashing `val` with `FxHasher`. #[inline] - pub fn get_shard_by_value(&self, val: &K) -> &Lock { - self.get_shard_by_hash(if SHARDS == 1 { 0 } else { make_hash(val) }) + #[track_caller] + pub fn lock_shard_by_value(&self, _val: &K) -> LockGuard<'_, T> { + match self { + Self::Single(single) => { + // Syncronization is disabled so use the `lock_assume_no_sync` method optimized + // for that case. + + // SAFETY: We know `is_dyn_thread_safe` was false when creating the lock thus + // `might_be_dyn_thread_safe` was also false. + unsafe { single.lock_assume(Mode::NoSync) } + } + #[cfg(parallel_compiler)] + Self::Shards(..) => self.lock_shard_by_hash(make_hash(_val)), + } + } + + #[inline] + #[track_caller] + pub fn lock_shard_by_hash(&self, hash: u64) -> LockGuard<'_, T> { + self.lock_shard_by_index(get_shard_hash(hash)) } #[inline] - pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { - self.get_shard_by_index(get_shard_hash(hash)) + #[track_caller] + pub fn lock_shard_by_index(&self, _i: usize) -> LockGuard<'_, T> { + match self { + Self::Single(single) => { + // Syncronization is disabled so use the `lock_assume_no_sync` method optimized + // for that case. + + // SAFETY: We know `is_dyn_thread_safe` was false when creating the lock thus + // `might_be_dyn_thread_safe` was also false. + unsafe { single.lock_assume(Mode::NoSync) } + } + #[cfg(parallel_compiler)] + Self::Shards(shards) => { + // Syncronization is enabled so use the `lock_assume_sync` method optimized + // for that case. + + // SAFETY (get_unchecked): The index gets ANDed with the shard mask, ensuring it is + // always inbounds. + // SAFETY (lock_assume_sync): We know `is_dyn_thread_safe` was true when creating + // the lock thus `might_be_dyn_thread_safe` was also true. + unsafe { shards.get_unchecked(_i & (SHARDS - 1)).0.lock_assume(Mode::Sync) } + } + } } #[inline] - pub fn get_shard_by_index(&self, i: usize) -> &Lock { - // SAFETY: The index get ANDed with the mask, ensuring it is always inbounds. - unsafe { &self.shards.get_unchecked(i & self.mask()).0 } + pub fn lock_shards(&self) -> impl Iterator> { + match self { + #[cfg(not(parallel_compiler))] + Self::Single(single) => iter::once(single.lock()), + #[cfg(parallel_compiler)] + Self::Single(single) => Either::Left(iter::once(single.lock())), + #[cfg(parallel_compiler)] + Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.lock())), + } } - pub fn lock_shards(&self) -> Vec> { - (0..self.count()).map(|i| self.get_shard_by_index(i).lock()).collect() + #[inline] + pub fn try_lock_shards(&self) -> impl Iterator>> { + match self { + #[cfg(not(parallel_compiler))] + Self::Single(single) => iter::once(single.try_lock()), + #[cfg(parallel_compiler)] + Self::Single(single) => Either::Left(iter::once(single.try_lock())), + #[cfg(parallel_compiler)] + Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.try_lock())), + } } +} - pub fn try_lock_shards(&self) -> Option>> { - (0..self.count()).map(|i| self.get_shard_by_index(i).try_lock()).collect() +#[inline] +pub fn shards() -> usize { + #[cfg(parallel_compiler)] + if is_dyn_thread_safe() { + return SHARDS; } + + 1 } pub type ShardedHashMap = Sharded>; impl ShardedHashMap { pub fn len(&self) -> usize { - self.lock_shards().iter().map(|shard| shard.len()).sum() + self.lock_shards().map(|shard| shard.len()).sum() } } @@ -105,7 +174,7 @@ impl ShardedHashMap { Q: Hash + Eq, { let hash = make_hash(value); - let mut shard = self.get_shard_by_hash(hash).lock(); + let mut shard = self.lock_shard_by_hash(hash); let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); match entry { @@ -125,7 +194,7 @@ impl ShardedHashMap { Q: Hash + Eq, { let hash = make_hash(&value); - let mut shard = self.get_shard_by_hash(hash).lock(); + let mut shard = self.lock_shard_by_hash(hash); let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); match entry { @@ -147,7 +216,7 @@ pub trait IntoPointer { impl ShardedHashMap { pub fn contains_pointer_to(&self, value: &T) -> bool { let hash = make_hash(&value); - let shard = self.get_shard_by_hash(hash).lock(); + let shard = self.lock_shard_by_hash(hash); let value = value.into_pointer(); shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() } diff --git a/compiler/rustc_data_structures/src/small_c_str.rs b/compiler/rustc_data_structures/src/small_c_str.rs index 719e4e3d9..349fd7f97 100644 --- a/compiler/rustc_data_structures/src/small_c_str.rs +++ b/compiler/rustc_data_structures/src/small_c_str.rs @@ -79,3 +79,9 @@ impl<'a> FromIterator<&'a str> for SmallCStr { Self { data } } } + +impl From<&ffi::CStr> for SmallCStr { + fn from(s: &ffi::CStr) -> Self { + Self { data: SmallVec::from_slice(s.to_bytes()) } + } +} diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 25a082373..cca043ba0 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -26,7 +26,8 @@ //! | `AtomicU64` | `Cell` | `atomic::AtomicU64` | //! | `AtomicUsize` | `Cell` | `atomic::AtomicUsize` | //! | | | | -//! | `Lock` | `RefCell` | `parking_lot::Mutex` | +//! | `Lock` | `RefCell` | `RefCell` or | +//! | | | `parking_lot::Mutex` | //! | `RwLock` | `RefCell` | `parking_lot::RwLock` | //! | `MTLock` [^1] | `T` | `Lock` | //! | `MTLockRef<'a, T>` [^2] | `&'a mut MTLock` | `&'a MTLock` | @@ -43,11 +44,18 @@ pub use crate::marker::*; use std::collections::HashMap; use std::hash::{BuildHasher, Hash}; use std::ops::{Deref, DerefMut}; -use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + +mod lock; +pub use lock::{Lock, LockGuard, Mode}; mod worker_local; pub use worker_local::{Registry, WorkerLocal}; +mod parallel; +#[cfg(parallel_compiler)] +pub use parallel::scope; +pub use parallel::{join, par_for_each_in, par_map, parallel_guard}; + pub use std::sync::atomic::Ordering; pub use std::sync::atomic::Ordering::SeqCst; @@ -55,6 +63,9 @@ pub use vec::{AppendOnlyIndexVec, AppendOnlyVec}; mod vec; +mod freeze; +pub use freeze::{FreezeLock, FreezeReadGuard, FreezeWriteGuard}; + mod mode { use super::Ordering; use std::sync::atomic::AtomicU8; @@ -75,6 +86,12 @@ mod mode { } } + // Whether thread safety might be enabled. + #[inline] + pub fn might_be_dyn_thread_safe() -> bool { + DYN_THREAD_SAFE_MODE.load(Ordering::Relaxed) != DYN_NOT_THREAD_SAFE + } + // Only set by the `-Z threads` compile option pub fn set_dyn_thread_safe_mode(mode: bool) { let set: u8 = if mode { DYN_THREAD_SAFE } else { DYN_NOT_THREAD_SAFE }; @@ -94,14 +111,15 @@ pub use mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode}; cfg_if! { if #[cfg(not(parallel_compiler))] { + use std::ops::Add; + use std::cell::Cell; + pub unsafe auto trait Send {} pub unsafe auto trait Sync {} unsafe impl Send for T {} unsafe impl Sync for T {} - use std::ops::Add; - /// This is a single threaded variant of `AtomicU64`, `AtomicUsize`, etc. /// It has explicit ordering arguments and is only intended for use with /// the native atomic types. @@ -182,88 +200,17 @@ cfg_if! { pub type AtomicU32 = Atomic; pub type AtomicU64 = Atomic; - pub fn join(oper_a: A, oper_b: B) -> (RA, RB) - where A: FnOnce() -> RA, - B: FnOnce() -> RB - { - (oper_a(), oper_b()) - } - - #[macro_export] - macro_rules! parallel { - ($($blocks:block),*) => { - // We catch panics here ensuring that all the blocks execute. - // This makes behavior consistent with the parallel compiler. - let mut panic = None; - $( - if let Err(p) = ::std::panic::catch_unwind( - ::std::panic::AssertUnwindSafe(|| $blocks) - ) { - if panic.is_none() { - panic = Some(p); - } - } - )* - if let Some(panic) = panic { - ::std::panic::resume_unwind(panic); - } - } - } - - pub fn par_for_each_in(t: T, mut for_each: impl FnMut(T::Item) + Sync + Send) { - // We catch panics here ensuring that all the loop iterations execute. - // This makes behavior consistent with the parallel compiler. - let mut panic = None; - t.into_iter().for_each(|i| { - if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) { - if panic.is_none() { - panic = Some(p); - } - } - }); - if let Some(panic) = panic { - resume_unwind(panic); - } - } - - pub fn par_map>( - t: T, - mut map: impl FnMut(<::IntoIter as Iterator>::Item) -> R, - ) -> C { - // We catch panics here ensuring that all the loop iterations execute. - let mut panic = None; - let r = t.into_iter().filter_map(|i| { - match catch_unwind(AssertUnwindSafe(|| map(i))) { - Ok(r) => Some(r), - Err(p) => { - if panic.is_none() { - panic = Some(p); - } - None - } - } - }).collect(); - if let Some(panic) = panic { - resume_unwind(panic); - } - r - } - pub use std::rc::Rc as Lrc; pub use std::rc::Weak as Weak; pub use std::cell::Ref as ReadGuard; pub use std::cell::Ref as MappedReadGuard; pub use std::cell::RefMut as WriteGuard; pub use std::cell::RefMut as MappedWriteGuard; - pub use std::cell::RefMut as LockGuard; pub use std::cell::RefMut as MappedLockGuard; - pub use std::cell::OnceCell; + pub use std::cell::OnceCell as OnceLock; use std::cell::RefCell as InnerRwLock; - use std::cell::RefCell as InnerLock; - - use std::cell::Cell; pub type MTLockRef<'a, T> = &'a mut MTLock; @@ -313,10 +260,9 @@ cfg_if! { pub use parking_lot::RwLockWriteGuard as WriteGuard; pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard; - pub use parking_lot::MutexGuard as LockGuard; pub use parking_lot::MappedMutexGuard as MappedLockGuard; - pub use std::sync::OnceLock as OnceCell; + pub use std::sync::OnceLock; pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; @@ -355,171 +301,10 @@ cfg_if! { } } - use parking_lot::Mutex as InnerLock; use parking_lot::RwLock as InnerRwLock; use std::thread; - #[inline] - pub fn join(oper_a: A, oper_b: B) -> (RA, RB) - where - A: FnOnce() -> RA + DynSend, - B: FnOnce() -> RB + DynSend, - { - if mode::is_dyn_thread_safe() { - let oper_a = FromDyn::from(oper_a); - let oper_b = FromDyn::from(oper_b); - let (a, b) = rayon::join(move || FromDyn::from(oper_a.into_inner()()), move || FromDyn::from(oper_b.into_inner()())); - (a.into_inner(), b.into_inner()) - } else { - (oper_a(), oper_b()) - } - } - - // This function only works when `mode::is_dyn_thread_safe()`. - pub fn scope<'scope, OP, R>(op: OP) -> R - where - OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend, - R: DynSend, - { - let op = FromDyn::from(op); - rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner() - } - - /// Runs a list of blocks in parallel. The first block is executed immediately on - /// the current thread. Use that for the longest running block. - #[macro_export] - macro_rules! parallel { - (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => { - parallel!(impl $fblock [$block, $($c,)*] [$($rest),*]) - }; - (impl $fblock:block [$($blocks:expr,)*] []) => { - ::rustc_data_structures::sync::scope(|s| { - $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks); - s.spawn(move |_| block.into_inner()());)* - (|| $fblock)(); - }); - }; - ($fblock:block, $($blocks:block),*) => { - if rustc_data_structures::sync::is_dyn_thread_safe() { - // Reverse the order of the later blocks since Rayon executes them in reverse order - // when using a single thread. This ensures the execution order matches that - // of a single threaded rustc. - parallel!(impl $fblock [] [$($blocks),*]); - } else { - // We catch panics here ensuring that all the blocks execute. - // This makes behavior consistent with the parallel compiler. - let mut panic = None; - if let Err(p) = ::std::panic::catch_unwind( - ::std::panic::AssertUnwindSafe(|| $fblock) - ) { - if panic.is_none() { - panic = Some(p); - } - } - $( - if let Err(p) = ::std::panic::catch_unwind( - ::std::panic::AssertUnwindSafe(|| $blocks) - ) { - if panic.is_none() { - panic = Some(p); - } - } - )* - if let Some(panic) = panic { - ::std::panic::resume_unwind(panic); - } - } - }; - } - - use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; - - pub fn par_for_each_in + IntoParallelIterator>( - t: T, - for_each: impl Fn(I) + DynSync + DynSend - ) { - if mode::is_dyn_thread_safe() { - let for_each = FromDyn::from(for_each); - let panic: Lock> = Lock::new(None); - t.into_par_iter().for_each(|i| if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) { - let mut l = panic.lock(); - if l.is_none() { - *l = Some(p) - } - }); - - if let Some(panic) = panic.into_inner() { - resume_unwind(panic); - } - } else { - // We catch panics here ensuring that all the loop iterations execute. - // This makes behavior consistent with the parallel compiler. - let mut panic = None; - t.into_iter().for_each(|i| { - if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) { - if panic.is_none() { - panic = Some(p); - } - } - }); - if let Some(panic) = panic { - resume_unwind(panic); - } - } - } - - pub fn par_map< - I, - T: IntoIterator + IntoParallelIterator, - R: std::marker::Send, - C: FromIterator + FromParallelIterator - >( - t: T, - map: impl Fn(I) -> R + DynSync + DynSend - ) -> C { - if mode::is_dyn_thread_safe() { - let panic: Lock> = Lock::new(None); - let map = FromDyn::from(map); - // We catch panics here ensuring that all the loop iterations execute. - let r = t.into_par_iter().filter_map(|i| { - match catch_unwind(AssertUnwindSafe(|| map(i))) { - Ok(r) => Some(r), - Err(p) => { - let mut l = panic.lock(); - if l.is_none() { - *l = Some(p); - } - None - }, - } - }).collect(); - - if let Some(panic) = panic.into_inner() { - resume_unwind(panic); - } - r - } else { - // We catch panics here ensuring that all the loop iterations execute. - let mut panic = None; - let r = t.into_iter().filter_map(|i| { - match catch_unwind(AssertUnwindSafe(|| map(i))) { - Ok(r) => Some(r), - Err(p) => { - if panic.is_none() { - panic = Some(p); - } - None - } - } - }).collect(); - if let Some(panic) = panic { - resume_unwind(panic); - } - r - } - } - /// This makes locks panic if they are already held. /// It is only useful when you are running in a single thread const ERROR_CHECKING: bool = false; @@ -542,81 +327,6 @@ impl HashMapExt for HashMap } } -#[derive(Debug)] -pub struct Lock(InnerLock); - -impl Lock { - #[inline(always)] - pub fn new(inner: T) -> Self { - Lock(InnerLock::new(inner)) - } - - #[inline(always)] - pub fn into_inner(self) -> T { - self.0.into_inner() - } - - #[inline(always)] - pub fn get_mut(&mut self) -> &mut T { - self.0.get_mut() - } - - #[cfg(parallel_compiler)] - #[inline(always)] - pub fn try_lock(&self) -> Option> { - self.0.try_lock() - } - - #[cfg(not(parallel_compiler))] - #[inline(always)] - pub fn try_lock(&self) -> Option> { - self.0.try_borrow_mut().ok() - } - - #[cfg(parallel_compiler)] - #[inline(always)] - #[track_caller] - pub fn lock(&self) -> LockGuard<'_, T> { - if ERROR_CHECKING { - self.0.try_lock().expect("lock was already held") - } else { - self.0.lock() - } - } - - #[cfg(not(parallel_compiler))] - #[inline(always)] - #[track_caller] - pub fn lock(&self) -> LockGuard<'_, T> { - self.0.borrow_mut() - } - - #[inline(always)] - #[track_caller] - pub fn with_lock R, R>(&self, f: F) -> R { - f(&mut *self.lock()) - } - - #[inline(always)] - #[track_caller] - pub fn borrow(&self) -> LockGuard<'_, T> { - self.lock() - } - - #[inline(always)] - #[track_caller] - pub fn borrow_mut(&self) -> LockGuard<'_, T> { - self.lock() - } -} - -impl Default for Lock { - #[inline] - fn default() -> Self { - Lock::new(T::default()) - } -} - #[derive(Debug, Default)] pub struct RwLock(InnerRwLock); diff --git a/compiler/rustc_data_structures/src/sync/freeze.rs b/compiler/rustc_data_structures/src/sync/freeze.rs new file mode 100644 index 000000000..466c44f59 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/freeze.rs @@ -0,0 +1,200 @@ +use crate::sync::{AtomicBool, ReadGuard, RwLock, WriteGuard}; +#[cfg(parallel_compiler)] +use crate::sync::{DynSend, DynSync}; +use std::{ + cell::UnsafeCell, + intrinsics::likely, + marker::PhantomData, + ops::{Deref, DerefMut}, + ptr::NonNull, + sync::atomic::Ordering, +}; + +/// A type which allows mutation using a lock until +/// the value is frozen and can be accessed lock-free. +/// +/// Unlike `RwLock`, it can be used to prevent mutation past a point. +#[derive(Default)] +pub struct FreezeLock { + data: UnsafeCell, + frozen: AtomicBool, + + /// This lock protects writes to the `data` and `frozen` fields. + lock: RwLock<()>, +} + +#[cfg(parallel_compiler)] +unsafe impl DynSync for FreezeLock {} + +impl FreezeLock { + #[inline] + pub fn new(value: T) -> Self { + Self::with(value, false) + } + + #[inline] + pub fn frozen(value: T) -> Self { + Self::with(value, true) + } + + #[inline] + pub fn with(value: T, frozen: bool) -> Self { + Self { + data: UnsafeCell::new(value), + frozen: AtomicBool::new(frozen), + lock: RwLock::new(()), + } + } + + /// Clones the inner value along with the frozen state. + #[inline] + pub fn clone(&self) -> Self + where + T: Clone, + { + let lock = self.read(); + Self::with(lock.clone(), self.is_frozen()) + } + + #[inline] + pub fn is_frozen(&self) -> bool { + self.frozen.load(Ordering::Acquire) + } + + /// Get the inner value if frozen. + #[inline] + pub fn get(&self) -> Option<&T> { + if likely(self.frozen.load(Ordering::Acquire)) { + // SAFETY: This is frozen so the data cannot be modified. + unsafe { Some(&*self.data.get()) } + } else { + None + } + } + + #[inline] + pub fn read(&self) -> FreezeReadGuard<'_, T> { + FreezeReadGuard { + _lock_guard: if self.frozen.load(Ordering::Acquire) { + None + } else { + Some(self.lock.read()) + }, + data: unsafe { NonNull::new_unchecked(self.data.get()) }, + } + } + + #[inline] + pub fn borrow(&self) -> FreezeReadGuard<'_, T> { + self.read() + } + + #[inline] + #[track_caller] + pub fn write(&self) -> FreezeWriteGuard<'_, T> { + self.try_write().expect("still mutable") + } + + #[inline] + pub fn try_write(&self) -> Option> { + let _lock_guard = self.lock.write(); + // Use relaxed ordering since we're in the write lock. + if self.frozen.load(Ordering::Relaxed) { + None + } else { + Some(FreezeWriteGuard { + _lock_guard, + data: unsafe { NonNull::new_unchecked(self.data.get()) }, + frozen: &self.frozen, + marker: PhantomData, + }) + } + } + + #[inline] + pub fn freeze(&self) -> &T { + if !self.frozen.load(Ordering::Acquire) { + // Get the lock to ensure no concurrent writes and that we release the latest write. + let _lock = self.lock.write(); + self.frozen.store(true, Ordering::Release); + } + + // SAFETY: This is frozen so the data cannot be modified and shared access is sound. + unsafe { &*self.data.get() } + } +} + +/// A guard holding shared access to a `FreezeLock` which is in a locked state or frozen. +#[must_use = "if unused the FreezeLock may immediately unlock"] +pub struct FreezeReadGuard<'a, T: ?Sized> { + _lock_guard: Option>, + data: NonNull, +} + +impl<'a, T: ?Sized + 'a> Deref for FreezeReadGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + // SAFETY: If the lock is not frozen, `_lock_guard` holds the lock to the `UnsafeCell` so + // this has shared access until the `FreezeReadGuard` is dropped. If the lock is frozen, + // the data cannot be modified and shared access is sound. + unsafe { &*self.data.as_ptr() } + } +} + +impl<'a, T: ?Sized> FreezeReadGuard<'a, T> { + #[inline] + pub fn map(this: Self, f: impl FnOnce(&T) -> &U) -> FreezeReadGuard<'a, U> { + FreezeReadGuard { data: NonNull::from(f(&*this)), _lock_guard: this._lock_guard } + } +} + +/// A guard holding mutable access to a `FreezeLock` which is in a locked state or frozen. +#[must_use = "if unused the FreezeLock may immediately unlock"] +pub struct FreezeWriteGuard<'a, T: ?Sized> { + _lock_guard: WriteGuard<'a, ()>, + frozen: &'a AtomicBool, + data: NonNull, + marker: PhantomData<&'a mut T>, +} + +impl<'a, T> FreezeWriteGuard<'a, T> { + pub fn freeze(self) -> &'a T { + self.frozen.store(true, Ordering::Release); + + // SAFETY: This is frozen so the data cannot be modified and shared access is sound. + unsafe { &*self.data.as_ptr() } + } +} + +impl<'a, T: ?Sized> FreezeWriteGuard<'a, T> { + #[inline] + pub fn map( + mut this: Self, + f: impl FnOnce(&mut T) -> &mut U, + ) -> FreezeWriteGuard<'a, U> { + FreezeWriteGuard { + data: NonNull::from(f(&mut *this)), + _lock_guard: this._lock_guard, + frozen: this.frozen, + marker: PhantomData, + } + } +} + +impl<'a, T: ?Sized + 'a> Deref for FreezeWriteGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + // SAFETY: `self._lock_guard` holds the lock to the `UnsafeCell` so this has shared access. + unsafe { &*self.data.as_ptr() } + } +} + +impl<'a, T: ?Sized + 'a> DerefMut for FreezeWriteGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self._lock_guard` holds the lock to the `UnsafeCell` so this has mutable access. + unsafe { &mut *self.data.as_ptr() } + } +} diff --git a/compiler/rustc_data_structures/src/sync/lock.rs b/compiler/rustc_data_structures/src/sync/lock.rs new file mode 100644 index 000000000..339aebbf8 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/lock.rs @@ -0,0 +1,275 @@ +//! This module implements a lock which only uses synchronization if `might_be_dyn_thread_safe` is true. +//! It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync` traits. +//! +//! When `cfg(parallel_compiler)` is not set, the lock is instead a wrapper around `RefCell`. + +#![allow(dead_code)] + +use std::fmt; + +#[cfg(parallel_compiler)] +pub use maybe_sync::*; +#[cfg(not(parallel_compiler))] +pub use no_sync::*; + +#[derive(Clone, Copy, PartialEq)] +pub enum Mode { + NoSync, + Sync, +} + +mod maybe_sync { + use super::Mode; + use crate::sync::mode; + #[cfg(parallel_compiler)] + use crate::sync::{DynSend, DynSync}; + use parking_lot::lock_api::RawMutex as _; + use parking_lot::RawMutex; + use std::cell::Cell; + use std::cell::UnsafeCell; + use std::intrinsics::unlikely; + use std::marker::PhantomData; + use std::mem::ManuallyDrop; + use std::ops::{Deref, DerefMut}; + + /// A guard holding mutable access to a `Lock` which is in a locked state. + #[must_use = "if unused the Lock will immediately unlock"] + pub struct LockGuard<'a, T> { + lock: &'a Lock, + marker: PhantomData<&'a mut T>, + + /// The syncronization mode of the lock. This is explicitly passed to let LLVM relate it + /// to the original lock operation. + mode: Mode, + } + + impl<'a, T: 'a> Deref for LockGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + // SAFETY: We have shared access to the mutable access owned by this type, + // so we can give out a shared reference. + unsafe { &*self.lock.data.get() } + } + } + + impl<'a, T: 'a> DerefMut for LockGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + // SAFETY: We have mutable access to the data so we can give out a mutable reference. + unsafe { &mut *self.lock.data.get() } + } + } + + impl<'a, T: 'a> Drop for LockGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // SAFETY (union access): We get `self.mode` from the lock operation so it is consistent + // with the `lock.mode` state. This means we access the right union fields. + match self.mode { + Mode::NoSync => { + let cell = unsafe { &self.lock.mode_union.no_sync }; + debug_assert_eq!(cell.get(), true); + cell.set(false); + } + // SAFETY (unlock): We know that the lock is locked as this type is a proof of that. + Mode::Sync => unsafe { self.lock.mode_union.sync.unlock() }, + } + } + } + + union ModeUnion { + /// Indicates if the cell is locked. Only used if `Lock.mode` is `NoSync`. + no_sync: ManuallyDrop>, + + /// A lock implementation that's only used if `Lock.mode` is `Sync`. + sync: ManuallyDrop, + } + + /// The value representing a locked state for the `Cell`. + const LOCKED: bool = true; + + /// A lock which only uses synchronization if `might_be_dyn_thread_safe` is true. + /// It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync`. + pub struct Lock { + /// Indicates if synchronization is used via `mode_union.sync` if it's `Sync`, or if a + /// not thread safe cell is used via `mode_union.no_sync` if it's `NoSync`. + /// This is set on initialization and never changed. + mode: Mode, + + mode_union: ModeUnion, + data: UnsafeCell, + } + + impl Lock { + #[inline(always)] + pub fn new(inner: T) -> Self { + let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) { + // Create the lock with synchronization enabled using the `RawMutex` type. + (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) }) + } else { + // Create the lock with synchronization disabled. + (Mode::NoSync, ModeUnion { no_sync: ManuallyDrop::new(Cell::new(!LOCKED)) }) + }; + Lock { mode, mode_union, data: UnsafeCell::new(inner) } + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.data.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.data.get_mut() + } + + #[inline(always)] + pub fn try_lock(&self) -> Option> { + let mode = self.mode; + // SAFETY: This is safe since the union fields are used in accordance with `self.mode`. + match mode { + Mode::NoSync => { + let cell = unsafe { &self.mode_union.no_sync }; + let was_unlocked = cell.get() != LOCKED; + if was_unlocked { + cell.set(LOCKED); + } + was_unlocked + } + Mode::Sync => unsafe { self.mode_union.sync.try_lock() }, + } + .then(|| LockGuard { lock: self, marker: PhantomData, mode }) + } + + /// This acquires the lock assuming syncronization is in a specific mode. + /// + /// Safety + /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was + /// true on lock creation. + #[inline(always)] + #[track_caller] + pub unsafe fn lock_assume(&self, mode: Mode) -> LockGuard<'_, T> { + #[inline(never)] + #[track_caller] + #[cold] + fn lock_held() -> ! { + panic!("lock was already held") + } + + // SAFETY: This is safe since the union fields are used in accordance with `mode` + // which also must match `self.mode` due to the safety precondition. + unsafe { + match mode { + Mode::NoSync => { + if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) { + lock_held() + } + } + Mode::Sync => self.mode_union.sync.lock(), + } + } + LockGuard { lock: self, marker: PhantomData, mode } + } + + #[inline(always)] + #[track_caller] + pub fn lock(&self) -> LockGuard<'_, T> { + unsafe { self.lock_assume(self.mode) } + } + } + + #[cfg(parallel_compiler)] + unsafe impl DynSend for Lock {} + #[cfg(parallel_compiler)] + unsafe impl DynSync for Lock {} +} + +mod no_sync { + use super::Mode; + use std::cell::RefCell; + + pub use std::cell::RefMut as LockGuard; + + pub struct Lock(RefCell); + + impl Lock { + #[inline(always)] + pub fn new(inner: T) -> Self { + Lock(RefCell::new(inner)) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + + #[inline(always)] + pub fn try_lock(&self) -> Option> { + self.0.try_borrow_mut().ok() + } + + #[inline(always)] + #[track_caller] + // This is unsafe to match the API for the `parallel_compiler` case. + pub unsafe fn lock_assume(&self, _mode: Mode) -> LockGuard<'_, T> { + self.0.borrow_mut() + } + + #[inline(always)] + #[track_caller] + pub fn lock(&self) -> LockGuard<'_, T> { + self.0.borrow_mut() + } + } +} + +impl Lock { + #[inline(always)] + #[track_caller] + pub fn with_lock R, R>(&self, f: F) -> R { + f(&mut *self.lock()) + } + + #[inline(always)] + #[track_caller] + pub fn borrow(&self) -> LockGuard<'_, T> { + self.lock() + } + + #[inline(always)] + #[track_caller] + pub fn borrow_mut(&self) -> LockGuard<'_, T> { + self.lock() + } +} + +impl Default for Lock { + #[inline] + fn default() -> Self { + Lock::new(T::default()) + } +} + +impl fmt::Debug for Lock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.try_lock() { + Some(guard) => f.debug_struct("Lock").field("data", &&*guard).finish(), + None => { + struct LockedPlaceholder; + impl fmt::Debug for LockedPlaceholder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + + f.debug_struct("Lock").field("data", &LockedPlaceholder).finish() + } + } + } +} diff --git a/compiler/rustc_data_structures/src/sync/parallel.rs b/compiler/rustc_data_structures/src/sync/parallel.rs new file mode 100644 index 000000000..1944ddfb7 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/parallel.rs @@ -0,0 +1,188 @@ +//! This module defines parallel operations that are implemented in +//! one way for the serial compiler, and another way the parallel compiler. + +#![allow(dead_code)] + +use parking_lot::Mutex; +use std::any::Any; +use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + +#[cfg(not(parallel_compiler))] +pub use disabled::*; +#[cfg(parallel_compiler)] +pub use enabled::*; + +/// A guard used to hold panics that occur during a parallel section to later by unwound. +/// This is used for the parallel compiler to prevent fatal errors from non-deterministically +/// hiding errors by ensuring that everything in the section has completed executing before +/// continuing with unwinding. It's also used for the non-parallel code to ensure error message +/// output match the parallel compiler for testing purposes. +pub struct ParallelGuard { + panic: Mutex>>, +} + +impl ParallelGuard { + pub fn run(&self, f: impl FnOnce() -> R) -> Option { + catch_unwind(AssertUnwindSafe(f)) + .map_err(|err| { + *self.panic.lock() = Some(err); + }) + .ok() + } +} + +/// This gives access to a fresh parallel guard in the closure and will unwind any panics +/// caught in it after the closure returns. +#[inline] +pub fn parallel_guard(f: impl FnOnce(&ParallelGuard) -> R) -> R { + let guard = ParallelGuard { panic: Mutex::new(None) }; + let ret = f(&guard); + if let Some(panic) = guard.panic.into_inner() { + resume_unwind(panic); + } + ret +} + +mod disabled { + use crate::sync::parallel_guard; + + #[macro_export] + #[cfg(not(parallel_compiler))] + macro_rules! parallel { + ($($blocks:block),*) => {{ + $crate::sync::parallel_guard(|guard| { + $(guard.run(|| $blocks);)* + }); + }} + } + + pub fn join(oper_a: A, oper_b: B) -> (RA, RB) + where + A: FnOnce() -> RA, + B: FnOnce() -> RB, + { + let (a, b) = parallel_guard(|guard| { + let a = guard.run(oper_a); + let b = guard.run(oper_b); + (a, b) + }); + (a.unwrap(), b.unwrap()) + } + + pub fn par_for_each_in(t: T, mut for_each: impl FnMut(T::Item)) { + parallel_guard(|guard| { + t.into_iter().for_each(|i| { + guard.run(|| for_each(i)); + }); + }) + } + + pub fn par_map>( + t: T, + mut map: impl FnMut(<::IntoIter as Iterator>::Item) -> R, + ) -> C { + parallel_guard(|guard| t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()) + } +} + +#[cfg(parallel_compiler)] +mod enabled { + use crate::sync::{mode, parallel_guard, DynSend, DynSync, FromDyn}; + + /// Runs a list of blocks in parallel. The first block is executed immediately on + /// the current thread. Use that for the longest running block. + #[macro_export] + macro_rules! parallel { + (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => { + parallel!(impl $fblock [$block, $($c,)*] [$($rest),*]) + }; + (impl $fblock:block [$($blocks:expr,)*] []) => { + ::rustc_data_structures::sync::scope(|s| { + $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks); + s.spawn(move |_| block.into_inner()());)* + (|| $fblock)(); + }); + }; + ($fblock:block, $($blocks:block),*) => { + if rustc_data_structures::sync::is_dyn_thread_safe() { + // Reverse the order of the later blocks since Rayon executes them in reverse order + // when using a single thread. This ensures the execution order matches that + // of a single threaded rustc. + parallel!(impl $fblock [] [$($blocks),*]); + } else { + $crate::sync::parallel_guard(|guard| { + guard.run(|| $fblock); + $(guard.run(|| $blocks);)* + }); + } + }; + } + + // This function only works when `mode::is_dyn_thread_safe()`. + pub fn scope<'scope, OP, R>(op: OP) -> R + where + OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend, + R: DynSend, + { + let op = FromDyn::from(op); + rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner() + } + + #[inline] + pub fn join(oper_a: A, oper_b: B) -> (RA, RB) + where + A: FnOnce() -> RA + DynSend, + B: FnOnce() -> RB + DynSend, + { + if mode::is_dyn_thread_safe() { + let oper_a = FromDyn::from(oper_a); + let oper_b = FromDyn::from(oper_b); + let (a, b) = rayon::join( + move || FromDyn::from(oper_a.into_inner()()), + move || FromDyn::from(oper_b.into_inner()()), + ); + (a.into_inner(), b.into_inner()) + } else { + super::disabled::join(oper_a, oper_b) + } + } + + use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; + + pub fn par_for_each_in + IntoParallelIterator>( + t: T, + for_each: impl Fn(I) + DynSync + DynSend, + ) { + parallel_guard(|guard| { + if mode::is_dyn_thread_safe() { + let for_each = FromDyn::from(for_each); + t.into_par_iter().for_each(|i| { + guard.run(|| for_each(i)); + }); + } else { + t.into_iter().for_each(|i| { + guard.run(|| for_each(i)); + }); + } + }); + } + + pub fn par_map< + I, + T: IntoIterator + IntoParallelIterator, + R: std::marker::Send, + C: FromIterator + FromParallelIterator, + >( + t: T, + map: impl Fn(I) -> R + DynSync + DynSend, + ) -> C { + parallel_guard(|guard| { + if mode::is_dyn_thread_safe() { + let map = FromDyn::from(map); + t.into_par_iter().filter_map(|i| guard.run(|| map(i))).collect() + } else { + t.into_iter().filter_map(|i| guard.run(|| map(i))).collect() + } + }) + } +} diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs index 8c84daf4f..ffafdba13 100644 --- a/compiler/rustc_data_structures/src/sync/worker_local.rs +++ b/compiler/rustc_data_structures/src/sync/worker_local.rs @@ -1,4 +1,4 @@ -use crate::sync::Lock; +use parking_lot::Mutex; use std::cell::Cell; use std::cell::OnceCell; use std::ops::Deref; @@ -6,7 +6,7 @@ use std::ptr; use std::sync::Arc; #[cfg(parallel_compiler)] -use {crate::cold_path, crate::sync::CacheAligned}; +use {crate::outline, crate::sync::CacheAligned}; /// A pointer to the `RegistryData` which uniquely identifies a registry. /// This identifier can be reused if the registry gets freed. @@ -25,17 +25,13 @@ impl RegistryId { fn verify(self) -> usize { let (id, index) = THREAD_DATA.with(|data| (data.registry_id.get(), data.index.get())); - if id == self { - index - } else { - cold_path(|| panic!("Unable to verify registry association")) - } + if id == self { index } else { outline(|| panic!("Unable to verify registry association")) } } } struct RegistryData { thread_limit: usize, - threads: Lock, + threads: Mutex, } /// Represents a list of threads which can access worker locals. @@ -65,7 +61,7 @@ thread_local! { impl Registry { /// Creates a registry which can hold up to `thread_limit` threads. pub fn new(thread_limit: usize) -> Self { - Registry(Arc::new(RegistryData { thread_limit, threads: Lock::new(0) })) + Registry(Arc::new(RegistryData { thread_limit, threads: Mutex::new(0) })) } /// Gets the registry associated with the current thread. Panics if there's no such registry. @@ -171,3 +167,9 @@ impl Deref for WorkerLocal { unsafe { &self.locals.get_unchecked(self.registry.id().verify()).0 } } } + +impl Default for WorkerLocal { + fn default() -> Self { + WorkerLocal::new(|_| T::default()) + } +} -- cgit v1.2.3