summaryrefslogtreecommitdiffstats
path: root/library/std/src/thread
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:20:39 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:20:39 +0000
commit1376c5a617be5c25655d0d7cb63e3beaa5a6e026 (patch)
tree3bb8d61aee02bc7a15eab3f36e3b921afc2075d0 /library/std/src/thread
parentReleasing progress-linux version 1.69.0+dfsg1-1~progress7.99u1. (diff)
downloadrustc-1376c5a617be5c25655d0d7cb63e3beaa5a6e026.tar.xz
rustc-1376c5a617be5c25655d0d7cb63e3beaa5a6e026.zip
Merging upstream version 1.70.0+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'library/std/src/thread')
-rw-r--r--library/std/src/thread/local.rs567
-rw-r--r--library/std/src/thread/mod.rs40
2 files changed, 3 insertions, 604 deletions
diff --git a/library/std/src/thread/local.rs b/library/std/src/thread/local.rs
index cf7c2e05a..7fdf03acc 100644
--- a/library/std/src/thread/local.rs
+++ b/library/std/src/thread/local.rs
@@ -173,200 +173,6 @@ macro_rules! thread_local {
);
}
-#[doc(hidden)]
-#[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")]
-#[macro_export]
-#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
-#[allow_internal_unsafe]
-macro_rules! __thread_local_inner {
- // used to generate the `LocalKey` value for const-initialized thread locals
- (@key $t:ty, const $init:expr) => {{
- #[cfg_attr(not(windows), inline)] // see comments below
- #[deny(unsafe_op_in_unsafe_fn)]
- unsafe fn __getit(
- _init: $crate::option::Option<&mut $crate::option::Option<$t>>,
- ) -> $crate::option::Option<&'static $t> {
- const INIT_EXPR: $t = $init;
-
- // wasm without atomics maps directly to `static mut`, and dtors
- // aren't implemented because thread dtors aren't really a thing
- // on wasm right now
- //
- // FIXME(#84224) this should come after the `target_thread_local`
- // block.
- #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
- {
- static mut VAL: $t = INIT_EXPR;
- unsafe { $crate::option::Option::Some(&VAL) }
- }
-
- // If the platform has support for `#[thread_local]`, use it.
- #[cfg(all(
- target_thread_local,
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
- ))]
- {
- #[thread_local]
- static mut VAL: $t = INIT_EXPR;
-
- // If a dtor isn't needed we can do something "very raw" and
- // just get going.
- if !$crate::mem::needs_drop::<$t>() {
- unsafe {
- return $crate::option::Option::Some(&VAL)
- }
- }
-
- // 0 == dtor not registered
- // 1 == dtor registered, dtor not run
- // 2 == dtor registered and is running or has run
- #[thread_local]
- static mut STATE: $crate::primitive::u8 = 0;
-
- unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
- let ptr = ptr as *mut $t;
-
- unsafe {
- $crate::debug_assert_eq!(STATE, 1);
- STATE = 2;
- $crate::ptr::drop_in_place(ptr);
- }
- }
-
- unsafe {
- match STATE {
- // 0 == we haven't registered a destructor, so do
- // so now.
- 0 => {
- $crate::thread::__FastLocalKeyInner::<$t>::register_dtor(
- $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
- destroy,
- );
- STATE = 1;
- $crate::option::Option::Some(&VAL)
- }
- // 1 == the destructor is registered and the value
- // is valid, so return the pointer.
- 1 => $crate::option::Option::Some(&VAL),
- // otherwise the destructor has already run, so we
- // can't give access.
- _ => $crate::option::Option::None,
- }
- }
- }
-
- // On platforms without `#[thread_local]` we fall back to the
- // same implementation as below for os thread locals.
- #[cfg(all(
- not(target_thread_local),
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
- ))]
- {
- #[inline]
- const fn __init() -> $t { INIT_EXPR }
- static __KEY: $crate::thread::__OsLocalKeyInner<$t> =
- $crate::thread::__OsLocalKeyInner::new();
- #[allow(unused_unsafe)]
- unsafe {
- __KEY.get(move || {
- if let $crate::option::Option::Some(init) = _init {
- if let $crate::option::Option::Some(value) = init.take() {
- return value;
- } else if $crate::cfg!(debug_assertions) {
- $crate::unreachable!("missing initial value");
- }
- }
- __init()
- })
- }
- }
- }
-
- unsafe {
- $crate::thread::LocalKey::new(__getit)
- }
- }};
-
- // used to generate the `LocalKey` value for `thread_local!`
- (@key $t:ty, $init:expr) => {
- {
- #[inline]
- fn __init() -> $t { $init }
-
- // When reading this function you might ask "why is this inlined
- // everywhere other than Windows?", and that's a very reasonable
- // question to ask. The short story is that it segfaults rustc if
- // this function is inlined. The longer story is that Windows looks
- // to not support `extern` references to thread locals across DLL
- // boundaries. This appears to at least not be supported in the ABI
- // that LLVM implements.
- //
- // Because of this we never inline on Windows, but we do inline on
- // other platforms (where external references to thread locals
- // across DLLs are supported). A better fix for this would be to
- // inline this function on Windows, but only for "statically linked"
- // components. For example if two separately compiled rlibs end up
- // getting linked into a DLL then it's fine to inline this function
- // across that boundary. It's only not fine to inline this function
- // across a DLL boundary. Unfortunately rustc doesn't currently
- // have this sort of logic available in an attribute, and it's not
- // clear that rustc is even equipped to answer this (it's more of a
- // Cargo question kinda). This means that, unfortunately, Windows
- // gets the pessimistic path for now where it's never inlined.
- //
- // The issue of "should enable on Windows sometimes" is #84933
- #[cfg_attr(not(windows), inline)]
- unsafe fn __getit(
- init: $crate::option::Option<&mut $crate::option::Option<$t>>,
- ) -> $crate::option::Option<&'static $t> {
- #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
- static __KEY: $crate::thread::__StaticLocalKeyInner<$t> =
- $crate::thread::__StaticLocalKeyInner::new();
-
- #[thread_local]
- #[cfg(all(
- target_thread_local,
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
- ))]
- static __KEY: $crate::thread::__FastLocalKeyInner<$t> =
- $crate::thread::__FastLocalKeyInner::new();
-
- #[cfg(all(
- not(target_thread_local),
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
- ))]
- static __KEY: $crate::thread::__OsLocalKeyInner<$t> =
- $crate::thread::__OsLocalKeyInner::new();
-
- // FIXME: remove the #[allow(...)] marker when macros don't
- // raise warning for missing/extraneous unsafe blocks anymore.
- // See https://github.com/rust-lang/rust/issues/74838.
- #[allow(unused_unsafe)]
- unsafe {
- __KEY.get(move || {
- if let $crate::option::Option::Some(init) = init {
- if let $crate::option::Option::Some(value) = init.take() {
- return value;
- } else if $crate::cfg!(debug_assertions) {
- $crate::unreachable!("missing default value");
- }
- }
- __init()
- })
- }
- }
-
- unsafe {
- $crate::thread::LocalKey::new(__getit)
- }
- }
- };
- ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
- $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
- $crate::__thread_local_inner!(@key $t, $($init)*);
- }
-}
-
/// An error returned by [`LocalKey::try_with`](struct.LocalKey.html#method.try_with).
#[stable(feature = "thread_local_try_with", since = "1.26.0")]
#[non_exhaustive]
@@ -779,376 +585,3 @@ impl<T: 'static> LocalKey<RefCell<T>> {
self.with(|cell| cell.replace(value))
}
}
-
-mod lazy {
- use crate::cell::UnsafeCell;
- use crate::hint;
- use crate::mem;
-
- pub struct LazyKeyInner<T> {
- inner: UnsafeCell<Option<T>>,
- }
-
- impl<T> LazyKeyInner<T> {
- pub const fn new() -> LazyKeyInner<T> {
- LazyKeyInner { inner: UnsafeCell::new(None) }
- }
-
- pub unsafe fn get(&self) -> Option<&'static T> {
- // SAFETY: The caller must ensure no reference is ever handed out to
- // the inner cell nor mutable reference to the Option<T> inside said
- // cell. This make it safe to hand a reference, though the lifetime
- // of 'static is itself unsafe, making the get method unsafe.
- unsafe { (*self.inner.get()).as_ref() }
- }
-
- /// The caller must ensure that no reference is active: this method
- /// needs unique access.
- pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T {
- // Execute the initialization up front, *then* move it into our slot,
- // just in case initialization fails.
- let value = init();
- let ptr = self.inner.get();
-
- // SAFETY:
- //
- // note that this can in theory just be `*ptr = Some(value)`, but due to
- // the compiler will currently codegen that pattern with something like:
- //
- // ptr::drop_in_place(ptr)
- // ptr::write(ptr, Some(value))
- //
- // Due to this pattern it's possible for the destructor of the value in
- // `ptr` (e.g., if this is being recursively initialized) to re-access
- // TLS, in which case there will be a `&` and `&mut` pointer to the same
- // value (an aliasing violation). To avoid setting the "I'm running a
- // destructor" flag we just use `mem::replace` which should sequence the
- // operations a little differently and make this safe to call.
- //
- // The precondition also ensures that we are the only one accessing
- // `self` at the moment so replacing is fine.
- unsafe {
- let _ = mem::replace(&mut *ptr, Some(value));
- }
-
- // SAFETY: With the call to `mem::replace` it is guaranteed there is
- // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked`
- // will never be reached.
- unsafe {
- // After storing `Some` we want to get a reference to the contents of
- // what we just stored. While we could use `unwrap` here and it should
- // always work it empirically doesn't seem to always get optimized away,
- // which means that using something like `try_with` can pull in
- // panicking code and cause a large size bloat.
- match *ptr {
- Some(ref x) => x,
- None => hint::unreachable_unchecked(),
- }
- }
- }
-
- /// The other methods hand out references while taking &self.
- /// As such, callers of this method must ensure no `&` and `&mut` are
- /// available and used at the same time.
- #[allow(unused)]
- pub unsafe fn take(&mut self) -> Option<T> {
- // SAFETY: See doc comment for this method.
- unsafe { (*self.inner.get()).take() }
- }
- }
-}
-
-/// On some targets like wasm there's no threads, so no need to generate
-/// thread locals and we can instead just use plain statics!
-#[doc(hidden)]
-#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
-pub mod statik {
- use super::lazy::LazyKeyInner;
- use crate::fmt;
-
- pub struct Key<T> {
- inner: LazyKeyInner<T>,
- }
-
- unsafe impl<T> Sync for Key<T> {}
-
- impl<T> fmt::Debug for Key<T> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("Key").finish_non_exhaustive()
- }
- }
-
- impl<T> Key<T> {
- pub const fn new() -> Key<T> {
- Key { inner: LazyKeyInner::new() }
- }
-
- pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> {
- // SAFETY: The caller must ensure no reference is ever handed out to
- // the inner cell nor mutable reference to the Option<T> inside said
- // cell. This make it safe to hand a reference, though the lifetime
- // of 'static is itself unsafe, making the get method unsafe.
- let value = unsafe {
- match self.inner.get() {
- Some(ref value) => value,
- None => self.inner.initialize(init),
- }
- };
-
- Some(value)
- }
- }
-}
-
-#[doc(hidden)]
-#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics"))),))]
-pub mod fast {
- use super::lazy::LazyKeyInner;
- use crate::cell::Cell;
- use crate::sys::thread_local_dtor::register_dtor;
- use crate::{fmt, mem, panic};
-
- #[derive(Copy, Clone)]
- enum DtorState {
- Unregistered,
- Registered,
- RunningOrHasRun,
- }
-
- // This data structure has been carefully constructed so that the fast path
- // only contains one branch on x86. That optimization is necessary to avoid
- // duplicated tls lookups on OSX.
- //
- // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
- pub struct Key<T> {
- // If `LazyKeyInner::get` returns `None`, that indicates either:
- // * The value has never been initialized
- // * The value is being recursively initialized
- // * The value has already been destroyed or is being destroyed
- // To determine which kind of `None`, check `dtor_state`.
- //
- // This is very optimizer friendly for the fast path - initialized but
- // not yet dropped.
- inner: LazyKeyInner<T>,
-
- // Metadata to keep track of the state of the destructor. Remember that
- // this variable is thread-local, not global.
- dtor_state: Cell<DtorState>,
- }
-
- impl<T> fmt::Debug for Key<T> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("Key").finish_non_exhaustive()
- }
- }
-
- impl<T> Key<T> {
- pub const fn new() -> Key<T> {
- Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
- }
-
- // note that this is just a publicly-callable function only for the
- // const-initialized form of thread locals, basically a way to call the
- // free `register_dtor` function defined elsewhere in std.
- pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
- unsafe {
- register_dtor(a, dtor);
- }
- }
-
- pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
- // SAFETY: See the definitions of `LazyKeyInner::get` and
- // `try_initialize` for more information.
- //
- // The caller must ensure no mutable references are ever active to
- // the inner cell or the inner T when this is called.
- // The `try_initialize` is dependant on the passed `init` function
- // for this.
- unsafe {
- match self.inner.get() {
- Some(val) => Some(val),
- None => self.try_initialize(init),
- }
- }
- }
-
- // `try_initialize` is only called once per fast thread local variable,
- // except in corner cases where thread_local dtors reference other
- // thread_local's, or it is being recursively initialized.
- //
- // Macos: Inlining this function can cause two `tlv_get_addr` calls to
- // be performed for every call to `Key::get`.
- // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
- #[inline(never)]
- unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
- // SAFETY: See comment above (this function doc).
- if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
- // SAFETY: See comment above (this function doc).
- Some(unsafe { self.inner.initialize(init) })
- } else {
- None
- }
- }
-
- // `try_register_dtor` is only called once per fast thread local
- // variable, except in corner cases where thread_local dtors reference
- // other thread_local's, or it is being recursively initialized.
- unsafe fn try_register_dtor(&self) -> bool {
- match self.dtor_state.get() {
- DtorState::Unregistered => {
- // SAFETY: dtor registration happens before initialization.
- // Passing `self` as a pointer while using `destroy_value<T>`
- // is safe because the function will build a pointer to a
- // Key<T>, which is the type of self and so find the correct
- // size.
- unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
- self.dtor_state.set(DtorState::Registered);
- true
- }
- DtorState::Registered => {
- // recursively initialized
- true
- }
- DtorState::RunningOrHasRun => false,
- }
- }
- }
-
- unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
- let ptr = ptr as *mut Key<T>;
-
- // SAFETY:
- //
- // The pointer `ptr` has been built just above and comes from
- // `try_register_dtor` where it is originally a Key<T> coming from `self`,
- // making it non-NUL and of the correct type.
- //
- // Right before we run the user destructor be sure to set the
- // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
- // causes future calls to `get` to run `try_initialize_drop` again,
- // which will now fail, and return `None`.
- //
- // Wrap the call in a catch to ensure unwinding is caught in the event
- // a panic takes place in a destructor.
- if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
- let value = (*ptr).inner.take();
- (*ptr).dtor_state.set(DtorState::RunningOrHasRun);
- drop(value);
- })) {
- rtabort!("thread local panicked on drop");
- }
- }
-}
-
-#[doc(hidden)]
-#[cfg(all(
- not(target_thread_local),
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
-))]
-pub mod os {
- use super::lazy::LazyKeyInner;
- use crate::cell::Cell;
- use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
- use crate::{fmt, marker, panic, ptr};
-
- /// Use a regular global static to store this key; the state provided will then be
- /// thread-local.
- pub struct Key<T> {
- // OS-TLS key that we'll use to key off.
- os: OsStaticKey,
- marker: marker::PhantomData<Cell<T>>,
- }
-
- impl<T> fmt::Debug for Key<T> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("Key").finish_non_exhaustive()
- }
- }
-
- unsafe impl<T> Sync for Key<T> {}
-
- struct Value<T: 'static> {
- inner: LazyKeyInner<T>,
- key: &'static Key<T>,
- }
-
- impl<T: 'static> Key<T> {
- #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
- pub const fn new() -> Key<T> {
- Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
- }
-
- /// It is a requirement for the caller to ensure that no mutable
- /// reference is active when this method is called.
- pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
- // SAFETY: See the documentation for this method.
- let ptr = unsafe { self.os.get() as *mut Value<T> };
- if ptr.addr() > 1 {
- // SAFETY: the check ensured the pointer is safe (its destructor
- // is not running) + it is coming from a trusted source (self).
- if let Some(ref value) = unsafe { (*ptr).inner.get() } {
- return Some(value);
- }
- }
- // SAFETY: At this point we are sure we have no value and so
- // initializing (or trying to) is safe.
- unsafe { self.try_initialize(init) }
- }
-
- // `try_initialize` is only called once per os thread local variable,
- // except in corner cases where thread_local dtors reference other
- // thread_local's, or it is being recursively initialized.
- unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
- // SAFETY: No mutable references are ever handed out meaning getting
- // the value is ok.
- let ptr = unsafe { self.os.get() as *mut Value<T> };
- if ptr.addr() == 1 {
- // destructor is running
- return None;
- }
-
- let ptr = if ptr.is_null() {
- // If the lookup returned null, we haven't initialized our own
- // local copy, so do that now.
- let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
- // SAFETY: At this point we are sure there is no value inside
- // ptr so setting it will not affect anyone else.
- unsafe {
- self.os.set(ptr as *mut u8);
- }
- ptr
- } else {
- // recursive initialization
- ptr
- };
-
- // SAFETY: ptr has been ensured as non-NUL just above an so can be
- // dereferenced safely.
- unsafe { Some((*ptr).inner.initialize(init)) }
- }
- }
-
- unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) {
- // SAFETY:
- //
- // The OS TLS ensures that this key contains a null value when this
- // destructor starts to run. We set it back to a sentinel value of 1 to
- // ensure that any future calls to `get` for this thread will return
- // `None`.
- //
- // Note that to prevent an infinite loop we reset it back to null right
- // before we return from the destructor ourselves.
- //
- // Wrap the call in a catch to ensure unwinding is caught in the event
- // a panic takes place in a destructor.
- if let Err(_) = panic::catch_unwind(|| unsafe {
- let ptr = Box::from_raw(ptr as *mut Value<T>);
- let key = ptr.key;
- key.os.set(ptr::invalid_mut(1));
- drop(ptr);
- key.os.set(ptr::null_mut());
- }) {
- rtabort!("thread local panicked on drop");
- }
- }
-}
diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs
index 489af7767..13b845b25 100644
--- a/library/std/src/thread/mod.rs
+++ b/library/std/src/thread/mod.rs
@@ -131,7 +131,8 @@
//!
//! * Build the thread with [`Builder`] and pass the desired stack size to [`Builder::stack_size`].
//! * Set the `RUST_MIN_STACK` environment variable to an integer representing the desired stack
-//! size (in bytes). Note that setting [`Builder::stack_size`] will override this.
+//! size (in bytes). Note that setting [`Builder::stack_size`] will override this. Be aware that
+//! changes to `RUST_MIN_STACK` may be ignored after program start.
//!
//! Note that the stack size of the main thread is *not* determined by Rust.
//!
@@ -203,44 +204,9 @@ pub use self::local::{AccessError, LocalKey};
// by the elf linker. "static" is for single-threaded platforms where a global
// static is sufficient.
-#[unstable(feature = "libstd_thread_internals", issue = "none")]
-#[cfg(not(test))]
-#[cfg(all(
- target_thread_local,
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
-))]
-#[doc(hidden)]
-pub use self::local::fast::Key as __FastLocalKeyInner;
-// when building for tests, use real std's type
-#[unstable(feature = "libstd_thread_internals", issue = "none")]
-#[cfg(test)]
-#[cfg(all(
- target_thread_local,
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
-))]
-pub use realstd::thread::__FastLocalKeyInner;
-
-#[unstable(feature = "libstd_thread_internals", issue = "none")]
-#[cfg(not(test))]
-#[cfg(all(
- not(target_thread_local),
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
-))]
#[doc(hidden)]
-pub use self::local::os::Key as __OsLocalKeyInner;
-// when building for tests, use real std's type
-#[unstable(feature = "libstd_thread_internals", issue = "none")]
-#[cfg(test)]
-#[cfg(all(
- not(target_thread_local),
- not(all(target_family = "wasm", not(target_feature = "atomics"))),
-))]
-pub use realstd::thread::__OsLocalKeyInner;
-
#[unstable(feature = "libstd_thread_internals", issue = "none")]
-#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
-#[doc(hidden)]
-pub use self::local::statik::Key as __StaticLocalKeyInner;
+pub use crate::sys::common::thread_local::Key as __LocalKeyInner;
////////////////////////////////////////////////////////////////////////////////
// Builder