use super::lazy::LazyKeyInner; use crate::cell::Cell; use crate::sys::thread_local_dtor::register_dtor; use crate::{fmt, mem, panic}; #[doc(hidden)] #[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] #[allow_internal_unsafe] #[unstable(feature = "thread_local_internals", issue = "none")] #[rustc_macro_transparency = "semitransparent"] pub macro thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals (@key $t:ty, const $init:expr) => {{ #[inline] #[deny(unsafe_op_in_unsafe_fn)] unsafe fn __getit( _init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { const INIT_EXPR: $t = $init; // If the platform has support for `#[thread_local]`, use it. #[thread_local] static mut VAL: $t = INIT_EXPR; // If a dtor isn't needed we can do something "very raw" and // just get going. if !$crate::mem::needs_drop::<$t>() { unsafe { return $crate::option::Option::Some(&VAL) } } // 0 == dtor not registered // 1 == dtor registered, dtor not run // 2 == dtor registered and is running or has run #[thread_local] static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0); // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires // all that comes with it. unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { $crate::thread::local_impl::abort_on_dtor_unwind(|| { let old_state = STATE.replace(2); $crate::debug_assert_eq!(old_state, 1); // Safety: safety requirement is passed on to caller. unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); } }); } unsafe { match STATE.get() { // 0 == we haven't registered a destructor, so do // so now. 0 => { $crate::thread::local_impl::Key::<$t>::register_dtor( $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8, destroy, ); STATE.set(1); $crate::option::Option::Some(&VAL) } // 1 == the destructor is registered and the value // is valid, so return the pointer. 1 => $crate::option::Option::Some(&VAL), // otherwise the destructor has already run, so we // can't give access. _ => $crate::option::Option::None, } } } unsafe { $crate::thread::LocalKey::new(__getit) } }}, // used to generate the `LocalKey` value for `thread_local!` (@key $t:ty, $init:expr) => { { #[inline] fn __init() -> $t { $init } #[inline] unsafe fn __getit( init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { #[thread_local] static __KEY: $crate::thread::local_impl::Key<$t> = $crate::thread::local_impl::Key::<$t>::new(); unsafe { __KEY.get(move || { if let $crate::option::Option::Some(init) = init { if let $crate::option::Option::Some(value) = init.take() { return value; } else if $crate::cfg!(debug_assertions) { $crate::unreachable!("missing default value"); } } __init() }) } } unsafe { $crate::thread::LocalKey::new(__getit) } } }, ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); }, } #[derive(Copy, Clone)] enum DtorState { Unregistered, Registered, RunningOrHasRun, } // This data structure has been carefully constructed so that the fast path // only contains one branch on x86. That optimization is necessary to avoid // duplicated tls lookups on OSX. // // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 pub struct Key { // If `LazyKeyInner::get` returns `None`, that indicates either: // * The value has never been initialized // * The value is being recursively initialized // * The value has already been destroyed or is being destroyed // To determine which kind of `None`, check `dtor_state`. // // This is very optimizer friendly for the fast path - initialized but // not yet dropped. inner: LazyKeyInner, // Metadata to keep track of the state of the destructor. Remember that // this variable is thread-local, not global. dtor_state: Cell, } impl fmt::Debug for Key { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Key").finish_non_exhaustive() } } impl Key { pub const fn new() -> Key { Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } } // note that this is just a publicly-callable function only for the // const-initialized form of thread locals, basically a way to call the // free `register_dtor` function defined elsewhere in std. pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { unsafe { register_dtor(a, dtor); } } pub unsafe fn get T>(&self, init: F) -> Option<&'static T> { // SAFETY: See the definitions of `LazyKeyInner::get` and // `try_initialize` for more information. // // The caller must ensure no mutable references are ever active to // the inner cell or the inner T when this is called. // The `try_initialize` is dependant on the passed `init` function // for this. unsafe { match self.inner.get() { Some(val) => Some(val), None => self.try_initialize(init), } } } // `try_initialize` is only called once per fast thread local variable, // except in corner cases where thread_local dtors reference other // thread_local's, or it is being recursively initialized. // // Macos: Inlining this function can cause two `tlv_get_addr` calls to // be performed for every call to `Key::get`. // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 #[inline(never)] unsafe fn try_initialize T>(&self, init: F) -> Option<&'static T> { // SAFETY: See comment above (this function doc). if !mem::needs_drop::() || unsafe { self.try_register_dtor() } { // SAFETY: See comment above (this function doc). Some(unsafe { self.inner.initialize(init) }) } else { None } } // `try_register_dtor` is only called once per fast thread local // variable, except in corner cases where thread_local dtors reference // other thread_local's, or it is being recursively initialized. unsafe fn try_register_dtor(&self) -> bool { match self.dtor_state.get() { DtorState::Unregistered => { // SAFETY: dtor registration happens before initialization. // Passing `self` as a pointer while using `destroy_value` // is safe because the function will build a pointer to a // Key, which is the type of self and so find the correct // size. unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::) }; self.dtor_state.set(DtorState::Registered); true } DtorState::Registered => { // recursively initialized true } DtorState::RunningOrHasRun => false, } } } unsafe extern "C" fn destroy_value(ptr: *mut u8) { let ptr = ptr as *mut Key; // SAFETY: // // The pointer `ptr` has been built just above and comes from // `try_register_dtor` where it is originally a Key coming from `self`, // making it non-NUL and of the correct type. // // Right before we run the user destructor be sure to set the // `Option` to `None`, and `dtor_state` to `RunningOrHasRun`. This // causes future calls to `get` to run `try_initialize_drop` again, // which will now fail, and return `None`. // // Wrap the call in a catch to ensure unwinding is caught in the event // a panic takes place in a destructor. if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { let value = (*ptr).inner.take(); (*ptr).dtor_state.set(DtorState::RunningOrHasRun); drop(value); })) { rtabort!("thread local panicked on drop"); } }