// Copyright 2016 Amanieu d'Antras // // Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. use core::cmp; use core::mem; use core::num::Wrapping; use core::ops; use core::sync::atomic::Ordering; use fallback; #[cfg(all(feature = "nightly", target_has_atomic = "8"))] use core::sync::atomic::{ AtomicI8, AtomicU8 }; #[cfg(all(feature = "nightly", target_has_atomic = "16"))] use core::sync::atomic::{ AtomicI16, AtomicU16 }; #[cfg(all(feature = "nightly", target_has_atomic = "32"))] use core::sync::atomic::{ AtomicI32, AtomicU32 }; #[cfg(all(feature = "nightly", target_has_atomic = "64"))] use core::sync::atomic::{ AtomicI64, AtomicU64 }; #[cfg(not(feature = "nightly"))] use core::sync::atomic::AtomicUsize; #[cfg(not(feature = "nightly"))] const SIZEOF_USIZE: usize = mem::size_of::(); #[cfg(not(feature = "nightly"))] const ALIGNOF_USIZE: usize = mem::align_of::(); #[cfg(feature = "nightly")] #[inline] pub const fn atomic_is_lock_free() -> bool { let size = mem::size_of::(); // FIXME: switch to … && … && … once that operator is supported in const functions (1 == size.count_ones()) & (8 >= size) & (mem::align_of::() >= size) } #[cfg(not(feature = "nightly"))] #[inline] pub fn atomic_is_lock_free() -> bool { let size = mem::size_of::(); 1 == size.count_ones() && SIZEOF_USIZE >= size && mem::align_of::() >= ALIGNOF_USIZE } #[inline] pub unsafe fn atomic_load(dst: *mut T, order: Ordering) -> T { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy(&(*(dst as *const AtomicU8)).load(order)) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy(&(*(dst as *const AtomicU16)).load(order)) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy(&(*(dst as *const AtomicU32)).load(order)) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy(&(*(dst as *const AtomicU64)).load(order)) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy(&(*(dst as *const AtomicUsize)).load(order)) } _ => fallback::atomic_load(dst), } } #[inline] pub unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { (*(dst as *const AtomicU8)).store(mem::transmute_copy(&val), order) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { (*(dst as *const AtomicU16)).store(mem::transmute_copy(&val), order) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { (*(dst as *const AtomicU32)).store(mem::transmute_copy(&val), order) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { (*(dst as *const AtomicU64)).store(mem::transmute_copy(&val), order) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { (*(dst as *const AtomicUsize)).store(mem::transmute_copy(&val), order) } _ => fallback::atomic_store(dst, val), } } #[inline] pub unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy(&(*(dst as *const AtomicU8)).swap(mem::transmute_copy(&val), order)) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).swap(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).swap(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).swap(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).swap(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_swap(dst, val), } } #[inline] unsafe fn map_result(r: Result) -> Result { match r { Ok(x) => Ok(mem::transmute_copy(&x)), Err(x) => Err(mem::transmute_copy(&x)), } } #[inline] pub unsafe fn atomic_compare_exchange( dst: *mut T, current: T, new: T, success: Ordering, failure: Ordering, ) -> Result { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { map_result((*(dst as *const AtomicU8)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { map_result((*(dst as *const AtomicU16)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { map_result((*(dst as *const AtomicU32)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { map_result((*(dst as *const AtomicU64)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { map_result((*(dst as *const AtomicUsize)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } _ => fallback::atomic_compare_exchange(dst, current, new), } } #[inline] pub unsafe fn atomic_compare_exchange_weak( dst: *mut T, current: T, new: T, success: Ordering, failure: Ordering, ) -> Result { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { map_result((*(dst as *const AtomicU8)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { map_result((*(dst as *const AtomicU16)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { map_result((*(dst as *const AtomicU32)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { map_result((*(dst as *const AtomicU64)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { map_result((*(dst as *const AtomicUsize)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )) } _ => fallback::atomic_compare_exchange(dst, current, new), } } #[inline] pub unsafe fn atomic_add(dst: *mut T, val: T, order: Ordering) -> T where Wrapping: ops::Add>, { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_add(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_add(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_add(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_add(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).fetch_add(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_add(dst, val), } } #[inline] pub unsafe fn atomic_sub(dst: *mut T, val: T, order: Ordering) -> T where Wrapping: ops::Sub>, { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_sub(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_sub(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_sub(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_sub(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).fetch_sub(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_sub(dst, val), } } #[inline] pub unsafe fn atomic_and>( dst: *mut T, val: T, order: Ordering, ) -> T { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_and(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_and(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_and(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_and(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).fetch_and(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_and(dst, val), } } #[inline] pub unsafe fn atomic_or>( dst: *mut T, val: T, order: Ordering, ) -> T { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_or(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_or(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_or(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_or(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).fetch_or(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_or(dst, val), } } #[inline] pub unsafe fn atomic_xor>( dst: *mut T, val: T, order: Ordering, ) -> T { match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_xor(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_xor(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_xor(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_xor(mem::transmute_copy(&val), order), ) } #[cfg(not(feature = "nightly"))] SIZEOF_USIZE if mem::align_of::() >= ALIGNOF_USIZE => { mem::transmute_copy( &(*(dst as *const AtomicUsize)).fetch_xor(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_xor(dst, val), } } #[inline] pub unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { // Silence warning, fetch_min is not stable yet #[cfg(not(feature = "nightly"))] let _ = order; match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicI8)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicI16)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicI32)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicI64)).fetch_min(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_min(dst, val), } } #[inline] pub unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { // Silence warning, fetch_min is not stable yet #[cfg(not(feature = "nightly"))] let _ = order; match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicI8)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicI16)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicI32)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicI64)).fetch_max(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_max(dst, val), } } #[inline] pub unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { // Silence warning, fetch_min is not stable yet #[cfg(not(feature = "nightly"))] let _ = order; match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_min(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_min(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_min(dst, val), } } #[inline] pub unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { // Silence warning, fetch_min is not stable yet #[cfg(not(feature = "nightly"))] let _ = order; match mem::size_of::() { #[cfg(all(feature = "nightly", target_has_atomic = "8"))] 1 if mem::align_of::() >= 1 => { mem::transmute_copy( &(*(dst as *const AtomicU8)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "16"))] 2 if mem::align_of::() >= 2 => { mem::transmute_copy( &(*(dst as *const AtomicU16)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "32"))] 4 if mem::align_of::() >= 4 => { mem::transmute_copy( &(*(dst as *const AtomicU32)).fetch_max(mem::transmute_copy(&val), order), ) } #[cfg(all(feature = "nightly", target_has_atomic = "64"))] 8 if mem::align_of::() >= 8 => { mem::transmute_copy( &(*(dst as *const AtomicU64)).fetch_max(mem::transmute_copy(&val), order), ) } _ => fallback::atomic_max(dst, val), } }