summaryrefslogtreecommitdiffstats
path: root/mfbt/Atomics.h
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--mfbt/Atomics.h521
1 files changed, 521 insertions, 0 deletions
diff --git a/mfbt/Atomics.h b/mfbt/Atomics.h
new file mode 100644
index 0000000000..4373e08af7
--- /dev/null
+++ b/mfbt/Atomics.h
@@ -0,0 +1,521 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Implements (almost always) lock-free atomic operations. The operations here
+ * are a subset of that which can be found in C++11's <atomic> header, with a
+ * different API to enforce consistent memory ordering constraints.
+ *
+ * Anyone caught using |volatile| for inter-thread memory safety needs to be
+ * sent a copy of this header and the C++11 standard.
+ */
+
+#ifndef mozilla_Atomics_h
+#define mozilla_Atomics_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Compiler.h"
+
+#ifdef __wasi__
+# include "mozilla/WasiAtomic.h"
+#else
+# include <atomic>
+#endif // __wasi__
+
+#include <stdint.h>
+#include <type_traits>
+
+namespace mozilla {
+
+/**
+ * An enum of memory ordering possibilities for atomics.
+ *
+ * Memory ordering is the observable state of distinct values in memory.
+ * (It's a separate concept from atomicity, which concerns whether an
+ * operation can ever be observed in an intermediate state. Don't
+ * conflate the two!) Given a sequence of operations in source code on
+ * memory, it is *not* always the case that, at all times and on all
+ * cores, those operations will appear to have occurred in that exact
+ * sequence. First, the compiler might reorder that sequence, if it
+ * thinks another ordering will be more efficient. Second, the CPU may
+ * not expose so consistent a view of memory. CPUs will often perform
+ * their own instruction reordering, above and beyond that performed by
+ * the compiler. And each core has its own memory caches, and accesses
+ * (reads and writes both) to "memory" may only resolve to out-of-date
+ * cache entries -- not to the "most recently" performed operation in
+ * some global sense. Any access to a value that may be used by
+ * multiple threads, potentially across multiple cores, must therefore
+ * have a memory ordering imposed on it, for all code on all
+ * threads/cores to have a sufficiently coherent worldview.
+ *
+ * http://gcc.gnu.org/wiki/Atomic/GCCMM/AtomicSync and
+ * http://en.cppreference.com/w/cpp/atomic/memory_order go into more
+ * detail on all this, including examples of how each mode works.
+ *
+ * Note that for simplicity and practicality, not all of the modes in
+ * C++11 are supported. The missing C++11 modes are either subsumed by
+ * the modes we provide below, or not relevant for the CPUs we support
+ * in Gecko. These three modes are confusing enough as it is!
+ */
+enum MemoryOrdering {
+ /*
+ * Relaxed ordering is the simplest memory ordering: none at all.
+ * When the result of a write is observed, nothing may be inferred
+ * about other memory. Writes ostensibly performed "before" on the
+ * writing thread may not yet be visible. Writes performed "after" on
+ * the writing thread may already be visible, if the compiler or CPU
+ * reordered them. (The latter can happen if reads and/or writes get
+ * held up in per-processor caches.) Relaxed ordering means
+ * operations can always use cached values (as long as the actual
+ * updates to atomic values actually occur, correctly, eventually), so
+ * it's usually the fastest sort of atomic access. For this reason,
+ * *it's also the most dangerous kind of access*.
+ *
+ * Relaxed ordering is good for things like process-wide statistics
+ * counters that don't need to be consistent with anything else, so
+ * long as updates themselves are atomic. (And so long as any
+ * observations of that value can tolerate being out-of-date -- if you
+ * need some sort of up-to-date value, you need some sort of other
+ * synchronizing operation.) It's *not* good for locks, mutexes,
+ * reference counts, etc. that mediate access to other memory, or must
+ * be observably consistent with other memory.
+ *
+ * x86 architectures don't take advantage of the optimization
+ * opportunities that relaxed ordering permits. Thus it's possible
+ * that using relaxed ordering will "work" on x86 but fail elsewhere
+ * (ARM, say, which *does* implement non-sequentially-consistent
+ * relaxed ordering semantics). Be extra-careful using relaxed
+ * ordering if you can't easily test non-x86 architectures!
+ */
+ Relaxed,
+
+ /*
+ * When an atomic value is updated with ReleaseAcquire ordering, and
+ * that new value is observed with ReleaseAcquire ordering, prior
+ * writes (atomic or not) are also observable. What ReleaseAcquire
+ * *doesn't* give you is any observable ordering guarantees for
+ * ReleaseAcquire-ordered operations on different objects. For
+ * example, if there are two cores that each perform ReleaseAcquire
+ * operations on separate objects, each core may or may not observe
+ * the operations made by the other core. The only way the cores can
+ * be synchronized with ReleaseAcquire is if they both
+ * ReleaseAcquire-access the same object. This implies that you can't
+ * necessarily describe some global total ordering of ReleaseAcquire
+ * operations.
+ *
+ * ReleaseAcquire ordering is good for (as the name implies) atomic
+ * operations on values controlling ownership of things: reference
+ * counts, mutexes, and the like. However, if you are thinking about
+ * using these to implement your own locks or mutexes, you should take
+ * a good, hard look at actual lock or mutex primitives first.
+ */
+ ReleaseAcquire,
+
+ /*
+ * When an atomic value is updated with SequentiallyConsistent
+ * ordering, all writes observable when the update is observed, just
+ * as with ReleaseAcquire ordering. But, furthermore, a global total
+ * ordering of SequentiallyConsistent operations *can* be described.
+ * For example, if two cores perform SequentiallyConsistent operations
+ * on separate objects, one core will observably perform its update
+ * (and all previous operations will have completed), then the other
+ * core will observably perform its update (and all previous
+ * operations will have completed). (Although those previous
+ * operations aren't themselves ordered -- they could be intermixed,
+ * or ordered if they occur on atomic values with ordering
+ * requirements.) SequentiallyConsistent is the *simplest and safest*
+ * ordering of atomic operations -- it's always as if one operation
+ * happens, then another, then another, in some order -- and every
+ * core observes updates to happen in that single order. Because it
+ * has the most synchronization requirements, operations ordered this
+ * way also tend to be slowest.
+ *
+ * SequentiallyConsistent ordering can be desirable when multiple
+ * threads observe objects, and they all have to agree on the
+ * observable order of changes to them. People expect
+ * SequentiallyConsistent ordering, even if they shouldn't, when
+ * writing code, atomic or otherwise. SequentiallyConsistent is also
+ * the ordering of choice when designing lockless data structures. If
+ * you don't know what order to use, use this one.
+ */
+ SequentiallyConsistent,
+};
+
+namespace detail {
+
+/*
+ * We provide CompareExchangeFailureOrder to work around a bug in some
+ * versions of GCC's <atomic> header. See bug 898491.
+ */
+template <MemoryOrdering Order>
+struct AtomicOrderConstraints;
+
+template <>
+struct AtomicOrderConstraints<Relaxed> {
+ static const std::memory_order AtomicRMWOrder = std::memory_order_relaxed;
+ static const std::memory_order LoadOrder = std::memory_order_relaxed;
+ static const std::memory_order StoreOrder = std::memory_order_relaxed;
+ static const std::memory_order CompareExchangeFailureOrder =
+ std::memory_order_relaxed;
+};
+
+template <>
+struct AtomicOrderConstraints<ReleaseAcquire> {
+ static const std::memory_order AtomicRMWOrder = std::memory_order_acq_rel;
+ static const std::memory_order LoadOrder = std::memory_order_acquire;
+ static const std::memory_order StoreOrder = std::memory_order_release;
+ static const std::memory_order CompareExchangeFailureOrder =
+ std::memory_order_acquire;
+};
+
+template <>
+struct AtomicOrderConstraints<SequentiallyConsistent> {
+ static const std::memory_order AtomicRMWOrder = std::memory_order_seq_cst;
+ static const std::memory_order LoadOrder = std::memory_order_seq_cst;
+ static const std::memory_order StoreOrder = std::memory_order_seq_cst;
+ static const std::memory_order CompareExchangeFailureOrder =
+ std::memory_order_seq_cst;
+};
+
+template <typename T, MemoryOrdering Order>
+struct IntrinsicBase {
+ typedef std::atomic<T> ValueType;
+ typedef AtomicOrderConstraints<Order> OrderedOp;
+};
+
+template <typename T, MemoryOrdering Order>
+struct IntrinsicMemoryOps : public IntrinsicBase<T, Order> {
+ typedef IntrinsicBase<T, Order> Base;
+
+ static T load(const typename Base::ValueType& aPtr) {
+ return aPtr.load(Base::OrderedOp::LoadOrder);
+ }
+
+ static void store(typename Base::ValueType& aPtr, T aVal) {
+ aPtr.store(aVal, Base::OrderedOp::StoreOrder);
+ }
+
+ static T exchange(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.exchange(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+
+ static bool compareExchange(typename Base::ValueType& aPtr, T aOldVal,
+ T aNewVal) {
+ return aPtr.compare_exchange_strong(
+ aOldVal, aNewVal, Base::OrderedOp::AtomicRMWOrder,
+ Base::OrderedOp::CompareExchangeFailureOrder);
+ }
+};
+
+template <typename T, MemoryOrdering Order>
+struct IntrinsicAddSub : public IntrinsicBase<T, Order> {
+ typedef IntrinsicBase<T, Order> Base;
+
+ static T add(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+
+ static T sub(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+};
+
+template <typename T, MemoryOrdering Order>
+struct IntrinsicAddSub<T*, Order> : public IntrinsicBase<T*, Order> {
+ typedef IntrinsicBase<T*, Order> Base;
+
+ static T* add(typename Base::ValueType& aPtr, ptrdiff_t aVal) {
+ return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+
+ static T* sub(typename Base::ValueType& aPtr, ptrdiff_t aVal) {
+ return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+};
+
+template <typename T, MemoryOrdering Order>
+struct IntrinsicIncDec : public IntrinsicAddSub<T, Order> {
+ typedef IntrinsicBase<T, Order> Base;
+
+ static T inc(typename Base::ValueType& aPtr) {
+ return IntrinsicAddSub<T, Order>::add(aPtr, 1);
+ }
+
+ static T dec(typename Base::ValueType& aPtr) {
+ return IntrinsicAddSub<T, Order>::sub(aPtr, 1);
+ }
+};
+
+template <typename T, MemoryOrdering Order>
+struct AtomicIntrinsics : public IntrinsicMemoryOps<T, Order>,
+ public IntrinsicIncDec<T, Order> {
+ typedef IntrinsicBase<T, Order> Base;
+
+ static T or_(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.fetch_or(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+
+ static T xor_(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.fetch_xor(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+
+ static T and_(typename Base::ValueType& aPtr, T aVal) {
+ return aPtr.fetch_and(aVal, Base::OrderedOp::AtomicRMWOrder);
+ }
+};
+
+template <typename T, MemoryOrdering Order>
+struct AtomicIntrinsics<T*, Order> : public IntrinsicMemoryOps<T*, Order>,
+ public IntrinsicIncDec<T*, Order> {};
+
+template <typename T>
+struct ToStorageTypeArgument {
+ static constexpr T convert(T aT) { return aT; }
+};
+
+template <typename T, MemoryOrdering Order>
+class AtomicBase {
+ static_assert(sizeof(T) == 4 || sizeof(T) == 8,
+ "mozilla/Atomics.h only supports 32-bit and 64-bit types");
+
+ protected:
+ typedef typename detail::AtomicIntrinsics<T, Order> Intrinsics;
+ typedef typename Intrinsics::ValueType ValueType;
+ ValueType mValue;
+
+ public:
+ constexpr AtomicBase() : mValue() {}
+ explicit constexpr AtomicBase(T aInit)
+ : mValue(ToStorageTypeArgument<T>::convert(aInit)) {}
+
+ // Note: we can't provide operator T() here because Atomic<bool> inherits
+ // from AtomcBase with T=uint32_t and not T=bool. If we implemented
+ // operator T() here, it would cause errors when comparing Atomic<bool> with
+ // a regular bool.
+
+ T operator=(T aVal) {
+ Intrinsics::store(mValue, aVal);
+ return aVal;
+ }
+
+ /**
+ * Performs an atomic swap operation. aVal is stored and the previous
+ * value of this variable is returned.
+ */
+ T exchange(T aVal) { return Intrinsics::exchange(mValue, aVal); }
+
+ /**
+ * Performs an atomic compare-and-swap operation and returns true if it
+ * succeeded. This is equivalent to atomically doing
+ *
+ * if (mValue == aOldValue) {
+ * mValue = aNewValue;
+ * return true;
+ * } else {
+ * return false;
+ * }
+ */
+ bool compareExchange(T aOldValue, T aNewValue) {
+ return Intrinsics::compareExchange(mValue, aOldValue, aNewValue);
+ }
+
+ private:
+ AtomicBase(const AtomicBase& aCopy) = delete;
+};
+
+template <typename T, MemoryOrdering Order>
+class AtomicBaseIncDec : public AtomicBase<T, Order> {
+ typedef typename detail::AtomicBase<T, Order> Base;
+
+ public:
+ constexpr AtomicBaseIncDec() : Base() {}
+ explicit constexpr AtomicBaseIncDec(T aInit) : Base(aInit) {}
+
+ using Base::operator=;
+
+ operator T() const { return Base::Intrinsics::load(Base::mValue); }
+ T operator++(int) { return Base::Intrinsics::inc(Base::mValue); }
+ T operator--(int) { return Base::Intrinsics::dec(Base::mValue); }
+ T operator++() { return Base::Intrinsics::inc(Base::mValue) + 1; }
+ T operator--() { return Base::Intrinsics::dec(Base::mValue) - 1; }
+
+ private:
+ AtomicBaseIncDec(const AtomicBaseIncDec& aCopy) = delete;
+};
+
+} // namespace detail
+
+/**
+ * A wrapper for a type that enforces that all memory accesses are atomic.
+ *
+ * In general, where a variable |T foo| exists, |Atomic<T> foo| can be used in
+ * its place. Implementations for integral and pointer types are provided
+ * below.
+ *
+ * Atomic accesses are sequentially consistent by default. You should
+ * use the default unless you are tall enough to ride the
+ * memory-ordering roller coaster (if you're not sure, you aren't) and
+ * you have a compelling reason to do otherwise.
+ *
+ * There is one exception to the case of atomic memory accesses: providing an
+ * initial value of the atomic value is not guaranteed to be atomic. This is a
+ * deliberate design choice that enables static atomic variables to be declared
+ * without introducing extra static constructors.
+ */
+template <typename T, MemoryOrdering Order = SequentiallyConsistent,
+ typename Enable = void>
+class Atomic;
+
+/**
+ * Atomic<T> implementation for integral types.
+ *
+ * In addition to atomic store and load operations, compound assignment and
+ * increment/decrement operators are implemented which perform the
+ * corresponding read-modify-write operation atomically. Finally, an atomic
+ * swap method is provided.
+ */
+template <typename T, MemoryOrdering Order>
+class Atomic<
+ T, Order,
+ std::enable_if_t<std::is_integral_v<T> && !std::is_same_v<T, bool>>>
+ : public detail::AtomicBaseIncDec<T, Order> {
+ typedef typename detail::AtomicBaseIncDec<T, Order> Base;
+
+ public:
+ constexpr Atomic() : Base() {}
+ explicit constexpr Atomic(T aInit) : Base(aInit) {}
+
+ using Base::operator=;
+
+ T operator+=(T aDelta) {
+ return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
+ }
+
+ T operator-=(T aDelta) {
+ return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
+ }
+
+ T operator|=(T aVal) {
+ return Base::Intrinsics::or_(Base::mValue, aVal) | aVal;
+ }
+
+ T operator^=(T aVal) {
+ return Base::Intrinsics::xor_(Base::mValue, aVal) ^ aVal;
+ }
+
+ T operator&=(T aVal) {
+ return Base::Intrinsics::and_(Base::mValue, aVal) & aVal;
+ }
+
+ private:
+ Atomic(Atomic& aOther) = delete;
+};
+
+/**
+ * Atomic<T> implementation for pointer types.
+ *
+ * An atomic compare-and-swap primitive for pointer variables is provided, as
+ * are atomic increment and decement operators. Also provided are the compound
+ * assignment operators for addition and subtraction. Atomic swap (via
+ * exchange()) is included as well.
+ */
+template <typename T, MemoryOrdering Order>
+class Atomic<T*, Order> : public detail::AtomicBaseIncDec<T*, Order> {
+ typedef typename detail::AtomicBaseIncDec<T*, Order> Base;
+
+ public:
+ constexpr Atomic() : Base() {}
+ explicit constexpr Atomic(T* aInit) : Base(aInit) {}
+
+ using Base::operator=;
+
+ T* operator+=(ptrdiff_t aDelta) {
+ return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
+ }
+
+ T* operator-=(ptrdiff_t aDelta) {
+ return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
+ }
+
+ private:
+ Atomic(Atomic& aOther) = delete;
+};
+
+/**
+ * Atomic<T> implementation for enum types.
+ *
+ * The atomic store and load operations and the atomic swap method is provided.
+ */
+template <typename T, MemoryOrdering Order>
+class Atomic<T, Order, std::enable_if_t<std::is_enum_v<T>>>
+ : public detail::AtomicBase<T, Order> {
+ typedef typename detail::AtomicBase<T, Order> Base;
+
+ public:
+ constexpr Atomic() : Base() {}
+ explicit constexpr Atomic(T aInit) : Base(aInit) {}
+
+ operator T() const { return T(Base::Intrinsics::load(Base::mValue)); }
+
+ using Base::operator=;
+
+ private:
+ Atomic(Atomic& aOther) = delete;
+};
+
+/**
+ * Atomic<T> implementation for boolean types.
+ *
+ * The atomic store and load operations and the atomic swap method is provided.
+ *
+ * Note:
+ *
+ * - sizeof(Atomic<bool>) != sizeof(bool) for some implementations of
+ * bool and/or some implementations of std::atomic. This is allowed in
+ * [atomic.types.generic]p9.
+ *
+ * - It's not obvious whether the 8-bit atomic functions on Windows are always
+ * inlined or not. If they are not inlined, the corresponding functions in the
+ * runtime library are not available on Windows XP. This is why we implement
+ * Atomic<bool> with an underlying type of uint32_t.
+ */
+template <MemoryOrdering Order>
+class Atomic<bool, Order> : protected detail::AtomicBase<uint32_t, Order> {
+ typedef typename detail::AtomicBase<uint32_t, Order> Base;
+
+ public:
+ constexpr Atomic() : Base() {}
+ explicit constexpr Atomic(bool aInit) : Base(aInit) {}
+
+ // We provide boolean wrappers for the underlying AtomicBase methods.
+ MOZ_IMPLICIT operator bool() const {
+ return Base::Intrinsics::load(Base::mValue);
+ }
+
+ bool operator=(bool aVal) { return Base::operator=(aVal); }
+
+ bool exchange(bool aVal) { return Base::exchange(aVal); }
+
+ bool compareExchange(bool aOldValue, bool aNewValue) {
+ return Base::compareExchange(aOldValue, aNewValue);
+ }
+
+ private:
+ Atomic(Atomic& aOther) = delete;
+};
+
+} // namespace mozilla
+
+namespace std {
+
+// If you want to atomically swap two atomic values, use exchange().
+template <typename T, mozilla::MemoryOrdering Order>
+void swap(mozilla::Atomic<T, Order>&, mozilla::Atomic<T, Order>&) = delete;
+
+} // namespace std
+
+#endif /* mozilla_Atomics_h */