summaryrefslogtreecommitdiffstats
path: root/js/public/HeapAPI.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /js/public/HeapAPI.h
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'js/public/HeapAPI.h')
-rw-r--r--js/public/HeapAPI.h838
1 files changed, 838 insertions, 0 deletions
diff --git a/js/public/HeapAPI.h b/js/public/HeapAPI.h
new file mode 100644
index 0000000000..3dfe00bd0d
--- /dev/null
+++ b/js/public/HeapAPI.h
@@ -0,0 +1,838 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef js_HeapAPI_h
+#define js_HeapAPI_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/BitSet.h"
+
+#include <limits.h>
+#include <type_traits>
+
+#include "js/AllocPolicy.h"
+#include "js/GCAnnotations.h"
+#include "js/HashTable.h"
+#include "js/shadow/String.h" // JS::shadow::String
+#include "js/shadow/Symbol.h" // JS::shadow::Symbol
+#include "js/shadow/Zone.h" // JS::shadow::Zone
+#include "js/TraceKind.h"
+#include "js/TypeDecls.h"
+
+/* These values are private to the JS engine. */
+namespace js {
+
+class NurseryDecommitTask;
+
+JS_PUBLIC_API bool CurrentThreadCanAccessZone(JS::Zone* zone);
+
+// To prevent false sharing, some data structures are aligned to a typical cache
+// line size.
+static constexpr size_t TypicalCacheLineSize = 64;
+
+namespace gc {
+
+class Arena;
+struct Cell;
+class TenuredChunk;
+class StoreBuffer;
+class TenuredCell;
+
+const size_t ArenaShift = 12;
+const size_t ArenaSize = size_t(1) << ArenaShift;
+const size_t ArenaMask = ArenaSize - 1;
+
+#if defined(XP_MACOSX) && defined(__aarch64__)
+const size_t PageShift = 14;
+#else
+const size_t PageShift = 12;
+#endif
+// Expected page size, so we could initialze ArenasPerPage at compile-time.
+// The actual system page size should be queried by SystemPageSize().
+const size_t PageSize = size_t(1) << PageShift;
+constexpr size_t ArenasPerPage = PageSize / ArenaSize;
+
+const size_t ChunkShift = 20;
+const size_t ChunkSize = size_t(1) << ChunkShift;
+const size_t ChunkMask = ChunkSize - 1;
+
+const size_t CellAlignShift = 3;
+const size_t CellAlignBytes = size_t(1) << CellAlignShift;
+const size_t CellAlignMask = CellAlignBytes - 1;
+
+const size_t CellBytesPerMarkBit = CellAlignBytes;
+const size_t MarkBitsPerCell = 2;
+
+/*
+ * The minimum cell size ends up as twice the cell alignment because the mark
+ * bitmap contains one bit per CellBytesPerMarkBit bytes (which is equal to
+ * CellAlignBytes) and we need two mark bits per cell.
+ */
+const size_t MinCellSize = CellBytesPerMarkBit * MarkBitsPerCell;
+
+/*
+ * The mark bitmap has one bit per each possible cell start position. This
+ * wastes some space for larger GC things but allows us to avoid division by the
+ * cell's size when accessing the bitmap.
+ */
+const size_t ArenaBitmapBits = ArenaSize / CellBytesPerMarkBit;
+const size_t ArenaBitmapBytes = HowMany(ArenaBitmapBits, 8);
+const size_t ArenaBitmapWords = HowMany(ArenaBitmapBits, JS_BITS_PER_WORD);
+
+// The base class for all GC chunks, either in the nursery or in the tenured
+// heap memory. This structure is locatable from any GC pointer by aligning to
+// the chunk size.
+class alignas(CellAlignBytes) ChunkBase {
+ protected:
+ ChunkBase(JSRuntime* rt, StoreBuffer* sb) {
+ MOZ_ASSERT((uintptr_t(this) & ChunkMask) == 0);
+ initBase(rt, sb);
+ }
+
+ void initBase(JSRuntime* rt, StoreBuffer* sb) {
+ runtime = rt;
+ storeBuffer = sb;
+ }
+
+ public:
+ // The store buffer for pointers from tenured things to things in this
+ // chunk. Will be non-null if and only if this is a nursery chunk.
+ StoreBuffer* storeBuffer;
+
+ // Provide quick access to the runtime from absolutely anywhere.
+ JSRuntime* runtime;
+};
+
+// Information about tenured heap chunks.
+struct TenuredChunkInfo {
+ private:
+ friend class ChunkPool;
+ TenuredChunk* next = nullptr;
+ TenuredChunk* prev = nullptr;
+
+ public:
+ /* Number of free arenas, either committed or decommitted. */
+ uint32_t numArenasFree;
+
+ /* Number of free, committed arenas. */
+ uint32_t numArenasFreeCommitted;
+};
+
+/*
+ * Calculating ArenasPerChunk:
+ *
+ * To figure out how many Arenas will fit in a chunk we need to know how much
+ * extra space is available after we allocate the header data. This is a problem
+ * because the header size depends on the number of arenas in the chunk.
+ *
+ * The dependent fields are markBits, decommittedPages and
+ * freeCommittedArenas. markBits needs ArenaBitmapBytes bytes per arena,
+ * decommittedPages needs one bit per page and freeCommittedArenas needs one
+ * bit per arena.
+ *
+ * We can calculate an approximate value by dividing the number of bits of free
+ * space in the chunk by the number of bits needed per arena. This is an
+ * approximation because it doesn't take account of the fact that the variable
+ * sized fields must be rounded up to a whole number of words, or any padding
+ * the compiler adds between fields.
+ *
+ * Fortunately, for the chunk and arena size parameters we use this
+ * approximation turns out to be correct. If it were not we might need to adjust
+ * the arena count down by one to allow more space for the padding.
+ */
+const size_t BitsPerPageWithHeaders =
+ (ArenaSize + ArenaBitmapBytes) * ArenasPerPage * CHAR_BIT + ArenasPerPage +
+ 1;
+const size_t ChunkBitsAvailable =
+ (ChunkSize - sizeof(ChunkBase) - sizeof(TenuredChunkInfo)) * CHAR_BIT;
+const size_t PagesPerChunk = ChunkBitsAvailable / BitsPerPageWithHeaders;
+const size_t ArenasPerChunk = PagesPerChunk * ArenasPerPage;
+const size_t FreeCommittedBits = ArenasPerChunk;
+const size_t DecommitBits = PagesPerChunk;
+const size_t BitsPerArenaWithHeaders =
+ (ArenaSize + ArenaBitmapBytes) * CHAR_BIT +
+ (DecommitBits / ArenasPerChunk) + 1;
+
+const size_t CalculatedChunkSizeRequired =
+ sizeof(ChunkBase) + sizeof(TenuredChunkInfo) +
+ RoundUp(ArenasPerChunk * ArenaBitmapBytes, sizeof(uintptr_t)) +
+ RoundUp(FreeCommittedBits, sizeof(uint32_t) * CHAR_BIT) / CHAR_BIT +
+ RoundUp(DecommitBits, sizeof(uint32_t) * CHAR_BIT) / CHAR_BIT +
+ ArenasPerChunk * ArenaSize;
+static_assert(CalculatedChunkSizeRequired <= ChunkSize,
+ "Calculated ArenasPerChunk is too large");
+
+const size_t CalculatedChunkPadSize = ChunkSize - CalculatedChunkSizeRequired;
+static_assert(CalculatedChunkPadSize * CHAR_BIT < BitsPerArenaWithHeaders,
+ "Calculated ArenasPerChunk is too small");
+
+static_assert(ArenasPerChunk == 252,
+ "Do not accidentally change our heap's density.");
+
+// Mark bitmaps are atomic because they can be written by gray unmarking on the
+// main thread while read by sweeping on a background thread. The former does
+// not affect the result of the latter.
+using MarkBitmapWord = mozilla::Atomic<uintptr_t, mozilla::Relaxed>;
+
+/*
+ * Live objects are marked black or gray. Everything reachable from a JS root is
+ * marked black. Objects marked gray are eligible for cycle collection.
+ *
+ * BlackBit: GrayOrBlackBit: Color:
+ * 0 0 white
+ * 0 1 gray
+ * 1 0 black
+ * 1 1 black
+ */
+enum class ColorBit : uint32_t { BlackBit = 0, GrayOrBlackBit = 1 };
+
+// Mark colors. Order is important here: the greater value the 'more marked' a
+// cell is.
+enum class MarkColor : uint8_t { Gray = 1, Black = 2 };
+
+// Mark bitmap for a tenured heap chunk.
+struct alignas(TypicalCacheLineSize) MarkBitmap {
+ static constexpr size_t WordCount = ArenaBitmapWords * ArenasPerChunk;
+ MarkBitmapWord bitmap[WordCount];
+
+ inline void getMarkWordAndMask(const TenuredCell* cell, ColorBit colorBit,
+ MarkBitmapWord** wordp, uintptr_t* maskp);
+
+ // The following are not exported and are defined in gc/Heap.h:
+ inline bool markBit(const TenuredCell* cell, ColorBit colorBit);
+ inline bool isMarkedAny(const TenuredCell* cell);
+ inline bool isMarkedBlack(const TenuredCell* cell);
+ inline bool isMarkedGray(const TenuredCell* cell);
+ inline bool markIfUnmarked(const TenuredCell* cell, MarkColor color);
+ inline bool markIfUnmarkedAtomic(const TenuredCell* cell, MarkColor color);
+ inline void markBlack(const TenuredCell* cell);
+ inline void markBlackAtomic(const TenuredCell* cell);
+ inline void copyMarkBit(TenuredCell* dst, const TenuredCell* src,
+ ColorBit colorBit);
+ inline void unmark(const TenuredCell* cell);
+ inline MarkBitmapWord* arenaBits(Arena* arena);
+};
+
+static_assert(ArenaBitmapBytes * ArenasPerChunk == sizeof(MarkBitmap),
+ "Ensure our MarkBitmap actually covers all arenas.");
+
+// Bitmap with one bit per page used for decommitted page set.
+using ChunkPageBitmap = mozilla::BitSet<PagesPerChunk, uint32_t>;
+
+// Bitmap with one bit per arena used for free committed arena set.
+using ChunkArenaBitmap = mozilla::BitSet<ArenasPerChunk, uint32_t>;
+
+// Base class containing data members for a tenured heap chunk.
+class TenuredChunkBase : public ChunkBase {
+ public:
+ TenuredChunkInfo info;
+ MarkBitmap markBits;
+ ChunkArenaBitmap freeCommittedArenas;
+ ChunkPageBitmap decommittedPages;
+
+ protected:
+ explicit TenuredChunkBase(JSRuntime* runtime) : ChunkBase(runtime, nullptr) {
+ info.numArenasFree = ArenasPerChunk;
+ }
+
+ void initAsDecommitted();
+};
+
+/*
+ * We sometimes use an index to refer to a cell in an arena. The index for a
+ * cell is found by dividing by the cell alignment so not all indices refer to
+ * valid cells.
+ */
+const size_t ArenaCellIndexBytes = CellAlignBytes;
+const size_t MaxArenaCellIndex = ArenaSize / CellAlignBytes;
+
+const size_t MarkBitmapWordBits = sizeof(MarkBitmapWord) * CHAR_BIT;
+
+constexpr size_t FirstArenaAdjustmentBits =
+ RoundUp(sizeof(gc::TenuredChunkBase), ArenaSize) / gc::CellBytesPerMarkBit;
+
+static_assert((FirstArenaAdjustmentBits % MarkBitmapWordBits) == 0);
+constexpr size_t FirstArenaAdjustmentWords =
+ FirstArenaAdjustmentBits / MarkBitmapWordBits;
+
+const size_t ChunkStoreBufferOffset = offsetof(ChunkBase, storeBuffer);
+const size_t ChunkMarkBitmapOffset = offsetof(TenuredChunkBase, markBits);
+
+// Hardcoded offsets into Arena class.
+const size_t ArenaZoneOffset = 2 * sizeof(uint32_t);
+const size_t ArenaHeaderSize = ArenaZoneOffset + 2 * sizeof(uintptr_t) +
+ sizeof(size_t) + sizeof(uintptr_t);
+
+// The first word of a GC thing has certain requirements from the GC and is used
+// to store flags in the low bits.
+const size_t CellFlagBitsReservedForGC = 3;
+
+// The first word can be used to store JSClass pointers for some thing kinds, so
+// these must be suitably aligned.
+const size_t JSClassAlignBytes = size_t(1) << CellFlagBitsReservedForGC;
+
+#ifdef JS_DEBUG
+/* When downcasting, ensure we are actually the right type. */
+extern JS_PUBLIC_API void AssertGCThingHasType(js::gc::Cell* cell,
+ JS::TraceKind kind);
+#else
+inline void AssertGCThingHasType(js::gc::Cell* cell, JS::TraceKind kind) {}
+#endif
+
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const js::gc::Cell* cell);
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const js::gc::TenuredCell* cell);
+
+} /* namespace gc */
+} /* namespace js */
+
+namespace JS {
+
+enum class HeapState {
+ Idle, // doing nothing with the GC heap
+ Tracing, // tracing the GC heap without collecting, e.g.
+ // IterateCompartments()
+ MajorCollecting, // doing a GC of the major heap
+ MinorCollecting, // doing a GC of the minor heap (nursery)
+ CycleCollecting // in the "Unlink" phase of cycle collection
+};
+
+JS_PUBLIC_API HeapState RuntimeHeapState();
+
+static inline bool RuntimeHeapIsBusy() {
+ return RuntimeHeapState() != HeapState::Idle;
+}
+
+static inline bool RuntimeHeapIsTracing() {
+ return RuntimeHeapState() == HeapState::Tracing;
+}
+
+static inline bool RuntimeHeapIsMajorCollecting() {
+ return RuntimeHeapState() == HeapState::MajorCollecting;
+}
+
+static inline bool RuntimeHeapIsMinorCollecting() {
+ return RuntimeHeapState() == HeapState::MinorCollecting;
+}
+
+static inline bool RuntimeHeapIsCollecting(HeapState state) {
+ return state == HeapState::MajorCollecting ||
+ state == HeapState::MinorCollecting;
+}
+
+static inline bool RuntimeHeapIsCollecting() {
+ return RuntimeHeapIsCollecting(RuntimeHeapState());
+}
+
+static inline bool RuntimeHeapIsCycleCollecting() {
+ return RuntimeHeapState() == HeapState::CycleCollecting;
+}
+
+/*
+ * This list enumerates the different types of conceptual stacks we have in
+ * SpiderMonkey. In reality, they all share the C stack, but we allow different
+ * stack limits depending on the type of code running.
+ */
+enum StackKind {
+ StackForSystemCode, // C++, such as the GC, running on behalf of the VM.
+ StackForTrustedScript, // Script running with trusted principals.
+ StackForUntrustedScript, // Script running with untrusted principals.
+ StackKindCount
+};
+
+/*
+ * Default maximum size for the generational nursery in bytes. This is the
+ * initial value. In the browser this configured by the
+ * javascript.options.mem.nursery.max_kb pref.
+ */
+const uint32_t DefaultNurseryMaxBytes = 16 * js::gc::ChunkSize;
+
+/* Default maximum heap size in bytes to pass to JS_NewContext(). */
+const uint32_t DefaultHeapMaxBytes = 32 * 1024 * 1024;
+
+/**
+ * A GC pointer, tagged with the trace kind.
+ *
+ * In general, a GC pointer should be stored with an exact type. This class
+ * is for use when that is not possible because a single pointer must point
+ * to several kinds of GC thing.
+ */
+class JS_PUBLIC_API GCCellPtr {
+ public:
+ GCCellPtr() : GCCellPtr(nullptr) {}
+
+ // Construction from a void* and trace kind.
+ GCCellPtr(void* gcthing, JS::TraceKind traceKind)
+ : ptr(checkedCast(gcthing, traceKind)) {}
+
+ // Automatically construct a null GCCellPtr from nullptr.
+ MOZ_IMPLICIT GCCellPtr(decltype(nullptr))
+ : ptr(checkedCast(nullptr, JS::TraceKind::Null)) {}
+
+ // Construction from an explicit type.
+ template <typename T>
+ explicit GCCellPtr(T* p)
+ : ptr(checkedCast(p, JS::MapTypeToTraceKind<T>::kind)) {}
+ explicit GCCellPtr(JSFunction* p)
+ : ptr(checkedCast(p, JS::TraceKind::Object)) {}
+ explicit GCCellPtr(JSScript* p)
+ : ptr(checkedCast(p, JS::TraceKind::Script)) {}
+ explicit GCCellPtr(const Value& v);
+
+ JS::TraceKind kind() const {
+ uintptr_t kindBits = ptr & OutOfLineTraceKindMask;
+ if (kindBits != OutOfLineTraceKindMask) {
+ return JS::TraceKind(kindBits);
+ }
+ return outOfLineKind();
+ }
+
+ // Allow GCCellPtr to be used in a boolean context.
+ explicit operator bool() const {
+ MOZ_ASSERT(bool(asCell()) == (kind() != JS::TraceKind::Null));
+ return asCell();
+ }
+
+ // Simplify checks to the kind.
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ bool is() const {
+ return kind() == JS::MapTypeToTraceKind<T>::kind;
+ }
+
+ // Conversions to more specific types must match the kind. Access to
+ // further refined types is not allowed directly from a GCCellPtr.
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ T& as() const {
+ MOZ_ASSERT(kind() == JS::MapTypeToTraceKind<T>::kind);
+ // We can't use static_cast here, because the fact that JSObject
+ // inherits from js::gc::Cell is not part of the public API.
+ return *reinterpret_cast<T*>(asCell());
+ }
+
+ // Return a pointer to the cell this |GCCellPtr| refers to, or |nullptr|.
+ // (It would be more symmetrical with |to| for this to return a |Cell&|, but
+ // the result can be |nullptr|, and null references are undefined behavior.)
+ js::gc::Cell* asCell() const {
+ return reinterpret_cast<js::gc::Cell*>(ptr & ~OutOfLineTraceKindMask);
+ }
+
+ // The CC's trace logger needs an identity that is XPIDL serializable.
+ uint64_t unsafeAsInteger() const {
+ return static_cast<uint64_t>(unsafeAsUIntPtr());
+ }
+ // Inline mark bitmap access requires direct pointer arithmetic.
+ uintptr_t unsafeAsUIntPtr() const {
+ MOZ_ASSERT(asCell());
+ MOZ_ASSERT(!js::gc::IsInsideNursery(asCell()));
+ return reinterpret_cast<uintptr_t>(asCell());
+ }
+
+ MOZ_ALWAYS_INLINE bool mayBeOwnedByOtherRuntime() const {
+ if (!is<JSString>() && !is<JS::Symbol>()) {
+ return false;
+ }
+ if (is<JSString>()) {
+ return JS::shadow::String::isPermanentAtom(asCell());
+ }
+ MOZ_ASSERT(is<JS::Symbol>());
+ return JS::shadow::Symbol::isWellKnownSymbol(asCell());
+ }
+
+ private:
+ static uintptr_t checkedCast(void* p, JS::TraceKind traceKind) {
+ auto* cell = static_cast<js::gc::Cell*>(p);
+ MOZ_ASSERT((uintptr_t(p) & OutOfLineTraceKindMask) == 0);
+ AssertGCThingHasType(cell, traceKind);
+ // Store trace in the bottom bits of pointer for common kinds.
+ uintptr_t kindBits = uintptr_t(traceKind);
+ if (kindBits >= OutOfLineTraceKindMask) {
+ kindBits = OutOfLineTraceKindMask;
+ }
+ return uintptr_t(p) | kindBits;
+ }
+
+ JS::TraceKind outOfLineKind() const;
+
+ uintptr_t ptr;
+} JS_HAZ_GC_POINTER;
+
+// Unwraps the given GCCellPtr, calls the functor |f| with a template argument
+// of the actual type of the pointer, and returns the result.
+template <typename F>
+auto MapGCThingTyped(GCCellPtr thing, F&& f) {
+ switch (thing.kind()) {
+#define JS_EXPAND_DEF(name, type, _, _1) \
+ case JS::TraceKind::name: \
+ return f(&thing.as<type>());
+ JS_FOR_EACH_TRACEKIND(JS_EXPAND_DEF);
+#undef JS_EXPAND_DEF
+ default:
+ MOZ_CRASH("Invalid trace kind in MapGCThingTyped for GCCellPtr.");
+ }
+}
+
+// Unwraps the given GCCellPtr and calls the functor |f| with a template
+// argument of the actual type of the pointer. Doesn't return anything.
+template <typename F>
+void ApplyGCThingTyped(GCCellPtr thing, F&& f) {
+ // This function doesn't do anything but is supplied for symmetry with other
+ // MapGCThingTyped/ApplyGCThingTyped implementations that have to wrap the
+ // functor to return a dummy value that is ignored.
+ MapGCThingTyped(thing, f);
+}
+
+} /* namespace JS */
+
+// These are defined in the toplevel namespace instead of within JS so that
+// they won't shadow other operator== overloads (see bug 1456512.)
+
+inline bool operator==(JS::GCCellPtr ptr1, JS::GCCellPtr ptr2) {
+ return ptr1.asCell() == ptr2.asCell();
+}
+
+inline bool operator!=(JS::GCCellPtr ptr1, JS::GCCellPtr ptr2) {
+ return !(ptr1 == ptr2);
+}
+
+namespace js {
+namespace gc {
+
+/* static */
+MOZ_ALWAYS_INLINE void MarkBitmap::getMarkWordAndMask(const TenuredCell* cell,
+ ColorBit colorBit,
+ MarkBitmapWord** wordp,
+ uintptr_t* maskp) {
+ // Note: the JIT pre-barrier trampolines inline this code. Update
+ // MacroAssembler::emitPreBarrierFastPath code too when making changes here!
+
+ MOZ_ASSERT(size_t(colorBit) < MarkBitsPerCell);
+
+ size_t offset = uintptr_t(cell) & ChunkMask;
+ const size_t bit = offset / CellBytesPerMarkBit + size_t(colorBit);
+ size_t word = bit / MarkBitmapWordBits - FirstArenaAdjustmentWords;
+ MOZ_ASSERT(word < WordCount);
+ *wordp = &bitmap[word];
+ *maskp = uintptr_t(1) << (bit % MarkBitmapWordBits);
+}
+
+namespace detail {
+
+static MOZ_ALWAYS_INLINE ChunkBase* GetCellChunkBase(const Cell* cell) {
+ MOZ_ASSERT(cell);
+ auto* chunk = reinterpret_cast<ChunkBase*>(uintptr_t(cell) & ~ChunkMask);
+ MOZ_ASSERT(chunk->runtime);
+ return chunk;
+}
+
+static MOZ_ALWAYS_INLINE TenuredChunkBase* GetCellChunkBase(
+ const TenuredCell* cell) {
+ MOZ_ASSERT(cell);
+ auto* chunk =
+ reinterpret_cast<TenuredChunkBase*>(uintptr_t(cell) & ~ChunkMask);
+ MOZ_ASSERT(chunk->runtime);
+ return chunk;
+}
+
+static MOZ_ALWAYS_INLINE JS::Zone* GetTenuredGCThingZone(const uintptr_t addr) {
+ MOZ_ASSERT(addr);
+ const uintptr_t zone_addr = (addr & ~ArenaMask) | ArenaZoneOffset;
+ return *reinterpret_cast<JS::Zone**>(zone_addr);
+}
+
+static MOZ_ALWAYS_INLINE bool TenuredCellIsMarkedBlack(
+ const TenuredCell* cell) {
+ // Return true if BlackBit is set.
+
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(!js::gc::IsInsideNursery(cell));
+
+ MarkBitmapWord* blackWord;
+ uintptr_t blackMask;
+ TenuredChunkBase* chunk = GetCellChunkBase(cell);
+ chunk->markBits.getMarkWordAndMask(cell, js::gc::ColorBit::BlackBit,
+ &blackWord, &blackMask);
+ return *blackWord & blackMask;
+}
+
+static MOZ_ALWAYS_INLINE bool NonBlackCellIsMarkedGray(
+ const TenuredCell* cell) {
+ // Return true if GrayOrBlackBit is set. Callers should check BlackBit first.
+
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(!js::gc::IsInsideNursery(cell));
+ MOZ_ASSERT(!TenuredCellIsMarkedBlack(cell));
+
+ MarkBitmapWord* grayWord;
+ uintptr_t grayMask;
+ TenuredChunkBase* chunk = GetCellChunkBase(cell);
+ chunk->markBits.getMarkWordAndMask(cell, js::gc::ColorBit::GrayOrBlackBit,
+ &grayWord, &grayMask);
+ return *grayWord & grayMask;
+}
+
+static MOZ_ALWAYS_INLINE bool TenuredCellIsMarkedGray(const TenuredCell* cell) {
+ return !TenuredCellIsMarkedBlack(cell) && NonBlackCellIsMarkedGray(cell);
+}
+
+static MOZ_ALWAYS_INLINE bool CellIsMarkedGray(const Cell* cell) {
+ MOZ_ASSERT(cell);
+ if (js::gc::IsInsideNursery(cell)) {
+ return false;
+ }
+ return TenuredCellIsMarkedGray(reinterpret_cast<const TenuredCell*>(cell));
+}
+
+extern JS_PUBLIC_API bool CanCheckGrayBits(const TenuredCell* cell);
+
+extern JS_PUBLIC_API bool CellIsMarkedGrayIfKnown(const TenuredCell* cell);
+
+#ifdef DEBUG
+extern JS_PUBLIC_API void AssertCellIsNotGray(const Cell* cell);
+
+extern JS_PUBLIC_API bool ObjectIsMarkedBlack(const JSObject* obj);
+#endif
+
+MOZ_ALWAYS_INLINE bool CellHasStoreBuffer(const Cell* cell) {
+ return GetCellChunkBase(cell)->storeBuffer;
+}
+
+} /* namespace detail */
+
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const Cell* cell) {
+ MOZ_ASSERT(cell);
+ return detail::CellHasStoreBuffer(cell);
+}
+
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const TenuredCell* cell) {
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(!IsInsideNursery(reinterpret_cast<const Cell*>(cell)));
+ return false;
+}
+
+// Allow use before the compiler knows the derivation of JSObject, JSString, and
+// JS::BigInt.
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const JSObject* obj) {
+ return IsInsideNursery(reinterpret_cast<const Cell*>(obj));
+}
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const JSString* str) {
+ return IsInsideNursery(reinterpret_cast<const Cell*>(str));
+}
+MOZ_ALWAYS_INLINE bool IsInsideNursery(const JS::BigInt* bi) {
+ return IsInsideNursery(reinterpret_cast<const Cell*>(bi));
+}
+
+MOZ_ALWAYS_INLINE bool IsCellPointerValid(const void* ptr) {
+ auto addr = uintptr_t(ptr);
+ if (addr < ChunkSize || addr % CellAlignBytes != 0) {
+ return false;
+ }
+
+ auto* cell = reinterpret_cast<const Cell*>(ptr);
+ if (!IsInsideNursery(cell)) {
+ return detail::GetTenuredGCThingZone(addr) != nullptr;
+ }
+
+ return true;
+}
+
+MOZ_ALWAYS_INLINE bool IsCellPointerValidOrNull(const void* cell) {
+ if (!cell) {
+ return true;
+ }
+ return IsCellPointerValid(cell);
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+namespace JS {
+
+static MOZ_ALWAYS_INLINE Zone* GetTenuredGCThingZone(GCCellPtr thing) {
+ MOZ_ASSERT(!js::gc::IsInsideNursery(thing.asCell()));
+ return js::gc::detail::GetTenuredGCThingZone(thing.unsafeAsUIntPtr());
+}
+
+extern JS_PUBLIC_API Zone* GetNurseryCellZone(js::gc::Cell* cell);
+
+static MOZ_ALWAYS_INLINE Zone* GetGCThingZone(GCCellPtr thing) {
+ if (!js::gc::IsInsideNursery(thing.asCell())) {
+ return js::gc::detail::GetTenuredGCThingZone(thing.unsafeAsUIntPtr());
+ }
+
+ return GetNurseryCellZone(thing.asCell());
+}
+
+static MOZ_ALWAYS_INLINE Zone* GetStringZone(JSString* str) {
+ if (!js::gc::IsInsideNursery(str)) {
+ return js::gc::detail::GetTenuredGCThingZone(
+ reinterpret_cast<uintptr_t>(str));
+ }
+ return GetNurseryCellZone(reinterpret_cast<js::gc::Cell*>(str));
+}
+
+extern JS_PUBLIC_API Zone* GetObjectZone(JSObject* obj);
+
+static MOZ_ALWAYS_INLINE bool GCThingIsMarkedGray(GCCellPtr thing) {
+ js::gc::Cell* cell = thing.asCell();
+ if (IsInsideNursery(cell)) {
+ return false;
+ }
+
+ auto* tenuredCell = reinterpret_cast<js::gc::TenuredCell*>(cell);
+ return js::gc::detail::CellIsMarkedGrayIfKnown(tenuredCell);
+}
+
+// Specialised gray marking check for use by the cycle collector. This is not
+// called during incremental GC or when the gray bits are invalid.
+static MOZ_ALWAYS_INLINE bool GCThingIsMarkedGrayInCC(GCCellPtr thing) {
+ js::gc::Cell* cell = thing.asCell();
+ if (IsInsideNursery(cell)) {
+ return false;
+ }
+
+ auto* tenuredCell = reinterpret_cast<js::gc::TenuredCell*>(cell);
+ if (!js::gc::detail::TenuredCellIsMarkedGray(tenuredCell)) {
+ return false;
+ }
+
+ MOZ_ASSERT(js::gc::detail::CanCheckGrayBits(tenuredCell));
+
+ return true;
+}
+
+extern JS_PUBLIC_API JS::TraceKind GCThingTraceKind(void* thing);
+
+extern JS_PUBLIC_API void EnableNurseryStrings(JSContext* cx);
+
+extern JS_PUBLIC_API void DisableNurseryStrings(JSContext* cx);
+
+extern JS_PUBLIC_API void EnableNurseryBigInts(JSContext* cx);
+
+extern JS_PUBLIC_API void DisableNurseryBigInts(JSContext* cx);
+
+/*
+ * Returns true when writes to GC thing pointers (and reads from weak pointers)
+ * must call an incremental barrier. This is generally only true when running
+ * mutator code in-between GC slices. At other times, the barrier may be elided
+ * for performance.
+ */
+extern JS_PUBLIC_API bool IsIncrementalBarrierNeeded(JSContext* cx);
+
+/*
+ * Notify the GC that a reference to a JSObject is about to be overwritten.
+ * This method must be called if IsIncrementalBarrierNeeded.
+ */
+extern JS_PUBLIC_API void IncrementalPreWriteBarrier(JSObject* obj);
+
+/*
+ * Notify the GC that a reference to a tenured GC cell is about to be
+ * overwritten. This method must be called if IsIncrementalBarrierNeeded.
+ */
+extern JS_PUBLIC_API void IncrementalPreWriteBarrier(GCCellPtr thing);
+
+/**
+ * Unsets the gray bit for anything reachable from |thing|. |kind| should not be
+ * JS::TraceKind::Shape. |thing| should be non-null. The return value indicates
+ * if anything was unmarked.
+ */
+extern JS_PUBLIC_API bool UnmarkGrayGCThingRecursively(GCCellPtr thing);
+
+} // namespace JS
+
+namespace js {
+namespace gc {
+
+extern JS_PUBLIC_API void PerformIncrementalReadBarrier(JS::GCCellPtr thing);
+
+static MOZ_ALWAYS_INLINE void ExposeGCThingToActiveJS(JS::GCCellPtr thing) {
+ // TODO: I'd like to assert !RuntimeHeapIsBusy() here but this gets
+ // called while we are tracing the heap, e.g. during memory reporting
+ // (see bug 1313318).
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+
+ // GC things residing in the nursery cannot be gray: they have no mark bits.
+ // All live objects in the nursery are moved to tenured at the beginning of
+ // each GC slice, so the gray marker never sees nursery things.
+ if (IsInsideNursery(thing.asCell())) {
+ return;
+ }
+
+ auto* cell = reinterpret_cast<TenuredCell*>(thing.asCell());
+ if (detail::TenuredCellIsMarkedBlack(cell)) {
+ return;
+ }
+
+ // GC things owned by other runtimes are always black.
+ MOZ_ASSERT(!thing.mayBeOwnedByOtherRuntime());
+
+ auto* zone = JS::shadow::Zone::from(JS::GetTenuredGCThingZone(thing));
+ if (zone->needsIncrementalBarrier()) {
+ PerformIncrementalReadBarrier(thing);
+ } else if (!zone->isGCPreparing() && detail::NonBlackCellIsMarkedGray(cell)) {
+ MOZ_ALWAYS_TRUE(JS::UnmarkGrayGCThingRecursively(thing));
+ }
+
+ MOZ_ASSERT_IF(!zone->isGCPreparing(), !detail::TenuredCellIsMarkedGray(cell));
+}
+
+static MOZ_ALWAYS_INLINE void IncrementalReadBarrier(JS::GCCellPtr thing) {
+ // This is a lighter version of ExposeGCThingToActiveJS that doesn't do gray
+ // unmarking.
+
+ if (IsInsideNursery(thing.asCell())) {
+ return;
+ }
+
+ auto* zone = JS::shadow::Zone::from(JS::GetTenuredGCThingZone(thing));
+ auto* cell = reinterpret_cast<TenuredCell*>(thing.asCell());
+ if (zone->needsIncrementalBarrier() &&
+ !detail::TenuredCellIsMarkedBlack(cell)) {
+ // GC things owned by other runtimes are always black.
+ MOZ_ASSERT(!thing.mayBeOwnedByOtherRuntime());
+ PerformIncrementalReadBarrier(thing);
+ }
+}
+
+template <typename T>
+extern JS_PUBLIC_API bool EdgeNeedsSweepUnbarrieredSlow(T* thingp);
+
+static MOZ_ALWAYS_INLINE bool EdgeNeedsSweepUnbarriered(JSObject** objp) {
+ // This function does not handle updating nursery pointers. Raw JSObject
+ // pointers should be updated separately or replaced with
+ // JS::Heap<JSObject*> which handles this automatically.
+ MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting());
+ if (IsInsideNursery(*objp)) {
+ return false;
+ }
+
+ auto zone =
+ JS::shadow::Zone::from(detail::GetTenuredGCThingZone(uintptr_t(*objp)));
+ if (!zone->isGCSweepingOrCompacting()) {
+ return false;
+ }
+
+ return EdgeNeedsSweepUnbarrieredSlow(objp);
+}
+
+} // namespace gc
+} // namespace js
+
+namespace JS {
+
+/*
+ * This should be called when an object that is marked gray is exposed to the JS
+ * engine (by handing it to running JS code or writing it into live JS
+ * data). During incremental GC, since the gray bits haven't been computed yet,
+ * we conservatively mark the object black.
+ */
+static MOZ_ALWAYS_INLINE void ExposeObjectToActiveJS(JSObject* obj) {
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(!js::gc::EdgeNeedsSweepUnbarrieredSlow(&obj));
+ js::gc::ExposeGCThingToActiveJS(GCCellPtr(obj));
+}
+
+} /* namespace JS */
+
+#endif /* js_HeapAPI_h */