From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- js/src/gc/GCRuntime.h | 1486 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1486 insertions(+) create mode 100644 js/src/gc/GCRuntime.h (limited to 'js/src/gc/GCRuntime.h') diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h new file mode 100644 index 0000000000..a7198f5bbc --- /dev/null +++ b/js/src/gc/GCRuntime.h @@ -0,0 +1,1486 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCRuntime_h +#define gc_GCRuntime_h + +#include "mozilla/Atomics.h" +#include "mozilla/DoublyLinkedList.h" +#include "mozilla/EnumSet.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "gc/ArenaList.h" +#include "gc/AtomMarking.h" +#include "gc/GCContext.h" +#include "gc/GCMarker.h" +#include "gc/GCParallelTask.h" +#include "gc/IteratorUtils.h" +#include "gc/Nursery.h" +#include "gc/Scheduling.h" +#include "gc/Statistics.h" +#include "gc/StoreBuffer.h" +#include "gc/SweepingAPI.h" +#include "js/friend/PerformanceHint.h" +#include "js/GCAnnotations.h" +#include "js/UniquePtr.h" +#include "vm/AtomsTable.h" + +namespace js { + +class AutoLockGC; +class AutoLockGCBgAlloc; +class AutoLockHelperThreadState; +class FinalizationRegistryObject; +class FinalizationRecordObject; +class FinalizationQueueObject; +class GlobalObject; +class VerifyPreTracer; +class WeakRefObject; + +namespace gc { + +using BlackGrayEdgeVector = Vector; +using ZoneVector = Vector; + +class AutoCallGCCallbacks; +class AutoGCSession; +class AutoHeapSession; +class AutoTraceSession; +struct FinalizePhase; +class MarkingValidator; +struct MovingTracer; +class ParallelMarkTask; +enum class ShouldCheckThresholds; +class SweepGroupsIter; + +// Interface to a sweep action. +struct SweepAction { + // The arguments passed to each action. + struct Args { + GCRuntime* gc; + JS::GCContext* gcx; + SliceBudget& budget; + }; + + virtual ~SweepAction() = default; + virtual IncrementalProgress run(Args& state) = 0; + virtual void assertFinished() const = 0; + virtual bool shouldSkip() { return false; } +}; + +class ChunkPool { + TenuredChunk* head_; + size_t count_; + + public: + ChunkPool() : head_(nullptr), count_(0) {} + ChunkPool(const ChunkPool& other) = delete; + ChunkPool(ChunkPool&& other) { *this = std::move(other); } + + ~ChunkPool() { + MOZ_ASSERT(!head_); + MOZ_ASSERT(count_ == 0); + } + + ChunkPool& operator=(const ChunkPool& other) = delete; + ChunkPool& operator=(ChunkPool&& other) { + head_ = other.head_; + other.head_ = nullptr; + count_ = other.count_; + other.count_ = 0; + return *this; + } + + bool empty() const { return !head_; } + size_t count() const { return count_; } + + TenuredChunk* head() { + MOZ_ASSERT(head_); + return head_; + } + TenuredChunk* pop(); + void push(TenuredChunk* chunk); + TenuredChunk* remove(TenuredChunk* chunk); + + void sort(); + + private: + TenuredChunk* mergeSort(TenuredChunk* list, size_t count); + bool isSorted() const; + +#ifdef DEBUG + public: + bool contains(TenuredChunk* chunk) const; + bool verify() const; + void verifyChunks() const; +#endif + + public: + // Pool mutation does not invalidate an Iter unless the mutation + // is of the TenuredChunk currently being visited by the Iter. + class Iter { + public: + explicit Iter(ChunkPool& pool) : current_(pool.head_) {} + bool done() const { return !current_; } + void next(); + TenuredChunk* get() const { return current_; } + operator TenuredChunk*() const { return get(); } + TenuredChunk* operator->() const { return get(); } + + private: + TenuredChunk* current_; + }; +}; + +class BackgroundMarkTask : public GCParallelTask { + public: + explicit BackgroundMarkTask(GCRuntime* gc); + void setBudget(const SliceBudget& budget) { this->budget = budget; } + void run(AutoLockHelperThreadState& lock) override; + + private: + SliceBudget budget; +}; + +class BackgroundUnmarkTask : public GCParallelTask { + public: + explicit BackgroundUnmarkTask(GCRuntime* gc); + void initZones(); + void run(AutoLockHelperThreadState& lock) override; + + ZoneVector zones; +}; + +class BackgroundSweepTask : public GCParallelTask { + public: + explicit BackgroundSweepTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +class BackgroundFreeTask : public GCParallelTask { + public: + explicit BackgroundFreeTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +// Performs extra allocation off thread so that when memory is required on the +// main thread it will already be available and waiting. +class BackgroundAllocTask : public GCParallelTask { + // Guarded by the GC lock. + GCLockData chunkPool_; + + const bool enabled_; + + public: + BackgroundAllocTask(GCRuntime* gc, ChunkPool& pool); + bool enabled() const { return enabled_; } + + void run(AutoLockHelperThreadState& lock) override; +}; + +// Search the provided chunks for free arenas and decommit them. +class BackgroundDecommitTask : public GCParallelTask { + public: + explicit BackgroundDecommitTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +template +struct Callback { + F op; + void* data; + + Callback() : op(nullptr), data(nullptr) {} + Callback(F op, void* data) : op(op), data(data) {} +}; + +template +using CallbackVector = Vector, 4, SystemAllocPolicy>; + +using RootedValueMap = + HashMap, SystemAllocPolicy>; + +using AllocKinds = mozilla::EnumSet; + +// A singly linked list of zones. +class ZoneList { + static Zone* const End; + + Zone* head; + Zone* tail; + + public: + ZoneList(); + ~ZoneList(); + + bool isEmpty() const; + Zone* front() const; + + void prepend(Zone* zone); + void append(Zone* zone); + void prependList(ZoneList&& other); + void appendList(ZoneList&& other); + Zone* removeFront(); + void clear(); + + private: + explicit ZoneList(Zone* singleZone); + void check() const; + + ZoneList(const ZoneList& other) = delete; + ZoneList& operator=(const ZoneList& other) = delete; +}; + +struct WeakCacheToSweep { + WeakCacheBase* cache; + JS::Zone* zone; +}; + +class WeakCacheSweepIterator { + JS::Zone* sweepZone; + WeakCacheBase* sweepCache; + + public: + explicit WeakCacheSweepIterator(JS::Zone* sweepGroup); + + bool done() const; + WeakCacheToSweep get() const; + void next(); + + private: + void settle(); +}; + +struct SweepingTracer final : public GenericTracerImpl { + explicit SweepingTracer(JSRuntime* rt); + + private: + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl; +}; + +class GCRuntime { + public: + explicit GCRuntime(JSRuntime* rt); + [[nodiscard]] bool init(uint32_t maxbytes); + bool wasInitialized() const { return initialized; } + void finishRoots(); + void finish(); + + Zone* atomsZone() { + Zone* zone = zones()[0]; + MOZ_ASSERT(JS::shadow::Zone::from(zone)->isAtomsZone()); + return zone; + } + Zone* maybeSharedAtomsZone() { return sharedAtomsZone_; } + + [[nodiscard]] bool freezeSharedAtomsZone(); + void restoreSharedAtomsZone(); + + JS::HeapState heapState() const { return heapState_; } + + inline bool hasZealMode(ZealMode mode); + inline void clearZealMode(ZealMode mode); + inline bool needZealousGC(); + inline bool hasIncrementalTwoSliceZealMode(); + + [[nodiscard]] bool addRoot(Value* vp, const char* name); + void removeRoot(Value* vp); + + [[nodiscard]] bool setParameter(JSContext* cx, JSGCParamKey key, + uint32_t value); + void resetParameter(JSContext* cx, JSGCParamKey key); + uint32_t getParameter(JSGCParamKey key); + + void setPerformanceHint(PerformanceHint hint); + bool isInPageLoad() const { return inPageLoadCount != 0; } + + [[nodiscard]] bool triggerGC(JS::GCReason reason); + // Check whether to trigger a zone GC after allocating GC cells. + void maybeTriggerGCAfterAlloc(Zone* zone); + // Check whether to trigger a zone GC after malloc memory. + void maybeTriggerGCAfterMalloc(Zone* zone); + bool maybeTriggerGCAfterMalloc(Zone* zone, const HeapSize& heap, + const HeapThreshold& threshold, + JS::GCReason reason); + // The return value indicates if we were able to do the GC. + bool triggerZoneGC(Zone* zone, JS::GCReason reason, size_t usedBytes, + size_t thresholdBytes); + + void maybeGC(); + + // Return whether we want to run a major GC. If eagerOk is true, include eager + // triggers (eg EAGER_ALLOC_TRIGGER) in this determination, and schedule all + // zones that exceed the eager thresholds. + JS::GCReason wantMajorGC(bool eagerOk); + bool checkEagerAllocTrigger(const HeapSize& size, + const HeapThreshold& threshold); + + // Do a minor GC if requested, followed by a major GC if requested. The return + // value indicates whether a major GC was performed. + bool gcIfRequested() { return gcIfRequestedImpl(false); } + + // Internal function to do a GC if previously requested. But if not and + // eagerOk, do an eager GC for all Zones that have exceeded the eager + // thresholds. + // + // Return whether a major GC was performed or started. + bool gcIfRequestedImpl(bool eagerOk); + + void gc(JS::GCOptions options, JS::GCReason reason); + void startGC(JS::GCOptions options, JS::GCReason reason, + const SliceBudget& budget); + void gcSlice(JS::GCReason reason, const SliceBudget& budget); + void finishGC(JS::GCReason reason); + void abortGC(); + void startDebugGC(JS::GCOptions options, const SliceBudget& budget); + void debugGCSlice(const SliceBudget& budget); + + void runDebugGC(); + void notifyRootsRemoved(); + + enum TraceOrMarkRuntime { TraceRuntime, MarkRuntime }; + void traceRuntime(JSTracer* trc, AutoTraceSession& session); + void traceRuntimeForMinorGC(JSTracer* trc, AutoGCSession& session); + + void purgeRuntimeForMinorGC(); + + void shrinkBuffers(); + void onOutOfMallocMemory(); + void onOutOfMallocMemory(const AutoLockGC& lock); + + Nursery& nursery() { return nursery_.ref(); } + gc::StoreBuffer& storeBuffer() { return storeBuffer_.ref(); } + + void minorGC(JS::GCReason reason, + gcstats::PhaseKind phase = gcstats::PhaseKind::MINOR_GC) + JS_HAZ_GC_CALL; + void evictNursery(JS::GCReason reason = JS::GCReason::EVICT_NURSERY) { + minorGC(reason, gcstats::PhaseKind::EVICT_NURSERY); + } + + void* addressOfNurseryPosition() { + return nursery_.refNoCheck().addressOfPosition(); + } + + const void* addressOfLastBufferedWholeCell() { + return storeBuffer_.refNoCheck().addressOfLastBufferedWholeCell(); + } + +#ifdef JS_GC_ZEAL + const uint32_t* addressOfZealModeBits() { return &zealModeBits.refNoCheck(); } + void getZealBits(uint32_t* zealBits, uint32_t* frequency, + uint32_t* nextScheduled); + void setZeal(uint8_t zeal, uint32_t frequency); + void unsetZeal(uint8_t zeal); + bool parseAndSetZeal(const char* str); + void setNextScheduled(uint32_t count); + void verifyPreBarriers(); + void maybeVerifyPreBarriers(bool always); + bool selectForMarking(JSObject* object); + void clearSelectedForMarking(); + void setDeterministic(bool enable); + void setMarkStackLimit(size_t limit, AutoLockGC& lock); +#endif + + uint64_t nextCellUniqueId() { + MOZ_ASSERT(nextCellUniqueId_ > 0); + uint64_t uid = ++nextCellUniqueId_; + return uid; + } + + void setLowMemoryState(bool newState) { lowMemoryState = newState; } + bool systemHasLowMemory() const { return lowMemoryState; } + + public: + // Internal public interface + ZoneVector& zones() { return zones_.ref(); } + gcstats::Statistics& stats() { return stats_.ref(); } + const gcstats::Statistics& stats() const { return stats_.ref(); } + State state() const { return incrementalState; } + bool isHeapCompacting() const { return state() == State::Compact; } + bool isForegroundSweeping() const { return state() == State::Sweep; } + bool isBackgroundSweeping() const { return sweepTask.wasStarted(); } + bool isBackgroundMarking() const { return markTask.wasStarted(); } + void waitBackgroundSweepEnd(); + void waitBackgroundAllocEnd() { allocTask.cancelAndWait(); } + void waitBackgroundFreeEnd(); + void waitForBackgroundTasks(); + bool isWaitingOnBackgroundTask() const; + + void lockGC() { lock.lock(); } + void unlockGC() { lock.unlock(); } + + void lockStoreBuffer() { storeBufferLock.lock(); } + void unlockStoreBuffer() { storeBufferLock.unlock(); } + +#ifdef DEBUG + void assertCurrentThreadHasLockedGC() const { + lock.assertOwnedByCurrentThread(); + } + void assertCurrentThreadHasLockedStoreBuffer() const { + storeBufferLock.assertOwnedByCurrentThread(); + } +#endif // DEBUG + + void setAlwaysPreserveCode() { alwaysPreserveCode = true; } + + bool isIncrementalGCAllowed() const { return incrementalAllowed; } + void disallowIncrementalGC() { incrementalAllowed = false; } + + void setIncrementalGCEnabled(bool enabled); + + bool isIncrementalGCEnabled() const { return incrementalGCEnabled; } + bool isPerZoneGCEnabled() const { return perZoneGCEnabled; } + bool isCompactingGCEnabled() const; + bool isParallelMarkingEnabled() const { return parallelMarkingEnabled; } + + bool isIncrementalGCInProgress() const { + return state() != State::NotActive && !isVerifyPreBarriersEnabled(); + } + + bool hasForegroundWork() const; + + bool isShrinkingGC() const { return gcOptions() == JS::GCOptions::Shrink; } + + bool isShutdownGC() const { return gcOptions() == JS::GCOptions::Shutdown; } + +#ifdef DEBUG + bool isShuttingDown() const { return hadShutdownGC; } +#endif + + bool initSweepActions(); + + void setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data); + [[nodiscard]] bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); + void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); + void clearBlackAndGrayRootTracers(); + + void setGCCallback(JSGCCallback callback, void* data); + void callGCCallback(JSGCStatus status, JS::GCReason reason) const; + void setObjectsTenuredCallback(JSObjectsTenuredCallback callback, void* data); + void callObjectsTenuredCallback(); + [[nodiscard]] bool addFinalizeCallback(JSFinalizeCallback callback, + void* data); + void removeFinalizeCallback(JSFinalizeCallback callback); + void setHostCleanupFinalizationRegistryCallback( + JSHostCleanupFinalizationRegistryCallback callback, void* data); + void callHostCleanupFinalizationRegistryCallback( + JSFunction* doCleanup, GlobalObject* incumbentGlobal); + [[nodiscard]] bool addWeakPointerZonesCallback( + JSWeakPointerZonesCallback callback, void* data); + void removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback); + [[nodiscard]] bool addWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback, void* data); + void removeWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback); + JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); + bool addNurseryCollectionCallback(JS::GCNurseryCollectionCallback callback, + void* data); + void removeNurseryCollectionCallback(JS::GCNurseryCollectionCallback callback, + void* data); + JS::DoCycleCollectionCallback setDoCycleCollectionCallback( + JS::DoCycleCollectionCallback callback); + void callNurseryCollectionCallbacks(JS::GCNurseryProgress progress, + JS::GCReason reason); + + bool addFinalizationRegistry(JSContext* cx, + Handle registry); + bool registerWithFinalizationRegistry(JSContext* cx, HandleObject target, + HandleObject record); + void queueFinalizationRegistryForCleanup(FinalizationQueueObject* queue); + + void nukeFinalizationRecordWrapper(JSObject* wrapper, + FinalizationRecordObject* record); + void nukeWeakRefWrapper(JSObject* wrapper, WeakRefObject* weakRef); + + void setFullCompartmentChecks(bool enable); + + // Get the main marking tracer. + GCMarker& marker() { return *markers[0]; } + + JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; } + unsigned getCurrentSweepGroupIndex() { + MOZ_ASSERT_IF(unsigned(state()) < unsigned(State::Sweep), + sweepGroupIndex == 0); + return sweepGroupIndex; + } + + uint64_t gcNumber() const { return number; } + void incGcNumber() { ++number; } + + uint64_t minorGCCount() const { return minorGCNumber; } + void incMinorGcNumber() { ++minorGCNumber; } + + uint64_t majorGCCount() const { return majorGCNumber; } + void incMajorGcNumber() { ++majorGCNumber; } + + uint64_t gcSliceCount() const { return sliceNumber; } + void incGcSliceNumber() { ++sliceNumber; } + + int64_t defaultSliceBudgetMS() const { return defaultTimeBudgetMS_; } + + bool isIncrementalGc() const { return isIncremental; } + bool isFullGc() const { return isFull; } + bool isCompactingGc() const { return isCompacting; } + bool didCompactZones() const { return isCompacting && zonesCompacted; } + + bool areGrayBitsValid() const { return grayBitsValid; } + void setGrayBitsInvalid() { grayBitsValid = false; } + + mozilla::TimeStamp lastGCStartTime() const { return lastGCStartTime_; } + mozilla::TimeStamp lastGCEndTime() const { return lastGCEndTime_; } + + bool majorGCRequested() const { + return majorGCTriggerReason != JS::GCReason::NO_REASON; + } + + double computeHeapGrowthFactor(size_t lastBytes); + size_t computeTriggerBytes(double growthFactor, size_t lastBytes); + + inline void updateOnFreeArenaAlloc(const TenuredChunkInfo& info); + void updateOnArenaFree() { ++numArenasFreeCommitted; } + + ChunkPool& fullChunks(const AutoLockGC& lock) { return fullChunks_.ref(); } + ChunkPool& availableChunks(const AutoLockGC& lock) { + return availableChunks_.ref(); + } + ChunkPool& emptyChunks(const AutoLockGC& lock) { return emptyChunks_.ref(); } + const ChunkPool& fullChunks(const AutoLockGC& lock) const { + return fullChunks_.ref(); + } + const ChunkPool& availableChunks(const AutoLockGC& lock) const { + return availableChunks_.ref(); + } + const ChunkPool& emptyChunks(const AutoLockGC& lock) const { + return emptyChunks_.ref(); + } + using NonEmptyChunksIter = ChainedIterator; + NonEmptyChunksIter allNonEmptyChunks(const AutoLockGC& lock) { + return NonEmptyChunksIter(availableChunks(lock), fullChunks(lock)); + } + uint32_t minEmptyChunkCount(const AutoLockGC& lock) const { + return minEmptyChunkCount_; + } + uint32_t maxEmptyChunkCount(const AutoLockGC& lock) const { + return maxEmptyChunkCount_; + } +#ifdef DEBUG + void verifyAllChunks(); +#endif + + TenuredChunk* getOrAllocChunk(AutoLockGCBgAlloc& lock); + void recycleChunk(TenuredChunk* chunk, const AutoLockGC& lock); + +#ifdef JS_GC_ZEAL + void startVerifyPreBarriers(); + void endVerifyPreBarriers(); + void finishVerifier(); + bool isVerifyPreBarriersEnabled() const { return verifyPreData.refNoCheck(); } + bool shouldYieldForZeal(ZealMode mode); +#else + bool isVerifyPreBarriersEnabled() const { return false; } +#endif + +#ifdef JSGC_HASH_TABLE_CHECKS + void checkHashTablesAfterMovingGC(); +#endif + + // Crawl the heap to check whether an arbitary pointer is within a cell of + // the given kind. (TraceKind::Null means to ignore the kind.) + bool isPointerWithinTenuredCell( + void* ptr, JS::TraceKind traceKind = JS::TraceKind::Null); + +#ifdef DEBUG + bool hasZone(Zone* target); +#endif + + // Queue memory memory to be freed on a background thread if possible. + void queueUnusedLifoBlocksForFree(LifoAlloc* lifo); + void queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo); + void queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers); + + // Public here for ReleaseArenaLists and FinalizeTypedArenas. + void releaseArena(Arena* arena, const AutoLockGC& lock); + + // Allocator internals. + static void* refillFreeListInGC(Zone* zone, AllocKind thingKind); + + // Delayed marking. + void delayMarkingChildren(gc::Cell* cell, MarkColor color); + bool hasDelayedMarking() const; + void markAllDelayedChildren(ShouldReportMarkTime reportTime); + + // If we have yielded to the mutator while foreground finalizing arenas from + // zone |zone| with kind |kind| then return a list of the arenas finalized so + // far. These will have been removed from the main arena lists at this + // point. Otherwise return nullptr. + SortedArenaList* maybeGetForegroundFinalizedArenas(Zone* zone, + AllocKind kind); + + /* + * Concurrent sweep infrastructure. + */ + void startTask(GCParallelTask& task, AutoLockHelperThreadState& lock); + void joinTask(GCParallelTask& task, AutoLockHelperThreadState& lock); + void updateHelperThreadCount(); + size_t parallelWorkerCount() const; + + // GC parallel task dispatch infrastructure. + size_t getMaxParallelThreads() const; + void dispatchOrQueueParallelTask(GCParallelTask* task, + const AutoLockHelperThreadState& lock); + void maybeDispatchParallelTasks(const AutoLockHelperThreadState& lock); + void onParallelTaskEnd(bool wasDispatched, + const AutoLockHelperThreadState& lock); + + // Parallel marking. + bool initOrDisableParallelMarking(); + [[nodiscard]] bool updateMarkersVector(); + size_t markingWorkerCount() const; + + // WeakRefs + bool registerWeakRef(HandleObject target, HandleObject weakRef); + void traceKeptObjects(JSTracer* trc); + + JS::GCReason lastStartReason() const { return initialReason; } + + void updateAllocationRates(); + + // Allocator internals + static void* refillFreeList(JSContext* cx, AllocKind thingKind); + void attemptLastDitchGC(JSContext* cx); + + // Test mark queue. +#ifdef DEBUG + const GCVector, 0, SystemAllocPolicy>& getTestMarkQueue() + const; + [[nodiscard]] bool appendTestMarkQueue(const JS::Value& value); + void clearTestMarkQueue(); + size_t testMarkQueuePos() const; +#endif + + private: + enum IncrementalResult { ResetIncremental = 0, Ok }; + + [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, + AutoLockGC& lock); + void resetParameter(JSGCParamKey key, AutoLockGC& lock); + uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); + bool setThreadParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); + void resetThreadParameter(JSGCParamKey key, AutoLockGC& lock); + void updateThreadDataStructures(AutoLockGC& lock); + + JS::GCOptions gcOptions() const { return maybeGcOptions.ref().ref(); } + + TriggerResult checkHeapThreshold(Zone* zone, const HeapSize& heapSize, + const HeapThreshold& heapThreshold); + + void updateSchedulingStateOnGCStart(); + void updateSchedulingStateAfterCollection(mozilla::TimeStamp currentTime); + void updateAllGCStartThresholds(); + + // For ArenaLists::allocateFromArena() + friend class ArenaLists; + TenuredChunk* pickChunk(AutoLockGCBgAlloc& lock); + Arena* allocateArena(TenuredChunk* chunk, Zone* zone, AllocKind kind, + ShouldCheckThresholds checkThresholds, + const AutoLockGC& lock); + + /* + * Return the list of chunks that can be released outside the GC lock. + * Must be called either during the GC or with the GC lock taken. + */ + friend class BackgroundDecommitTask; + bool tooManyEmptyChunks(const AutoLockGC& lock); + ChunkPool expireEmptyChunkPool(const AutoLockGC& lock); + void freeEmptyChunks(const AutoLockGC& lock); + void prepareToFreeChunk(TenuredChunkInfo& info); + void setMinEmptyChunkCount(uint32_t value, const AutoLockGC& lock); + void setMaxEmptyChunkCount(uint32_t value, const AutoLockGC& lock); + + friend class BackgroundAllocTask; + bool wantBackgroundAllocation(const AutoLockGC& lock) const; + void startBackgroundAllocTaskIfIdle(); + + void requestMajorGC(JS::GCReason reason); + SliceBudget defaultBudget(JS::GCReason reason, int64_t millis); + bool maybeIncreaseSliceBudget(SliceBudget& budget); + bool maybeIncreaseSliceBudgetForLongCollections(SliceBudget& budget); + bool maybeIncreaseSliceBudgetForUrgentCollections(SliceBudget& budget); + IncrementalResult budgetIncrementalGC(bool nonincrementalByAPI, + JS::GCReason reason, + SliceBudget& budget); + void checkZoneIsScheduled(Zone* zone, JS::GCReason reason, + const char* trigger); + IncrementalResult resetIncrementalGC(GCAbortReason reason); + + // Assert if the system state is such that we should never + // receive a request to do GC work. + void checkCanCallAPI(); + + // Check if the system state is such that GC has been supressed + // or otherwise delayed. + [[nodiscard]] bool checkIfGCAllowedInCurrentState(JS::GCReason reason); + + gcstats::ZoneGCStats scanZonesBeforeGC(); + + void setGCOptions(JS::GCOptions options); + + void collect(bool nonincrementalByAPI, const SliceBudget& budget, + JS::GCReason reason) JS_HAZ_GC_CALL; + + /* + * Run one GC "cycle" (either a slice of incremental GC or an entire + * non-incremental GC). + * + * Returns: + * * ResetIncremental if we "reset" an existing incremental GC, which would + * force us to run another cycle or + * * Ok otherwise. + */ + [[nodiscard]] IncrementalResult gcCycle(bool nonincrementalByAPI, + const SliceBudget& budgetArg, + JS::GCReason reason); + bool shouldRepeatForDeadZone(JS::GCReason reason); + + void incrementalSlice(SliceBudget& budget, JS::GCReason reason, + bool budgetWasIncreased); + + bool mightSweepInThisSlice(bool nonIncremental); + void collectNurseryFromMajorGC(JS::GCReason reason); + void collectNursery(JS::GCOptions options, JS::GCReason reason, + gcstats::PhaseKind phase); + + friend class AutoCallGCCallbacks; + void maybeCallGCCallback(JSGCStatus status, JS::GCReason reason); + + void startCollection(JS::GCReason reason); + + void purgeRuntime(); + [[nodiscard]] bool beginPreparePhase(JS::GCReason reason, + AutoGCSession& session); + bool prepareZonesForCollection(JS::GCReason reason, bool* isFullOut); + void unmarkWeakMaps(); + void endPreparePhase(JS::GCReason reason); + void beginMarkPhase(AutoGCSession& session); + bool shouldPreserveJITCode(JS::Realm* realm, + const mozilla::TimeStamp& currentTime, + JS::GCReason reason, bool canAllocateMoreCode, + bool isActiveCompartment); + void discardJITCodeForGC(); + void startBackgroundFreeAfterMinorGC(); + void relazifyFunctionsForShrinkingGC(); + void purgePropMapTablesForShrinkingGC(); + void purgeSourceURLsForShrinkingGC(); + void traceRuntimeForMajorGC(JSTracer* trc, AutoGCSession& session); + void traceRuntimeAtoms(JSTracer* trc); + void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark); + void traceEmbeddingBlackRoots(JSTracer* trc); + void traceEmbeddingGrayRoots(JSTracer* trc); + IncrementalProgress traceEmbeddingGrayRoots(JSTracer* trc, + SliceBudget& budget); + void checkNoRuntimeRoots(AutoGCSession& session); + void maybeDoCycleCollection(); + void findDeadCompartments(); + + friend class BackgroundMarkTask; + enum ParallelMarking : bool { + SingleThreadedMarking = false, + AllowParallelMarking = true + }; + IncrementalProgress markUntilBudgetExhausted( + SliceBudget& sliceBudget, + ParallelMarking allowParallelMarking = SingleThreadedMarking, + ShouldReportMarkTime reportTime = ReportMarkTime); + bool canMarkInParallel() const; + bool initParallelMarkers(); + void finishParallelMarkers(); + + bool hasMarkingWork(MarkColor color) const; + + void drainMarkStack(); + +#ifdef DEBUG + void assertNoMarkingWork() const; +#else + void assertNoMarkingWork() const {} +#endif + + void markDelayedChildren(gc::Arena* arena, MarkColor color); + void processDelayedMarkingList(gc::MarkColor color); + void rebuildDelayedMarkingList(); + void appendToDelayedMarkingList(gc::Arena** listTail, gc::Arena* arena); + void resetDelayedMarking(); + template + void forEachDelayedMarkingArena(F&& f); + + template + IncrementalProgress markWeakReferences(SliceBudget& budget); + IncrementalProgress markWeakReferencesInCurrentGroup(SliceBudget& budget); + template + IncrementalProgress markGrayRoots(SliceBudget& budget, + gcstats::PhaseKind phase); + void markBufferedGrayRoots(JS::Zone* zone); + IncrementalProgress markAllWeakReferences(); + void markAllGrayReferences(gcstats::PhaseKind phase); + + // The mark queue is a testing-only feature for controlling mark ordering and + // yield timing. + enum MarkQueueProgress { + QueueYielded, // End this incremental GC slice, if possible + QueueComplete, // Done with the queue + QueueSuspended // Continue the GC without ending the slice + }; + MarkQueueProgress processTestMarkQueue(); + + // GC Sweeping. Implemented in Sweeping.cpp. + void beginSweepPhase(JS::GCReason reason, AutoGCSession& session); + void dropStringWrappers(); + void groupZonesForSweeping(JS::GCReason reason); + [[nodiscard]] bool findSweepGroupEdges(); + [[nodiscard]] bool addEdgesForMarkQueue(); + void getNextSweepGroup(); + void resetGrayList(Compartment* comp); + IncrementalProgress beginMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress markGrayRootsInCurrentGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress markGray(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress endMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + void markIncomingGrayCrossCompartmentPointers(); + IncrementalProgress beginSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + void initBackgroundSweep(Zone* zone, JS::GCContext* gcx, + const FinalizePhase& phase); + IncrementalProgress markDuringSweeping(JS::GCContext* gcx, + SliceBudget& budget); + void updateAtomsBitmap(); + void sweepCCWrappers(); + void sweepRealmGlobals(); + void sweepEmbeddingWeakPointers(JS::GCContext* gcx); + void sweepMisc(); + void sweepCompressionTasks(); + void sweepWeakMaps(); + void sweepUniqueIds(); + void sweepObjectsWithWeakPointers(); + void sweepDebuggerOnMainThread(JS::GCContext* gcx); + void sweepJitDataOnMainThread(JS::GCContext* gcx); + void sweepFinalizationObserversOnMainThread(); + void traceWeakFinalizationObserverEdges(JSTracer* trc, Zone* zone); + void sweepWeakRefs(); + IncrementalProgress endSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress performSweepActions(SliceBudget& sliceBudget); + void startSweepingAtomsTable(); + IncrementalProgress sweepAtomsTable(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress sweepWeakCaches(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress finalizeAllocKind(JS::GCContext* gcx, + SliceBudget& budget); + bool foregroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind thingKind, + js::SliceBudget& sliceBudget, + SortedArenaList& sweepList); + IncrementalProgress sweepPropMapTree(JS::GCContext* gcx, SliceBudget& budget); + void endSweepPhase(bool destroyingRuntime); + void queueZonesAndStartBackgroundSweep(ZoneList&& zones); + void sweepFromBackgroundThread(AutoLockHelperThreadState& lock); + void startBackgroundFree(); + void freeFromBackgroundThread(AutoLockHelperThreadState& lock); + void sweepBackgroundThings(ZoneList& zones); + void backgroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind kind, + Arena** empty); + void assertBackgroundSweepingFinished(); + + bool allCCVisibleZonesWereCollected(); + void sweepZones(JS::GCContext* gcx, bool destroyingRuntime); + bool shouldDecommit() const; + void startDecommit(); + void decommitEmptyChunks(const bool& cancel, AutoLockGC& lock); + void decommitFreeArenas(const bool& cancel, AutoLockGC& lock); + void decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock); + + // Compacting GC. Implemented in Compacting.cpp. + bool shouldCompact(); + void beginCompactPhase(); + IncrementalProgress compactPhase(JS::GCReason reason, + SliceBudget& sliceBudget, + AutoGCSession& session); + void endCompactPhase(); + void sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone); + bool canRelocateZone(Zone* zone) const; + [[nodiscard]] bool relocateArenas(Zone* zone, JS::GCReason reason, + Arena*& relocatedListOut, + SliceBudget& sliceBudget); + void updateCellPointers(Zone* zone, AllocKinds kinds); + void updateAllCellPointers(MovingTracer* trc, Zone* zone); + void updateZonePointersToRelocatedCells(Zone* zone); + void updateRuntimePointersToRelocatedCells(AutoGCSession& session); + void clearRelocatedArenas(Arena* arenaList, JS::GCReason reason); + void clearRelocatedArenasWithoutUnlocking(Arena* arenaList, + JS::GCReason reason, + const AutoLockGC& lock); + void releaseRelocatedArenas(Arena* arenaList); + void releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, + const AutoLockGC& lock); +#ifdef DEBUG + void protectOrReleaseRelocatedArenas(Arena* arenaList, JS::GCReason reason); + void protectAndHoldArenas(Arena* arenaList); + void unprotectHeldRelocatedArenas(const AutoLockGC& lock); + void releaseHeldRelocatedArenas(); + void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock); +#endif + + /* + * Whether to immediately trigger a slice after a background task + * finishes. This may not happen at a convenient time, so the consideration is + * whether the slice will run quickly or may take a long time. + */ + enum ShouldTriggerSliceWhenFinished : bool { + DontTriggerSliceWhenFinished = false, + TriggerSliceWhenFinished = true + }; + + IncrementalProgress waitForBackgroundTask( + GCParallelTask& task, const SliceBudget& budget, bool shouldPauseMutator, + ShouldTriggerSliceWhenFinished triggerSlice); + + void maybeRequestGCAfterBackgroundTask(const AutoLockHelperThreadState& lock); + void cancelRequestedGCAfterBackgroundTask(); + void finishCollection(JS::GCReason reason); + void maybeStopPretenuring(); + void checkGCStateNotInUse(); + IncrementalProgress joinBackgroundMarkTask(); + +#ifdef JS_GC_ZEAL + void computeNonIncrementalMarkingForValidation(AutoGCSession& session); + void validateIncrementalMarking(); + void finishMarkingValidation(); +#endif + +#ifdef DEBUG + void checkForCompartmentMismatches(); +#endif + + void callFinalizeCallbacks(JS::GCContext* gcx, JSFinalizeStatus status) const; + void callWeakPointerZonesCallbacks(JSTracer* trc) const; + void callWeakPointerCompartmentCallbacks(JSTracer* trc, + JS::Compartment* comp) const; + void callDoCycleCollectionCallback(JSContext* cx); + + public: + JSRuntime* const rt; + + // Embedders can use this zone however they wish. + MainThreadData systemZone; + + MainThreadData mainThreadContext; + + private: + // For parent runtimes, a zone containing atoms that is shared by child + // runtimes. + MainThreadData sharedAtomsZone_; + + // All zones in the runtime. The first element is always the atoms zone. + MainThreadOrGCTaskData zones_; + + // Any activity affecting the heap. + MainThreadOrGCTaskData heapState_; + friend class AutoHeapSession; + friend class JS::AutoEnterCycleCollection; + + UnprotectedData stats_; + + public: + js::StringStats stringStats; + + Vector, 1, SystemAllocPolicy> markers; + + // Delayed marking support in case we OOM pushing work onto the mark stack. + MainThreadOrGCTaskData delayedMarkingList; + MainThreadOrGCTaskData delayedMarkingWorkAdded; +#ifdef DEBUG + /* Count of arenas that are currently in the stack. */ + MainThreadOrGCTaskData markLaterArenas; +#endif + + SweepingTracer sweepingTracer; + + /* Track total GC heap size for this runtime. */ + HeapSize heapSize; + + /* GC scheduling state and parameters. */ + GCSchedulingTunables tunables; + GCSchedulingState schedulingState; + MainThreadData fullGCRequested; + + // Helper thread configuration. + MainThreadData helperThreadRatio; + MainThreadData maxHelperThreads; + MainThreadOrGCTaskData helperThreadCount; + MainThreadData markingThreadCount; + + // Per-runtime helper thread task queue. Can be accessed from helper threads + // in maybeDispatchParallelTasks(). + HelperThreadLockData maxParallelThreads; + HelperThreadLockData dispatchedParallelTasks; + HelperThreadLockData queuedParallelTasks; + + // State used for managing atom mark bitmaps in each zone. + AtomMarkingRuntime atomMarking; + + /* + * Pointer to a callback that, if set, will be used to create a + * budget for internally-triggered GCs. + */ + MainThreadData createBudgetCallback; + + private: + // Arenas used for permanent things created at startup and shared by child + // runtimes. + MainThreadData permanentAtoms; + MainThreadData permanentWellKnownSymbols; + + // When chunks are empty, they reside in the emptyChunks pool and are + // re-used as needed or eventually expired if not re-used. The emptyChunks + // pool gets refilled from the background allocation task heuristically so + // that empty chunks should always be available for immediate allocation + // without syscalls. + GCLockData emptyChunks_; + + // Chunks which have had some, but not all, of their arenas allocated live + // in the available chunk lists. When all available arenas in a chunk have + // been allocated, the chunk is removed from the available list and moved + // to the fullChunks pool. During a GC, if all arenas are free, the chunk + // is moved back to the emptyChunks pool and scheduled for eventual + // release. + GCLockData availableChunks_; + + // When all arenas in a chunk are used, it is moved to the fullChunks pool + // so as to reduce the cost of operations on the available lists. + GCLockData fullChunks_; + + /* + * JSGC_MIN_EMPTY_CHUNK_COUNT + * JSGC_MAX_EMPTY_CHUNK_COUNT + * + * Controls the number of empty chunks reserved for future allocation. + * + * They can be read off main thread by the background allocation task and the + * background decommit task. + */ + GCLockData minEmptyChunkCount_; + GCLockData maxEmptyChunkCount_; + + MainThreadData rootsHash; + + // An incrementing id used to assign unique ids to cells that require one. + MainThreadData nextCellUniqueId_; + + /* + * Number of the committed arenas in all GC chunks including empty chunks. + */ + mozilla::Atomic numArenasFreeCommitted; + MainThreadData verifyPreData; + + MainThreadData lastGCStartTime_; + MainThreadData lastGCEndTime_; + + WriteOnceData initialized; + MainThreadData incrementalGCEnabled; + MainThreadData perZoneGCEnabled; + + mozilla::Atomic numActiveZoneIters; + + /* During shutdown, the GC needs to clean up every possible object. */ + MainThreadData cleanUpEverything; + + /* + * The gray bits can become invalid if UnmarkGray overflows the stack. A + * full GC will reset this bit, since it fills in all the gray bits. + */ + UnprotectedData grayBitsValid; + + mozilla::Atomic majorGCTriggerReason; + + /* Incremented at the start of every minor GC. */ + MainThreadData minorGCNumber; + + /* Incremented at the start of every major GC. */ + MainThreadData majorGCNumber; + + /* Incremented on every GC slice or minor collection. */ + MainThreadData number; + + /* Incremented on every GC slice. */ + MainThreadData sliceNumber; + + /* Whether the currently running GC can finish in multiple slices. */ + MainThreadOrGCTaskData isIncremental; + + /* Whether all zones are being collected in first GC slice. */ + MainThreadData isFull; + + /* Whether the heap will be compacted at the end of GC. */ + MainThreadData isCompacting; + + /* Whether to use parallel marking. */ + MainThreadData useParallelMarking; + + /* The invocation kind of the current GC, set at the start of collection. */ + MainThreadOrGCTaskData> maybeGcOptions; + + /* The initial GC reason, taken from the first slice. */ + MainThreadData initialReason; + + /* + * The current incremental GC phase. This is also used internally in + * non-incremental GC. + */ + MainThreadOrGCTaskData incrementalState; + + /* The incremental state at the start of this slice. */ + MainThreadOrGCTaskData initialState; + + /* Whether to pay attention the zeal settings in this incremental slice. */ +#ifdef JS_GC_ZEAL + MainThreadData useZeal; +#else + const bool useZeal; +#endif + + /* Indicates that the last incremental slice exhausted the mark stack. */ + MainThreadData lastMarkSlice; + + // Whether it's currently safe to yield to the mutator in an incremental GC. + MainThreadData safeToYield; + + // Whether to do any marking caused by barriers on a background thread during + // incremental sweeping, while also sweeping zones which have finished + // marking. + MainThreadData markOnBackgroundThreadDuringSweeping; + + // Whether any sweeping and decommitting will run on a separate GC helper + // thread. + MainThreadData useBackgroundThreads; + + // Whether we have already discarded JIT code for all collected zones in this + // slice. + MainThreadData haveDiscardedJITCodeThisSlice; + +#ifdef DEBUG + /* Shutdown has started. Further collections must be shutdown collections. */ + MainThreadData hadShutdownGC; +#endif + + /* Singly linked list of zones to be swept in the background. */ + HelperThreadLockData backgroundSweepZones; + + /* + * Whether to trigger a GC slice after a background task is complete, so that + * the collector can continue or finsish collecting. This is only used for the + * tasks that run concurrently with the mutator, which are background + * finalization and background decommit. + */ + HelperThreadLockData requestSliceAfterBackgroundTask; + + /* + * Free LIFO blocks are transferred to these allocators before being freed on + * a background thread. + */ + HelperThreadLockData lifoBlocksToFree; + MainThreadData lifoBlocksToFreeAfterMinorGC; + HelperThreadLockData buffersToFreeAfterMinorGC; + + /* Index of current sweep group (for stats). */ + MainThreadData sweepGroupIndex; + + /* + * Incremental sweep state. + */ + MainThreadData sweepGroups; + MainThreadOrGCTaskData currentSweepGroup; + MainThreadData> sweepActions; + MainThreadOrGCTaskData sweepZone; + MainThreadOrGCTaskData sweepAllocKind; + MainThreadData> maybeAtomsToSweep; + MainThreadOrGCTaskData> + weakCachesToSweep; + MainThreadData abortSweepAfterCurrentGroup; + MainThreadOrGCTaskData sweepMarkResult; + + /* + * During incremental foreground finalization, we may have a list of arenas of + * the current AllocKind and Zone whose contents have been finalized but which + * have not yet been merged back into the main arena lists. + */ + MainThreadOrGCTaskData foregroundFinalizedZone; + MainThreadOrGCTaskData foregroundFinalizedAllocKind; + MainThreadData> foregroundFinalizedArenas; + +#ifdef DEBUG + /* + * List of objects to mark at the beginning of a GC for testing purposes. May + * also contain string directives to change mark color or wait until different + * phases of the GC. + * + * This is a WeakCache because not everything in this list is guaranteed to + * end up marked (eg if you insert an object from an already-processed sweep + * group in the middle of an incremental GC). Also, the mark queue is not + * used during shutdown GCs. In either case, unmarked objects may need to be + * discarded. + */ + WeakCache, 0, SystemAllocPolicy>> testMarkQueue; + + /* Position within the test mark queue. */ + size_t queuePos = 0; + + /* The test marking queue might want to be marking a particular color. */ + mozilla::Maybe queueMarkColor; + + // During gray marking, delay AssertCellIsNotGray checks by + // recording the cell pointers here and checking after marking has + // finished. + MainThreadData> + cellsToAssertNotGray; + friend void js::gc::detail::AssertCellIsNotGray(const Cell*); +#endif + + friend class SweepGroupsIter; + + /* + * Incremental compacting state. + */ + MainThreadData startedCompacting; + MainThreadData zonesToMaybeCompact; + MainThreadData zonesCompacted; +#ifdef DEBUG + GCLockData relocatedArenasToRelease; +#endif + +#ifdef JS_GC_ZEAL + MainThreadData markingValidator; +#endif + + /* + * Default budget for incremental GC slice. See js/SliceBudget.h. + * + * JSGC_SLICE_TIME_BUDGET_MS + * pref: javascript.options.mem.gc_incremental_slice_ms, + */ + MainThreadData defaultTimeBudgetMS_; + + /* + * We disable incremental GC if we encounter a Class with a trace hook + * that does not implement write barriers. + */ + MainThreadData incrementalAllowed; + + /* + * Whether compacting GC can is enabled globally. + * + * JSGC_COMPACTING_ENABLED + * pref: javascript.options.mem.gc_compacting + */ + MainThreadData compactingEnabled; + + /* + * Whether parallel marking is enabled globally. + * + * JSGC_PARALLEL_MARKING_ENABLED + * pref: javascript.options.mem.gc_parallel_marking + */ + MainThreadData parallelMarkingEnabled; + + MainThreadData rootsRemoved; + + /* + * These options control the zealousness of the GC. At every allocation, + * nextScheduled is decremented. When it reaches zero we do a full GC. + * + * At this point, if zeal_ is one of the types that trigger periodic + * collection, then nextScheduled is reset to the value of zealFrequency. + * Otherwise, no additional GCs take place. + * + * You can control these values in several ways: + * - Set the JS_GC_ZEAL environment variable + * - Call gczeal() or schedulegc() from inside shell-executed JS code + * (see the help for details) + * + * If gcZeal_ == 1 then we perform GCs in select places (during MaybeGC and + * whenever we are notified that GC roots have been removed). This option is + * mainly useful to embedders. + * + * We use zeal_ == 4 to enable write barrier verification. See the comment + * in gc/Verifier.cpp for more information about this. + * + * zeal_ values from 8 to 10 periodically run different types of + * incremental GC. + * + * zeal_ value 14 performs periodic shrinking collections. + */ +#ifdef JS_GC_ZEAL + static_assert(size_t(ZealMode::Count) <= 32, + "Too many zeal modes to store in a uint32_t"); + MainThreadData zealModeBits; + MainThreadData zealFrequency; + MainThreadData nextScheduled; + MainThreadData deterministicOnly; + MainThreadData zealSliceBudget; + MainThreadData maybeMarkStackLimit; + + MainThreadData>> + selectedForMarking; +#endif + + MainThreadData fullCompartmentChecks; + + MainThreadData gcCallbackDepth; + + MainThreadData> gcCallback; + MainThreadData> + gcDoCycleCollectionCallback; + MainThreadData> tenuredCallback; + MainThreadData> finalizeCallbacks; + MainThreadOrGCTaskData> + hostCleanupFinalizationRegistryCallback; + MainThreadData> + updateWeakPointerZonesCallbacks; + MainThreadData> + updateWeakPointerCompartmentCallbacks; + MainThreadData> + nurseryCollectionCallbacks; + + /* + * The trace operations to trace embedding-specific GC roots. One is for + * tracing through black roots and the other is for tracing through gray + * roots. The black/gray distinction is only relevant to the cycle + * collector. + */ + MainThreadData> blackRootTracers; + MainThreadOrGCTaskData> grayRootTracer; + + /* Always preserve JIT code during GCs, for testing. */ + MainThreadData alwaysPreserveCode; + + /* Count of the number of zones that are currently in page load. */ + MainThreadData inPageLoadCount; + + MainThreadData lowMemoryState; + + /* + * General purpose GC lock, used for synchronising operations on + * arenas and during parallel marking. + */ + friend class js::AutoLockGC; + friend class js::AutoLockGCBgAlloc; + Mutex lock MOZ_UNANNOTATED; + + /* + * Lock used to synchronise access to the store buffer during parallel + * sweeping. + */ + Mutex storeBufferLock MOZ_UNANNOTATED; + + /* Lock used to synchronise access to delayed marking state. */ + Mutex delayedMarkingLock MOZ_UNANNOTATED; + + friend class BackgroundSweepTask; + friend class BackgroundFreeTask; + + BackgroundAllocTask allocTask; + BackgroundUnmarkTask unmarkTask; + BackgroundMarkTask markTask; + BackgroundSweepTask sweepTask; + BackgroundFreeTask freeTask; + BackgroundDecommitTask decommitTask; + + MainThreadData nursery_; + + // The store buffer used to track tenured to nursery edges for generational + // GC. This is accessed off main thread when sweeping WeakCaches. + MainThreadOrGCTaskData storeBuffer_; + + mozilla::TimeStamp lastLastDitchTime; + + // The last time per-zone allocation rates were updated. + MainThreadData lastAllocRateUpdateTime; + + // Total collector time since per-zone allocation rates were last updated. + MainThreadData collectorTimeSinceAllocRateUpdate; + + friend class MarkingValidator; + friend class AutoEnterIteration; +}; + +/* Prevent compartments and zones from being collected during iteration. */ +class MOZ_RAII AutoEnterIteration { + GCRuntime* gc; + + public: + explicit AutoEnterIteration(GCRuntime* gc_) : gc(gc_) { + ++gc->numActiveZoneIters; + } + + ~AutoEnterIteration() { + MOZ_ASSERT(gc->numActiveZoneIters); + --gc->numActiveZoneIters; + } +}; + +#ifdef JS_GC_ZEAL + +inline bool GCRuntime::hasZealMode(ZealMode mode) { + static_assert(size_t(ZealMode::Limit) < sizeof(zealModeBits) * 8, + "Zeal modes must fit in zealModeBits"); + return zealModeBits & (1 << uint32_t(mode)); +} + +inline void GCRuntime::clearZealMode(ZealMode mode) { + zealModeBits &= ~(1 << uint32_t(mode)); + MOZ_ASSERT(!hasZealMode(mode)); +} + +inline bool GCRuntime::needZealousGC() { + if (nextScheduled > 0 && --nextScheduled == 0) { + if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::GenerationalGC) || + hasZealMode(ZealMode::IncrementalMultipleSlices) || + hasZealMode(ZealMode::Compact) || hasIncrementalTwoSliceZealMode()) { + nextScheduled = zealFrequency; + } + return true; + } + return false; +} + +inline bool GCRuntime::hasIncrementalTwoSliceZealMode() { + return hasZealMode(ZealMode::YieldBeforeRootMarking) || + hasZealMode(ZealMode::YieldBeforeMarking) || + hasZealMode(ZealMode::YieldBeforeSweeping) || + hasZealMode(ZealMode::YieldBeforeSweepingAtoms) || + hasZealMode(ZealMode::YieldBeforeSweepingCaches) || + hasZealMode(ZealMode::YieldBeforeSweepingObjects) || + hasZealMode(ZealMode::YieldBeforeSweepingNonObjects) || + hasZealMode(ZealMode::YieldBeforeSweepingPropMapTrees) || + hasZealMode(ZealMode::YieldWhileGrayMarking); +} + +#else +inline bool GCRuntime::hasZealMode(ZealMode mode) { return false; } +inline void GCRuntime::clearZealMode(ZealMode mode) {} +inline bool GCRuntime::needZealousGC() { return false; } +inline bool GCRuntime::hasIncrementalTwoSliceZealMode() { return false; } +#endif + +bool IsCurrentlyAnimating(const mozilla::TimeStamp& lastAnimationTime, + const mozilla::TimeStamp& currentTime); + +} /* namespace gc */ +} /* namespace js */ + +#endif -- cgit v1.2.3