diff options
Diffstat (limited to 'js/src/gc/Nursery.cpp')
| -rw-r--r-- | js/src/gc/Nursery.cpp | 2114 |
1 files changed, 2114 insertions, 0 deletions
diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp new file mode 100644 index 0000000000..a78db5cc9a --- /dev/null +++ b/js/src/gc/Nursery.cpp @@ -0,0 +1,2114 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Nursery-inl.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Sprintf.h" +#include "mozilla/TimeStamp.h" + +#include <algorithm> +#include <cmath> +#include <utility> + +#include "builtin/MapObject.h" +#include "debugger/DebugAPI.h" +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/GCParallelTask.h" +#include "gc/Memory.h" +#include "gc/PublicIterators.h" +#include "gc/Tenuring.h" +#include "jit/JitFrames.h" +#include "jit/JitZone.h" +#include "js/Printer.h" +#include "util/DifferentialTesting.h" +#include "util/GetPidProvider.h" // getpid() +#include "util/Poison.h" +#include "vm/JSONPrinter.h" +#include "vm/Realm.h" +#include "vm/Time.h" + +#include "gc/Heap-inl.h" +#include "gc/Marking-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "vm/GeckoProfiler-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::DebugOnly; +using mozilla::PodCopy; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +namespace js { + +struct NurseryChunk : public ChunkBase { + char data[Nursery::NurseryChunkUsableSize]; + + static NurseryChunk* fromChunk(gc::TenuredChunk* chunk); + + explicit NurseryChunk(JSRuntime* runtime) + : ChunkBase(runtime, &runtime->gc.storeBuffer()) {} + + void poisonAndInit(JSRuntime* rt, size_t size = ChunkSize); + void poisonRange(size_t start, size_t end, uint8_t value, + MemCheckKind checkKind); + void poisonAfterEvict(size_t extent = ChunkSize); + + // Mark pages from startOffset to the end of the chunk as unused. The start + // offset must be after the first page, which contains the chunk header and is + // not marked as unused. + void markPagesUnusedHard(size_t startOffset); + + // Mark pages from the second page of the chunk to endOffset as in use, + // following a call to markPagesUnusedHard. + [[nodiscard]] bool markPagesInUseHard(size_t endOffset); + + uintptr_t start() const { return uintptr_t(&data); } + uintptr_t end() const { return uintptr_t(this) + ChunkSize; } +}; +static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize, + "Nursery chunk size must match gc::Chunk size."); + +class NurseryDecommitTask : public GCParallelTask { + public: + explicit NurseryDecommitTask(gc::GCRuntime* gc); + bool reserveSpaceForBytes(size_t nbytes); + + bool isEmpty(const AutoLockHelperThreadState& lock) const; + + void queueChunk(NurseryChunk* chunk, const AutoLockHelperThreadState& lock); + void queueRange(size_t newCapacity, NurseryChunk& chunk, + const AutoLockHelperThreadState& lock); + + private: + using NurseryChunkVector = Vector<NurseryChunk*, 0, SystemAllocPolicy>; + + void run(AutoLockHelperThreadState& lock) override; + + NurseryChunkVector& chunksToDecommit() { return chunksToDecommit_.ref(); } + const NurseryChunkVector& chunksToDecommit() const { + return chunksToDecommit_.ref(); + } + + MainThreadOrGCTaskData<NurseryChunkVector> chunksToDecommit_; + + MainThreadOrGCTaskData<NurseryChunk*> partialChunk; + MainThreadOrGCTaskData<size_t> partialCapacity; +}; + +} // namespace js + +inline void js::NurseryChunk::poisonAndInit(JSRuntime* rt, size_t size) { + MOZ_ASSERT(size >= sizeof(ChunkBase)); + MOZ_ASSERT(size <= ChunkSize); + poisonRange(0, size, JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined); + new (this) NurseryChunk(rt); +} + +inline void js::NurseryChunk::poisonRange(size_t start, size_t end, + uint8_t value, + MemCheckKind checkKind) { + MOZ_ASSERT((start % gc::CellAlignBytes) == 0); + MOZ_ASSERT((end % gc::CellAlignBytes) == 0); + MOZ_ASSERT(end >= start); + MOZ_ASSERT(end <= ChunkSize); + + auto* ptr = reinterpret_cast<uint8_t*>(this) + start; + size_t size = end - start; + + // We can poison the same chunk more than once, so first make sure memory + // sanitizers will let us poison it. + MOZ_MAKE_MEM_UNDEFINED(ptr, size); + Poison(ptr, value, size, checkKind); +} + +inline void js::NurseryChunk::poisonAfterEvict(size_t extent) { + poisonRange(sizeof(ChunkBase), extent, JS_SWEPT_NURSERY_PATTERN, + MemCheckKind::MakeNoAccess); +} + +inline void js::NurseryChunk::markPagesUnusedHard(size_t startOffset) { + MOZ_ASSERT(startOffset >= sizeof(ChunkBase)); // Don't touch the header. + MOZ_ASSERT(startOffset >= SystemPageSize()); + MOZ_ASSERT(startOffset <= ChunkSize); + uintptr_t start = uintptr_t(this) + startOffset; + size_t length = ChunkSize - startOffset; + MarkPagesUnusedHard(reinterpret_cast<void*>(start), length); +} + +inline bool js::NurseryChunk::markPagesInUseHard(size_t endOffset) { + MOZ_ASSERT(endOffset >= sizeof(ChunkBase)); + MOZ_ASSERT(endOffset >= SystemPageSize()); + MOZ_ASSERT(endOffset <= ChunkSize); + uintptr_t start = uintptr_t(this) + SystemPageSize(); + size_t length = endOffset - SystemPageSize(); + return MarkPagesInUseHard(reinterpret_cast<void*>(start), length); +} + +// static +inline js::NurseryChunk* js::NurseryChunk::fromChunk(TenuredChunk* chunk) { + return reinterpret_cast<NurseryChunk*>(chunk); +} + +js::NurseryDecommitTask::NurseryDecommitTask(gc::GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::NONE) { + // This can occur outside GCs so doesn't have a stats phase. +} + +bool js::NurseryDecommitTask::isEmpty( + const AutoLockHelperThreadState& lock) const { + return chunksToDecommit().empty() && !partialChunk; +} + +bool js::NurseryDecommitTask::reserveSpaceForBytes(size_t nbytes) { + MOZ_ASSERT(isIdle()); + size_t nchunks = HowMany(nbytes, ChunkSize); + return chunksToDecommit().reserve(nchunks); +} + +void js::NurseryDecommitTask::queueChunk( + NurseryChunk* chunk, const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isIdle(lock)); + MOZ_ALWAYS_TRUE(chunksToDecommit().append(chunk)); +} + +void js::NurseryDecommitTask::queueRange( + size_t newCapacity, NurseryChunk& newChunk, + const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isIdle(lock)); + MOZ_ASSERT(!partialChunk); + MOZ_ASSERT(newCapacity < ChunkSize); + MOZ_ASSERT(newCapacity % SystemPageSize() == 0); + + partialChunk = &newChunk; + partialCapacity = newCapacity; +} + +void js::NurseryDecommitTask::run(AutoLockHelperThreadState& lock) { + while (!chunksToDecommit().empty()) { + NurseryChunk* nurseryChunk = chunksToDecommit().popCopy(); + AutoUnlockHelperThreadState unlock(lock); + nurseryChunk->~NurseryChunk(); + TenuredChunk* tenuredChunk = TenuredChunk::emplace( + nurseryChunk, gc, /* allMemoryCommitted = */ false); + AutoLockGC lock(gc); + gc->recycleChunk(tenuredChunk, lock); + } + + if (partialChunk) { + { + AutoUnlockHelperThreadState unlock(lock); + partialChunk->markPagesUnusedHard(partialCapacity); + } + partialChunk = nullptr; + partialCapacity = 0; + } +} + +js::Nursery::Nursery(GCRuntime* gc) + : position_(0), + currentEnd_(0), + gc(gc), + currentChunk_(0), + startChunk_(0), + startPosition_(0), + capacity_(0), + enableProfiling_(false), + canAllocateStrings_(true), + canAllocateBigInts_(true), + reportDeduplications_(false), + reportPretenuring_(false), + reportPretenuringThreshold_(0), + minorGCTriggerReason_(JS::GCReason::NO_REASON), + prevPosition_(0), + hasRecentGrowthData(false), + smoothedTargetSize(0.0) { + const char* env = getenv("MOZ_NURSERY_STRINGS"); + if (env && *env) { + canAllocateStrings_ = (*env == '1'); + } + env = getenv("MOZ_NURSERY_BIGINTS"); + if (env && *env) { + canAllocateBigInts_ = (*env == '1'); + } +} + +static void PrintAndExit(const char* message) { + fprintf(stderr, "%s", message); + exit(0); +} + +static const char* GetEnvVar(const char* name, const char* helpMessage) { + const char* value = getenv(name); + if (!value) { + return nullptr; + } + + if (strcmp(value, "help") == 0) { + PrintAndExit(helpMessage); + } + + return value; +} + +static bool GetBoolEnvVar(const char* name, const char* helpMessage) { + const char* env = GetEnvVar(name, helpMessage); + return env && bool(atoi(env)); +} + +static void ReadReportPretenureEnv(const char* name, const char* helpMessage, + bool* enabled, size_t* threshold) { + *enabled = false; + *threshold = 0; + + const char* env = GetEnvVar(name, helpMessage); + if (!env) { + return; + } + + char* end; + *threshold = strtol(env, &end, 10); + if (end == env || *end) { + PrintAndExit(helpMessage); + } + + *enabled = true; +} + +bool js::Nursery::init(AutoLockGCBgAlloc& lock) { + ReadProfileEnv("JS_GC_PROFILE_NURSERY", + "Report minor GCs taking at least N microseconds.\n", + &enableProfiling_, &profileWorkers_, &profileThreshold_); + + reportDeduplications_ = GetBoolEnvVar( + "JS_GC_REPORT_STATS", + "JS_GC_REPORT_STATS=1\n" + "\tAfter a minor GC, report how many strings were deduplicated.\n"); + + ReadReportPretenureEnv( + "JS_GC_REPORT_PRETENURE", + "JS_GC_REPORT_PRETENURE=N\n" + "\tAfter a minor GC, report information about pretenuring, including\n" + "\tallocation sites with at least N allocations.\n", + &reportPretenuring_, &reportPretenuringThreshold_); + + decommitTask = MakeUnique<NurseryDecommitTask>(gc); + if (!decommitTask) { + return false; + } + + if (!gc->storeBuffer().enable()) { + return false; + } + + return initFirstChunk(lock); +} + +js::Nursery::~Nursery() { disable(); } + +void js::Nursery::enable() { + MOZ_ASSERT(isEmpty()); + MOZ_ASSERT(!gc->isVerifyPreBarriersEnabled()); + if (isEnabled()) { + return; + } + + { + AutoLockGCBgAlloc lock(gc); + if (!initFirstChunk(lock)) { + // If we fail to allocate memory, the nursery will not be enabled. + return; + } + } + +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + enterZealMode(); + } +#endif + + updateAllZoneAllocFlags(); + + // This should always succeed after the first time it's called. + MOZ_ALWAYS_TRUE(gc->storeBuffer().enable()); +} + +bool js::Nursery::initFirstChunk(AutoLockGCBgAlloc& lock) { + MOZ_ASSERT(!isEnabled()); + + capacity_ = tunables().gcMinNurseryBytes(); + + if (!decommitTask->reserveSpaceForBytes(capacity_) || + !allocateNextChunk(0, lock)) { + capacity_ = 0; + return false; + } + + moveToStartOfChunk(0); + setStartToCurrentPosition(); + poisonAndInitCurrentChunk(); + + // Clear any information about previous collections. + clearRecentGrowthData(); + + return true; +} + +void js::Nursery::disable() { + MOZ_ASSERT(isEmpty()); + if (!isEnabled()) { + return; + } + + // Free all chunks. + decommitTask->join(); + freeChunksFrom(0); + decommitTask->runFromMainThread(); + + capacity_ = 0; + + // We must reset currentEnd_ so that there is no space for anything in the + // nursery. JIT'd code uses this even if the nursery is disabled. + currentEnd_ = 0; + position_ = 0; + gc->storeBuffer().disable(); + + if (gc->wasInitialized()) { + // This assumes there is an atoms zone. + updateAllZoneAllocFlags(); + } +} + +void js::Nursery::enableStrings() { + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = true; + updateAllZoneAllocFlags(); +} + +void js::Nursery::disableStrings() { + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = false; + updateAllZoneAllocFlags(); +} + +void js::Nursery::enableBigInts() { + MOZ_ASSERT(isEmpty()); + canAllocateBigInts_ = true; + updateAllZoneAllocFlags(); +} + +void js::Nursery::disableBigInts() { + MOZ_ASSERT(isEmpty()); + canAllocateBigInts_ = false; + updateAllZoneAllocFlags(); +} + +void js::Nursery::updateAllZoneAllocFlags() { + // The alloc flags are not relevant for the atoms zone, and flushing + // jit-related information can be problematic for the atoms zone. + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + updateAllocFlagsForZone(zone); + } +} + +void js::Nursery::getAllocFlagsForZone(JS::Zone* zone, bool* allocObjectsOut, + bool* allocStringsOut, + bool* allocBigIntsOut) { + *allocObjectsOut = isEnabled(); + *allocStringsOut = + isEnabled() && canAllocateStrings() && !zone->nurseryStringsDisabled; + *allocBigIntsOut = + isEnabled() && canAllocateBigInts() && !zone->nurseryBigIntsDisabled; +} + +void js::Nursery::setAllocFlagsForZone(JS::Zone* zone) { + bool allocObjects; + bool allocStrings; + bool allocBigInts; + + getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts); + zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts); +} + +void js::Nursery::updateAllocFlagsForZone(JS::Zone* zone) { + bool allocObjects; + bool allocStrings; + bool allocBigInts; + + getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts); + + if (allocObjects != zone->allocNurseryObjects() || + allocStrings != zone->allocNurseryStrings() || + allocBigInts != zone->allocNurseryBigInts()) { + CancelOffThreadIonCompile(zone); + zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts); + discardCodeAndSetJitFlagsForZone(zone); + } +} + +void js::Nursery::discardCodeAndSetJitFlagsForZone(JS::Zone* zone) { + zone->forceDiscardJitCode(runtime()->gcContext()); + + if (jit::JitZone* jitZone = zone->jitZone()) { + jitZone->discardStubs(); + jitZone->setStringsCanBeInNursery(zone->allocNurseryStrings()); + } +} + +bool js::Nursery::isEmpty() const { + if (!isEnabled()) { + return true; + } + + if (!gc->hasZealMode(ZealMode::GenerationalGC)) { + MOZ_ASSERT(startChunk_ == 0); + MOZ_ASSERT(startPosition_ == chunk(0).start()); + } + return position() == startPosition_; +} + +#ifdef JS_GC_ZEAL +void js::Nursery::enterZealMode() { + if (!isEnabled()) { + return; + } + + MOZ_ASSERT(isEmpty()); + + decommitTask->join(); + + AutoEnterOOMUnsafeRegion oomUnsafe; + + if (isSubChunkMode()) { + { + if (!chunk(0).markPagesInUseHard(ChunkSize)) { + oomUnsafe.crash("Out of memory trying to extend chunk for zeal mode"); + } + } + + // It'd be simpler to poison the whole chunk, but we can't do that + // because the nursery might be partially used. + chunk(0).poisonRange(capacity_, ChunkSize, JS_FRESH_NURSERY_PATTERN, + MemCheckKind::MakeUndefined); + } + + capacity_ = RoundUp(tunables().gcMaxNurseryBytes(), ChunkSize); + + if (!decommitTask->reserveSpaceForBytes(capacity_)) { + oomUnsafe.crash("Nursery::enterZealMode"); + } + + setCurrentEnd(); +} + +void js::Nursery::leaveZealMode() { + if (!isEnabled()) { + return; + } + + MOZ_ASSERT(isEmpty()); + + moveToStartOfChunk(0); + setStartToCurrentPosition(); + poisonAndInitCurrentChunk(); +} +#endif // JS_GC_ZEAL + +void* js::Nursery::allocateCell(gc::AllocSite* site, size_t size, + JS::TraceKind kind) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + + void* ptr = tryAllocateCell(site, size, kind); + if (MOZ_LIKELY(ptr)) { + return ptr; + } + + if (handleAllocationFailure() != JS::GCReason::NO_REASON) { + return nullptr; + } + + ptr = tryAllocateCell(site, size, kind); + MOZ_ASSERT(ptr); + return ptr; +} + +inline void* js::Nursery::allocate(size_t size) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + + void* ptr = tryAllocate(size); + if (MOZ_LIKELY(ptr)) { + return ptr; + } + + if (handleAllocationFailure() != JS::GCReason::NO_REASON) { + return nullptr; + } + + ptr = tryAllocate(size); + MOZ_ASSERT(ptr); + return ptr; +} + +MOZ_NEVER_INLINE JS::GCReason Nursery::handleAllocationFailure() { + if (minorGCRequested()) { + // If a minor GC was requested then fail the allocation. The collection is + // then run in GCRuntime::tryNewNurseryCell. + return minorGCTriggerReason_; + } + + if (!moveToNextChunk()) { + return JS::GCReason::OUT_OF_NURSERY; + } + + return JS::GCReason::NO_REASON; +} + +bool Nursery::moveToNextChunk() { + unsigned chunkno = currentChunk_ + 1; + MOZ_ASSERT(chunkno <= maxChunkCount()); + MOZ_ASSERT(chunkno <= allocatedChunkCount()); + if (chunkno == maxChunkCount()) { + return false; + } + + if (chunkno == allocatedChunkCount()) { + TimeStamp start = TimeStamp::Now(); + { + AutoLockGCBgAlloc lock(gc); + if (!allocateNextChunk(chunkno, lock)) { + return false; + } + } + timeInChunkAlloc_ += TimeStamp::Now() - start; + MOZ_ASSERT(chunkno < allocatedChunkCount()); + } + + moveToStartOfChunk(chunkno); + poisonAndInitCurrentChunk(); + return true; +} + +std::tuple<void*, bool> js::Nursery::allocateBuffer(Zone* zone, size_t nbytes, + arena_id_t arenaId) { + MOZ_ASSERT(nbytes > 0); + MOZ_ASSERT(nbytes <= SIZE_MAX - gc::CellAlignBytes); + nbytes = RoundUp(nbytes, gc::CellAlignBytes); + + if (nbytes <= MaxNurseryBufferSize) { + void* buffer = allocate(nbytes); + if (buffer) { + return {buffer, false}; + } + } + + void* buffer = zone->pod_arena_malloc<uint8_t>(arenaId, nbytes); + return {buffer, bool(buffer)}; +} + +void* js::Nursery::allocateBuffer(Zone* zone, Cell* owner, size_t nbytes, + arena_id_t arenaId) { + MOZ_ASSERT(owner); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(owner)) { + return zone->pod_arena_malloc<uint8_t>(arenaId, nbytes); + } + + auto [buffer, isMalloced] = allocateBuffer(zone, nbytes, arenaId); + if (isMalloced && !registerMallocedBuffer(buffer, nbytes)) { + js_free(buffer); + return nullptr; + } + return buffer; +} + +void* js::Nursery::allocateBufferSameLocation(Cell* owner, size_t nbytes, + arena_id_t arenaId) { + MOZ_ASSERT(owner); + MOZ_ASSERT(nbytes > 0); + MOZ_ASSERT(nbytes <= MaxNurseryBufferSize); + + if (!IsInsideNursery(owner)) { + return owner->asTenured().zone()->pod_arena_malloc<uint8_t>(arenaId, + nbytes); + } + + return allocate(nbytes); +} + +std::tuple<void*, bool> js::Nursery::allocateZeroedBuffer(Zone* zone, + size_t nbytes, + arena_id_t arena) { + MOZ_ASSERT(nbytes > 0); + + if (nbytes <= MaxNurseryBufferSize) { + void* buffer = allocate(nbytes); + if (buffer) { + memset(buffer, 0, nbytes); + return {buffer, false}; + } + } + + void* buffer = zone->pod_arena_calloc<uint8_t>(arena, nbytes); + return {buffer, bool(buffer)}; +} + +void* js::Nursery::allocateZeroedBuffer(Cell* owner, size_t nbytes, + arena_id_t arena) { + MOZ_ASSERT(owner); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(owner)) { + return owner->asTenured().zone()->pod_arena_calloc<uint8_t>(arena, nbytes); + } + auto [buffer, isMalloced] = + allocateZeroedBuffer(owner->nurseryZone(), nbytes, arena); + if (isMalloced && !registerMallocedBuffer(buffer, nbytes)) { + js_free(buffer); + return nullptr; + } + return buffer; +} + +void* js::Nursery::reallocateBuffer(Zone* zone, Cell* cell, void* oldBuffer, + size_t oldBytes, size_t newBytes, + arena_id_t arena) { + if (!IsInsideNursery(cell)) { + MOZ_ASSERT(!isInside(oldBuffer)); + return zone->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes); + } + + if (!isInside(oldBuffer)) { + MOZ_ASSERT(mallocedBufferBytes >= oldBytes); + void* newBuffer = + zone->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes); + if (newBuffer) { + if (oldBuffer != newBuffer) { + MOZ_ALWAYS_TRUE( + mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer)); + } + mallocedBufferBytes -= oldBytes; + mallocedBufferBytes += newBytes; + } + return newBuffer; + } + + // The nursery cannot make use of the returned slots data. + if (newBytes < oldBytes) { + return oldBuffer; + } + + auto newBuffer = allocateBuffer(zone, cell, newBytes, js::MallocArena); + if (newBuffer) { + PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes); + } + return newBuffer; +} + +void js::Nursery::freeBuffer(void* buffer, size_t nbytes) { + if (!isInside(buffer)) { + removeMallocedBuffer(buffer, nbytes); + js_free(buffer); + } +} + +#ifdef DEBUG +/* static */ +inline bool Nursery::checkForwardingPointerLocation(void* ptr, + bool expectedInside) { + if (isInside(ptr) == expectedInside) { + return true; + } + + // If a zero-capacity elements header lands right at the end of a chunk then + // elements data will appear to be in the next chunk. If we have a pointer to + // the very start of a chunk, check the previous chunk. + if ((uintptr_t(ptr) & ChunkMask) == 0 && + isInside(reinterpret_cast<uint8_t*>(ptr) - 1) == expectedInside) { + return true; + } + + return false; +} +#endif + +void Nursery::setIndirectForwardingPointer(void* oldData, void* newData) { + MOZ_ASSERT(checkForwardingPointerLocation(oldData, true)); + MOZ_ASSERT(checkForwardingPointerLocation(newData, false)); + + AutoEnterOOMUnsafeRegion oomUnsafe; +#ifdef DEBUG + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData)) { + MOZ_ASSERT(p->value() == newData); + } +#endif + if (!forwardedBuffers.put(oldData, newData)) { + oomUnsafe.crash("Nursery::setForwardingPointer"); + } +} + +#ifdef DEBUG +static bool IsWriteableAddress(void* ptr) { + auto* vPtr = reinterpret_cast<volatile uint64_t*>(ptr); + *vPtr = *vPtr; + return true; +} +#endif + +void js::Nursery::forwardBufferPointer(uintptr_t* pSlotsElems) { + // Read the current pointer value which may be one of: + // - Non-nursery pointer + // - Nursery-allocated buffer + // - A BufferRelocationOverlay inside the nursery + // + // Note: The buffer has already be relocated. We are just patching stale + // pointers now. + auto* buffer = reinterpret_cast<void*>(*pSlotsElems); + + if (!isInside(buffer)) { + return; + } + + // The new location for this buffer is either stored inline with it or in + // the forwardedBuffers table. + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(buffer)) { + buffer = p->value(); + // It's not valid to assert IsWriteableAddress for indirect forwarding + // pointers because the size of the allocation could be less than a word. + } else { + BufferRelocationOverlay* reloc = + static_cast<BufferRelocationOverlay*>(buffer); + buffer = *reloc; + MOZ_ASSERT(IsWriteableAddress(buffer)); + } + + MOZ_ASSERT(!isInside(buffer)); + *pSlotsElems = reinterpret_cast<uintptr_t>(buffer); +} + +inline double js::Nursery::calcPromotionRate(bool* validForTenuring) const { + MOZ_ASSERT(validForTenuring); + + if (previousGC.nurseryUsedBytes == 0) { + *validForTenuring = false; + return 0.0; + } + + double used = double(previousGC.nurseryUsedBytes); + double capacity = double(previousGC.nurseryCapacity); + double tenured = double(previousGC.tenuredBytes); + + // We should only use the promotion rate to make tenuring decisions if it's + // likely to be valid. The criterion we use is that the nursery was at least + // 90% full. + *validForTenuring = used > capacity * 0.9; + + MOZ_ASSERT(tenured <= used); + return tenured / used; +} + +void js::Nursery::renderProfileJSON(JSONPrinter& json) const { + if (!isEnabled()) { + json.beginObject(); + json.property("status", "nursery disabled"); + json.endObject(); + return; + } + + if (previousGC.reason == JS::GCReason::NO_REASON) { + // If the nursery was empty when the last minorGC was requested, then + // no nursery collection will have been performed but JSON may still be + // requested. (And as a public API, this function should not crash in + // such a case.) + json.beginObject(); + json.property("status", "nursery empty"); + json.endObject(); + return; + } + + json.beginObject(); + + json.property("status", "complete"); + + json.property("reason", JS::ExplainGCReason(previousGC.reason)); + json.property("bytes_tenured", previousGC.tenuredBytes); + json.property("cells_tenured", previousGC.tenuredCells); + json.property("strings_tenured", + stats().getStat(gcstats::STAT_STRINGS_TENURED)); + json.property("strings_deduplicated", + stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED)); + json.property("bigints_tenured", + stats().getStat(gcstats::STAT_BIGINTS_TENURED)); + json.property("bytes_used", previousGC.nurseryUsedBytes); + json.property("cur_capacity", previousGC.nurseryCapacity); + const size_t newCapacity = capacity(); + if (newCapacity != previousGC.nurseryCapacity) { + json.property("new_capacity", newCapacity); + } + if (previousGC.nurseryCommitted != previousGC.nurseryCapacity) { + json.property("lazy_capacity", previousGC.nurseryCommitted); + } + if (!timeInChunkAlloc_.IsZero()) { + json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS); + } + + // These counters only contain consistent data if the profiler is enabled, + // and then there's no guarentee. + if (runtime()->geckoProfiler().enabled()) { + json.property("cells_allocated_nursery", + pretenuringNursery.totalAllocCount()); + json.property("cells_allocated_tenured", + stats().allocsSinceMinorGCTenured()); + } + + json.beginObjectProperty("phase_times"); + +#define EXTRACT_NAME(name, text) #name, + static const char* const names[] = { + FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME) +#undef EXTRACT_NAME + ""}; + + size_t i = 0; + for (auto time : profileDurations_) { + json.property(names[i++], time, json.MICROSECONDS); + } + + json.endObject(); // timings value + + json.endObject(); +} + +// The following macros define nursery GC profile metadata fields that are +// printed before the timing information defined by +// FOR_EACH_NURSERY_PROFILE_TIME. + +#define FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \ + _("PID", 7, "%7zu", pid) \ + _("Runtime", 14, "0x%12p", runtime) + +#define FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_) \ + _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \ + _("Reason", 20, "%-20.20s", reasonStr) \ + _("PRate", 6, "%5.1f%%", promotionRatePercent) \ + _("OldKB", 6, "%6zu", oldSizeKB) \ + _("NewKB", 6, "%6zu", newSizeKB) \ + _("Dedup", 6, "%6zu", dedupCount) + +#define FOR_EACH_NURSERY_PROFILE_METADATA(_) \ + FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \ + FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_) + +void js::Nursery::printCollectionProfile(JS::GCReason reason, + double promotionRate) { + stats().maybePrintProfileHeaders(); + + Sprinter sprinter; + if (!sprinter.init()) { + return; + } + sprinter.put(gcstats::MinorGCProfilePrefix); + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + TimeDuration timestamp = collectionStartTime() - stats().creationTime(); + const char* reasonStr = ExplainGCReason(reason); + double promotionRatePercent = promotionRate * 100; + size_t oldSizeKB = previousGC.nurseryCapacity / 1024; + size_t newSizeKB = capacity() / 1024; + size_t dedupCount = stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED); + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + sprinter.printf(" " format, value); + + FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + printProfileDurations(profileDurations_, sprinter); + + JS::UniqueChars str = sprinter.release(); + if (!str) { + return; + } + fputs(str.get(), stats().profileFile()); +} + +void js::Nursery::printProfileHeader() { + Sprinter sprinter; + if (!sprinter.init()) { + return; + } + sprinter.put(gcstats::MinorGCProfilePrefix); + +#define PRINT_FIELD_NAME(name, width, _1, _2) \ + sprinter.printf(" %-*s", width, name); + + FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_NAME) +#undef PRINT_FIELD_NAME + +#define PRINT_PROFILE_NAME(_1, text) sprinter.printf(" %-6.6s", text); + + FOR_EACH_NURSERY_PROFILE_TIME(PRINT_PROFILE_NAME) +#undef PRINT_PROFILE_NAME + + sprinter.put("\n"); + + JS::UniqueChars str = sprinter.release(); + if (!str) { + return; + } + fputs(str.get(), stats().profileFile()); +} + +// static +void js::Nursery::printProfileDurations(const ProfileDurations& times, + Sprinter& sprinter) { + for (auto time : times) { + int64_t micros = int64_t(time.ToMicroseconds()); + sprinter.printf(" %6" PRIi64, micros); + } + + sprinter.put("\n"); +} + +static constexpr size_t NurserySliceMetadataFormatWidth() { + size_t fieldCount = 0; + size_t totalWidth = 0; + +#define UPDATE_COUNT_AND_WIDTH(_1, width, _2, _3) \ + fieldCount++; \ + totalWidth += width; + FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(UPDATE_COUNT_AND_WIDTH) +#undef UPDATE_COUNT_AND_WIDTH + + // Add padding between fields. + totalWidth += fieldCount - 1; + + return totalWidth; +} + +void js::Nursery::printTotalProfileTimes() { + if (!enableProfiling_) { + return; + } + + Sprinter sprinter; + if (!sprinter.init()) { + return; + } + sprinter.put(gcstats::MinorGCProfilePrefix); + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + + char collections[32]; + DebugOnly<int> r = SprintfLiteral( + collections, "TOTALS: %7" PRIu64 " collections:", gc->minorGCCount()); + MOZ_ASSERT(r > 0 && r < int(sizeof(collections))); + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + sprinter.printf(" " format, value); + + FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + // Use whole width of per-slice metadata to print total slices so the profile + // totals that follow line up. + size_t width = NurserySliceMetadataFormatWidth(); + sprinter.printf(" %-*s", int(width), collections); + + printProfileDurations(totalDurations_, sprinter); + + JS::UniqueChars str = sprinter.release(); + if (!str) { + return; + } + fputs(str.get(), stats().profileFile()); +} + +void js::Nursery::maybeClearProfileDurations() { + for (auto& duration : profileDurations_) { + duration = mozilla::TimeDuration::Zero(); + } +} + +inline void js::Nursery::startProfile(ProfileKey key) { + startTimes_[key] = TimeStamp::Now(); +} + +inline void js::Nursery::endProfile(ProfileKey key) { + profileDurations_[key] = TimeStamp::Now() - startTimes_[key]; + totalDurations_[key] += profileDurations_[key]; +} + +inline TimeStamp js::Nursery::collectionStartTime() const { + return startTimes_[ProfileKey::Total]; +} + +TimeStamp js::Nursery::lastCollectionEndTime() const { + return previousGC.endTime; +} + +bool js::Nursery::shouldCollect() const { + if (!isEnabled()) { + return false; + } + + if (isEmpty() && capacity() == tunables().gcMinNurseryBytes()) { + return false; + } + + if (minorGCRequested()) { + return true; + } + + // Eagerly collect the nursery in idle time if it's nearly full. + if (isNearlyFull()) { + return true; + } + + // If the nursery is not being collected often then it may be taking up more + // space than necessary. + return isUnderused(); +} + +inline bool js::Nursery::isNearlyFull() const { + bool belowBytesThreshold = + freeSpace() < tunables().nurseryFreeThresholdForIdleCollection(); + bool belowFractionThreshold = + double(freeSpace()) / double(capacity()) < + tunables().nurseryFreeThresholdForIdleCollectionFraction(); + + // We want to use belowBytesThreshold when the nursery is sufficiently large, + // and belowFractionThreshold when it's small. + // + // When the nursery is small then belowBytesThreshold is a lower threshold + // (triggered earlier) than belowFractionThreshold. So if the fraction + // threshold is true, the bytes one will be true also. The opposite is true + // when the nursery is large. + // + // Therefore, by the time we cross the threshold we care about, we've already + // crossed the other one, and we can boolean AND to use either condition + // without encoding any "is the nursery big/small" test/threshold. The point + // at which they cross is when the nursery is: BytesThreshold / + // FractionThreshold large. + // + // With defaults that's: + // + // 1MB = 256KB / 0.25 + // + return belowBytesThreshold && belowFractionThreshold; +} + +inline bool js::Nursery::isUnderused() const { + if (js::SupportDifferentialTesting() || !previousGC.endTime) { + return false; + } + + if (capacity() == tunables().gcMinNurseryBytes()) { + return false; + } + + // If the nursery is above its minimum size, collect it every so often if we + // have idle time. This allows the nursery to shrink when it's not being + // used. There are other heuristics we could use for this, but this is the + // simplest. + TimeDuration timeSinceLastCollection = + TimeStamp::NowLoRes() - previousGC.endTime; + return timeSinceLastCollection > tunables().nurseryTimeoutForIdleCollection(); +} + +void js::Nursery::collect(JS::GCOptions options, JS::GCReason reason) { + JSRuntime* rt = runtime(); + MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC); + + if (minorGCRequested()) { + MOZ_ASSERT(position_ == chunk(currentChunk_).end()); + position_ = prevPosition_; + prevPosition_ = 0; + minorGCTriggerReason_ = JS::GCReason::NO_REASON; + rt->mainContextFromOwnThread()->clearPendingInterrupt( + InterruptReason::MinorGC); + } + + if (!isEnabled() || isEmpty()) { + // Our barriers are not always exact, and there may be entries in the + // storebuffer even when the nursery is disabled or empty. It's not safe + // to keep these entries as they may refer to tenured cells which may be + // freed after this point. + gc->storeBuffer().clear(); + + MOZ_ASSERT(!pretenuringNursery.hasAllocatedSites()); + } + + if (!isEnabled()) { + return; + } + + AutoGCSession session(gc, JS::HeapState::MinorCollecting); + + stats().beginNurseryCollection(); + gcprobes::MinorGCStart(); + + gc->callNurseryCollectionCallbacks( + JS::GCNurseryProgress::GC_NURSERY_COLLECTION_START, reason); + + maybeClearProfileDurations(); + startProfile(ProfileKey::Total); + + previousGC.reason = JS::GCReason::NO_REASON; + previousGC.nurseryUsedBytes = usedSpace(); + previousGC.nurseryCapacity = capacity(); + previousGC.nurseryCommitted = committed(); + previousGC.nurseryUsedChunkCount = currentChunk_ + 1; + previousGC.tenuredBytes = 0; + previousGC.tenuredCells = 0; + + // If it isn't empty, it will call doCollection, and possibly after that + // isEmpty() will become true, so use another variable to keep track of the + // old empty state. + bool wasEmpty = isEmpty(); + if (!wasEmpty) { + CollectionResult result = doCollection(session, options, reason); + // Don't include chunk headers when calculating nursery space, since this + // space does not represent data that can be tenured + MOZ_ASSERT(result.tenuredBytes <= + (previousGC.nurseryUsedBytes - + (sizeof(ChunkBase) * previousGC.nurseryUsedChunkCount))); + + previousGC.reason = reason; + previousGC.tenuredBytes = result.tenuredBytes; + previousGC.tenuredCells = result.tenuredCells; + previousGC.nurseryUsedChunkCount = currentChunk_ + 1; + } + + // Resize the nursery. + maybeResizeNursery(options, reason); + + // Poison/initialise the first chunk. + if (previousGC.nurseryUsedBytes) { + // In most cases Nursery::clear() has not poisoned this chunk or marked it + // as NoAccess; so we only need to poison the region used during the last + // cycle. Also, if the heap was recently expanded we don't want to + // re-poison the new memory. In both cases we only need to poison until + // previousGC.nurseryUsedBytes. + // + // In cases where this is not true, like generational zeal mode or subchunk + // mode, poisonAndInitCurrentChunk() will ignore its parameter. It will + // also clamp the parameter. + poisonAndInitCurrentChunk(previousGC.nurseryUsedBytes); + } + + bool validPromotionRate; + const double promotionRate = calcPromotionRate(&validPromotionRate); + + startProfile(ProfileKey::Pretenure); + size_t sitesPretenured = 0; + if (!wasEmpty) { + sitesPretenured = + doPretenuring(rt, reason, validPromotionRate, promotionRate); + } + endProfile(ProfileKey::Pretenure); + + // We ignore gcMaxBytes when allocating for minor collection. However, if we + // overflowed, we disable the nursery. The next time we allocate, we'll fail + // because bytes >= gcMaxBytes. + if (gc->heapSize.bytes() >= tunables().gcMaxBytes()) { + disable(); + } + + previousGC.endTime = + TimeStamp::Now(); // Must happen after maybeResizeNursery. + endProfile(ProfileKey::Total); + gc->incMinorGcNumber(); + + TimeDuration totalTime = profileDurations_[ProfileKey::Total]; + sendTelemetry(reason, totalTime, wasEmpty, promotionRate, sitesPretenured); + + gc->callNurseryCollectionCallbacks( + JS::GCNurseryProgress::GC_NURSERY_COLLECTION_END, reason); + + stats().endNurseryCollection(); + gcprobes::MinorGCEnd(); + + timeInChunkAlloc_ = mozilla::TimeDuration::Zero(); + + js::StringStats prevStats = gc->stringStats; + js::StringStats& currStats = gc->stringStats; + currStats = js::StringStats(); + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + currStats += zone->stringStats; + zone->previousGCStringStats = zone->stringStats; + } + stats().setStat( + gcstats::STAT_STRINGS_DEDUPLICATED, + currStats.deduplicatedStrings - prevStats.deduplicatedStrings); + if (ShouldPrintProfile(runtime(), enableProfiling_, profileWorkers_, + profileThreshold_, totalTime)) { + printCollectionProfile(reason, promotionRate); + } + + if (reportDeduplications_) { + printDeduplicationData(prevStats, currStats); + } +} + +void js::Nursery::sendTelemetry(JS::GCReason reason, TimeDuration totalTime, + bool wasEmpty, double promotionRate, + size_t sitesPretenured) { + JSRuntime* rt = runtime(); + rt->metrics().GC_MINOR_REASON(uint32_t(reason)); + + // Long minor GCs are those that take more than 1ms. + bool wasLongMinorGC = totalTime.ToMilliseconds() > 1.0; + if (wasLongMinorGC) { + rt->metrics().GC_MINOR_REASON_LONG(uint32_t(reason)); + } + rt->metrics().GC_MINOR_US(totalTime); + rt->metrics().GC_NURSERY_BYTES_2(committed()); + + if (!wasEmpty) { + rt->metrics().GC_PRETENURE_COUNT_2(sitesPretenured); + rt->metrics().GC_NURSERY_PROMOTION_RATE(promotionRate * 100); + } +} + +void js::Nursery::printDeduplicationData(js::StringStats& prev, + js::StringStats& curr) { + if (curr.deduplicatedStrings > prev.deduplicatedStrings) { + fprintf(stderr, + "pid %zu: deduplicated %" PRIi64 " strings, %" PRIu64 + " chars, %" PRIu64 " malloc bytes\n", + size_t(getpid()), + curr.deduplicatedStrings - prev.deduplicatedStrings, + curr.deduplicatedChars - prev.deduplicatedChars, + curr.deduplicatedBytes - prev.deduplicatedBytes); + } +} + +void js::Nursery::freeTrailerBlocks(void) { + // This routine frees those blocks denoted by the set + // + // trailersAdded_ (all of it) + // - trailersRemoved_ (entries with index below trailersRemovedUsed_) + // + // For each block, places it back on the nursery's small-malloced-block pool + // by calling mallocedBlockCache_.free. + + MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length()); + MOZ_ASSERT(trailersRemovedUsed_ <= trailersRemoved_.length()); + + // Sort the removed entries. + std::sort(trailersRemoved_.begin(), + trailersRemoved_.begin() + trailersRemovedUsed_, + [](const void* block1, const void* block2) { + return uintptr_t(block1) < uintptr_t(block2); + }); + + // Use one of two schemes to enumerate the set subtraction. + if (trailersRemovedUsed_ < 1000) { + // If the number of removed items is relatively small, it isn't worth the + // cost of sorting `trailersAdded_`. Instead, walk through the vector in + // whatever order it is and use binary search to establish whether each + // item is present in trailersRemoved_[0 .. trailersRemovedUsed_ - 1]. + const size_t nAdded = trailersAdded_.length(); + for (size_t i = 0; i < nAdded; i++) { + const PointerAndUint7 block = trailersAdded_[i]; + const void* blockPointer = block.pointer(); + if (!std::binary_search(trailersRemoved_.begin(), + trailersRemoved_.begin() + trailersRemovedUsed_, + blockPointer)) { + mallocedBlockCache_.free(block); + } + } + } else { + // The general case, which is algorithmically safer for large inputs. + // Sort the added entries, and then walk through both them and the removed + // entries in lockstep. + std::sort(trailersAdded_.begin(), trailersAdded_.end(), + [](const PointerAndUint7& block1, const PointerAndUint7& block2) { + return uintptr_t(block1.pointer()) < + uintptr_t(block2.pointer()); + }); + // Enumerate the set subtraction. This is somewhat simplified by the fact + // that all elements of the removed set must also be present in the added + // set. (the "inclusion property"). + const size_t nAdded = trailersAdded_.length(); + const size_t nRemoved = trailersRemovedUsed_; + size_t iAdded; + size_t iRemoved = 0; + for (iAdded = 0; iAdded < nAdded; iAdded++) { + if (iRemoved == nRemoved) { + // We've run out of items to skip, so move on to the next loop. + break; + } + const PointerAndUint7 blockAdded = trailersAdded_[iAdded]; + const void* blockRemoved = trailersRemoved_[iRemoved]; + if (blockAdded.pointer() < blockRemoved) { + mallocedBlockCache_.free(blockAdded); + continue; + } + // If this doesn't hold + // (that is, if `blockAdded.pointer() > blockRemoved`), + // then the abovementioned inclusion property doesn't hold. + MOZ_RELEASE_ASSERT(blockAdded.pointer() == blockRemoved); + iRemoved++; + } + MOZ_ASSERT(iRemoved == nRemoved); + // We've used up the removed set, so now finish up the remainder of the + // added set. + for (/*keep going*/; iAdded < nAdded; iAdded++) { + const PointerAndUint7 block = trailersAdded_[iAdded]; + mallocedBlockCache_.free(block); + } + } + + // And empty out both sets, but preserve the underlying storage. + trailersAdded_.clear(); + trailersRemoved_.clear(); + trailersRemovedUsed_ = 0; + trailerBytes_ = 0; + + // Discard blocks from the cache at 0.05% per megabyte of nursery capacity, + // that is, 0.8% of blocks for a 16-megabyte nursery. This allows the cache + // to gradually discard unneeded blocks in long running applications. + mallocedBlockCache_.preen(0.05 * double(capacity()) / (1024.0 * 1024.0)); +} + +size_t Nursery::sizeOfTrailerBlockSets( + mozilla::MallocSizeOf mallocSizeOf) const { + return trailersAdded_.sizeOfExcludingThis(mallocSizeOf) + + trailersRemoved_.sizeOfExcludingThis(mallocSizeOf); +} + +js::Nursery::CollectionResult js::Nursery::doCollection(AutoGCSession& session, + JS::GCOptions options, + JS::GCReason reason) { + JSRuntime* rt = runtime(); + AutoSetThreadIsPerformingGC performingGC(rt->gcContext()); + AutoStopVerifyingBarriers av(rt, false); + AutoDisableProxyCheck disableStrictProxyChecking; + mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion; + + // Move objects pointed to by roots from the nursery to the major heap. + TenuringTracer mover(rt, this); + + // Trace everything considered as a root by a minor GC. + traceRoots(session, mover); + + startProfile(ProfileKey::SweepCaches); + gc->purgeRuntimeForMinorGC(); + endProfile(ProfileKey::SweepCaches); + + // Most of the work is done here. This loop iterates over objects that have + // been moved to the major heap. If these objects have any outgoing pointers + // to the nursery, then those nursery objects get moved as well, until no + // objects are left to move. That is, we iterate to a fixed point. + startProfile(ProfileKey::CollectToObjFP); + mover.collectToObjectFixedPoint(); + endProfile(ProfileKey::CollectToObjFP); + + startProfile(ProfileKey::CollectToStrFP); + mover.collectToStringFixedPoint(); + endProfile(ProfileKey::CollectToStrFP); + + // Sweep to update any pointers to nursery objects that have now been + // tenured. + startProfile(ProfileKey::Sweep); + sweep(); + endProfile(ProfileKey::Sweep); + + // Update any slot or element pointers whose destination has been tenured. + startProfile(ProfileKey::UpdateJitActivations); + js::jit::UpdateJitActivationsForMinorGC(rt); + forwardedBuffers.clearAndCompact(); + endProfile(ProfileKey::UpdateJitActivations); + + startProfile(ProfileKey::ObjectsTenuredCallback); + gc->callObjectsTenuredCallback(); + endProfile(ProfileKey::ObjectsTenuredCallback); + + // Sweep. + startProfile(ProfileKey::FreeMallocedBuffers); + gc->queueBuffersForFreeAfterMinorGC(mallocedBuffers); + mallocedBufferBytes = 0; + endProfile(ProfileKey::FreeMallocedBuffers); + + // Give trailer blocks associated with non-tenured Wasm{Struct,Array}Objects + // back to our `mallocedBlockCache_`. + startProfile(ProfileKey::FreeTrailerBlocks); + freeTrailerBlocks(); + if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason)) { + mallocedBlockCache_.clear(); + } + endProfile(ProfileKey::FreeTrailerBlocks); + + startProfile(ProfileKey::ClearNursery); + clear(); + endProfile(ProfileKey::ClearNursery); + + // Purge the StringToAtomCache. This has to happen at the end because the + // cache is used when tenuring strings. + startProfile(ProfileKey::PurgeStringToAtomCache); + runtime()->caches().stringToAtomCache.purge(); + endProfile(ProfileKey::PurgeStringToAtomCache); + + // Make sure hashtables have been updated after the collection. + startProfile(ProfileKey::CheckHashTables); +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) { + runtime()->caches().checkEvalCacheAfterMinorGC(); + gc->checkHashTablesAfterMovingGC(); + } +#endif + endProfile(ProfileKey::CheckHashTables); + + return {mover.getTenuredSize(), mover.getTenuredCells()}; +} + +void js::Nursery::traceRoots(AutoGCSession& session, TenuringTracer& mover) { + { + // Suppress the sampling profiler to prevent it observing moved functions. + AutoSuppressProfilerSampling suppressProfiler( + runtime()->mainContextFromOwnThread()); + + // Trace the store buffer, which must happen first. + + // Create an empty store buffer on the stack and swap it with the main store + // buffer, clearing it. + StoreBuffer sb(runtime()); + { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!sb.enable()) { + oomUnsafe.crash("Nursery::traceRoots"); + } + } + std::swap(sb, gc->storeBuffer()); + MOZ_ASSERT(gc->storeBuffer().isEnabled()); + MOZ_ASSERT(gc->storeBuffer().isEmpty()); + + // Strings in the whole cell buffer must be traced first, in order to mark + // tenured dependent strings' bases as non-deduplicatable. The rest of + // nursery collection (whole non-string cells, edges, etc.) can happen + // later. + startProfile(ProfileKey::TraceWholeCells); + sb.traceWholeCells(mover); + endProfile(ProfileKey::TraceWholeCells); + + startProfile(ProfileKey::TraceValues); + sb.traceValues(mover); + endProfile(ProfileKey::TraceValues); + + startProfile(ProfileKey::TraceWasmAnyRefs); + sb.traceWasmAnyRefs(mover); + endProfile(ProfileKey::TraceWasmAnyRefs); + + startProfile(ProfileKey::TraceCells); + sb.traceCells(mover); + endProfile(ProfileKey::TraceCells); + + startProfile(ProfileKey::TraceSlots); + sb.traceSlots(mover); + endProfile(ProfileKey::TraceSlots); + + startProfile(ProfileKey::TraceGenericEntries); + sb.traceGenericEntries(&mover); + endProfile(ProfileKey::TraceGenericEntries); + + startProfile(ProfileKey::MarkRuntime); + gc->traceRuntimeForMinorGC(&mover, session); + endProfile(ProfileKey::MarkRuntime); + } + + MOZ_ASSERT(gc->storeBuffer().isEmpty()); + + startProfile(ProfileKey::MarkDebugger); + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS); + DebugAPI::traceAllForMovingGC(&mover); + } + endProfile(ProfileKey::MarkDebugger); +} + +size_t js::Nursery::doPretenuring(JSRuntime* rt, JS::GCReason reason, + bool validPromotionRate, + double promotionRate) { + size_t sitesPretenured = pretenuringNursery.doPretenuring( + gc, reason, validPromotionRate, promotionRate, reportPretenuring_, + reportPretenuringThreshold_); + + size_t zonesWhereStringsDisabled = 0; + size_t zonesWhereBigIntsDisabled = 0; + + uint32_t numStringsTenured = 0; + uint32_t numBigIntsTenured = 0; + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + bool disableNurseryStrings = + zone->allocNurseryStrings() && + zone->unknownAllocSite(JS::TraceKind::String)->state() == + AllocSite::State::LongLived; + + bool disableNurseryBigInts = + zone->allocNurseryBigInts() && + zone->unknownAllocSite(JS::TraceKind::BigInt)->state() == + AllocSite::State::LongLived; + + if (disableNurseryStrings || disableNurseryBigInts) { + if (disableNurseryStrings) { + zone->nurseryStringsDisabled = true; + zonesWhereStringsDisabled++; + } + if (disableNurseryBigInts) { + zone->nurseryBigIntsDisabled = true; + zonesWhereStringsDisabled++; + } + updateAllocFlagsForZone(zone); + } + } + + stats().setStat(gcstats::STAT_STRINGS_TENURED, numStringsTenured); + stats().setStat(gcstats::STAT_BIGINTS_TENURED, numBigIntsTenured); + + if (reportPretenuring_ && zonesWhereStringsDisabled) { + fprintf(stderr, + "Pretenuring disabled nursery string allocation in %zu zones\n", + zonesWhereStringsDisabled); + } + if (reportPretenuring_ && zonesWhereBigIntsDisabled) { + fprintf(stderr, + "Pretenuring disabled nursery big int allocation in %zu zones\n", + zonesWhereBigIntsDisabled); + } + + return sitesPretenured; +} + +bool js::Nursery::registerMallocedBuffer(void* buffer, size_t nbytes) { + MOZ_ASSERT(buffer); + MOZ_ASSERT(nbytes > 0); + MOZ_ASSERT(!isInside(buffer)); + if (!mallocedBuffers.putNew(buffer)) { + return false; + } + + mallocedBufferBytes += nbytes; + if (MOZ_UNLIKELY(mallocedBufferBytes > capacity() * 8)) { + requestMinorGC(JS::GCReason::NURSERY_MALLOC_BUFFERS); + } + + return true; +} + +/* + * Several things may need to happen when a nursery allocated cell with an + * external buffer is promoted: + * - the buffer may need to be moved if it is currently in the nursery + * - the buffer may need to be removed from the list of buffers that will be + * freed after nursery collection if it is malloced + * - memory accounting for the buffer needs to be updated + */ +Nursery::WasBufferMoved js::Nursery::maybeMoveRawBufferOnPromotion( + void** bufferp, gc::Cell* owner, size_t nbytes, MemoryUse use, + arena_id_t arena) { + MOZ_ASSERT(!IsInsideNursery(owner)); + + void* buffer = *bufferp; + if (!isInside(buffer)) { + // This is a malloced buffer. Remove it from the nursery's list of buffers + // so we don't free it and add it to the memory accounting for the zone + removeMallocedBufferDuringMinorGC(buffer); + AddCellMemory(owner, nbytes, use); + return BufferNotMoved; + } + + // Copy the nursery-allocated buffer into a new malloc allocation. + + AutoEnterOOMUnsafeRegion oomUnsafe; + Zone* zone = owner->asTenured().zone(); + void* movedBuffer = zone->pod_arena_malloc<uint8_t>(arena, nbytes); + if (!movedBuffer) { + oomUnsafe.crash("Nursery::updateBufferOnPromotion"); + } + + memcpy(movedBuffer, buffer, nbytes); + + AddCellMemory(owner, nbytes, use); + + *bufferp = movedBuffer; + return BufferMoved; +} + +void Nursery::requestMinorGC(JS::GCReason reason) { + MOZ_ASSERT(reason != JS::GCReason::NO_REASON); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + MOZ_ASSERT(isEnabled()); + + if (minorGCRequested()) { + return; + } + + // Set position to end of chunk to block further allocation. + MOZ_ASSERT(prevPosition_ == 0); + prevPosition_ = position_; + position_ = chunk(currentChunk_).end(); + + minorGCTriggerReason_ = reason; + runtime()->mainContextFromOwnThread()->requestInterrupt( + InterruptReason::MinorGC); +} + +size_t Nursery::sizeOfMallocedBuffers( + mozilla::MallocSizeOf mallocSizeOf) const { + size_t total = 0; + for (BufferSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront()) { + total += mallocSizeOf(r.front()); + } + total += mallocedBuffers.shallowSizeOfExcludingThis(mallocSizeOf); + return total; +} + +void js::Nursery::sweep() { + // It's important that the context's GCUse is not Finalizing at this point, + // otherwise we will miscount memory attached to nursery objects with + // CellAllocPolicy. + AutoSetThreadIsSweeping setThreadSweeping(runtime()->gcContext()); + + MinorSweepingTracer trc(runtime()); + + // Sweep unique IDs first before we sweep any tables that may be keyed based + // on them. + for (Cell* cell : cellsWithUid_) { + auto* obj = static_cast<JSObject*>(cell); + if (!IsForwarded(obj)) { + gc::RemoveUniqueId(obj); + } else { + JSObject* dst = Forwarded(obj); + gc::TransferUniqueId(dst, obj); + } + } + cellsWithUid_.clear(); + + for (ZonesIter zone(runtime(), SkipAtoms); !zone.done(); zone.next()) { + zone->sweepAfterMinorGC(&trc); + } + + sweepMapAndSetObjects(); + + runtime()->caches().sweepAfterMinorGC(&trc); +} + +void js::Nursery::clear() { + // Poison the nursery contents so touching a freed object will crash. + unsigned firstClearChunk; + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + // Poison all the chunks used in this cycle. The new start chunk is + // reposioned in Nursery::collect() but there's no point optimising that in + // this case. + firstClearChunk = startChunk_; + } else { + // In normal mode we start at the second chunk, the first one will be used + // in the next cycle and poisoned in Nusery::collect(); + MOZ_ASSERT(startChunk_ == 0); + firstClearChunk = 1; + } + for (unsigned i = firstClearChunk; i < currentChunk_; ++i) { + chunk(i).poisonAfterEvict(); + } + // Clear only the used part of the chunk because that's the part we touched, + // but only if it's not going to be re-used immediately (>= firstClearChunk). + if (currentChunk_ >= firstClearChunk) { + chunk(currentChunk_) + .poisonAfterEvict(position() - chunk(currentChunk_).start()); + } + + // Reset the start chunk & position if we're not in this zeal mode, or we're + // in it and close to the end of the nursery. + MOZ_ASSERT(maxChunkCount() > 0); + if (!gc->hasZealMode(ZealMode::GenerationalGC) || + (gc->hasZealMode(ZealMode::GenerationalGC) && + currentChunk_ + 1 == maxChunkCount())) { + moveToStartOfChunk(0); + } + + // Set current start position for isEmpty checks. + setStartToCurrentPosition(); +} + +MOZ_ALWAYS_INLINE void js::Nursery::moveToStartOfChunk(unsigned chunkno) { + MOZ_ASSERT(chunkno < allocatedChunkCount()); + + currentChunk_ = chunkno; + position_ = chunk(chunkno).start(); + setCurrentEnd(); + + MOZ_ASSERT(position_ != 0); + MOZ_ASSERT(currentEnd_ > position_); // Check this cannot wrap. +} + +void js::Nursery::poisonAndInitCurrentChunk(size_t extent) { + if (gc->hasZealMode(ZealMode::GenerationalGC) || !isSubChunkMode()) { + chunk(currentChunk_).poisonAndInit(runtime()); + } else { + extent = std::min(capacity_, extent); + chunk(currentChunk_).poisonAndInit(runtime(), extent); + } +} + +MOZ_ALWAYS_INLINE void js::Nursery::setCurrentEnd() { + MOZ_ASSERT_IF(isSubChunkMode(), + currentChunk_ == 0 && currentEnd_ <= chunk(0).end()); + currentEnd_ = + uintptr_t(&chunk(currentChunk_)) + std::min(capacity_, ChunkSize); + + MOZ_ASSERT_IF(!isSubChunkMode(), currentEnd_ == chunk(currentChunk_).end()); + MOZ_ASSERT(currentEnd_ != chunk(currentChunk_).start()); +} + +bool js::Nursery::allocateNextChunk(const unsigned chunkno, + AutoLockGCBgAlloc& lock) { + const unsigned priorCount = allocatedChunkCount(); + const unsigned newCount = priorCount + 1; + + MOZ_ASSERT((chunkno == currentChunk_ + 1) || + (chunkno == 0 && allocatedChunkCount() == 0)); + MOZ_ASSERT(chunkno == allocatedChunkCount()); + MOZ_ASSERT(chunkno < HowMany(capacity(), ChunkSize)); + + if (!chunks_.resize(newCount)) { + return false; + } + + TenuredChunk* newChunk; + newChunk = gc->getOrAllocChunk(lock); + if (!newChunk) { + chunks_.shrinkTo(priorCount); + return false; + } + + chunks_[chunkno] = NurseryChunk::fromChunk(newChunk); + return true; +} + +MOZ_ALWAYS_INLINE void js::Nursery::setStartToCurrentPosition() { + startChunk_ = currentChunk_; + startPosition_ = position(); +} + +void js::Nursery::maybeResizeNursery(JS::GCOptions options, + JS::GCReason reason) { +#ifdef JS_GC_ZEAL + // This zeal mode disabled nursery resizing. + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + return; + } +#endif + + decommitTask->join(); + + size_t newCapacity = mozilla::Clamp(targetSize(options, reason), + tunables().gcMinNurseryBytes(), + tunables().gcMaxNurseryBytes()); + + MOZ_ASSERT(roundSize(newCapacity) == newCapacity); + MOZ_ASSERT(newCapacity >= SystemPageSize()); + + if (newCapacity > capacity()) { + growAllocableSpace(newCapacity); + } else if (newCapacity < capacity()) { + shrinkAllocableSpace(newCapacity); + } + + AutoLockHelperThreadState lock; + if (!decommitTask->isEmpty(lock)) { + decommitTask->startOrRunIfIdle(lock); + } +} + +static inline bool ClampDouble(double* value, double min, double max) { + MOZ_ASSERT(!std::isnan(*value) && !std::isnan(min) && !std::isnan(max)); + MOZ_ASSERT(max >= min); + + if (*value <= min) { + *value = min; + return true; + } + + if (*value >= max) { + *value = max; + return true; + } + + return false; +} + +size_t js::Nursery::targetSize(JS::GCOptions options, JS::GCReason reason) { + // Shrink the nursery as much as possible if purging was requested or in low + // memory situations. + if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason) || + gc->systemHasLowMemory()) { + clearRecentGrowthData(); + return 0; + } + + // Don't resize the nursery during shutdown. + if (options == JS::GCOptions::Shutdown) { + clearRecentGrowthData(); + return capacity(); + } + + TimeStamp now = TimeStamp::Now(); + + if (reason == JS::GCReason::PREPARE_FOR_PAGELOAD) { + return roundSize(tunables().gcMaxNurseryBytes()); + } + + // If the nursery is completely unused then minimise it. + if (hasRecentGrowthData && previousGC.nurseryUsedBytes == 0 && + now - lastCollectionEndTime() > + tunables().nurseryTimeoutForIdleCollection() && + !js::SupportDifferentialTesting()) { + clearRecentGrowthData(); + return 0; + } + + // Calculate the fraction of the nursery promoted out of its entire + // capacity. This gives better results than using the promotion rate (based on + // the amount of nursery used) in cases where we collect before the nursery is + // full. + double fractionPromoted = + double(previousGC.tenuredBytes) / double(previousGC.nurseryCapacity); + + // Calculate the duty factor, the fraction of time spent collecting the + // nursery. + double dutyFactor = 0.0; + TimeDuration collectorTime = now - collectionStartTime(); + if (hasRecentGrowthData && !js::SupportDifferentialTesting()) { + TimeDuration totalTime = now - lastCollectionEndTime(); + dutyFactor = collectorTime.ToSeconds() / totalTime.ToSeconds(); + } + + // Calculate a growth factor to try to achieve target promotion rate and duty + // factor goals. + static const double PromotionGoal = 0.02; + static const double DutyFactorGoal = 0.01; + double promotionGrowth = fractionPromoted / PromotionGoal; + double dutyGrowth = dutyFactor / DutyFactorGoal; + double growthFactor = std::max(promotionGrowth, dutyGrowth); + +#ifndef DEBUG + // In optimized builds, decrease the growth factor to try to keep collections + // shorter than a target maximum time. Don't do this during page load. + // + // Debug builds are so much slower and more unpredictable that doing this + // would cause very different nursery behaviour to an equivalent release + // build. + static const double MaxTimeGoalMs = 4.0; + if (!gc->isInPageLoad() && !js::SupportDifferentialTesting()) { + double timeGrowth = MaxTimeGoalMs / collectorTime.ToMilliseconds(); + growthFactor = std::min(growthFactor, timeGrowth); + } +#endif + + // Limit the range of the growth factor to prevent transient high promotion + // rates from affecting the nursery size too far into the future. + static const double GrowthRange = 2.0; + bool wasClamped = ClampDouble(&growthFactor, 1.0 / GrowthRange, GrowthRange); + + // Calculate the target size based on data from this collection. + double target = double(capacity()) * growthFactor; + + // Use exponential smoothing on the target size to take into account data from + // recent previous collections. + if (hasRecentGrowthData && + now - lastCollectionEndTime() < TimeDuration::FromMilliseconds(200) && + !js::SupportDifferentialTesting()) { + // Pay more attention to large changes. + double fraction = wasClamped ? 0.5 : 0.25; + smoothedTargetSize = + (1 - fraction) * smoothedTargetSize + fraction * target; + } else { + smoothedTargetSize = target; + } + hasRecentGrowthData = true; + + // Leave size untouched if we are close to the target. + static const double GoalWidth = 1.5; + growthFactor = smoothedTargetSize / double(capacity()); + if (growthFactor > (1.0 / GoalWidth) && growthFactor < GoalWidth) { + return capacity(); + } + + return roundSize(size_t(smoothedTargetSize)); +} + +void js::Nursery::clearRecentGrowthData() { + if (js::SupportDifferentialTesting()) { + return; + } + + hasRecentGrowthData = false; + smoothedTargetSize = 0.0; +} + +/* static */ +size_t js::Nursery::roundSize(size_t size) { + size_t step = size >= ChunkSize ? ChunkSize : SystemPageSize(); + return Round(size, step); +} + +void js::Nursery::growAllocableSpace(size_t newCapacity) { + MOZ_ASSERT_IF(!isSubChunkMode(), newCapacity > currentChunk_ * ChunkSize); + MOZ_ASSERT(newCapacity <= tunables().gcMaxNurseryBytes()); + MOZ_ASSERT(newCapacity > capacity()); + + if (!decommitTask->reserveSpaceForBytes(newCapacity)) { + return; + } + + if (isSubChunkMode()) { + MOZ_ASSERT(currentChunk_ == 0); + + // The remainder of the chunk may have been decommitted. + if (!chunk(0).markPagesInUseHard(std::min(newCapacity, ChunkSize))) { + // The OS won't give us the memory we need, we can't grow. + return; + } + + // The capacity has changed and since we were in sub-chunk mode we need to + // update the poison values / asan information for the now-valid region of + // this chunk. + size_t end = std::min(newCapacity, ChunkSize); + chunk(0).poisonRange(capacity(), end, JS_FRESH_NURSERY_PATTERN, + MemCheckKind::MakeUndefined); + } + + capacity_ = newCapacity; + + setCurrentEnd(); +} + +void js::Nursery::freeChunksFrom(const unsigned firstFreeChunk) { + MOZ_ASSERT(firstFreeChunk < chunks_.length()); + + // The loop below may need to skip the first chunk, so we may use this so we + // can modify it. + unsigned firstChunkToDecommit = firstFreeChunk; + + if ((firstChunkToDecommit == 0) && isSubChunkMode()) { + // Part of the first chunk may be hard-decommitted, un-decommit it so that + // the GC's normal chunk-handling doesn't segfault. + MOZ_ASSERT(currentChunk_ == 0); + if (!chunk(0).markPagesInUseHard(ChunkSize)) { + // Free the chunk if we can't allocate its pages. + UnmapPages(static_cast<void*>(&chunk(0)), ChunkSize); + firstChunkToDecommit = 1; + } + } + + { + AutoLockHelperThreadState lock; + for (size_t i = firstChunkToDecommit; i < chunks_.length(); i++) { + decommitTask->queueChunk(chunks_[i], lock); + } + } + + chunks_.shrinkTo(firstFreeChunk); +} + +void js::Nursery::shrinkAllocableSpace(size_t newCapacity) { +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + return; + } +#endif + + // Don't shrink the nursery to zero (use Nursery::disable() instead) + // This can't happen due to the rounding-down performed above because of the + // clamping in maybeResizeNursery(). + MOZ_ASSERT(newCapacity != 0); + // Don't attempt to shrink it to the same size. + if (newCapacity == capacity_) { + return; + } + MOZ_ASSERT(newCapacity < capacity_); + + unsigned newCount = HowMany(newCapacity, ChunkSize); + if (newCount < allocatedChunkCount()) { + freeChunksFrom(newCount); + } + + size_t oldCapacity = capacity_; + capacity_ = newCapacity; + + setCurrentEnd(); + + if (isSubChunkMode()) { + MOZ_ASSERT(currentChunk_ == 0); + size_t end = std::min(oldCapacity, ChunkSize); + chunk(0).poisonRange(newCapacity, end, JS_SWEPT_NURSERY_PATTERN, + MemCheckKind::MakeNoAccess); + + AutoLockHelperThreadState lock; + decommitTask->queueRange(capacity_, chunk(0), lock); + } +} + +gcstats::Statistics& js::Nursery::stats() const { return gc->stats(); } + +MOZ_ALWAYS_INLINE const js::gc::GCSchedulingTunables& js::Nursery::tunables() + const { + return gc->tunables; +} + +bool js::Nursery::isSubChunkMode() const { + return capacity() <= NurseryChunkUsableSize; +} + +void js::Nursery::sweepMapAndSetObjects() { + auto* gcx = runtime()->gcContext(); + + for (auto* mapobj : mapsWithNurseryMemory_) { + MapObject::sweepAfterMinorGC(gcx, mapobj); + } + mapsWithNurseryMemory_.clearAndFree(); + + for (auto* setobj : setsWithNurseryMemory_) { + SetObject::sweepAfterMinorGC(gcx, setobj); + } + setsWithNurseryMemory_.clearAndFree(); +} + +void js::Nursery::joinDecommitTask() { decommitTask->join(); } + +JS_PUBLIC_API void JS::EnableNurseryStrings(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().enableStrings(); +} + +JS_PUBLIC_API void JS::DisableNurseryStrings(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().disableStrings(); +} + +JS_PUBLIC_API void JS::EnableNurseryBigInts(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().enableBigInts(); +} + +JS_PUBLIC_API void JS::DisableNurseryBigInts(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().disableBigInts(); +} |