From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- js/src/gc/AllocKind.h | 326 +++ js/src/gc/Allocator.cpp | 672 +++++ js/src/gc/Allocator.h | 173 ++ js/src/gc/ArenaList-inl.h | 327 +++ js/src/gc/ArenaList.h | 386 +++ js/src/gc/AtomMarking-inl.h | 96 + js/src/gc/AtomMarking.cpp | 294 ++ js/src/gc/AtomMarking.h | 86 + js/src/gc/Barrier.cpp | 144 + js/src/gc/Barrier.h | 1257 +++++++++ js/src/gc/Cell.h | 930 +++++++ js/src/gc/ClearEdgesTracer.h | 27 + js/src/gc/Compacting.cpp | 967 +++++++ js/src/gc/FinalizationObservers.cpp | 509 ++++ js/src/gc/FinalizationObservers.h | 129 + js/src/gc/FindSCCs.h | 207 ++ js/src/gc/GC-inl.h | 344 +++ js/src/gc/GC.cpp | 5101 +++++++++++++++++++++++++++++++++++ js/src/gc/GC.h | 243 ++ js/src/gc/GCAPI.cpp | 798 ++++++ js/src/gc/GCContext-inl.h | 40 + js/src/gc/GCContext.h | 257 ++ js/src/gc/GCEnum.h | 160 ++ js/src/gc/GCInternals.h | 344 +++ js/src/gc/GCLock.h | 110 + js/src/gc/GCMarker.h | 598 ++++ js/src/gc/GCParallelTask.cpp | 231 ++ js/src/gc/GCParallelTask.h | 246 ++ js/src/gc/GCProbes.h | 49 + js/src/gc/GCRuntime.h | 1444 ++++++++++ js/src/gc/GenerateStatsPhases.py | 404 +++ js/src/gc/HashUtil.h | 84 + js/src/gc/Heap-inl.h | 70 + js/src/gc/Heap.cpp | 635 +++++ js/src/gc/Heap.h | 846 ++++++ js/src/gc/IteratorUtils.h | 121 + js/src/gc/MallocedBlockCache.cpp | 144 + js/src/gc/MallocedBlockCache.h | 91 + js/src/gc/Marking-inl.h | 196 ++ js/src/gc/Marking.cpp | 2774 +++++++++++++++++++ js/src/gc/Marking.h | 150 + js/src/gc/MaybeRooted.h | 141 + js/src/gc/Memory.cpp | 1050 +++++++ js/src/gc/Memory.h | 84 + js/src/gc/Nursery-inl.h | 172 ++ js/src/gc/Nursery.cpp | 2083 ++++++++++++++ js/src/gc/Nursery.h | 645 +++++ js/src/gc/NurseryAwareHashMap.h | 209 ++ js/src/gc/ObjectKind-inl.h | 162 ++ js/src/gc/ParallelMarking.cpp | 359 +++ js/src/gc/ParallelMarking.h | 123 + js/src/gc/ParallelWork.h | 136 + js/src/gc/Policy.h | 101 + js/src/gc/Pretenuring.cpp | 459 ++++ js/src/gc/Pretenuring.h | 348 +++ js/src/gc/PrivateIterators-inl.h | 164 ++ js/src/gc/PublicIterators.cpp | 272 ++ js/src/gc/PublicIterators.h | 158 ++ js/src/gc/RelocationOverlay.h | 66 + js/src/gc/RootMarking.cpp | 467 ++++ js/src/gc/Scheduling.cpp | 873 ++++++ js/src/gc/Scheduling.h | 917 +++++++ js/src/gc/StableCellHasher-inl.h | 245 ++ js/src/gc/StableCellHasher.h | 46 + js/src/gc/Statistics.cpp | 1811 +++++++++++++ js/src/gc/Statistics.h | 606 +++++ js/src/gc/StoreBuffer-inl.h | 97 + js/src/gc/StoreBuffer.cpp | 255 ++ js/src/gc/StoreBuffer.h | 662 +++++ js/src/gc/Sweeping.cpp | 2383 ++++++++++++++++ js/src/gc/Tenuring.cpp | 1016 +++++++ js/src/gc/Tenuring.h | 169 ++ js/src/gc/TraceKind.h | 59 + js/src/gc/TraceMethods-inl.h | 384 +++ js/src/gc/Tracer.cpp | 297 ++ js/src/gc/Tracer.h | 401 +++ js/src/gc/Verifier.cpp | 1135 ++++++++ js/src/gc/WeakMap-inl.h | 413 +++ js/src/gc/WeakMap.cpp | 175 ++ js/src/gc/WeakMap.h | 354 +++ js/src/gc/WeakMapPtr.cpp | 114 + js/src/gc/Zone.cpp | 979 +++++++ js/src/gc/Zone.h | 653 +++++ js/src/gc/ZoneAllocator.h | 354 +++ js/src/gc/moz.build | 61 + 85 files changed, 43668 insertions(+) create mode 100644 js/src/gc/AllocKind.h create mode 100644 js/src/gc/Allocator.cpp create mode 100644 js/src/gc/Allocator.h create mode 100644 js/src/gc/ArenaList-inl.h create mode 100644 js/src/gc/ArenaList.h create mode 100644 js/src/gc/AtomMarking-inl.h create mode 100644 js/src/gc/AtomMarking.cpp create mode 100644 js/src/gc/AtomMarking.h create mode 100644 js/src/gc/Barrier.cpp create mode 100644 js/src/gc/Barrier.h create mode 100644 js/src/gc/Cell.h create mode 100644 js/src/gc/ClearEdgesTracer.h create mode 100644 js/src/gc/Compacting.cpp create mode 100644 js/src/gc/FinalizationObservers.cpp create mode 100644 js/src/gc/FinalizationObservers.h create mode 100644 js/src/gc/FindSCCs.h create mode 100644 js/src/gc/GC-inl.h create mode 100644 js/src/gc/GC.cpp create mode 100644 js/src/gc/GC.h create mode 100644 js/src/gc/GCAPI.cpp create mode 100644 js/src/gc/GCContext-inl.h create mode 100644 js/src/gc/GCContext.h create mode 100644 js/src/gc/GCEnum.h create mode 100644 js/src/gc/GCInternals.h create mode 100644 js/src/gc/GCLock.h create mode 100644 js/src/gc/GCMarker.h create mode 100644 js/src/gc/GCParallelTask.cpp create mode 100644 js/src/gc/GCParallelTask.h create mode 100644 js/src/gc/GCProbes.h create mode 100644 js/src/gc/GCRuntime.h create mode 100644 js/src/gc/GenerateStatsPhases.py create mode 100644 js/src/gc/HashUtil.h create mode 100644 js/src/gc/Heap-inl.h create mode 100644 js/src/gc/Heap.cpp create mode 100644 js/src/gc/Heap.h create mode 100644 js/src/gc/IteratorUtils.h create mode 100644 js/src/gc/MallocedBlockCache.cpp create mode 100644 js/src/gc/MallocedBlockCache.h create mode 100644 js/src/gc/Marking-inl.h create mode 100644 js/src/gc/Marking.cpp create mode 100644 js/src/gc/Marking.h create mode 100644 js/src/gc/MaybeRooted.h create mode 100644 js/src/gc/Memory.cpp create mode 100644 js/src/gc/Memory.h create mode 100644 js/src/gc/Nursery-inl.h create mode 100644 js/src/gc/Nursery.cpp create mode 100644 js/src/gc/Nursery.h create mode 100644 js/src/gc/NurseryAwareHashMap.h create mode 100644 js/src/gc/ObjectKind-inl.h create mode 100644 js/src/gc/ParallelMarking.cpp create mode 100644 js/src/gc/ParallelMarking.h create mode 100644 js/src/gc/ParallelWork.h create mode 100644 js/src/gc/Policy.h create mode 100644 js/src/gc/Pretenuring.cpp create mode 100644 js/src/gc/Pretenuring.h create mode 100644 js/src/gc/PrivateIterators-inl.h create mode 100644 js/src/gc/PublicIterators.cpp create mode 100644 js/src/gc/PublicIterators.h create mode 100644 js/src/gc/RelocationOverlay.h create mode 100644 js/src/gc/RootMarking.cpp create mode 100644 js/src/gc/Scheduling.cpp create mode 100644 js/src/gc/Scheduling.h create mode 100644 js/src/gc/StableCellHasher-inl.h create mode 100644 js/src/gc/StableCellHasher.h create mode 100644 js/src/gc/Statistics.cpp create mode 100644 js/src/gc/Statistics.h create mode 100644 js/src/gc/StoreBuffer-inl.h create mode 100644 js/src/gc/StoreBuffer.cpp create mode 100644 js/src/gc/StoreBuffer.h create mode 100644 js/src/gc/Sweeping.cpp create mode 100644 js/src/gc/Tenuring.cpp create mode 100644 js/src/gc/Tenuring.h create mode 100644 js/src/gc/TraceKind.h create mode 100644 js/src/gc/TraceMethods-inl.h create mode 100644 js/src/gc/Tracer.cpp create mode 100644 js/src/gc/Tracer.h create mode 100644 js/src/gc/Verifier.cpp create mode 100644 js/src/gc/WeakMap-inl.h create mode 100644 js/src/gc/WeakMap.cpp create mode 100644 js/src/gc/WeakMap.h create mode 100644 js/src/gc/WeakMapPtr.cpp create mode 100644 js/src/gc/Zone.cpp create mode 100644 js/src/gc/Zone.h create mode 100644 js/src/gc/ZoneAllocator.h create mode 100644 js/src/gc/moz.build (limited to 'js/src/gc') diff --git a/js/src/gc/AllocKind.h b/js/src/gc/AllocKind.h new file mode 100644 index 0000000000..3427d0aba6 --- /dev/null +++ b/js/src/gc/AllocKind.h @@ -0,0 +1,326 @@ + +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definition of GC cell kinds. + */ + +#ifndef gc_AllocKind_h +#define gc_AllocKind_h + +#include "mozilla/EnumeratedArray.h" +#include "mozilla/EnumeratedRange.h" + +#include +#include + +#include "js/TraceKind.h" + +class JSDependentString; +class JSExternalString; +class JSFatInlineString; +class JSLinearString; +class JSRope; +class JSThinInlineString; + +namespace js { + +class CompactPropMap; +class FatInlineAtom; +class NormalAtom; +class NormalPropMap; +class DictionaryPropMap; +class DictionaryShape; +class SharedShape; +class ProxyShape; +class WasmGCShape; + +namespace gc { + +// The GC allocation kinds. +// +// These are defined by macros which enumerate the different allocation kinds +// and supply the following information: +// +// - the corresponding AllocKind +// - their JS::TraceKind +// - their C++ base type +// - a C++ type of the correct size +// - whether they can be finalized on the background thread +// - whether they can be allocated in the nursery (this is true for foreground +// finalized objects but these will can only actually be allocated in the +// nursery if JSCLASS_SKIP_NURSERY_FINALIZE is set) +// - whether they can be compacted + +// clang-format off +#define FOR_EACH_OBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \ + D(FUNCTION, Object, JSObject, JSObject_Slots4, true, true, true) \ + D(FUNCTION_EXTENDED, Object, JSObject, JSObject_Slots6, true, true, true) \ + D(OBJECT0, Object, JSObject, JSObject_Slots0, false, true, true) \ + D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0, true, true, true) \ + D(OBJECT2, Object, JSObject, JSObject_Slots2, false, true, true) \ + D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2, true, true, true) \ + D(ARRAYBUFFER4, Object, JSObject, JSObject_Slots4, true, true, true) \ + D(OBJECT4, Object, JSObject, JSObject_Slots4, false, true, true) \ + D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4, true, true, true) \ + D(ARRAYBUFFER8, Object, JSObject, JSObject_Slots8, true, true, true) \ + D(OBJECT8, Object, JSObject, JSObject_Slots8, false, true, true) \ + D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8, true, true, true) \ + D(ARRAYBUFFER12, Object, JSObject, JSObject_Slots12, true, true, true) \ + D(OBJECT12, Object, JSObject, JSObject_Slots12, false, true, true) \ + D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12, true, true, true) \ + D(ARRAYBUFFER16, Object, JSObject, JSObject_Slots16, true, true, true) \ + D(OBJECT16, Object, JSObject, JSObject_Slots16, false, true, true) \ + D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16, true, true, true) + +#define FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \ + D(SCRIPT, Script, js::BaseScript, js::BaseScript, false, false, true) \ + D(SHAPE, Shape, js::Shape, js::SizedShape, true, false, true) \ + D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape, true, false, true) \ + D(GETTER_SETTER, GetterSetter, js::GetterSetter, js::GetterSetter, true, false, true) \ + D(COMPACT_PROP_MAP, PropMap, js::CompactPropMap, js::CompactPropMap, true, false, true) \ + D(NORMAL_PROP_MAP, PropMap, js::NormalPropMap, js::NormalPropMap, true, false, true) \ + D(DICT_PROP_MAP, PropMap, js::DictionaryPropMap, js::DictionaryPropMap, true, false, true) \ + D(EXTERNAL_STRING, String, JSExternalString, JSExternalString, true, false, true) \ + D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom, true, false, false) \ + D(ATOM, String, js::NormalAtom, js::NormalAtom, true, false, false) \ + D(SYMBOL, Symbol, JS::Symbol, JS::Symbol, true, false, false) \ + D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode, false, false, false) \ + D(SCOPE, Scope, js::Scope, js::Scope, true, false, true) \ + D(REGEXP_SHARED, RegExpShared, js::RegExpShared, js::RegExpShared, true, false, true) + +#define FOR_EACH_NONOBJECT_NURSERY_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \ + D(BIGINT, BigInt, JS::BigInt, JS::BigInt, true, true, true) + +#define FOR_EACH_NURSERY_STRING_ALLOCKIND(D) \ + D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString, true, true, true) \ + D(STRING, String, JSString, JSString, true, true, true) +// clang-format on + +#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_NURSERY_ALLOCKIND(D) \ + FOR_EACH_NURSERY_STRING_ALLOCKIND(D) + +#define FOR_EACH_ALLOCKIND(D) \ + FOR_EACH_OBJECT_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_ALLOCKIND(D) + +#define DEFINE_ALLOC_KIND(allocKind, _1, _2, _3, _4, _5, _6) allocKind, +enum class AllocKind : uint8_t { + // clang-format off + FOR_EACH_OBJECT_ALLOCKIND(DEFINE_ALLOC_KIND) + + OBJECT_LIMIT, + OBJECT_LAST = OBJECT_LIMIT - 1, + + FOR_EACH_NONOBJECT_ALLOCKIND(DEFINE_ALLOC_KIND) + + LIMIT, + LAST = LIMIT - 1, + + FIRST = 0, + OBJECT_FIRST = FUNCTION // Hardcoded to first object kind. + // clang-format on +}; +#undef DEFINE_ALLOC_KIND + +static_assert(int(AllocKind::FIRST) == 0, + "Various places depend on AllocKind starting at 0"); +static_assert(int(AllocKind::OBJECT_FIRST) == 0, + "OBJECT_FIRST must be defined as the first object kind"); + +constexpr size_t AllocKindCount = size_t(AllocKind::LIMIT); + +/* + * A flag specifying either the tenured heap or a default heap (which may be + * either the nursery or the tenured heap). + * + * This allows an allocation site to request a heap based upon the estimated + * lifetime or lifetime requirements of objects allocated from that site. + * + * Order is important as these are numerically compared. + */ +enum class Heap : uint8_t { Default = 0, Tenured = 1 }; + +inline bool IsAllocKind(AllocKind kind) { + return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT; +} + +inline bool IsValidAllocKind(AllocKind kind) { + return kind >= AllocKind::FIRST && kind <= AllocKind::LAST; +} + +const char* AllocKindName(AllocKind kind); + +inline bool IsObjectAllocKind(AllocKind kind) { + return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST; +} + +inline bool IsShapeAllocKind(AllocKind kind) { + return kind == AllocKind::SHAPE; +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all alloc kinds. +inline auto AllAllocKinds() { + return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all object alloc kinds. +inline auto ObjectAllocKinds() { + return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, + AllocKind::OBJECT_LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over alloc kinds from |first| to |limit|, exclusive. +inline auto SomeAllocKinds(AllocKind first = AllocKind::FIRST, + AllocKind limit = AllocKind::LIMIT) { + MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!"); + MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!"); + return mozilla::MakeEnumeratedRange(first, limit); +} + +// AllAllocKindArray gives an enumerated array of ValueTypes, +// with each index corresponding to a particular alloc kind. +template +using AllAllocKindArray = + mozilla::EnumeratedArray; + +// ObjectAllocKindArray gives an enumerated array of ValueTypes, +// with each index corresponding to a particular object alloc kind. +template +using ObjectAllocKindArray = + mozilla::EnumeratedArray; + +/* + * Map from C++ type to alloc kind for non-object types. JSObject does not have + * a 1:1 mapping, so must use Arena::thingSize. + * + * The AllocKind is available as MapTypeToAllocKind::kind. + * + * There are specializations for strings and shapes since more than one derived + * type shares the same alloc kind. + */ +template +struct MapTypeToAllocKind {}; +#define EXPAND_MAPTYPETOALLOCKIND(allocKind, traceKind, type, sizedType, \ + bgFinal, nursery, compact) \ + template <> \ + struct MapTypeToAllocKind { \ + static const AllocKind kind = AllocKind::allocKind; \ + }; +FOR_EACH_NONOBJECT_ALLOCKIND(EXPAND_MAPTYPETOALLOCKIND) +#undef EXPAND_MAPTYPETOALLOCKIND + +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::STRING; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::STRING; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::STRING; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::STRING; +}; + +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::SHAPE; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::SHAPE; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::SHAPE; +}; +template <> +struct MapTypeToAllocKind { + static const AllocKind kind = AllocKind::SHAPE; +}; + +static inline JS::TraceKind MapAllocToTraceKind(AllocKind kind) { + static const JS::TraceKind map[] = { +#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType, bgFinal, \ + nursery, compact) \ + JS::TraceKind::traceKind, + FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) +#undef EXPAND_ELEMENT + }; + + static_assert(std::size(map) == AllocKindCount, + "AllocKind-to-TraceKind mapping must be in sync"); + return map[size_t(kind)]; +} + +static inline bool IsNurseryAllocable(AllocKind kind) { + MOZ_ASSERT(IsValidAllocKind(kind)); + + static const bool map[] = { +#define DEFINE_NURSERY_ALLOCABLE(_1, _2, _3, _4, _5, nursery, _6) nursery, + FOR_EACH_ALLOCKIND(DEFINE_NURSERY_ALLOCABLE) +#undef DEFINE_NURSERY_ALLOCABLE + }; + + static_assert(std::size(map) == AllocKindCount, + "IsNurseryAllocable sanity check"); + return map[size_t(kind)]; +} + +static inline bool IsBackgroundFinalized(AllocKind kind) { + MOZ_ASSERT(IsValidAllocKind(kind)); + + static const bool map[] = { +#define DEFINE_BACKGROUND_FINALIZED(_1, _2, _3, _4, bgFinal, _5, _6) bgFinal, + FOR_EACH_ALLOCKIND(DEFINE_BACKGROUND_FINALIZED) +#undef DEFINE_BACKGROUND_FINALIZED + }; + + static_assert(std::size(map) == AllocKindCount, + "IsBackgroundFinalized sanity check"); + return map[size_t(kind)]; +} + +static inline bool IsForegroundFinalized(AllocKind kind) { + return !IsBackgroundFinalized(kind); +} + +static inline bool IsCompactingKind(AllocKind kind) { + MOZ_ASSERT(IsValidAllocKind(kind)); + + static const bool map[] = { +#define DEFINE_COMPACTING_KIND(_1, _2, _3, _4, _5, _6, compact) compact, + FOR_EACH_ALLOCKIND(DEFINE_COMPACTING_KIND) +#undef DEFINE_COMPACTING_KIND + }; + + static_assert(std::size(map) == AllocKindCount, + "IsCompactingKind sanity check"); + return map[size_t(kind)]; +} + +static inline bool IsMovableKind(AllocKind kind) { + return IsNurseryAllocable(kind) || IsCompactingKind(kind); +} + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_AllocKind_h */ diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp new file mode 100644 index 0000000000..92d4653961 --- /dev/null +++ b/js/src/gc/Allocator.cpp @@ -0,0 +1,672 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Allocator.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/OperatorNewExtensions.h" +#include "mozilla/TimeStamp.h" + +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/GCProbes.h" +#include "gc/Nursery.h" +#include "threading/CpuCount.h" +#include "util/Poison.h" +#include "vm/BigIntType.h" +#include "vm/JSContext.h" +#include "vm/Runtime.h" +#include "vm/StringType.h" + +#include "gc/ArenaList-inl.h" +#include "gc/Heap-inl.h" +#include "gc/PrivateIterators-inl.h" +#include "vm/JSContext-inl.h" + +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +using namespace js; +using namespace js::gc; + +// Return a Heap value that can be compared numerically with an +// allocation's requested heap to determine whether to allocate in the nursery +// or the tenured heap. +// +// If nursery allocation is allowed this returns Heap::Tenured, meaning only +// Heap::Tenured allocations will be tenured. If nursery allocation is not +// allowed this returns Heap::Default, meaning all allocations are tenured. +static Heap MinHeapToTenure(bool allowNurseryAlloc) { + static_assert(Heap::Tenured > Heap::Default); + return allowNurseryAlloc ? Heap::Tenured : Heap::Default; +} + +void Zone::setNurseryAllocFlags(bool allocObjects, bool allocStrings, + bool allocBigInts) { + allocNurseryObjects_ = allocObjects; + allocNurseryStrings_ = allocStrings; + allocNurseryBigInts_ = allocBigInts; + + minObjectHeapToTenure_ = MinHeapToTenure(allocNurseryObjects()); + minStringHeapToTenure_ = MinHeapToTenure(allocNurseryStrings()); + minBigintHeapToTenure_ = MinHeapToTenure(allocNurseryBigInts()); +} + +template +void* gc::CellAllocator::AllocNurseryOrTenuredCell(JSContext* cx, + AllocKind allocKind, + gc::Heap heap, + AllocSite* site) { + MOZ_ASSERT(!cx->isHelperThreadContext()); + MOZ_ASSERT_IF(heap != gc::Heap::Tenured, IsNurseryAllocable(allocKind)); + MOZ_ASSERT(MapAllocToTraceKind(allocKind) == traceKind); + MOZ_ASSERT_IF(site && site->initialHeap() == Heap::Tenured, + heap == Heap::Tenured); + + size_t thingSize = Arena::thingSize(allocKind); + + JSRuntime* rt = cx->runtime(); + if (!rt->gc.checkAllocatorState(cx, allocKind)) { + return nullptr; + } + + if (heap < cx->zone()->minHeapToTenure(traceKind)) { + if (!site) { + site = cx->zone()->unknownAllocSite(traceKind); + } + + void* obj = + rt->gc.tryNewNurseryCell(cx, thingSize, site); + if (obj) { + return obj; + } + + // Our most common non-jit allocation path is NoGC; thus, if we fail the + // alloc and cannot GC, we *must* return nullptr here so that the caller + // will do a CanGC allocation to clear the nursery. Failing to do so will + // cause all allocations on this path to land in Tenured, and we will not + // get the benefit of the nursery. + if (!allowGC) { + return nullptr; + } + } + + return GCRuntime::tryNewTenuredThing(cx, allocKind, thingSize); +} + +#define INSTANTIATE_ALLOC_NURSERY_CELL(traceKind, allowGc) \ + template void* \ + gc::CellAllocator::AllocNurseryOrTenuredCell( \ + JSContext*, AllocKind, gc::Heap, AllocSite*); +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::Object, NoGC) +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::Object, CanGC) +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::String, NoGC) +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::String, CanGC) +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::BigInt, NoGC) +INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::BigInt, CanGC) +#undef INSTANTIATE_ALLOC_NURSERY_CELL + +// Attempt to allocate a new cell in the nursery. If there is not enough room in +// the nursery or there is an OOM, this method will return nullptr. +template +void* GCRuntime::tryNewNurseryCell(JSContext* cx, size_t thingSize, + AllocSite* site) { + MOZ_ASSERT(cx->isNurseryAllocAllowed()); + MOZ_ASSERT(cx->zone() == site->zone()); + MOZ_ASSERT(!cx->zone()->isAtomsZone()); + MOZ_ASSERT(cx->zone()->allocKindInNursery(kind)); + + void* ptr = cx->nursery().allocateCell(site, thingSize, kind); + if (ptr) { + return ptr; + } + + if constexpr (allowGC) { + if (!cx->suppressGC) { + cx->runtime()->gc.minorGC(JS::GCReason::OUT_OF_NURSERY); + + // Exceeding gcMaxBytes while tenuring can disable the Nursery. + if (cx->zone()->allocKindInNursery(kind)) { + return cx->nursery().allocateCell(site, thingSize, kind); + } + } + } + + return nullptr; +} + +template +void* gc::CellAllocator::AllocTenuredCell(JSContext* cx, gc::AllocKind kind, + size_t size) { + MOZ_ASSERT(!cx->isHelperThreadContext()); + MOZ_ASSERT(!IsNurseryAllocable(kind)); + MOZ_ASSERT(size == Arena::thingSize(kind)); + MOZ_ASSERT( + size >= gc::MinCellSize, + "All allocations must be at least the allocator-imposed minimum size."); + + if (!cx->runtime()->gc.checkAllocatorState(cx, kind)) { + return nullptr; + } + + return GCRuntime::tryNewTenuredThing(cx, kind, size); +} +template void* gc::CellAllocator::AllocTenuredCell(JSContext*, AllocKind, + size_t); +template void* gc::CellAllocator::AllocTenuredCell(JSContext*, AllocKind, + size_t); + +template +/* static */ +void* GCRuntime::tryNewTenuredThing(JSContext* cx, AllocKind kind, + size_t thingSize) { + // Bump allocate in the arena's current free-list span. + Zone* zone = cx->zone(); + void* ptr = zone->arenas.freeLists().allocate(kind); + if (MOZ_UNLIKELY(!ptr)) { + // Get the next available free list and allocate out of it. This may + // acquire a new arena, which will lock the chunk list. If there are no + // chunks available it may also allocate new memory directly. + ptr = refillFreeList(cx, kind); + + if (MOZ_UNLIKELY(!ptr)) { + if constexpr (allowGC) { + cx->runtime()->gc.attemptLastDitchGC(cx); + ptr = tryNewTenuredThing(cx, kind, thingSize); + if (ptr) { + return ptr; + } + ReportOutOfMemory(cx); + } + + return nullptr; + } + } + +#ifdef DEBUG + checkIncrementalZoneState(cx, ptr); +#endif + + gcprobes::TenuredAlloc(ptr, kind); + + // We count this regardless of the profiler's state, assuming that it costs + // just as much to count it, as to check the profiler's state and decide not + // to count it. + zone->noteTenuredAlloc(); + + return ptr; +} + +void GCRuntime::attemptLastDitchGC(JSContext* cx) { + // Either there was no memory available for a new chunk or the heap hit its + // size limit. Try to perform an all-compartments, non-incremental, shrinking + // GC and wait for it to finish. + + if (!lastLastDitchTime.IsNull() && + TimeStamp::Now() - lastLastDitchTime <= tunables.minLastDitchGCPeriod()) { + return; + } + + JS::PrepareForFullGC(cx); + gc(JS::GCOptions::Shrink, JS::GCReason::LAST_DITCH); + waitBackgroundAllocEnd(); + waitBackgroundFreeEnd(); + + lastLastDitchTime = mozilla::TimeStamp::Now(); +} + +template +bool GCRuntime::checkAllocatorState(JSContext* cx, AllocKind kind) { + MOZ_ASSERT_IF(cx->zone()->isAtomsZone(), + kind == AllocKind::ATOM || kind == AllocKind::FAT_INLINE_ATOM || + kind == AllocKind::SYMBOL || kind == AllocKind::JITCODE || + kind == AllocKind::SCOPE); + MOZ_ASSERT_IF(!cx->zone()->isAtomsZone(), + kind != AllocKind::ATOM && kind != AllocKind::FAT_INLINE_ATOM); + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + + if constexpr (allowGC) { + // Crash if we could perform a GC action when it is not safe. + if (!cx->suppressGC) { + cx->verifyIsSafeToGC(); + } + + gcIfNeededAtAllocation(cx); + } + + // For testing out of memory conditions. + if (js::oom::ShouldFailWithOOM()) { + // If we are doing a fallible allocation, percolate up the OOM instead of + // reporting it. + if constexpr (allowGC) { + ReportOutOfMemory(cx); + } + return false; + } + + return true; +} + +inline void GCRuntime::gcIfNeededAtAllocation(JSContext* cx) { +#ifdef JS_GC_ZEAL + if (needZealousGC()) { + runDebugGC(); + } +#endif + + // Invoking the interrupt callback can fail and we can't usefully + // handle that here. Just check in case we need to collect instead. + if (cx->hasAnyPendingInterrupt()) { + gcIfRequested(); + } +} + +#ifdef DEBUG +void GCRuntime::checkIncrementalZoneState(JSContext* cx, void* ptr) { + MOZ_ASSERT(ptr); + TenuredCell* cell = reinterpret_cast(ptr); + TenuredChunkBase* chunk = detail::GetCellChunkBase(cell); + if (cx->zone()->isGCMarkingOrSweeping()) { + MOZ_ASSERT(chunk->markBits.isMarkedBlack(cell)); + } else { + MOZ_ASSERT(!chunk->markBits.isMarkedAny(cell)); + } +} +#endif + +void* js::gc::AllocateCellInGC(Zone* zone, AllocKind thingKind) { + void* ptr = zone->arenas.allocateFromFreeList(thingKind); + if (!ptr) { + AutoEnterOOMUnsafeRegion oomUnsafe; + ptr = GCRuntime::refillFreeListInGC(zone, thingKind); + if (!ptr) { + oomUnsafe.crash(ChunkSize, "Failed to allocate new chunk during GC"); + } + } + return ptr; +} + +// /////////// Arena -> Thing Allocator ////////////////////////////////////// + +void GCRuntime::startBackgroundAllocTaskIfIdle() { + AutoLockHelperThreadState lock; + if (!allocTask.wasStarted(lock)) { + // Join the previous invocation of the task. This will return immediately + // if the thread has never been started. + allocTask.joinWithLockHeld(lock); + allocTask.startWithLockHeld(lock); + } +} + +/* static */ +void* GCRuntime::refillFreeList(JSContext* cx, AllocKind thingKind) { + MOZ_ASSERT(cx->zone()->arenas.freeLists().isEmpty(thingKind)); + MOZ_ASSERT(!cx->isHelperThreadContext()); + + // It should not be possible to allocate on the main thread while we are + // inside a GC. + MOZ_ASSERT(!JS::RuntimeHeapIsBusy(), "allocating while under GC"); + + return cx->zone()->arenas.refillFreeListAndAllocate( + thingKind, ShouldCheckThresholds::CheckThresholds); +} + +/* static */ +void* GCRuntime::refillFreeListInGC(Zone* zone, AllocKind thingKind) { + // Called by compacting GC to refill a free list while we are in a GC. + MOZ_ASSERT(JS::RuntimeHeapIsCollecting()); + MOZ_ASSERT_IF(!JS::RuntimeHeapIsMinorCollecting(), + !zone->runtimeFromMainThread()->gc.isBackgroundSweeping()); + + return zone->arenas.refillFreeListAndAllocate( + thingKind, ShouldCheckThresholds::DontCheckThresholds); +} + +void* ArenaLists::refillFreeListAndAllocate( + AllocKind thingKind, ShouldCheckThresholds checkThresholds) { + MOZ_ASSERT(freeLists().isEmpty(thingKind)); + + JSRuntime* rt = runtimeFromAnyThread(); + + mozilla::Maybe maybeLock; + + // See if we can proceed without taking the GC lock. + if (concurrentUse(thingKind) != ConcurrentUse::None) { + maybeLock.emplace(rt); + } + + Arena* arena = arenaList(thingKind).takeNextArena(); + if (arena) { + // Empty arenas should be immediately freed. + MOZ_ASSERT(!arena->isEmpty()); + + return freeLists().setArenaAndAllocate(arena, thingKind); + } + + // Parallel threads have their own ArenaLists, but chunks are shared; + // if we haven't already, take the GC lock now to avoid racing. + if (maybeLock.isNothing()) { + maybeLock.emplace(rt); + } + + TenuredChunk* chunk = rt->gc.pickChunk(maybeLock.ref()); + if (!chunk) { + return nullptr; + } + + // Although our chunk should definitely have enough space for another arena, + // there are other valid reasons why TenuredChunk::allocateArena() may fail. + arena = rt->gc.allocateArena(chunk, zone_, thingKind, checkThresholds, + maybeLock.ref()); + if (!arena) { + return nullptr; + } + + ArenaList& al = arenaList(thingKind); + MOZ_ASSERT(al.isCursorAtEnd()); + al.insertBeforeCursor(arena); + + return freeLists().setArenaAndAllocate(arena, thingKind); +} + +inline void* FreeLists::setArenaAndAllocate(Arena* arena, AllocKind kind) { +#ifdef DEBUG + auto old = freeLists_[kind]; + if (!old->isEmpty()) { + old->getArena()->checkNoMarkedFreeCells(); + } +#endif + + FreeSpan* span = arena->getFirstFreeSpan(); + freeLists_[kind] = span; + + Zone* zone = arena->zone; + if (MOZ_UNLIKELY(zone->isGCMarkingOrSweeping())) { + arena->arenaAllocatedDuringGC(); + } + + TenuredCell* thing = span->allocate(Arena::thingSize(kind)); + MOZ_ASSERT(thing); // This allocation is infallible. + + return thing; +} + +void Arena::arenaAllocatedDuringGC() { + // Ensure that anything allocated during the mark or sweep phases of an + // incremental GC will be marked black by pre-marking all free cells in the + // arena we are about to allocate from. + + MOZ_ASSERT(zone->isGCMarkingOrSweeping()); + for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) { + MOZ_ASSERT(!cell->isMarkedAny()); + cell->markBlack(); + } +} + +// /////////// TenuredChunk -> Arena Allocator /////////////////////////////// + +bool GCRuntime::wantBackgroundAllocation(const AutoLockGC& lock) const { + // To minimize memory waste, we do not want to run the background chunk + // allocation if we already have some empty chunks or when the runtime has + // a small heap size (and therefore likely has a small growth rate). + return allocTask.enabled() && + emptyChunks(lock).count() < minEmptyChunkCount(lock) && + (fullChunks(lock).count() + availableChunks(lock).count()) >= 4; +} + +Arena* GCRuntime::allocateArena(TenuredChunk* chunk, Zone* zone, + AllocKind thingKind, + ShouldCheckThresholds checkThresholds, + const AutoLockGC& lock) { + MOZ_ASSERT(chunk->hasAvailableArenas()); + + // Fail the allocation if we are over our heap size limits. + if ((checkThresholds != ShouldCheckThresholds::DontCheckThresholds) && + (heapSize.bytes() >= tunables.gcMaxBytes())) { + return nullptr; + } + + Arena* arena = chunk->allocateArena(this, zone, thingKind, lock); + zone->gcHeapSize.addGCArena(heapSize); + + // Trigger an incremental slice if needed. + if (checkThresholds != ShouldCheckThresholds::DontCheckThresholds) { + maybeTriggerGCAfterAlloc(zone); + } + + return arena; +} + +Arena* TenuredChunk::allocateArena(GCRuntime* gc, Zone* zone, + AllocKind thingKind, + const AutoLockGC& lock) { + if (info.numArenasFreeCommitted == 0) { + commitOnePage(gc); + MOZ_ASSERT(info.numArenasFreeCommitted == ArenasPerPage); + } + + MOZ_ASSERT(info.numArenasFreeCommitted > 0); + Arena* arena = fetchNextFreeArena(gc); + + arena->init(zone, thingKind, lock); + updateChunkListAfterAlloc(gc, lock); + + verify(); + + return arena; +} + +template +static inline size_t FindFirstBitSet( + const mozilla::BitSet& bitset) { + MOZ_ASSERT(!bitset.IsEmpty()); + + const auto& words = bitset.Storage(); + for (size_t i = 0; i < words.Length(); i++) { + uint32_t word = words[i]; + if (word) { + return i * 32 + mozilla::CountTrailingZeroes32(word); + } + } + + MOZ_CRASH("No bits found"); +} + +void TenuredChunk::commitOnePage(GCRuntime* gc) { + MOZ_ASSERT(info.numArenasFreeCommitted == 0); + MOZ_ASSERT(info.numArenasFree >= ArenasPerPage); + + uint32_t pageIndex = FindFirstBitSet(decommittedPages); + MOZ_ASSERT(decommittedPages[pageIndex]); + + if (DecommitEnabled()) { + MarkPagesInUseSoft(pageAddress(pageIndex), PageSize); + } + + decommittedPages[pageIndex] = false; + + for (size_t i = 0; i < ArenasPerPage; i++) { + size_t arenaIndex = pageIndex * ArenasPerPage + i; + MOZ_ASSERT(!freeCommittedArenas[arenaIndex]); + freeCommittedArenas[arenaIndex] = true; + arenas[arenaIndex].setAsNotAllocated(); + ++info.numArenasFreeCommitted; + gc->updateOnArenaFree(); + } + + verify(); +} + +inline void GCRuntime::updateOnFreeArenaAlloc(const TenuredChunkInfo& info) { + MOZ_ASSERT(info.numArenasFreeCommitted <= numArenasFreeCommitted); + --numArenasFreeCommitted; +} + +Arena* TenuredChunk::fetchNextFreeArena(GCRuntime* gc) { + MOZ_ASSERT(info.numArenasFreeCommitted > 0); + MOZ_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree); + + size_t index = FindFirstBitSet(freeCommittedArenas); + MOZ_ASSERT(freeCommittedArenas[index]); + + freeCommittedArenas[index] = false; + --info.numArenasFreeCommitted; + --info.numArenasFree; + gc->updateOnFreeArenaAlloc(info); + + return &arenas[index]; +} + +// /////////// System -> TenuredChunk Allocator ////////////////////////////// + +TenuredChunk* GCRuntime::getOrAllocChunk(AutoLockGCBgAlloc& lock) { + TenuredChunk* chunk = emptyChunks(lock).pop(); + if (chunk) { + // Reinitialize ChunkBase; arenas are all free and may or may not be + // committed. + SetMemCheckKind(chunk, sizeof(ChunkBase), MemCheckKind::MakeUndefined); + chunk->initBase(rt, nullptr); + MOZ_ASSERT(chunk->unused()); + } else { + void* ptr = TenuredChunk::allocate(this); + if (!ptr) { + return nullptr; + } + + chunk = TenuredChunk::emplace(ptr, this, /* allMemoryCommitted = */ true); + MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0); + } + + if (wantBackgroundAllocation(lock)) { + lock.tryToStartBackgroundAllocation(); + } + + return chunk; +} + +void GCRuntime::recycleChunk(TenuredChunk* chunk, const AutoLockGC& lock) { +#ifdef DEBUG + MOZ_ASSERT(chunk->unused()); + chunk->verify(); +#endif + + // Poison ChunkBase to catch use after free. + AlwaysPoison(chunk, JS_FREED_CHUNK_PATTERN, sizeof(ChunkBase), + MemCheckKind::MakeNoAccess); + + emptyChunks(lock).push(chunk); +} + +TenuredChunk* GCRuntime::pickChunk(AutoLockGCBgAlloc& lock) { + if (availableChunks(lock).count()) { + return availableChunks(lock).head(); + } + + TenuredChunk* chunk = getOrAllocChunk(lock); + if (!chunk) { + return nullptr; + } + +#ifdef DEBUG + chunk->verify(); + MOZ_ASSERT(chunk->unused()); + MOZ_ASSERT(!fullChunks(lock).contains(chunk)); + MOZ_ASSERT(!availableChunks(lock).contains(chunk)); +#endif + + availableChunks(lock).push(chunk); + + return chunk; +} + +BackgroundAllocTask::BackgroundAllocTask(GCRuntime* gc, ChunkPool& pool) + : GCParallelTask(gc, gcstats::PhaseKind::NONE), + chunkPool_(pool), + enabled_(CanUseExtraThreads() && GetCPUCount() >= 2) { + // This can occur outside GCs so doesn't have a stats phase. +} + +void BackgroundAllocTask::run(AutoLockHelperThreadState& lock) { + AutoUnlockHelperThreadState unlock(lock); + + AutoLockGC gcLock(gc); + while (!isCancelled() && gc->wantBackgroundAllocation(gcLock)) { + TenuredChunk* chunk; + { + AutoUnlockGC unlock(gcLock); + void* ptr = TenuredChunk::allocate(gc); + if (!ptr) { + break; + } + chunk = TenuredChunk::emplace(ptr, gc, /* allMemoryCommitted = */ true); + } + chunkPool_.ref().push(chunk); + } +} + +/* static */ +void* TenuredChunk::allocate(GCRuntime* gc) { + void* chunk = MapAlignedPages(ChunkSize, ChunkSize); + if (!chunk) { + return nullptr; + } + + gc->stats().count(gcstats::COUNT_NEW_CHUNK); + return chunk; +} + +static inline bool ShouldDecommitNewChunk(bool allMemoryCommitted, + const GCSchedulingState& state) { + if (!DecommitEnabled()) { + return false; + } + + return !allMemoryCommitted || !state.inHighFrequencyGCMode(); +} + +TenuredChunk* TenuredChunk::emplace(void* ptr, GCRuntime* gc, + bool allMemoryCommitted) { + /* The chunk may still have some regions marked as no-access. */ + MOZ_MAKE_MEM_UNDEFINED(ptr, ChunkSize); + + /* + * Poison the chunk. Note that decommitAllArenas() below will mark the + * arenas as inaccessible (for memory sanitizers). + */ + Poison(ptr, JS_FRESH_TENURED_PATTERN, ChunkSize, MemCheckKind::MakeUndefined); + + TenuredChunk* chunk = new (mozilla::KnownNotNull, ptr) TenuredChunk(gc->rt); + + if (ShouldDecommitNewChunk(allMemoryCommitted, gc->schedulingState)) { + // Decommit the arenas. We do this after poisoning so that if the OS does + // not have to recycle the pages, we still get the benefit of poisoning. + chunk->decommitAllArenas(); + } else { + // The chunk metadata is initialized as decommitted regardless, to avoid + // having to initialize the arenas at this time. + chunk->initAsDecommitted(); + } + + chunk->verify(); + + return chunk; +} + +void TenuredChunk::decommitAllArenas() { + MOZ_ASSERT(unused()); + MarkPagesUnusedSoft(&arenas[0], ArenasPerChunk * ArenaSize); + initAsDecommitted(); +} + +void TenuredChunkBase::initAsDecommitted() { + // Set the state of all arenas to free and decommitted. They might not + // actually be decommitted, but in that case the re-commit operation is a + // no-op so it doesn't matter. + decommittedPages.SetAll(); + freeCommittedArenas.ResetAll(); + info.numArenasFree = ArenasPerChunk; + info.numArenasFreeCommitted = 0; +} diff --git a/js/src/gc/Allocator.h b/js/src/gc/Allocator.h new file mode 100644 index 0000000000..1cd3ff6aad --- /dev/null +++ b/js/src/gc/Allocator.h @@ -0,0 +1,173 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Allocator_h +#define gc_Allocator_h + +#include "mozilla/OperatorNewExtensions.h" + +#include + +#include "gc/AllocKind.h" +#include "gc/Cell.h" +#include "js/Class.h" +#include "js/TypeDecls.h" + +namespace js { + +// [SMDOC] AllowGC template parameter +// +// AllowGC is a template parameter for functions that support both with and +// without GC operation. +// +// The CanGC variant of the function can trigger a garbage collection, and +// should set a pending exception on failure. +// +// The NoGC variant of the function cannot trigger a garbage collection, and +// should not set any pending exception on failure. This variant can be called +// in fast paths where the caller has unrooted pointers. The failure means we +// need to perform GC to allocate an object. The caller can fall back to a slow +// path that roots pointers before calling a CanGC variant of the function, +// without having to clear a pending exception. +enum AllowGC { NoGC = 0, CanGC = 1 }; + +namespace gc { + +class AllocSite; +struct Cell; +class TenuredCell; + +// Allocator implementation functions. SpiderMonkey code outside this file +// should use: +// +// cx->newCell(...) +// +// or optionally: +// +// cx->newCell(...) +// +// `friend` js::gc::CellAllocator in a subtype T of Cell in order to allow it to +// be allocated with cx->newCell(...). The friend declaration will allow +// calling T's constructor. +// +// The parameters will be passed to a type-specific function or constructor. For +// nursery-allocatable types, see e.g. the NewString, NewObject, and NewBigInt +// methods. For all other types, the parameters will be forwarded to the +// constructor. +class CellAllocator { + public: + template + static T* NewCell(JSContext* cx, Args&&... args); + + private: + // Allocate a cell in the nursery, unless |heap| is Heap::Tenured or nursery + // allocation is disabled for |traceKind| in the current zone. + template + static void* AllocNurseryOrTenuredCell(JSContext* cx, gc::AllocKind allocKind, + gc::Heap heap, AllocSite* site); + + // Allocate a cell in the tenured heap. + template + static void* AllocTenuredCell(JSContext* cx, gc::AllocKind kind, size_t size); + + // Allocate a string. Use cx->newCell([heap]). + // + // Use for nursery-allocatable strings. Returns a value cast to the correct + // type. Non-nursery-allocatable strings will go through the fallback + // tenured-only allocation path. + template + static T* NewString(JSContext* cx, gc::Heap heap, Args&&... args) { + static_assert(std::is_base_of_v); + gc::AllocKind kind = gc::MapTypeToAllocKind::kind; + void* ptr = AllocNurseryOrTenuredCell( + cx, kind, heap, nullptr); + if (!ptr) { + return nullptr; + } + return new (mozilla::KnownNotNull, ptr) T(std::forward(args)...); + } + + template + static T* NewBigInt(JSContext* cx, Heap heap) { + void* ptr = AllocNurseryOrTenuredCell( + cx, gc::AllocKind::BIGINT, heap, nullptr); + if (ptr) { + return new (mozilla::KnownNotNull, ptr) T(); + } + return nullptr; + } + + template + static T* NewObject(JSContext* cx, gc::AllocKind kind, gc::Heap heap, + const JSClass* clasp, gc::AllocSite* site = nullptr) { + MOZ_ASSERT(IsObjectAllocKind(kind)); + MOZ_ASSERT_IF(heap != gc::Heap::Tenured && clasp->hasFinalize() && + !clasp->isProxyObject(), + CanNurseryAllocateFinalizedClass(clasp)); + void* cell = AllocNurseryOrTenuredCell( + cx, kind, heap, site); + if (!cell) { + return nullptr; + } + return new (mozilla::KnownNotNull, cell) T(); + } + + // Allocate all other kinds of GC thing. + template + static T* NewTenuredCell(JSContext* cx, Args&&... args) { + gc::AllocKind kind = gc::MapTypeToAllocKind::kind; + void* cell = AllocTenuredCell(cx, kind, sizeof(T)); + if (!cell) { + return nullptr; + } + return new (mozilla::KnownNotNull, cell) T(std::forward(args)...); + } +}; + +} // namespace gc + +// This is the entry point for all allocation, though callers should still not +// use this directly. Use cx->newCell(...) instead. +// +// After a successful allocation the caller must fully initialize the thing +// before calling any function that can potentially trigger GC. This will +// ensure that GC tracing never sees junk values stored in the partially +// initialized thing. +template +T* gc::CellAllocator::NewCell(JSContext* cx, Args&&... args) { + static_assert(std::is_base_of_v); + + // Objects. See the valid parameter list in NewObject, above. + if constexpr (std::is_base_of_v) { + return NewObject(cx, std::forward(args)...); + } + + // BigInt + else if constexpr (std::is_base_of_v) { + return NewBigInt(cx, std::forward(args)...); + } + + // "Normal" strings (all of which can be nursery allocated). Atoms and + // external strings will fall through to the generic code below. All other + // strings go through NewString, which will forward the arguments to the + // appropriate string class's constructor. + else if constexpr (std::is_base_of_v && + !std::is_base_of_v && + !std::is_base_of_v) { + return NewString(cx, std::forward(args)...); + } + + else { + // Allocate a new tenured GC thing that's not nursery-allocatable. Use + // cx->newCell(...), where the parameters are forwarded to the type's + // constructor. + return NewTenuredCell(cx, std::forward(args)...); + } +} + +} // namespace js + +#endif // gc_Allocator_h diff --git a/js/src/gc/ArenaList-inl.h b/js/src/gc/ArenaList-inl.h new file mode 100644 index 0000000000..92925a227c --- /dev/null +++ b/js/src/gc/ArenaList-inl.h @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_ArenaList_inl_h +#define gc_ArenaList_inl_h + +#include "gc/ArenaList.h" + +#include "gc/Heap.h" +#include "gc/Zone.h" + +void js::gc::SortedArenaListSegment::append(Arena* arena) { + MOZ_ASSERT(arena); + MOZ_ASSERT_IF(head, head->getAllocKind() == arena->getAllocKind()); + *tailp = arena; + tailp = &arena->next; +} + +inline js::gc::ArenaList::ArenaList() { clear(); } + +inline js::gc::ArenaList::ArenaList(ArenaList&& other) { moveFrom(other); } + +inline js::gc::ArenaList::~ArenaList() { MOZ_ASSERT(isEmpty()); } + +void js::gc::ArenaList::moveFrom(ArenaList& other) { + other.check(); + + head_ = other.head_; + cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_; + other.clear(); + + check(); +} + +js::gc::ArenaList& js::gc::ArenaList::operator=(ArenaList&& other) { + MOZ_ASSERT(isEmpty()); + moveFrom(other); + return *this; +} + +inline js::gc::ArenaList::ArenaList(const SortedArenaListSegment& segment) { + head_ = segment.head; + cursorp_ = segment.isEmpty() ? &head_ : segment.tailp; + check(); +} + +// This does checking just of |head_| and |cursorp_|. +void js::gc::ArenaList::check() const { +#ifdef DEBUG + // If the list is empty, it must have this form. + MOZ_ASSERT_IF(!head_, cursorp_ == &head_); + + // If there's an arena following the cursor, it must not be full. + Arena* cursor = *cursorp_; + MOZ_ASSERT_IF(cursor, cursor->hasFreeThings()); +#endif +} + +void js::gc::ArenaList::clear() { + head_ = nullptr; + cursorp_ = &head_; + check(); +} + +bool js::gc::ArenaList::isEmpty() const { + check(); + return !head_; +} + +js::gc::Arena* js::gc::ArenaList::head() const { + check(); + return head_; +} + +bool js::gc::ArenaList::isCursorAtHead() const { + check(); + return cursorp_ == &head_; +} + +bool js::gc::ArenaList::isCursorAtEnd() const { + check(); + return !*cursorp_; +} + +js::gc::Arena* js::gc::ArenaList::arenaAfterCursor() const { + check(); + return *cursorp_; +} + +js::gc::Arena* js::gc::ArenaList::takeNextArena() { + check(); + Arena* arena = *cursorp_; + if (!arena) { + return nullptr; + } + cursorp_ = &arena->next; + check(); + return arena; +} + +void js::gc::ArenaList::insertAtCursor(Arena* a) { + check(); + a->next = *cursorp_; + *cursorp_ = a; + // At this point, the cursor is sitting before |a|. Move it after |a| + // if necessary. + if (!a->hasFreeThings()) { + cursorp_ = &a->next; + } + check(); +} + +void js::gc::ArenaList::insertBeforeCursor(Arena* a) { + check(); + a->next = *cursorp_; + *cursorp_ = a; + cursorp_ = &a->next; + check(); +} + +js::gc::ArenaList& js::gc::ArenaList::insertListWithCursorAtEnd( + ArenaList& other) { + check(); + other.check(); + MOZ_ASSERT(other.isCursorAtEnd()); + + if (other.isEmpty()) { + return *this; + } + + // Insert the full arenas of |other| after those of |this|. + *other.cursorp_ = *cursorp_; + *cursorp_ = other.head_; + cursorp_ = other.cursorp_; + check(); + + other.clear(); + return *this; +} + +js::gc::Arena* js::gc::ArenaList::takeFirstArena() { + check(); + Arena* arena = head_; + if (!arena) { + return nullptr; + } + + head_ = arena->next; + if (cursorp_ == &arena->next) { + cursorp_ = &head_; + } + + check(); + return arena; +} + +js::gc::SortedArenaList::SortedArenaList(size_t thingsPerArena) { + reset(thingsPerArena); +} + +void js::gc::SortedArenaList::setThingsPerArena(size_t thingsPerArena) { + MOZ_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena); + thingsPerArena_ = thingsPerArena; +} + +void js::gc::SortedArenaList::reset(size_t thingsPerArena) { + setThingsPerArena(thingsPerArena); + // Initialize the segments. + for (size_t i = 0; i <= thingsPerArena; ++i) { + segments[i].clear(); + } +} + +void js::gc::SortedArenaList::insertAt(Arena* arena, size_t nfree) { + MOZ_ASSERT(nfree <= thingsPerArena_); + segments[nfree].append(arena); +} + +void js::gc::SortedArenaList::extractEmpty(Arena** empty) { + SortedArenaListSegment& segment = segments[thingsPerArena_]; + if (segment.head) { + *segment.tailp = *empty; + *empty = segment.head; + segment.clear(); + } +} + +js::gc::ArenaList js::gc::SortedArenaList::toArenaList() { + // Link the non-empty segment tails up to the non-empty segment heads. + size_t tailIndex = 0; + for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) { + if (headAt(headIndex)) { + segments[tailIndex].linkTo(headAt(headIndex)); + tailIndex = headIndex; + } + } + // Point the tail of the final non-empty segment at null. Note that if + // the list is empty, this will just set segments[0].head to null. + segments[tailIndex].linkTo(nullptr); + // Create an ArenaList with head and cursor set to the head and tail of + // the first segment (if that segment is empty, only the head is used). + return ArenaList(segments[0]); +} + +#ifdef DEBUG + +bool js::gc::FreeLists::allEmpty() const { + for (auto i : AllAllocKinds()) { + if (!isEmpty(i)) { + return false; + } + } + return true; +} + +bool js::gc::FreeLists::isEmpty(AllocKind kind) const { + return freeLists_[kind]->isEmpty(); +} + +#endif + +void js::gc::FreeLists::clear() { + for (auto i : AllAllocKinds()) { +#ifdef DEBUG + auto old = freeLists_[i]; + if (!old->isEmpty()) { + old->getArena()->checkNoMarkedFreeCells(); + } +#endif + freeLists_[i] = &emptySentinel; + } +} + +js::gc::TenuredCell* js::gc::FreeLists::allocate(AllocKind kind) { + return freeLists_[kind]->allocate(Arena::thingSize(kind)); +} + +void js::gc::FreeLists::unmarkPreMarkedFreeCells(AllocKind kind) { + FreeSpan* freeSpan = freeLists_[kind]; + if (!freeSpan->isEmpty()) { + freeSpan->getArena()->unmarkPreMarkedFreeCells(); + } +} + +JSRuntime* js::gc::ArenaLists::runtime() { + return zone_->runtimeFromMainThread(); +} + +JSRuntime* js::gc::ArenaLists::runtimeFromAnyThread() { + return zone_->runtimeFromAnyThread(); +} + +js::gc::Arena* js::gc::ArenaLists::getFirstArena(AllocKind thingKind) const { + return arenaList(thingKind).head(); +} + +js::gc::Arena* js::gc::ArenaLists::getFirstCollectingArena( + AllocKind thingKind) const { + return collectingArenaList(thingKind).head(); +} + +js::gc::Arena* js::gc::ArenaLists::getFirstSweptArena( + AllocKind thingKind) const { + if (thingKind != incrementalSweptArenaKind.ref()) { + return nullptr; + } + return incrementalSweptArenas.ref().head(); +} + +js::gc::Arena* js::gc::ArenaLists::getArenaAfterCursor( + AllocKind thingKind) const { + return arenaList(thingKind).arenaAfterCursor(); +} + +bool js::gc::ArenaLists::arenaListsAreEmpty() const { + for (auto i : AllAllocKinds()) { + /* + * The arena cannot be empty if the background finalization is not yet + * done. + */ + if (concurrentUse(i) == ConcurrentUse::BackgroundFinalize) { + return false; + } + if (!arenaList(i).isEmpty()) { + return false; + } + } + return true; +} + +bool js::gc::ArenaLists::doneBackgroundFinalize(AllocKind kind) const { + return concurrentUse(kind) != ConcurrentUse::BackgroundFinalize; +} + +bool js::gc::ArenaLists::needBackgroundFinalizeWait(AllocKind kind) const { + return concurrentUse(kind) == ConcurrentUse::BackgroundFinalize; +} + +void js::gc::ArenaLists::clearFreeLists() { freeLists().clear(); } + +MOZ_ALWAYS_INLINE js::gc::TenuredCell* js::gc::ArenaLists::allocateFromFreeList( + AllocKind thingKind) { + return freeLists().allocate(thingKind); +} + +void js::gc::ArenaLists::unmarkPreMarkedFreeCells() { + for (auto i : AllAllocKinds()) { + freeLists().unmarkPreMarkedFreeCells(i); + } +} + +void js::gc::ArenaLists::checkEmptyFreeLists() { + MOZ_ASSERT(freeLists().allEmpty()); +} + +void js::gc::ArenaLists::checkEmptyArenaLists() { +#ifdef DEBUG + for (auto i : AllAllocKinds()) { + checkEmptyArenaList(i); + } +#endif +} + +#endif // gc_ArenaList_inl_h diff --git a/js/src/gc/ArenaList.h b/js/src/gc/ArenaList.h new file mode 100644 index 0000000000..4ae4afde71 --- /dev/null +++ b/js/src/gc/ArenaList.h @@ -0,0 +1,386 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definitions of ArenaList and associated heap data structures. + */ + +#ifndef gc_ArenaList_h +#define gc_ArenaList_h + +#include "gc/AllocKind.h" +#include "js/GCAPI.h" +#include "js/HeapAPI.h" +#include "js/TypeDecls.h" +#include "threading/ProtectedData.h" + +namespace js { + +class Nursery; +class SliceBudget; + +namespace gcstats { +struct Statistics; +} + +namespace gc { + +class Arena; +class BackgroundUnmarkTask; +struct FinalizePhase; +class FreeSpan; +class TenuredCell; +class TenuringTracer; + +/* + * A single segment of a SortedArenaList. Each segment has a head and a tail, + * which track the start and end of a segment for O(1) append and concatenation. + */ +struct SortedArenaListSegment { + Arena* head; + Arena** tailp; + + void clear() { + head = nullptr; + tailp = &head; + } + + bool isEmpty() const { return tailp == &head; } + + // Appends |arena| to this segment. + inline void append(Arena* arena); + + // Points the tail of this segment at |arena|, which may be null. Note + // that this does not change the tail itself, but merely which arena + // follows it. This essentially turns the tail into a cursor (see also the + // description of ArenaList), but from the perspective of a SortedArenaList + // this makes no difference. + void linkTo(Arena* arena) { *tailp = arena; } +}; + +/* + * Arena lists contain a singly linked lists of arenas starting from a head + * pointer. + * + * They also have a cursor, which conceptually lies on arena boundaries, + * i.e. before the first arena, between two arenas, or after the last arena. + * + * Arenas are usually sorted in order of increasing free space, with the cursor + * following the Arena currently being allocated from. This ordering should not + * be treated as an invariant, however, as the free lists may be cleared, + * leaving arenas previously used for allocation partially full. Sorting order + * is restored during sweeping. + * + * Arenas following the cursor should not be full. + */ +class ArenaList { + // The cursor is implemented via an indirect pointer, |cursorp_|, to allow + // for efficient list insertion at the cursor point and other list + // manipulations. + // + // - If the list is empty: |head| is null, |cursorp_| points to |head|, and + // therefore |*cursorp_| is null. + // + // - If the list is not empty: |head| is non-null, and... + // + // - If the cursor is at the start of the list: |cursorp_| points to + // |head|, and therefore |*cursorp_| points to the first arena. + // + // - If cursor is at the end of the list: |cursorp_| points to the |next| + // field of the last arena, and therefore |*cursorp_| is null. + // + // - If the cursor is at neither the start nor the end of the list: + // |cursorp_| points to the |next| field of the arena preceding the + // cursor, and therefore |*cursorp_| points to the arena following the + // cursor. + // + // |cursorp_| is never null. + // + Arena* head_; + Arena** cursorp_; + + // Transfers the contents of |other| to this list and clears |other|. + inline void moveFrom(ArenaList& other); + + public: + inline ArenaList(); + inline ArenaList(ArenaList&& other); + inline ~ArenaList(); + + inline ArenaList& operator=(ArenaList&& other); + + // It doesn't make sense for arenas to be present in more than one list, so + // list copy operations are not provided. + ArenaList(const ArenaList& other) = delete; + ArenaList& operator=(const ArenaList& other) = delete; + + inline explicit ArenaList(const SortedArenaListSegment& segment); + + inline void check() const; + + inline void clear(); + inline bool isEmpty() const; + + // This returns nullptr if the list is empty. + inline Arena* head() const; + + inline bool isCursorAtHead() const; + inline bool isCursorAtEnd() const; + + // This can return nullptr. + inline Arena* arenaAfterCursor() const; + + // This returns the arena after the cursor and moves the cursor past it. + inline Arena* takeNextArena(); + + // This does two things. + // - Inserts |a| at the cursor. + // - Leaves the cursor sitting just before |a|, if |a| is not full, or just + // after |a|, if |a| is full. + inline void insertAtCursor(Arena* a); + + // Inserts |a| at the cursor, then moves the cursor past it. + inline void insertBeforeCursor(Arena* a); + + // This inserts the contents of |other|, which must be full, at the cursor of + // |this| and clears |other|. + inline ArenaList& insertListWithCursorAtEnd(ArenaList& other); + + inline Arena* takeFirstArena(); + + Arena* removeRemainingArenas(Arena** arenap); + Arena** pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut); + Arena* relocateArenas(Arena* toRelocate, Arena* relocated, + js::SliceBudget& sliceBudget, + gcstats::Statistics& stats); + +#ifdef DEBUG + void dump(); +#endif +}; + +/* + * A class that holds arenas in sorted order by appending arenas to specific + * segments. Each segment has a head and a tail, which can be linked up to + * other segments to create a contiguous ArenaList. + */ +class SortedArenaList { + public: + // The minimum size, in bytes, of a GC thing. + static const size_t MinThingSize = 16; + + static_assert(ArenaSize <= 4096, + "When increasing the Arena size, please consider how" + " this will affect the size of a SortedArenaList."); + + static_assert(MinThingSize >= 16, + "When decreasing the minimum thing size, please consider" + " how this will affect the size of a SortedArenaList."); + + private: + // The maximum number of GC things that an arena can hold. + static const size_t MaxThingsPerArena = + (ArenaSize - ArenaHeaderSize) / MinThingSize; + + size_t thingsPerArena_; + SortedArenaListSegment segments[MaxThingsPerArena + 1]; + + // Convenience functions to get the nth head and tail. + Arena* headAt(size_t n) { return segments[n].head; } + Arena** tailAt(size_t n) { return segments[n].tailp; } + + public: + inline explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena); + + inline void setThingsPerArena(size_t thingsPerArena); + + // Resets the first |thingsPerArena| segments of this list for further use. + inline void reset(size_t thingsPerArena = MaxThingsPerArena); + + // Inserts an arena, which has room for |nfree| more things, in its segment. + inline void insertAt(Arena* arena, size_t nfree); + + // Remove all empty arenas, inserting them as a linked list. + inline void extractEmpty(Arena** empty); + + // Links up the tail of each non-empty segment to the head of the next + // non-empty segment, creating a contiguous list that is returned as an + // ArenaList. This is not a destructive operation: neither the head nor tail + // of any segment is modified. However, note that the Arenas in the + // resulting ArenaList should be treated as read-only unless the + // SortedArenaList is no longer needed: inserting or removing arenas would + // invalidate the SortedArenaList. + inline ArenaList toArenaList(); +}; + +enum class ShouldCheckThresholds { + DontCheckThresholds = 0, + CheckThresholds = 1 +}; + +// For each arena kind its free list is represented as the first span with free +// things. Initially all the spans are initialized as empty. After we find a new +// arena with available things we move its first free span into the list and set +// the arena as fully allocated. That way we do not need to update the arena +// after the initial allocation. When starting the GC we only move the head of +// the of the list of spans back to the arena only for the arena that was not +// fully allocated. +class FreeLists { + AllAllocKindArray freeLists_; + + public: + // Because the JITs can allocate from the free lists, they cannot be null. + // We use a placeholder FreeSpan that is empty (and wihout an associated + // Arena) so the JITs can fall back gracefully. + static FreeSpan emptySentinel; + + FreeLists(); + +#ifdef DEBUG + inline bool allEmpty() const; + inline bool isEmpty(AllocKind kind) const; +#endif + + inline void clear(); + + MOZ_ALWAYS_INLINE TenuredCell* allocate(AllocKind kind); + + inline void* setArenaAndAllocate(Arena* arena, AllocKind kind); + + inline void unmarkPreMarkedFreeCells(AllocKind kind); + + FreeSpan** addressOfFreeList(AllocKind thingKind) { + return &freeLists_[thingKind]; + } +}; + +class ArenaLists { + enum class ConcurrentUse : uint32_t { None, BackgroundFinalize }; + + using ConcurrentUseState = + mozilla::Atomic; + + JS::Zone* zone_; + + // Whether this structure can be accessed by other threads. + UnprotectedData> concurrentUseState_; + + MainThreadData freeLists_; + + /* The main list of arenas for each alloc kind. */ + MainThreadOrGCTaskData> arenaLists_; + + /* + * Arenas which are currently being collected. The collector can move arenas + * from arenaLists_ here and back again at various points in collection. + */ + MainThreadOrGCTaskData> collectingArenaLists_; + + /* During incremental sweeping, a list of the arenas already swept. */ + MainThreadOrGCTaskData incrementalSweptArenaKind; + MainThreadOrGCTaskData incrementalSweptArenas; + + // Arena lists which have yet to be swept, but need additional foreground + // processing before they are swept. + MainThreadData gcCompactPropMapArenasToUpdate; + MainThreadData gcNormalPropMapArenasToUpdate; + + // The list of empty arenas which are collected during the sweep phase and + // released at the end of sweeping every sweep group. + MainThreadOrGCTaskData savedEmptyArenas; + + public: + explicit ArenaLists(JS::Zone* zone); + ~ArenaLists(); + + FreeLists& freeLists() { return freeLists_.ref(); } + const FreeLists& freeLists() const { return freeLists_.ref(); } + + FreeSpan** addressOfFreeList(AllocKind thingKind) { + return freeLists_.refNoCheck().addressOfFreeList(thingKind); + } + + inline Arena* getFirstArena(AllocKind thingKind) const; + inline Arena* getFirstCollectingArena(AllocKind thingKind) const; + inline Arena* getFirstSweptArena(AllocKind thingKind) const; + inline Arena* getArenaAfterCursor(AllocKind thingKind) const; + + inline bool arenaListsAreEmpty() const; + + inline bool doneBackgroundFinalize(AllocKind kind) const; + inline bool needBackgroundFinalizeWait(AllocKind kind) const; + + /* Clear the free lists so we won't try to allocate from swept arenas. */ + inline void clearFreeLists(); + + inline void unmarkPreMarkedFreeCells(); + + MOZ_ALWAYS_INLINE TenuredCell* allocateFromFreeList(AllocKind thingKind); + + inline void checkEmptyFreeLists(); + inline void checkEmptyArenaLists(); + inline void checkEmptyFreeList(AllocKind kind); + + void checkEmptyArenaList(AllocKind kind); + + bool relocateArenas(Arena*& relocatedListOut, JS::GCReason reason, + js::SliceBudget& sliceBudget, gcstats::Statistics& stats); + + void queueForegroundObjectsForSweep(JS::GCContext* gcx); + void queueForegroundThingsForSweep(); + + Arena* takeSweptEmptyArenas(); + + void setIncrementalSweptArenas(AllocKind kind, SortedArenaList& arenas); + void clearIncrementalSweptArenas(); + + void mergeFinalizedArenas(AllocKind thingKind, + SortedArenaList& finalizedArenas); + + void moveArenasToCollectingLists(); + void mergeArenasFromCollectingLists(); + + void checkGCStateNotInUse(); + void checkSweepStateNotInUse(); + void checkNoArenasToUpdate(); + void checkNoArenasToUpdateForKind(AllocKind kind); + + private: + ArenaList& arenaList(AllocKind i) { return arenaLists_.ref()[i]; } + const ArenaList& arenaList(AllocKind i) const { return arenaLists_.ref()[i]; } + + ArenaList& collectingArenaList(AllocKind i) { + return collectingArenaLists_.ref()[i]; + } + const ArenaList& collectingArenaList(AllocKind i) const { + return collectingArenaLists_.ref()[i]; + } + + ConcurrentUseState& concurrentUse(AllocKind i) { + return concurrentUseState_.ref()[i]; + } + ConcurrentUse concurrentUse(AllocKind i) const { + return concurrentUseState_.ref()[i]; + } + + inline JSRuntime* runtime(); + inline JSRuntime* runtimeFromAnyThread(); + + void initBackgroundSweep(AllocKind thingKind); + + void* refillFreeListAndAllocate(AllocKind thingKind, + ShouldCheckThresholds checkThresholds); + + friend class BackgroundUnmarkTask; + friend class GCRuntime; + friend class js::Nursery; + friend class TenuringTracer; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_ArenaList_h */ diff --git a/js/src/gc/AtomMarking-inl.h b/js/src/gc/AtomMarking-inl.h new file mode 100644 index 0000000000..9b6e869e72 --- /dev/null +++ b/js/src/gc/AtomMarking-inl.h @@ -0,0 +1,96 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/AtomMarking.h" + +#include "mozilla/Assertions.h" + +#include + +#include "vm/JSContext.h" +#include "vm/StringType.h" +#include "vm/SymbolType.h" + +#include "gc/Heap-inl.h" + +namespace js { +namespace gc { + +inline size_t GetAtomBit(TenuredCell* thing) { + MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone()); + Arena* arena = thing->arena(); + size_t arenaBit = (reinterpret_cast(thing) - arena->address()) / + CellBytesPerMarkBit; + return arena->atomBitmapStart() * JS_BITS_PER_WORD + arenaBit; +} + +template +MOZ_ALWAYS_INLINE bool AtomMarkingRuntime::inlinedMarkAtomInternal( + JSContext* cx, T* thing) { + static_assert(std::is_same_v || std::is_same_v, + "Should only be called with JSAtom* or JS::Symbol* argument"); + + MOZ_ASSERT(cx->isMainThreadContext()); + MOZ_ASSERT(cx->zone()); + MOZ_ASSERT(!cx->zone()->isAtomsZone()); + + MOZ_ASSERT(thing); + js::gc::TenuredCell* cell = &thing->asTenured(); + MOZ_ASSERT(cell->zoneFromAnyThread()->isAtomsZone()); + + // This doesn't check for pinned atoms since that might require taking a + // lock. This is not required for correctness. + if (thing->isPermanentAndMayBeShared()) { + return true; + } + + size_t bit = GetAtomBit(cell); + MOZ_ASSERT(bit / JS_BITS_PER_WORD < allocatedWords); + + if (Fallible) { + if (!cx->zone()->markedAtoms().setBitFallible(bit)) { + return false; + } + } else { + cx->zone()->markedAtoms().setBit(bit); + } + + // Trigger a read barrier on the atom, in case there is an incremental + // GC in progress. This is necessary if the atom is being marked + // because a reference to it was obtained from another zone which is + // not being collected by the incremental GC. + ReadBarrier(thing); + + // Children of the thing also need to be marked in the context's zone. + // We don't have a JSTracer for this so manually handle the cases in which + // an atom can reference other atoms. + markChildren(cx, thing); + + return true; +} + +void AtomMarkingRuntime::markChildren(JSContext* cx, JSAtom*) {} + +void AtomMarkingRuntime::markChildren(JSContext* cx, JS::Symbol* symbol) { + if (JSAtom* description = symbol->description()) { + markAtom(cx, description); + } +} + +template +MOZ_ALWAYS_INLINE void AtomMarkingRuntime::inlinedMarkAtom(JSContext* cx, + T* thing) { + MOZ_ALWAYS_TRUE((inlinedMarkAtomInternal(cx, thing))); +} + +template +MOZ_ALWAYS_INLINE bool AtomMarkingRuntime::inlinedMarkAtomFallible( + JSContext* cx, T* thing) { + return inlinedMarkAtomInternal(cx, thing); +} + +} // namespace gc +} // namespace js diff --git a/js/src/gc/AtomMarking.cpp b/js/src/gc/AtomMarking.cpp new file mode 100644 index 0000000000..78beb742d0 --- /dev/null +++ b/js/src/gc/AtomMarking.cpp @@ -0,0 +1,294 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/AtomMarking-inl.h" + +#include + +#include "gc/PublicIterators.h" + +#include "gc/GC-inl.h" +#include "gc/Heap-inl.h" + +namespace js { +namespace gc { + +// [SMDOC] GC Atom Marking +// +// Things in the atoms zone (which includes atomized strings and other things, +// all of which we will refer to as 'atoms' here) may be pointed to freely by +// things in other zones. To avoid the need to perform garbage collections of +// the entire runtime to collect atoms, we compute a separate atom mark bitmap +// for each zone that is always an overapproximation of the atoms that zone is +// using. When an atom is not in the mark bitmap for any zone, it can be +// destroyed. +// +// To minimize interference with the rest of the GC, atom marking and sweeping +// is done by manipulating the mark bitmaps in the chunks used for the atoms. +// When the atoms zone is being collected, the mark bitmaps for the chunk(s) +// used by the atoms are updated normally during marking. After marking +// finishes, the chunk mark bitmaps are translated to a more efficient atom mark +// bitmap (see below) that is stored on the zones which the GC collected +// (computeBitmapFromChunkMarkBits). Before sweeping begins, the chunk mark +// bitmaps are updated with any atoms that might be referenced by zones which +// weren't collected (markAtomsUsedByUncollectedZones). The GC sweeping will +// then release all atoms which are not marked by any zone. +// +// The representation of atom mark bitmaps is as follows: +// +// Each arena in the atoms zone has an atomBitmapStart() value indicating the +// word index into the bitmap of the first thing in the arena. Each arena uses +// ArenaBitmapWords of data to store its bitmap, which uses the same +// representation as chunk mark bitmaps: one bit is allocated per Cell, with +// bits for space between things being unused when things are larger than a +// single Cell. + +void AtomMarkingRuntime::registerArena(Arena* arena, const AutoLockGC& lock) { + MOZ_ASSERT(arena->getThingSize() != 0); + MOZ_ASSERT(arena->getThingSize() % CellAlignBytes == 0); + MOZ_ASSERT(arena->zone->isAtomsZone()); + + // We need to find a range of bits from the atoms bitmap for this arena. + + // Look for a free range of bits compatible with this arena. + if (freeArenaIndexes.ref().length()) { + arena->atomBitmapStart() = freeArenaIndexes.ref().popCopy(); + return; + } + + // Allocate a range of bits from the end for this arena. + arena->atomBitmapStart() = allocatedWords; + allocatedWords += ArenaBitmapWords; +} + +void AtomMarkingRuntime::unregisterArena(Arena* arena, const AutoLockGC& lock) { + MOZ_ASSERT(arena->zone->isAtomsZone()); + + // Leak these atom bits if we run out of memory. + (void)freeArenaIndexes.ref().emplaceBack(arena->atomBitmapStart()); +} + +bool AtomMarkingRuntime::computeBitmapFromChunkMarkBits(JSRuntime* runtime, + DenseBitmap& bitmap) { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + + if (!bitmap.ensureSpace(allocatedWords)) { + return false; + } + + Zone* atomsZone = runtime->unsafeAtomsZone(); + for (auto thingKind : AllAllocKinds()) { + for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + MarkBitmapWord* chunkWords = arena->chunk()->markBits.arenaBits(arena); + bitmap.copyBitsFrom(arena->atomBitmapStart(), ArenaBitmapWords, + chunkWords); + } + } + + return true; +} + +void AtomMarkingRuntime::refineZoneBitmapForCollectedZone( + Zone* zone, const DenseBitmap& bitmap) { + MOZ_ASSERT(zone->isCollectingFromAnyThread()); + + if (zone->isAtomsZone()) { + return; + } + + // Take the bitwise and between the two mark bitmaps to get the best new + // overapproximation we can. |bitmap| might include bits that are not in + // the zone's mark bitmap, if additional zones were collected by the GC. + zone->markedAtoms().bitwiseAndWith(bitmap); +} + +// Set any bits in the chunk mark bitmaps for atoms which are marked in bitmap. +template +static void BitwiseOrIntoChunkMarkBits(JSRuntime* runtime, Bitmap& bitmap) { + // Make sure that by copying the mark bits for one arena in word sizes we + // do not affect the mark bits for other arenas. + static_assert(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD, + "ArenaBitmapWords must evenly divide ArenaBitmapBits"); + + Zone* atomsZone = runtime->unsafeAtomsZone(); + for (auto thingKind : AllAllocKinds()) { + for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + MarkBitmapWord* chunkWords = arena->chunk()->markBits.arenaBits(arena); + bitmap.bitwiseOrRangeInto(arena->atomBitmapStart(), ArenaBitmapWords, + chunkWords); + } + } +} + +void AtomMarkingRuntime::markAtomsUsedByUncollectedZones(JSRuntime* runtime) { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + + // Try to compute a simple union of the zone atom bitmaps before updating + // the chunk mark bitmaps. If this allocation fails then fall back to + // updating the chunk mark bitmaps separately for each zone. + DenseBitmap markedUnion; + if (markedUnion.ensureSpace(allocatedWords)) { + for (ZonesIter zone(runtime, SkipAtoms); !zone.done(); zone.next()) { + // We only need to update the chunk mark bits for zones which were + // not collected in the current GC. Atoms which are referenced by + // collected zones have already been marked. + if (!zone->isCollectingFromAnyThread()) { + zone->markedAtoms().bitwiseOrInto(markedUnion); + } + } + BitwiseOrIntoChunkMarkBits(runtime, markedUnion); + } else { + for (ZonesIter zone(runtime, SkipAtoms); !zone.done(); zone.next()) { + if (!zone->isCollectingFromAnyThread()) { + BitwiseOrIntoChunkMarkBits(runtime, zone->markedAtoms()); + } + } + } +} + +template +void AtomMarkingRuntime::markAtom(JSContext* cx, T* thing) { + return inlinedMarkAtom(cx, thing); +} + +template void AtomMarkingRuntime::markAtom(JSContext* cx, JSAtom* thing); +template void AtomMarkingRuntime::markAtom(JSContext* cx, JS::Symbol* thing); + +void AtomMarkingRuntime::markId(JSContext* cx, jsid id) { + if (id.isAtom()) { + markAtom(cx, id.toAtom()); + return; + } + if (id.isSymbol()) { + markAtom(cx, id.toSymbol()); + return; + } + MOZ_ASSERT(!id.isGCThing()); +} + +void AtomMarkingRuntime::markAtomValue(JSContext* cx, const Value& value) { + if (value.isString()) { + if (value.toString()->isAtom()) { + markAtom(cx, &value.toString()->asAtom()); + } + return; + } + if (value.isSymbol()) { + markAtom(cx, value.toSymbol()); + return; + } + MOZ_ASSERT_IF(value.isGCThing(), value.isObject() || + value.isPrivateGCThing() || + value.isBigInt()); +} + +#ifdef DEBUG +template +bool AtomMarkingRuntime::atomIsMarked(Zone* zone, T* thing) { + static_assert(std::is_same_v || std::is_same_v, + "Should only be called with JSAtom* or JS::Symbol* argument"); + + MOZ_ASSERT(thing); + MOZ_ASSERT(!IsInsideNursery(thing)); + MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone()); + + if (!zone->runtimeFromAnyThread()->permanentAtomsPopulated()) { + return true; + } + + if (thing->isPermanentAndMayBeShared()) { + return true; + } + + if constexpr (std::is_same_v) { + if (thing->isPinned()) { + return true; + } + } + + size_t bit = GetAtomBit(&thing->asTenured()); + return zone->markedAtoms().readonlyThreadsafeGetBit(bit); +} + +template bool AtomMarkingRuntime::atomIsMarked(Zone* zone, JSAtom* thing); +template bool AtomMarkingRuntime::atomIsMarked(Zone* zone, JS::Symbol* thing); + +template <> +bool AtomMarkingRuntime::atomIsMarked(Zone* zone, TenuredCell* thing) { + if (!thing) { + return true; + } + + if (thing->is()) { + JSString* str = thing->as(); + if (!str->isAtom()) { + return true; + } + return atomIsMarked(zone, &str->asAtom()); + } + + if (thing->is()) { + return atomIsMarked(zone, thing->as()); + } + + return true; +} + +bool AtomMarkingRuntime::idIsMarked(Zone* zone, jsid id) { + if (id.isAtom()) { + return atomIsMarked(zone, id.toAtom()); + } + + if (id.isSymbol()) { + return atomIsMarked(zone, id.toSymbol()); + } + + MOZ_ASSERT(!id.isGCThing()); + return true; +} + +bool AtomMarkingRuntime::valueIsMarked(Zone* zone, const Value& value) { + if (value.isString()) { + if (value.toString()->isAtom()) { + return atomIsMarked(zone, &value.toString()->asAtom()); + } + return true; + } + + if (value.isSymbol()) { + return atomIsMarked(zone, value.toSymbol()); + } + + MOZ_ASSERT_IF(value.isGCThing(), value.hasObjectPayload() || + value.isPrivateGCThing() || + value.isBigInt()); + return true; +} + +#endif // DEBUG + +} // namespace gc + +#ifdef DEBUG + +bool AtomIsMarked(Zone* zone, JSAtom* atom) { + return zone->runtimeFromAnyThread()->gc.atomMarking.atomIsMarked(zone, atom); +} + +bool AtomIsMarked(Zone* zone, jsid id) { + return zone->runtimeFromAnyThread()->gc.atomMarking.idIsMarked(zone, id); +} + +bool AtomIsMarked(Zone* zone, const Value& value) { + return zone->runtimeFromAnyThread()->gc.atomMarking.valueIsMarked(zone, + value); +} + +#endif // DEBUG + +} // namespace js diff --git a/js/src/gc/AtomMarking.h b/js/src/gc/AtomMarking.h new file mode 100644 index 0000000000..e7e97fb389 --- /dev/null +++ b/js/src/gc/AtomMarking.h @@ -0,0 +1,86 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_AtomMarking_h +#define gc_AtomMarking_h + +#include "NamespaceImports.h" +#include "js/Vector.h" +#include "threading/ProtectedData.h" + +namespace js { + +class AutoLockGC; +class DenseBitmap; + +namespace gc { + +class Arena; + +// This class manages state used for marking atoms during GCs. +// See AtomMarking.cpp for details. +class AtomMarkingRuntime { + // Unused arena atom bitmap indexes. Protected by the GC lock. + js::GCLockData> freeArenaIndexes; + + inline void markChildren(JSContext* cx, JSAtom*); + inline void markChildren(JSContext* cx, JS::Symbol* symbol); + + public: + // The extent of all allocated and free words in atom mark bitmaps. + // This monotonically increases and may be read from without locking. + mozilla::Atomic allocatedWords; + + AtomMarkingRuntime() : allocatedWords(0) {} + + // Mark an arena as holding things in the atoms zone. + void registerArena(Arena* arena, const AutoLockGC& lock); + + // Mark an arena as no longer holding things in the atoms zone. + void unregisterArena(Arena* arena, const AutoLockGC& lock); + + // Fill |bitmap| with an atom marking bitmap based on the things that are + // currently marked in the chunks used by atoms zone arenas. This returns + // false on an allocation failure (but does not report an exception). + bool computeBitmapFromChunkMarkBits(JSRuntime* runtime, DenseBitmap& bitmap); + + // Update the atom marking bitmap in |zone| according to another + // overapproximation of the reachable atoms in |bitmap|. + void refineZoneBitmapForCollectedZone(Zone* zone, const DenseBitmap& bitmap); + + // Set any bits in the chunk mark bitmaps for atoms which are marked in any + // uncollected zone in the runtime. + void markAtomsUsedByUncollectedZones(JSRuntime* runtime); + + // Mark an atom or id as being newly reachable by the context's zone. + template + void markAtom(JSContext* cx, T* thing); + + // Version of markAtom that's always inlined, for performance-sensitive + // callers. + template + MOZ_ALWAYS_INLINE bool inlinedMarkAtomInternal(JSContext* cx, T* thing); + template + MOZ_ALWAYS_INLINE void inlinedMarkAtom(JSContext* cx, T* thing); + template + MOZ_ALWAYS_INLINE bool inlinedMarkAtomFallible(JSContext* cx, T* thing); + + void markId(JSContext* cx, jsid id); + void markAtomValue(JSContext* cx, const Value& value); + +#ifdef DEBUG + // Return whether |thing/id| is in the atom marking bitmap for |zone|. + template + bool atomIsMarked(Zone* zone, T* thing); + bool idIsMarked(Zone* zone, jsid id); + bool valueIsMarked(Zone* zone, const Value& value); +#endif +}; + +} // namespace gc +} // namespace js + +#endif // gc_AtomMarking_h diff --git a/js/src/gc/Barrier.cpp b/js/src/gc/Barrier.cpp new file mode 100644 index 0000000000..c0b9d28798 --- /dev/null +++ b/js/src/gc/Barrier.cpp @@ -0,0 +1,144 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Barrier.h" + +#include "gc/Marking.h" +#include "jit/JitContext.h" +#include "js/HashTable.h" +#include "js/shadow/Zone.h" // JS::shadow::Zone +#include "js/Value.h" +#include "vm/BigIntType.h" // JS::BigInt +#include "vm/EnvironmentObject.h" +#include "vm/GeneratorObject.h" +#include "vm/JSObject.h" +#include "vm/PropMap.h" +#include "wasm/WasmJS.h" + +#include "gc/StableCellHasher-inl.h" + +namespace js { + +bool RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone) { + MOZ_ASSERT( + CurrentThreadCanAccessRuntime(shadowZone->runtimeFromMainThread())); + return JS::RuntimeHeapIsMajorCollecting(); +} + +#ifdef DEBUG + +bool IsMarkedBlack(JSObject* obj) { return obj->isMarkedBlack(); } + +bool HeapSlot::preconditionForSet(NativeObject* owner, Kind kind, + uint32_t slot) const { + if (kind == Slot) { + return &owner->getSlotRef(slot) == this; + } + + uint32_t numShifted = owner->getElementsHeader()->numShiftedElements(); + MOZ_ASSERT(slot >= numShifted); + return &owner->getDenseElement(slot - numShifted) == (const Value*)this; +} + +void HeapSlot::assertPreconditionForPostWriteBarrier( + NativeObject* obj, Kind kind, uint32_t slot, const Value& target) const { + if (kind == Slot) { + MOZ_ASSERT(obj->getSlotAddressUnchecked(slot)->get() == target); + } else { + uint32_t numShifted = obj->getElementsHeader()->numShiftedElements(); + MOZ_ASSERT(slot >= numShifted); + MOZ_ASSERT( + static_cast(obj->getDenseElements() + (slot - numShifted)) + ->get() == target); + } + + if (!obj->zone()->isGCPreparing()) { + AssertTargetIsNotGray(obj); + } +} + +bool CurrentThreadIsIonCompiling() { + jit::JitContext* jcx = jit::MaybeGetJitContext(); + return jcx && jcx->inIonBackend(); +} + +#endif // DEBUG + +#if !MOZ_IS_GCC +template struct JS_PUBLIC_API StableCellHasher; +#endif + +} // namespace js + +// Post-write barrier, used by the C++ Heap implementation. + +JS_PUBLIC_API void JS::HeapObjectPostWriteBarrier(JSObject** objp, + JSObject* prev, + JSObject* next) { + MOZ_ASSERT(objp); + js::InternalBarrierMethods::postBarrier(objp, prev, next); +} + +JS_PUBLIC_API void JS::HeapStringPostWriteBarrier(JSString** strp, + JSString* prev, + JSString* next) { + MOZ_ASSERT(strp); + js::InternalBarrierMethods::postBarrier(strp, prev, next); +} + +JS_PUBLIC_API void JS::HeapBigIntPostWriteBarrier(JS::BigInt** bip, + JS::BigInt* prev, + JS::BigInt* next) { + MOZ_ASSERT(bip); + js::InternalBarrierMethods::postBarrier(bip, prev, next); +} + +JS_PUBLIC_API void JS::HeapValuePostWriteBarrier(JS::Value* valuep, + const Value& prev, + const Value& next) { + MOZ_ASSERT(valuep); + js::InternalBarrierMethods::postBarrier(valuep, prev, next); +} + +// Combined pre- and post-write barriers, used by the rust Heap +// implementation. + +JS_PUBLIC_API void JS::HeapObjectWriteBarriers(JSObject** objp, JSObject* prev, + JSObject* next) { + MOZ_ASSERT(objp); + js::InternalBarrierMethods::preBarrier(prev); + js::InternalBarrierMethods::postBarrier(objp, prev, next); +} + +JS_PUBLIC_API void JS::HeapStringWriteBarriers(JSString** strp, JSString* prev, + JSString* next) { + MOZ_ASSERT(strp); + js::InternalBarrierMethods::preBarrier(prev); + js::InternalBarrierMethods::postBarrier(strp, prev, next); +} + +JS_PUBLIC_API void JS::HeapBigIntWriteBarriers(JS::BigInt** bip, + JS::BigInt* prev, + JS::BigInt* next) { + MOZ_ASSERT(bip); + js::InternalBarrierMethods::preBarrier(prev); + js::InternalBarrierMethods::postBarrier(bip, prev, next); +} + +JS_PUBLIC_API void JS::HeapScriptWriteBarriers(JSScript** scriptp, + JSScript* prev, JSScript* next) { + MOZ_ASSERT(scriptp); + js::InternalBarrierMethods::preBarrier(prev); + js::InternalBarrierMethods::postBarrier(scriptp, prev, next); +} + +JS_PUBLIC_API void JS::HeapValueWriteBarriers(JS::Value* valuep, + const Value& prev, + const Value& next) { + MOZ_ASSERT(valuep); + js::InternalBarrierMethods::preBarrier(prev); + js::InternalBarrierMethods::postBarrier(valuep, prev, next); +} diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h new file mode 100644 index 0000000000..d4451a60e5 --- /dev/null +++ b/js/src/gc/Barrier.h @@ -0,0 +1,1257 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Barrier_h +#define gc_Barrier_h + +#include // std::true_type + +#include "NamespaceImports.h" + +#include "gc/Cell.h" +#include "gc/GCContext.h" +#include "gc/StoreBuffer.h" +#include "js/ComparisonOperators.h" // JS::detail::DefineComparisonOps +#include "js/experimental/TypedData.h" // js::EnableIfABOVType +#include "js/HeapAPI.h" +#include "js/Id.h" +#include "js/RootingAPI.h" +#include "js/Value.h" +#include "util/Poison.h" + +/* + * [SMDOC] GC Barriers + * + * Several kinds of barrier are necessary to allow the GC to function correctly. + * These are triggered by reading or writing to GC pointers in the heap and + * serve to tell the collector about changes to the graph of reachable GC + * things. + * + * Since it would be awkward to change every write to memory into a function + * call, this file contains a bunch of C++ classes and templates that use + * operator overloading to take care of barriers automatically. In most cases, + * all that's necessary is to replace: + * + * Type* field; + * + * with: + * + * HeapPtr field; + * + * All heap-based GC pointers and tagged pointers must use one of these classes, + * except in a couple of exceptional cases. + * + * These classes are designed to be used by the internals of the JS engine. + * Barriers designed to be used externally are provided in js/RootingAPI.h. + * + * Overview + * ======== + * + * This file implements the following concrete classes: + * + * HeapPtr General wrapper for heap-based pointers that provides pre- and + * post-write barriers. Most clients should use this. + * + * GCPtr An optimisation of HeapPtr for objects which are only destroyed + * by GC finalization (this rules out use in Vector, for example). + * + * PreBarriered Provides a pre-barrier but not a post-barrier. Necessary when + * generational GC updates are handled manually, e.g. for hash + * table keys that don't use StableCellHasher. + * + * HeapSlot Provides pre and post-barriers, optimised for use in JSObject + * slots and elements. + * + * WeakHeapPtr Provides read and post-write barriers, for use with weak + * pointers. + * + * UnsafeBarePtr Provides no barriers. Don't add new uses of this, or only if + * you really know what you are doing. + * + * The following classes are implemented in js/RootingAPI.h (in the JS + * namespace): + * + * Heap General wrapper for external clients. Like HeapPtr but also + * handles cycle collector concerns. Most external clients should + * use this. + * + * Heap::Tenured Like Heap but doesn't allow nursery pointers. Allows storing + * flags in unused lower bits of the pointer. + * + * Which class to use? + * ------------------- + * + * Answer the following questions to decide which barrier class is right for + * your use case: + * + * Is your code part of the JS engine? + * Yes, it's internal => + * Is your pointer weak or strong? + * Strong => + * Do you want automatic handling of nursery pointers? + * Yes, of course => + * Can your object be destroyed outside of a GC? + * Yes => Use HeapPtr + * No => Use GCPtr (optimization) + * No, I'll do this myself => + * Do you want pre-barriers so incremental marking works? + * Yes, of course => Use PreBarriered + * No, and I'll fix all the bugs myself => Use UnsafeBarePtr + * Weak => Use WeakHeapPtr + * No, it's external => + * Can your pointer refer to nursery objects? + * Yes => Use JS::Heap + * Never => Use JS::Heap::Tenured (optimization) + * + * If in doubt, use HeapPtr. + * + * Write barriers + * ============== + * + * A write barrier is a mechanism used by incremental or generational GCs to + * ensure that every value that needs to be marked is marked. In general, the + * write barrier should be invoked whenever a write can cause the set of things + * traced through by the GC to change. This includes: + * + * - writes to object properties + * - writes to array slots + * - writes to fields like JSObject::shape_ that we trace through + * - writes to fields in private data + * - writes to non-markable fields like JSObject::private that point to + * markable data + * + * The last category is the trickiest. Even though the private pointer does not + * point to a GC thing, changing the private pointer may change the set of + * objects that are traced by the GC. Therefore it needs a write barrier. + * + * Every barriered write should have the following form: + * + * + * obj->field = value; // do the actual write + * + * + * The pre-barrier is used for incremental GC and the post-barrier is for + * generational GC. + * + * Pre-write barrier + * ----------------- + * + * To understand the pre-barrier, let's consider how incremental GC works. The + * GC itself is divided into "slices". Between each slice, JS code is allowed to + * run. Each slice should be short so that the user doesn't notice the + * interruptions. In our GC, the structure of the slices is as follows: + * + * 1. ... JS work, which leads to a request to do GC ... + * 2. [first GC slice, which performs all root marking and (maybe) more marking] + * 3. ... more JS work is allowed to run ... + * 4. [GC mark slice, which runs entirely in + * GCRuntime::markUntilBudgetExhausted] + * 5. ... more JS work ... + * 6. [GC mark slice, which runs entirely in + * GCRuntime::markUntilBudgetExhausted] + * 7. ... more JS work ... + * 8. [GC marking finishes; sweeping done non-incrementally; GC is done] + * 9. ... JS continues uninterrupted now that GC is finishes ... + * + * Of course, there may be a different number of slices depending on how much + * marking is to be done. + * + * The danger inherent in this scheme is that the JS code in steps 3, 5, and 7 + * might change the heap in a way that causes the GC to collect an object that + * is actually reachable. The write barrier prevents this from happening. We use + * a variant of incremental GC called "snapshot at the beginning." This approach + * guarantees the invariant that if an object is reachable in step 2, then we + * will mark it eventually. The name comes from the idea that we take a + * theoretical "snapshot" of all reachable objects in step 2; all objects in + * that snapshot should eventually be marked. (Note that the write barrier + * verifier code takes an actual snapshot.) + * + * The basic correctness invariant of a snapshot-at-the-beginning collector is + * that any object reachable at the end of the GC (step 9) must either: + * (1) have been reachable at the beginning (step 2) and thus in the snapshot + * (2) or must have been newly allocated, in steps 3, 5, or 7. + * To deal with case (2), any objects allocated during an incremental GC are + * automatically marked black. + * + * This strategy is actually somewhat conservative: if an object becomes + * unreachable between steps 2 and 8, it would be safe to collect it. We won't, + * mainly for simplicity. (Also, note that the snapshot is entirely + * theoretical. We don't actually do anything special in step 2 that we wouldn't + * do in a non-incremental GC. + * + * It's the pre-barrier's job to maintain the snapshot invariant. Consider the + * write "obj->field = value". Let the prior value of obj->field be + * value0. Since it's possible that value0 may have been what obj->field + * contained in step 2, when the snapshot was taken, the barrier marks + * value0. Note that it only does this if we're in the middle of an incremental + * GC. Since this is rare, the cost of the write barrier is usually just an + * extra branch. + * + * In practice, we implement the pre-barrier differently based on the type of + * value0. E.g., see JSObject::preWriteBarrier, which is used if obj->field is + * a JSObject*. It takes value0 as a parameter. + * + * Post-write barrier + * ------------------ + * + * For generational GC, we want to be able to quickly collect the nursery in a + * minor collection. Part of the way this is achieved is to only mark the + * nursery itself; tenured things, which may form the majority of the heap, are + * not traced through or marked. This leads to the problem of what to do about + * tenured objects that have pointers into the nursery: if such things are not + * marked, they may be discarded while there are still live objects which + * reference them. The solution is to maintain information about these pointers, + * and mark their targets when we start a minor collection. + * + * The pointers can be thought of as edges in an object graph, and the set of + * edges from the tenured generation into the nursery is known as the remembered + * set. Post barriers are used to track this remembered set. + * + * Whenever a slot which could contain such a pointer is written, we check + * whether the pointed-to thing is in the nursery (if storeBuffer() returns a + * buffer). If so we add the cell into the store buffer, which is the + * collector's representation of the remembered set. This means that when we + * come to do a minor collection we can examine the contents of the store buffer + * and mark any edge targets that are in the nursery. + * + * Read barriers + * ============= + * + * Weak pointer read barrier + * ------------------------- + * + * Weak pointers must have a read barrier to prevent the referent from being + * collected if it is read after the start of an incremental GC. + * + * The problem happens when, during an incremental GC, some code reads a weak + * pointer and writes it somewhere on the heap that has been marked black in a + * previous slice. Since the weak pointer will not otherwise be marked and will + * be swept and finalized in the last slice, this will leave the pointer just + * written dangling after the GC. To solve this, we immediately mark black all + * weak pointers that get read between slices so that it is safe to store them + * in an already marked part of the heap, e.g. in Rooted. + * + * Cycle collector read barrier + * ---------------------------- + * + * Heap pointers external to the engine may be marked gray. The JS API has an + * invariant that no gray pointers may be passed, and this maintained by a read + * barrier that calls ExposeGCThingToActiveJS on such pointers. This is + * implemented by JS::Heap in js/RootingAPI.h. + * + * Implementation Details + * ====================== + * + * One additional note: not all object writes need to be pre-barriered. Writes + * to newly allocated objects do not need a pre-barrier. In these cases, we use + * the "obj->field.init(value)" method instead of "obj->field = value". We use + * the init naming idiom in many places to signify that a field is being + * assigned for the first time. + * + * This file implements the following hierarchy of classes: + * + * BarrieredBase base class of all barriers + * | | + * | WriteBarriered base class which provides common write operations + * | | | | | + * | | | | PreBarriered provides pre-barriers only + * | | | | + * | | | GCPtr provides pre- and post-barriers + * | | | + * | | HeapPtr provides pre- and post-barriers; is relocatable + * | | and deletable for use inside C++ managed memory + * | | + * | HeapSlot similar to GCPtr, but tailored to slots storage + * | + * ReadBarriered base class which provides common read operations + * | + * WeakHeapPtr provides read barriers only + * + * + * The implementation of the barrier logic is implemented in the + * Cell/TenuredCell base classes, which are called via: + * + * WriteBarriered::pre + * -> InternalBarrierMethods::preBarrier + * -> Cell::preWriteBarrier + * -> InternalBarrierMethods::preBarrier + * -> InternalBarrierMethods::preBarrier + * -> InternalBarrierMethods::preBarrier + * -> Cell::preWriteBarrier + * + * GCPtr::post and HeapPtr::post + * -> InternalBarrierMethods::postBarrier + * -> gc::PostWriteBarrierImpl + * -> InternalBarrierMethods::postBarrier + * -> StoreBuffer::put + * + * Barriers for use outside of the JS engine call into the same barrier + * implementations at InternalBarrierMethods::post via an indirect call to + * Heap(.+)PostWriteBarrier. + * + * These clases are designed to be used to wrap GC thing pointers or values that + * act like them (i.e. JS::Value and jsid). It is possible to use them for + * other types by supplying the necessary barrier implementations but this + * is not usually necessary and should be done with caution. + */ + +namespace js { + +class NativeObject; + +namespace gc { + +inline void ValueReadBarrier(const Value& v) { + MOZ_ASSERT(v.isGCThing()); + ReadBarrierImpl(v.toGCThing()); +} + +inline void ValuePreWriteBarrier(const Value& v) { + MOZ_ASSERT(v.isGCThing()); + PreWriteBarrierImpl(v.toGCThing()); +} + +inline void IdPreWriteBarrier(jsid id) { + MOZ_ASSERT(id.isGCThing()); + PreWriteBarrierImpl(&id.toGCThing()->asTenured()); +} + +inline void CellPtrPreWriteBarrier(JS::GCCellPtr thing) { + MOZ_ASSERT(thing); + PreWriteBarrierImpl(thing.asCell()); +} + +} // namespace gc + +#ifdef DEBUG + +bool CurrentThreadIsTouchingGrayThings(); + +bool IsMarkedBlack(JSObject* obj); + +#endif + +template +struct InternalBarrierMethods {}; + +template +struct InternalBarrierMethods { + static_assert(std::is_base_of_v, "Expected a GC thing type"); + + static bool isMarkable(const T* v) { return v != nullptr; } + + static void preBarrier(T* v) { gc::PreWriteBarrier(v); } + + static void postBarrier(T** vp, T* prev, T* next) { + gc::PostWriteBarrier(vp, prev, next); + } + + static void readBarrier(T* v) { gc::ReadBarrier(v); } + +#ifdef DEBUG + static void assertThingIsNotGray(T* v) { return T::assertThingIsNotGray(v); } +#endif +}; + +template <> +struct InternalBarrierMethods { + static bool isMarkable(const Value& v) { return v.isGCThing(); } + + static void preBarrier(const Value& v) { + if (v.isGCThing()) { + gc::ValuePreWriteBarrier(v); + } + } + + static MOZ_ALWAYS_INLINE void postBarrier(Value* vp, const Value& prev, + const Value& next) { + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(vp); + + // If the target needs an entry, add it. + js::gc::StoreBuffer* sb; + if (next.isGCThing() && (sb = next.toGCThing()->storeBuffer())) { + // If we know that the prev has already inserted an entry, we can + // skip doing the lookup to add the new entry. Note that we cannot + // safely assert the presence of the entry because it may have been + // added via a different store buffer. + if (prev.isGCThing() && prev.toGCThing()->storeBuffer()) { + return; + } + sb->putValue(vp); + return; + } + // Remove the prev entry if the new value does not need it. + if (prev.isGCThing() && (sb = prev.toGCThing()->storeBuffer())) { + sb->unputValue(vp); + } + } + + static void readBarrier(const Value& v) { + if (v.isGCThing()) { + gc::ValueReadBarrier(v); + } + } + +#ifdef DEBUG + static void assertThingIsNotGray(const Value& v) { + JS::AssertValueIsNotGray(v); + } +#endif +}; + +template <> +struct InternalBarrierMethods { + static bool isMarkable(jsid id) { return id.isGCThing(); } + static void preBarrier(jsid id) { + if (id.isGCThing()) { + gc::IdPreWriteBarrier(id); + } + } + static void postBarrier(jsid* idp, jsid prev, jsid next) {} +#ifdef DEBUG + static void assertThingIsNotGray(jsid id) { JS::AssertIdIsNotGray(id); } +#endif +}; + +// Specialization for JS::ArrayBufferOrView subclasses. +template +struct InternalBarrierMethods> { + using BM = BarrierMethods; + + static bool isMarkable(const T& thing) { return bool(thing); } + static void preBarrier(const T& thing) { + gc::PreWriteBarrier(thing.asObjectUnbarriered()); + } + static void postBarrier(T* tp, const T& prev, const T& next) { + BM::postWriteBarrier(tp, prev, next); + } + static void readBarrier(const T& thing) { BM::readBarrier(thing); } +#ifdef DEBUG + static void assertThingIsNotGray(const T& thing) { + JSObject* obj = thing.asObjectUnbarriered(); + if (obj) { + JS::AssertValueIsNotGray(JS::ObjectValue(*obj)); + } + } +#endif +}; + +template +static inline void AssertTargetIsNotGray(const T& v) { +#ifdef DEBUG + if (!CurrentThreadIsTouchingGrayThings()) { + InternalBarrierMethods::assertThingIsNotGray(v); + } +#endif +} + +// Base class of all barrier types. +// +// This is marked non-memmovable since post barriers added by derived classes +// can add pointers to class instances to the store buffer. +template +class MOZ_NON_MEMMOVABLE BarrieredBase { + protected: + // BarrieredBase is not directly instantiable. + explicit BarrieredBase(const T& v) : value(v) {} + + // BarrieredBase subclasses cannot be copy constructed by default. + BarrieredBase(const BarrieredBase& other) = default; + + // Storage for all barrier classes. |value| must be a GC thing reference + // type: either a direct pointer to a GC thing or a supported tagged + // pointer that can reference GC things, such as JS::Value or jsid. Nested + // barrier types are NOT supported. See assertTypeConstraints. + T value; + + public: + using ElementType = T; + + // Note: this is public because C++ cannot friend to a specific template + // instantiation. Friending to the generic template leads to a number of + // unintended consequences, including template resolution ambiguity and a + // circular dependency with Tracing.h. + T* unbarrieredAddress() const { return const_cast(&value); } +}; + +// Base class for barriered pointer types that intercept only writes. +template +class WriteBarriered : public BarrieredBase, + public WrappedPtrOperations> { + protected: + using BarrieredBase::value; + + // WriteBarriered is not directly instantiable. + explicit WriteBarriered(const T& v) : BarrieredBase(v) {} + + public: + DECLARE_POINTER_CONSTREF_OPS(T); + + // Use this if the automatic coercion to T isn't working. + const T& get() const { return this->value; } + + // Use this if you want to change the value without invoking barriers. + // Obviously this is dangerous unless you know the barrier is not needed. + void unbarrieredSet(const T& v) { this->value = v; } + + // For users who need to manually barrier the raw types. + static void preWriteBarrier(const T& v) { + InternalBarrierMethods::preBarrier(v); + } + + protected: + void pre() { InternalBarrierMethods::preBarrier(this->value); } + MOZ_ALWAYS_INLINE void post(const T& prev, const T& next) { + InternalBarrierMethods::postBarrier(&this->value, prev, next); + } +}; + +#define DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(Wrapper, T) \ + DECLARE_POINTER_ASSIGN_OPS(Wrapper, T) \ + Wrapper& operator=(Wrapper&& other) { \ + setUnchecked(other.release()); \ + return *this; \ + } + +/* + * PreBarriered only automatically handles pre-barriers. Post-barriers must be + * manually implemented when using this class. GCPtr and HeapPtr should be used + * in all cases that do not require explicit low-level control of moving + * behavior. + * + * This class is useful for example for HashMap keys where automatically + * updating a moved nursery pointer would break the hash table. + */ +template +class PreBarriered : public WriteBarriered { + public: + PreBarriered() : WriteBarriered(JS::SafelyInitialized::create()) {} + /* + * Allow implicit construction for use in generic contexts. + */ + MOZ_IMPLICIT PreBarriered(const T& v) : WriteBarriered(v) {} + + explicit PreBarriered(const PreBarriered& other) + : WriteBarriered(other.value) {} + + PreBarriered(PreBarriered&& other) : WriteBarriered(other.release()) {} + + ~PreBarriered() { this->pre(); } + + void init(const T& v) { this->value = v; } + + /* Use to set the pointer to nullptr. */ + void clear() { set(JS::SafelyInitialized::create()); } + + DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(PreBarriered, T); + + void set(const T& v) { + AssertTargetIsNotGray(v); + setUnchecked(v); + } + + private: + void setUnchecked(const T& v) { + this->pre(); + this->value = v; + } + + T release() { + T tmp = this->value; + this->value = JS::SafelyInitialized::create(); + return tmp; + } +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::PreBarriered& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +/* + * A pre- and post-barriered heap pointer, for use inside the JS engine. + * + * It must only be stored in memory that has GC lifetime. GCPtr must not be + * used in contexts where it may be implicitly moved or deleted, e.g. most + * containers. + * + * The post-barriers implemented by this class are faster than those + * implemented by js::HeapPtr or JS::Heap at the cost of not + * automatically handling deletion or movement. + */ +template +class GCPtr : public WriteBarriered { + public: + GCPtr() : WriteBarriered(JS::SafelyInitialized::create()) {} + + explicit GCPtr(const T& v) : WriteBarriered(v) { + this->post(JS::SafelyInitialized::create(), v); + } + + explicit GCPtr(const GCPtr& v) : WriteBarriered(v) { + this->post(JS::SafelyInitialized::create(), v); + } + +#ifdef DEBUG + ~GCPtr() { + // No barriers are necessary as this only happens when the GC is sweeping. + // + // If this assertion fails you may need to make the containing object use a + // HeapPtr instead, as this can be deleted from outside of GC. + MOZ_ASSERT(CurrentThreadIsGCSweeping() || CurrentThreadIsGCFinalizing()); + + Poison(this, JS_FREED_HEAP_PTR_PATTERN, sizeof(*this), + MemCheckKind::MakeNoAccess); + } +#endif + + void init(const T& v) { + AssertTargetIsNotGray(v); + this->value = v; + this->post(JS::SafelyInitialized::create(), v); + } + + DECLARE_POINTER_ASSIGN_OPS(GCPtr, T); + + void set(const T& v) { + AssertTargetIsNotGray(v); + setUnchecked(v); + } + + private: + void setUnchecked(const T& v) { + this->pre(); + T tmp = this->value; + this->value = v; + this->post(tmp, this->value); + } + + /* + * Unlike HeapPtr, GCPtr must be managed with GC lifetimes. + * Specifically, the memory used by the pointer itself must be live until + * at least the next minor GC. For that reason, move semantics are invalid + * and are deleted here. Please note that not all containers support move + * semantics, so this does not completely prevent invalid uses. + */ + GCPtr(GCPtr&&) = delete; + GCPtr& operator=(GCPtr&&) = delete; +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::GCPtr& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +/* + * A pre- and post-barriered heap pointer, for use inside the JS engine. These + * heap pointers can be stored in C++ containers like GCVector and GCHashMap. + * + * The GC sometimes keeps pointers to pointers to GC things --- for example, to + * track references into the nursery. However, C++ containers like GCVector and + * GCHashMap usually reserve the right to relocate their elements any time + * they're modified, invalidating all pointers to the elements. HeapPtr + * has a move constructor which knows how to keep the GC up to date if it is + * moved to a new location. + * + * However, because of this additional communication with the GC, HeapPtr + * is somewhat slower, so it should only be used in contexts where this ability + * is necessary. + * + * Obviously, JSObjects, JSStrings, and the like get tenured and compacted, so + * whatever pointers they contain get relocated, in the sense used here. + * However, since the GC itself is moving those values, it takes care of its + * internal pointers to those pointers itself. HeapPtr is only necessary + * when the relocation would otherwise occur without the GC's knowledge. + */ +template +class HeapPtr : public WriteBarriered { + public: + HeapPtr() : WriteBarriered(JS::SafelyInitialized::create()) {} + + // Implicitly adding barriers is a reasonable default. + MOZ_IMPLICIT HeapPtr(const T& v) : WriteBarriered(v) { + this->post(JS::SafelyInitialized::create(), this->value); + } + + MOZ_IMPLICIT HeapPtr(const HeapPtr& other) : WriteBarriered(other) { + this->post(JS::SafelyInitialized::create(), this->value); + } + + HeapPtr(HeapPtr&& other) : WriteBarriered(other.release()) { + this->post(JS::SafelyInitialized::create(), this->value); + } + + ~HeapPtr() { + this->pre(); + this->post(this->value, JS::SafelyInitialized::create()); + } + + void init(const T& v) { + MOZ_ASSERT(this->value == JS::SafelyInitialized::create()); + AssertTargetIsNotGray(v); + this->value = v; + this->post(JS::SafelyInitialized::create(), this->value); + } + + DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(HeapPtr, T); + + void set(const T& v) { + AssertTargetIsNotGray(v); + setUnchecked(v); + } + + /* Make this friend so it can access pre() and post(). */ + template + friend inline void BarrieredSetPair(Zone* zone, HeapPtr& v1, T1* val1, + HeapPtr& v2, T2* val2); + + protected: + void setUnchecked(const T& v) { + this->pre(); + postBarrieredSet(v); + } + + void postBarrieredSet(const T& v) { + T tmp = this->value; + this->value = v; + this->post(tmp, this->value); + } + + T release() { + T tmp = this->value; + postBarrieredSet(JS::SafelyInitialized::create()); + return tmp; + } +}; + +/* + * A pre-barriered heap pointer, for use inside the JS engine. + * + * Similar to GCPtr, but used for a pointer to a malloc-allocated structure + * containing GC thing pointers. + * + * It must only be stored in memory that has GC lifetime. It must not be used in + * contexts where it may be implicitly moved or deleted, e.g. most containers. + * + * A post-barrier is unnecessary since malloc-allocated structures cannot be in + * the nursery. + */ +template +class GCStructPtr : public BarrieredBase { + public: + // This is sometimes used to hold tagged pointers. + static constexpr uintptr_t MaxTaggedPointer = 0x2; + + GCStructPtr() : BarrieredBase(JS::SafelyInitialized::create()) {} + + // Implicitly adding barriers is a reasonable default. + MOZ_IMPLICIT GCStructPtr(const T& v) : BarrieredBase(v) {} + + GCStructPtr(const GCStructPtr& other) : BarrieredBase(other) {} + + GCStructPtr(GCStructPtr&& other) : BarrieredBase(other.release()) {} + + ~GCStructPtr() { + // No barriers are necessary as this only happens when the GC is sweeping. + MOZ_ASSERT_IF(isTraceable(), + CurrentThreadIsGCSweeping() || CurrentThreadIsGCFinalizing()); + } + + void init(const T& v) { + MOZ_ASSERT(this->get() == JS::SafelyInitialized()); + AssertTargetIsNotGray(v); + this->value = v; + } + + void set(JS::Zone* zone, const T& v) { + pre(zone); + this->value = v; + } + + T get() const { return this->value; } + operator T() const { return get(); } + T operator->() const { return get(); } + + protected: + bool isTraceable() const { return uintptr_t(get()) > MaxTaggedPointer; } + + void pre(JS::Zone* zone) { + if (isTraceable()) { + PreWriteBarrier(zone, get()); + } + } +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::HeapPtr& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +// Base class for barriered pointer types that intercept reads and writes. +template +class ReadBarriered : public BarrieredBase { + protected: + // ReadBarriered is not directly instantiable. + explicit ReadBarriered(const T& v) : BarrieredBase(v) {} + + void read() const { InternalBarrierMethods::readBarrier(this->value); } + void post(const T& prev, const T& next) { + InternalBarrierMethods::postBarrier(&this->value, prev, next); + } +}; + +// Incremental GC requires that weak pointers have read barriers. See the block +// comment at the top of Barrier.h for a complete discussion of why. +// +// Note that this class also has post-barriers, so is safe to use with nursery +// pointers. However, when used as a hashtable key, care must still be taken to +// insert manual post-barriers on the table for rekeying if the key is based in +// any way on the address of the object. +template +class WeakHeapPtr : public ReadBarriered, + public WrappedPtrOperations> { + protected: + using ReadBarriered::value; + + public: + WeakHeapPtr() : ReadBarriered(JS::SafelyInitialized::create()) {} + + // It is okay to add barriers implicitly. + MOZ_IMPLICIT WeakHeapPtr(const T& v) : ReadBarriered(v) { + this->post(JS::SafelyInitialized::create(), v); + } + + // The copy constructor creates a new weak edge but the wrapped pointer does + // not escape, so no read barrier is necessary. + explicit WeakHeapPtr(const WeakHeapPtr& other) : ReadBarriered(other) { + this->post(JS::SafelyInitialized::create(), value); + } + + // Move retains the lifetime status of the source edge, so does not fire + // the read barrier of the defunct edge. + WeakHeapPtr(WeakHeapPtr&& other) : ReadBarriered(other.release()) { + this->post(JS::SafelyInitialized::create(), value); + } + + ~WeakHeapPtr() { + this->post(this->value, JS::SafelyInitialized::create()); + } + + WeakHeapPtr& operator=(const WeakHeapPtr& v) { + AssertTargetIsNotGray(v.value); + T prior = this->value; + this->value = v.value; + this->post(prior, v.value); + return *this; + } + + const T& get() const { + if (InternalBarrierMethods::isMarkable(this->value)) { + this->read(); + } + return this->value; + } + + const T& unbarrieredGet() const { return this->value; } + + explicit operator bool() const { return bool(this->value); } + + operator const T&() const { return get(); } + + const T& operator->() const { return get(); } + + void set(const T& v) { + AssertTargetIsNotGray(v); + setUnchecked(v); + } + + void unbarrieredSet(const T& v) { + AssertTargetIsNotGray(v); + this->value = v; + } + + private: + void setUnchecked(const T& v) { + T tmp = this->value; + this->value = v; + this->post(tmp, v); + } + + T release() { + T tmp = value; + set(JS::SafelyInitialized::create()); + return tmp; + } +}; + +// A wrapper for a bare pointer, with no barriers. +// +// This should only be necessary in a limited number of cases. Please don't add +// more uses of this if at all possible. +template +class UnsafeBarePtr : public BarrieredBase { + public: + UnsafeBarePtr() : BarrieredBase(JS::SafelyInitialized::create()) {} + MOZ_IMPLICIT UnsafeBarePtr(T v) : BarrieredBase(v) {} + const T& get() const { return this->value; } + void set(T newValue) { this->value = newValue; } + DECLARE_POINTER_CONSTREF_OPS(T); +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::WeakHeapPtr& v) { + return v.unbarrieredGet(); + } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +// A pre- and post-barriered Value that is specialized to be aware that it +// resides in a slots or elements vector. This allows it to be relocated in +// memory, but with substantially less overhead than a HeapPtr. +class HeapSlot : public WriteBarriered { + public: + enum Kind { Slot = 0, Element = 1 }; + + void init(NativeObject* owner, Kind kind, uint32_t slot, const Value& v) { + value = v; + post(owner, kind, slot, v); + } + + void initAsUndefined() { value.setUndefined(); } + + void destroy() { pre(); } + + void setUndefinedUnchecked() { + pre(); + value.setUndefined(); + } + +#ifdef DEBUG + bool preconditionForSet(NativeObject* owner, Kind kind, uint32_t slot) const; + void assertPreconditionForPostWriteBarrier(NativeObject* obj, Kind kind, + uint32_t slot, + const Value& target) const; +#endif + + MOZ_ALWAYS_INLINE void set(NativeObject* owner, Kind kind, uint32_t slot, + const Value& v) { + MOZ_ASSERT(preconditionForSet(owner, kind, slot)); + pre(); + value = v; + post(owner, kind, slot, v); + } + + private: + void post(NativeObject* owner, Kind kind, uint32_t slot, + const Value& target) { +#ifdef DEBUG + assertPreconditionForPostWriteBarrier(owner, kind, slot, target); +#endif + if (this->value.isGCThing()) { + gc::Cell* cell = this->value.toGCThing(); + if (cell->storeBuffer()) { + cell->storeBuffer()->putSlot(owner, kind, slot, 1); + } + } + } +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template <> +struct DefineComparisonOps : std::true_type { + static const Value& get(const js::HeapSlot& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +class HeapSlotArray { + HeapSlot* array; + + public: + explicit HeapSlotArray(HeapSlot* array) : array(array) {} + + HeapSlot* begin() const { return array; } + + operator const Value*() const { + static_assert(sizeof(GCPtr) == sizeof(Value)); + static_assert(sizeof(HeapSlot) == sizeof(Value)); + return reinterpret_cast(array); + } + operator HeapSlot*() const { return begin(); } + + HeapSlotArray operator+(int offset) const { + return HeapSlotArray(array + offset); + } + HeapSlotArray operator+(uint32_t offset) const { + return HeapSlotArray(array + offset); + } +}; + +/* + * This is a hack for RegExpStatics::updateFromMatch. It allows us to do two + * barriers with only one branch to check if we're in an incremental GC. + */ +template +static inline void BarrieredSetPair(Zone* zone, HeapPtr& v1, T1* val1, + HeapPtr& v2, T2* val2) { + AssertTargetIsNotGray(val1); + AssertTargetIsNotGray(val2); + if (T1::needPreWriteBarrier(zone)) { + v1.pre(); + v2.pre(); + } + v1.postBarrieredSet(val1); + v2.postBarrieredSet(val2); +} + +/* + * ImmutableTenuredPtr is designed for one very narrow case: replacing + * immutable raw pointers to GC-managed things, implicitly converting to a + * handle type for ease of use. Pointers encapsulated by this type must: + * + * be immutable (no incremental write barriers), + * never point into the nursery (no generational write barriers), and + * be traced via MarkRuntime (we use fromMarkedLocation). + * + * In short: you *really* need to know what you're doing before you use this + * class! + */ +template +class MOZ_HEAP_CLASS ImmutableTenuredPtr { + T value; + + public: + operator T() const { return value; } + T operator->() const { return value; } + + // `ImmutableTenuredPtr` is implicitly convertible to `Handle`. + // + // In case you need to convert to `Handle` where `U` is base class of `T`, + // convert this to `Handle` by `toHandle()` and then use implicit + // conversion from `Handle` to `Handle`. + operator Handle() const { return toHandle(); } + Handle toHandle() const { return Handle::fromMarkedLocation(&value); } + + void init(T ptr) { + MOZ_ASSERT(ptr->isTenured()); + AssertTargetIsNotGray(ptr); + value = ptr; + } + + T get() const { return value; } + const T* address() { return &value; } +}; + +// Template to remove any barrier wrapper and get the underlying type. +template +struct RemoveBarrier { + using Type = T; +}; +template +struct RemoveBarrier> { + using Type = T; +}; +template +struct RemoveBarrier> { + using Type = T; +}; +template +struct RemoveBarrier> { + using Type = T; +}; +template +struct RemoveBarrier> { + using Type = T; +}; + +#if MOZ_IS_GCC +template struct JS_PUBLIC_API StableCellHasher; +#endif + +template +struct StableCellHasher> { + using Key = PreBarriered; + using Lookup = T; + + static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::maybeGetHash(l, hashOut); + } + static bool ensureHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::ensureHash(l, hashOut); + } + static HashNumber hash(const Lookup& l) { + return StableCellHasher::hash(l); + } + static bool match(const Key& k, const Lookup& l) { + return StableCellHasher::match(k, l); + } +}; + +template +struct StableCellHasher> { + using Key = HeapPtr; + using Lookup = T; + + static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::maybeGetHash(l, hashOut); + } + static bool ensureHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::ensureHash(l, hashOut); + } + static HashNumber hash(const Lookup& l) { + return StableCellHasher::hash(l); + } + static bool match(const Key& k, const Lookup& l) { + return StableCellHasher::match(k, l); + } +}; + +template +struct StableCellHasher> { + using Key = WeakHeapPtr; + using Lookup = T; + + static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::maybeGetHash(l, hashOut); + } + static bool ensureHash(const Lookup& l, HashNumber* hashOut) { + return StableCellHasher::ensureHash(l, hashOut); + } + static HashNumber hash(const Lookup& l) { + return StableCellHasher::hash(l); + } + static bool match(const Key& k, const Lookup& l) { + return StableCellHasher::match(k.unbarrieredGet(), l); + } +}; + +/* Useful for hashtables with a HeapPtr as key. */ +template +struct HeapPtrHasher { + using Key = HeapPtr; + using Lookup = T; + + static HashNumber hash(Lookup obj) { return DefaultHasher::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.get() == l; } + static void rekey(Key& k, const Key& newKey) { k.unbarrieredSet(newKey); } +}; + +template +struct PreBarrieredHasher { + using Key = PreBarriered; + using Lookup = T; + + static HashNumber hash(Lookup obj) { return DefaultHasher::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.get() == l; } + static void rekey(Key& k, const Key& newKey) { k.unbarrieredSet(newKey); } +}; + +/* Useful for hashtables with a WeakHeapPtr as key. */ +template +struct WeakHeapPtrHasher { + using Key = WeakHeapPtr; + using Lookup = T; + + static HashNumber hash(Lookup obj) { return DefaultHasher::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.unbarrieredGet() == l; } + static void rekey(Key& k, const Key& newKey) { + k.set(newKey.unbarrieredGet()); + } +}; + +template +struct UnsafeBarePtrHasher { + using Key = UnsafeBarePtr; + using Lookup = T; + + static HashNumber hash(const Lookup& l) { return DefaultHasher::hash(l); } + static bool match(const Key& k, Lookup l) { return k.get() == l; } + static void rekey(Key& k, const Key& newKey) { k.set(newKey.get()); } +}; + +} // namespace js + +namespace mozilla { + +template +struct DefaultHasher> : js::HeapPtrHasher {}; + +template +struct DefaultHasher> { + // Not implemented. GCPtr can't be used as a hash table key because it has a + // post barrier but doesn't support relocation. +}; + +template +struct DefaultHasher> : js::PreBarrieredHasher {}; + +template +struct DefaultHasher> : js::WeakHeapPtrHasher {}; + +template +struct DefaultHasher> : js::UnsafeBarePtrHasher {}; + +} // namespace mozilla + +#endif /* gc_Barrier_h */ diff --git a/js/src/gc/Cell.h b/js/src/gc/Cell.h new file mode 100644 index 0000000000..bc2da39b28 --- /dev/null +++ b/js/src/gc/Cell.h @@ -0,0 +1,930 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Cell_h +#define gc_Cell_h + +#include "mozilla/Atomics.h" +#include "mozilla/EndianUtils.h" + +#include + +#include "gc/GCContext.h" +#include "gc/Heap.h" +#include "gc/TraceKind.h" +#include "js/GCAnnotations.h" +#include "js/shadow/Zone.h" // JS::shadow::Zone +#include "js/TypeDecls.h" + +namespace JS { +enum class TraceKind; +} /* namespace JS */ + +namespace js { + +class JS_PUBLIC_API GenericPrinter; + +extern bool RuntimeFromMainThreadIsHeapMajorCollecting( + JS::shadow::Zone* shadowZone); + +#ifdef DEBUG +// Barriers can't be triggered during backend Ion compilation, which may run on +// a helper thread. +extern bool CurrentThreadIsIonCompiling(); +#endif + +extern void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, + gc::Cell** thingp, + const char* name); + +namespace gc { + +enum class AllocKind : uint8_t; +class CellAllocator; // Declared so subtypes of Cell can friend it easily. +class StoreBuffer; +class TenuredCell; + +extern void PerformIncrementalReadBarrier(TenuredCell* cell); +extern void PerformIncrementalPreWriteBarrier(TenuredCell* cell); +extern void PerformIncrementalBarrierDuringFlattening(JSString* str); +extern void UnmarkGrayGCThingRecursively(TenuredCell* cell); + +// Like gc::MarkColor but allows the possibility of the cell being unmarked. +// +// This class mimics an enum class, but supports operator overloading. +class CellColor { + public: + enum Color { White = 0, Gray = 1, Black = 2 }; + + CellColor() : color(White) {} + + MOZ_IMPLICIT CellColor(MarkColor markColor) + : color(markColor == MarkColor::Black ? Black : Gray) {} + + MOZ_IMPLICIT constexpr CellColor(Color c) : color(c) {} + + MarkColor asMarkColor() const { + MOZ_ASSERT(color != White); + return color == Black ? MarkColor::Black : MarkColor::Gray; + } + + // Implement a total ordering for CellColor, with white being 'least marked' + // and black being 'most marked'. + bool operator<(const CellColor other) const { return color < other.color; } + bool operator>(const CellColor other) const { return color > other.color; } + bool operator<=(const CellColor other) const { return color <= other.color; } + bool operator>=(const CellColor other) const { return color >= other.color; } + bool operator!=(const CellColor other) const { return color != other.color; } + bool operator==(const CellColor other) const { return color == other.color; } + explicit operator bool() const { return color != White; } + +#if defined(JS_GC_ZEAL) || defined(DEBUG) + const char* name() const { + switch (color) { + case CellColor::White: + return "white"; + case CellColor::Black: + return "black"; + case CellColor::Gray: + return "gray"; + default: + MOZ_CRASH("Unexpected cell color"); + } + } +#endif + + private: + Color color; +}; + +// Cell header word. Stores GC flags and derived class data. +// +// Loads of GC flags + all stores are marked as (relaxed) atomic operations, +// to deal with the following benign data race during compacting GC: +// +// - Thread 1 checks isForwarded (which is always false in this situation). +// - Thread 2 updates the derived class data (without changing the forwarded +// flag). +// +// To improve performance, we don't use atomic operations for get() because +// atomic operations inhibit certain compiler optimizations: GCC and Clang are +// unable to fold multiple loads even if they're both relaxed atomics. This is +// especially a problem for chained loads such as obj->shape->base->clasp. +class HeaderWord { + // Indicates whether the cell has been forwarded (moved) by generational or + // compacting GC and is now a RelocationOverlay. + static constexpr uintptr_t FORWARD_BIT = Bit(0); + // Bits 1 and 2 are reserved for future use by the GC. + + uintptr_t value_; + + void setAtomic(uintptr_t value) { + __atomic_store_n(&value_, value, __ATOMIC_RELAXED); + } + + public: + static constexpr uintptr_t RESERVED_MASK = + BitMask(gc::CellFlagBitsReservedForGC); + static_assert(gc::CellFlagBitsReservedForGC >= 3, + "Not enough flag bits reserved for GC"); + + uintptr_t getAtomic() const { + return __atomic_load_n(&value_, __ATOMIC_RELAXED); + } + + // Accessors for derived class data. + uintptr_t get() const { + // Note: non-atomic load. See class comment. + uintptr_t value = value_; + MOZ_ASSERT((value & RESERVED_MASK) == 0); + return value; + } + void set(uintptr_t value) { + MOZ_ASSERT((value & RESERVED_MASK) == 0); + setAtomic(value); + } + + // Accessors for GC data. + uintptr_t flags() const { return getAtomic() & RESERVED_MASK; } + bool isForwarded() const { return flags() & FORWARD_BIT; } + void setForwardingAddress(uintptr_t ptr) { + MOZ_ASSERT((ptr & RESERVED_MASK) == 0); + setAtomic(ptr | FORWARD_BIT); + } + uintptr_t getForwardingAddress() const { + MOZ_ASSERT(isForwarded()); + return getAtomic() & ~RESERVED_MASK; + } +}; + +// [SMDOC] GC Cell +// +// A GC cell is the ultimate base class for all GC things. All types allocated +// on the GC heap extend either gc::Cell or gc::TenuredCell. If a type is always +// tenured, prefer the TenuredCell class as base. +// +// The first word of Cell is a HeaderWord (a uintptr_t) that reserves the low +// three bits for GC purposes. The remaining bits are available to sub-classes +// and can be used store a pointer to another gc::Cell. To make use of the +// remaining space, sub-classes derive from a helper class such as +// TenuredCellWithNonGCPointer. +// +// During moving GC operation a Cell may be marked as forwarded. This indicates +// that a gc::RelocationOverlay is currently stored in the Cell's memory and +// should be used to find the new location of the Cell. +struct Cell { + // Cell header word. Stores GC flags and derived class data. + HeaderWord header_; + + public: + Cell() = default; + + Cell(const Cell&) = delete; + void operator=(const Cell&) = delete; + + bool isForwarded() const { return header_.isForwarded(); } + uintptr_t flags() const { return header_.flags(); } + + MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); } + MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; + MOZ_ALWAYS_INLINE TenuredCell& asTenured(); + + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + MOZ_ALWAYS_INLINE bool isMarked(gc::MarkColor color) const; + MOZ_ALWAYS_INLINE bool isMarkedAtLeast(gc::MarkColor color) const; + MOZ_ALWAYS_INLINE CellColor color() const; + + inline JSRuntime* runtimeFromMainThread() const; + + // Note: Unrestricted access to the runtime of a GC thing from an arbitrary + // thread can easily lead to races. Use this method very carefully. + inline JSRuntime* runtimeFromAnyThread() const; + + // May be overridden by GC thing kinds that have a compartment pointer. + inline JS::Compartment* maybeCompartment() const { return nullptr; } + + // The StoreBuffer used to record incoming pointers from the tenured heap. + // This will return nullptr for a tenured cell. + inline StoreBuffer* storeBuffer() const; + + inline JS::TraceKind getTraceKind() const; + + static MOZ_ALWAYS_INLINE bool needPreWriteBarrier(JS::Zone* zone); + + template >> + inline bool is() const { + return getTraceKind() == JS::MapTypeToTraceKind::kind; + } + + template >> + inline T* as() { + // |this|-qualify the |is| call below to avoid compile errors with even + // fairly recent versions of gcc, e.g. 7.1.1 according to bz. + MOZ_ASSERT(this->is()); + return static_cast(this); + } + + template >> + inline const T* as() const { + // |this|-qualify the |is| call below to avoid compile errors with even + // fairly recent versions of gcc, e.g. 7.1.1 according to bz. + MOZ_ASSERT(this->is()); + return static_cast(this); + } + + inline JS::Zone* zone() const; + inline JS::Zone* zoneFromAnyThread() const; + + // Get the zone for a cell known to be in the nursery. + inline JS::Zone* nurseryZone() const; + inline JS::Zone* nurseryZoneFromAnyThread() const; + + // Default implementation for kinds that cannot be permanent. This may be + // overriden by derived classes. + MOZ_ALWAYS_INLINE bool isPermanentAndMayBeShared() const { return false; } + +#ifdef DEBUG + static inline void assertThingIsNotGray(Cell* cell); + inline bool isAligned() const; + void dump(GenericPrinter& out) const; + void dump() const; +#endif + + protected: + uintptr_t address() const; + inline ChunkBase* chunk() const; + + private: + // Cells are destroyed by the GC. Do not delete them directly. + void operator delete(void*) = delete; +} JS_HAZ_GC_THING; + +// A GC TenuredCell gets behaviors that are valid for things in the Tenured +// heap, such as access to the arena and mark bits. +class TenuredCell : public Cell { + public: + MOZ_ALWAYS_INLINE bool isTenured() const { + MOZ_ASSERT(!IsInsideNursery(this)); + return true; + } + + TenuredChunk* chunk() const { + return static_cast(Cell::chunk()); + } + + // Mark bit management. + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + MOZ_ALWAYS_INLINE CellColor color() const; + + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked( + MarkColor color = MarkColor::Black) const; + MOZ_ALWAYS_INLINE bool markIfUnmarkedAtomic(MarkColor color) const; + MOZ_ALWAYS_INLINE void markBlack() const; + MOZ_ALWAYS_INLINE void markBlackAtomic() const; + MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); + MOZ_ALWAYS_INLINE void unmark(); + + // Access to the arena. + inline Arena* arena() const; + inline AllocKind getAllocKind() const; + inline JS::TraceKind getTraceKind() const; + inline JS::Zone* zone() const; + inline JS::Zone* zoneFromAnyThread() const; + inline bool isInsideZone(JS::Zone* zone) const; + + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const { + return JS::shadow::Zone::from(zone()); + } + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const { + return JS::shadow::Zone::from(zoneFromAnyThread()); + } + + template >> + inline bool is() const { + return getTraceKind() == JS::MapTypeToTraceKind::kind; + } + + template >> + inline T* as() { + // |this|-qualify the |is| call below to avoid compile errors with even + // fairly recent versions of gcc, e.g. 7.1.1 according to bz. + MOZ_ASSERT(this->is()); + return static_cast(this); + } + + template >> + inline const T* as() const { + // |this|-qualify the |is| call below to avoid compile errors with even + // fairly recent versions of gcc, e.g. 7.1.1 according to bz. + MOZ_ASSERT(this->is()); + return static_cast(this); + } + + // Default implementation for kinds that don't require fixup. + void fixupAfterMovingGC() {} + + static inline CellColor getColor(MarkBitmap* bitmap, const TenuredCell* cell); + +#ifdef DEBUG + inline bool isAligned() const; +#endif +}; + +MOZ_ALWAYS_INLINE const TenuredCell& Cell::asTenured() const { + MOZ_ASSERT(isTenured()); + return *static_cast(this); +} + +MOZ_ALWAYS_INLINE TenuredCell& Cell::asTenured() { + MOZ_ASSERT(isTenured()); + return *static_cast(this); +} + +MOZ_ALWAYS_INLINE bool Cell::isMarkedAny() const { + return !isTenured() || asTenured().isMarkedAny(); +} + +MOZ_ALWAYS_INLINE bool Cell::isMarkedBlack() const { + return !isTenured() || asTenured().isMarkedBlack(); +} + +MOZ_ALWAYS_INLINE bool Cell::isMarkedGray() const { + return isTenured() && asTenured().isMarkedGray(); +} + +MOZ_ALWAYS_INLINE bool Cell::isMarked(gc::MarkColor color) const { + return color == MarkColor::Gray ? isMarkedGray() : isMarkedBlack(); +} + +MOZ_ALWAYS_INLINE bool Cell::isMarkedAtLeast(gc::MarkColor color) const { + return color == MarkColor::Gray ? isMarkedAny() : isMarkedBlack(); +} + +MOZ_ALWAYS_INLINE CellColor Cell::color() const { + return isTenured() ? asTenured().color() : CellColor::Black; +} + +inline JSRuntime* Cell::runtimeFromMainThread() const { + JSRuntime* rt = chunk()->runtime; + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + return rt; +} + +inline JSRuntime* Cell::runtimeFromAnyThread() const { + return chunk()->runtime; +} + +inline uintptr_t Cell::address() const { + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellAlignBytes == 0); + MOZ_ASSERT(TenuredChunk::withinValidRange(addr)); + return addr; +} + +ChunkBase* Cell::chunk() const { + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellAlignBytes == 0); + addr &= ~ChunkMask; + return reinterpret_cast(addr); +} + +inline StoreBuffer* Cell::storeBuffer() const { return chunk()->storeBuffer; } + +JS::Zone* Cell::zone() const { + if (isTenured()) { + return asTenured().zone(); + } + + return nurseryZone(); +} + +JS::Zone* Cell::zoneFromAnyThread() const { + if (isTenured()) { + return asTenured().zoneFromAnyThread(); + } + + return nurseryZoneFromAnyThread(); +} + +JS::Zone* Cell::nurseryZone() const { + JS::Zone* zone = nurseryZoneFromAnyThread(); + MOZ_ASSERT(CurrentThreadIsGCMarking() || CurrentThreadCanAccessZone(zone)); + return zone; +} + +JS::Zone* Cell::nurseryZoneFromAnyThread() const { + return NurseryCellHeader::from(this)->zone(); +} + +#ifdef DEBUG +extern Cell* UninlinedForwarded(const Cell* cell); +#endif + +inline JS::TraceKind Cell::getTraceKind() const { + if (isTenured()) { + MOZ_ASSERT_IF(isForwarded(), UninlinedForwarded(this)->getTraceKind() == + asTenured().getTraceKind()); + return asTenured().getTraceKind(); + } + + return NurseryCellHeader::from(this)->traceKind(); +} + +/* static */ MOZ_ALWAYS_INLINE bool Cell::needPreWriteBarrier(JS::Zone* zone) { + return JS::shadow::Zone::from(zone)->needsIncrementalBarrier(); +} + +MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedAny() const { + MOZ_ASSERT(arena()->allocated()); + return chunk()->markBits.isMarkedAny(this); +} + +MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedBlack() const { + MOZ_ASSERT(arena()->allocated()); + return chunk()->markBits.isMarkedBlack(this); +} + +MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedGray() const { + MOZ_ASSERT(arena()->allocated()); + return chunk()->markBits.isMarkedGray(this); +} + +MOZ_ALWAYS_INLINE CellColor TenuredCell::color() const { + return getColor(&chunk()->markBits, this); +} + +/* static */ +inline CellColor TenuredCell::getColor(MarkBitmap* bitmap, + const TenuredCell* cell) { + // Note that this method isn't synchronised so may give surprising results if + // the mark bitmap is being modified concurrently. + + if (bitmap->isMarkedBlack(cell)) { + return CellColor::Black; + } + + if (bitmap->isMarkedGray(cell)) { + return CellColor::Gray; + } + + return CellColor::White; +} + +bool TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const { + return chunk()->markBits.markIfUnmarked(this, color); +} + +bool TenuredCell::markIfUnmarkedAtomic(MarkColor color) const { + return chunk()->markBits.markIfUnmarkedAtomic(this, color); +} + +void TenuredCell::markBlack() const { chunk()->markBits.markBlack(this); } +void TenuredCell::markBlackAtomic() const { + chunk()->markBits.markBlackAtomic(this); +} + +void TenuredCell::copyMarkBitsFrom(const TenuredCell* src) { + MarkBitmap& markBits = chunk()->markBits; + markBits.copyMarkBit(this, src, ColorBit::BlackBit); + markBits.copyMarkBit(this, src, ColorBit::GrayOrBlackBit); +} + +void TenuredCell::unmark() { chunk()->markBits.unmark(this); } + +inline Arena* TenuredCell::arena() const { + MOZ_ASSERT(isTenured()); + uintptr_t addr = address(); + addr &= ~ArenaMask; + return reinterpret_cast(addr); +} + +AllocKind TenuredCell::getAllocKind() const { return arena()->getAllocKind(); } + +JS::TraceKind TenuredCell::getTraceKind() const { + return MapAllocToTraceKind(getAllocKind()); +} + +JS::Zone* TenuredCell::zone() const { + JS::Zone* zone = arena()->zone; + MOZ_ASSERT(CurrentThreadIsGCMarking() || CurrentThreadCanAccessZone(zone)); + return zone; +} + +JS::Zone* TenuredCell::zoneFromAnyThread() const { return arena()->zone; } + +bool TenuredCell::isInsideZone(JS::Zone* zone) const { + return zone == arena()->zone; +} + +// Read barrier and pre-write barrier implementation for GC cells. + +template +MOZ_ALWAYS_INLINE void ReadBarrier(T* thing) { + static_assert(std::is_base_of_v); + static_assert(!std::is_same_v && !std::is_same_v); + + if (thing) { + ReadBarrierImpl(thing); + } +} + +MOZ_ALWAYS_INLINE void ReadBarrierImpl(TenuredCell* thing) { + MOZ_ASSERT(CurrentThreadIsMainThread()); + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + MOZ_ASSERT(thing); + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + PerformIncrementalReadBarrier(thing); + return; + } + + if (thing->isMarkedGray()) { + UnmarkGrayGCThingRecursively(thing); + } +} + +MOZ_ALWAYS_INLINE void ReadBarrierImpl(Cell* thing) { + MOZ_ASSERT(!CurrentThreadIsGCMarking()); + MOZ_ASSERT(thing); + + if (thing->isTenured()) { + ReadBarrierImpl(&thing->asTenured()); + } +} + +MOZ_ALWAYS_INLINE void PreWriteBarrierImpl(TenuredCell* thing) { + MOZ_ASSERT(CurrentThreadIsMainThread() || CurrentThreadIsGCSweeping() || + CurrentThreadIsGCFinalizing()); + MOZ_ASSERT(thing); + + // Barriers can be triggered on the main thread while collecting, but are + // disabled. For example, this happens when sweeping HeapPtr wrappers. See + // AutoDisableBarriers. + + JS::shadow::Zone* zone = thing->shadowZoneFromAnyThread(); + if (zone->needsIncrementalBarrier()) { + PerformIncrementalPreWriteBarrier(thing); + } +} + +MOZ_ALWAYS_INLINE void PreWriteBarrierImpl(Cell* thing) { + MOZ_ASSERT(!CurrentThreadIsGCMarking()); + MOZ_ASSERT(thing); + + if (thing->isTenured()) { + PreWriteBarrierImpl(&thing->asTenured()); + } +} + +template +MOZ_ALWAYS_INLINE void PreWriteBarrier(T* thing) { + static_assert(std::is_base_of_v); + static_assert(!std::is_same_v && !std::is_same_v); + + if (thing) { + PreWriteBarrierImpl(thing); + } +} + +// Pre-write barrier implementation for structures containing GC cells, taking a +// functor to trace the structure. +template +MOZ_ALWAYS_INLINE void PreWriteBarrier(JS::Zone* zone, T* data, + const F& traceFn) { + MOZ_ASSERT(data); + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(!CurrentThreadIsGCMarking()); + + auto* shadowZone = JS::shadow::Zone::from(zone); + if (!shadowZone->needsIncrementalBarrier()) { + return; + } + + MOZ_ASSERT(CurrentThreadCanAccessRuntime(shadowZone->runtimeFromAnyThread())); + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); + + traceFn(shadowZone->barrierTracer(), data); +} + +// Pre-write barrier implementation for structures containing GC cells. T must +// support a |trace| method. +template +MOZ_ALWAYS_INLINE void PreWriteBarrier(JS::Zone* zone, T* data) { + MOZ_ASSERT(data); + PreWriteBarrier(zone, data, [](JSTracer* trc, T* data) { data->trace(trc); }); +} + +#ifdef DEBUG + +/* static */ void Cell::assertThingIsNotGray(Cell* cell) { + JS::AssertCellIsNotGray(cell); +} + +bool Cell::isAligned() const { + if (!isTenured()) { + return true; + } + return asTenured().isAligned(); +} + +bool TenuredCell::isAligned() const { + return Arena::isAligned(address(), arena()->getThingSize()); +} + +#endif + +// Base class for nusery-allocatable GC things that have 32-bit length and +// 32-bit flags (currently JSString and BigInt). +// +// This tries to store both in Cell::header_, but if that isn't large enough the +// length is stored separately. +// +// 32 0 +// ------------------ +// | Length | Flags | +// ------------------ +// +// The low bits of the flags word (see CellFlagBitsReservedForGC) are reserved +// for GC. Derived classes must ensure they don't use these flags for non-GC +// purposes. +class alignas(gc::CellAlignBytes) CellWithLengthAndFlags : public Cell { +#if JS_BITS_PER_WORD == 32 + // Additional storage for length if |header_| is too small to fit both. + uint32_t length_; +#endif + + protected: + uint32_t headerLengthField() const { +#if JS_BITS_PER_WORD == 32 + return length_; +#else + return uint32_t(header_.get() >> 32); +#endif + } + + uint32_t headerFlagsField() const { return uint32_t(header_.get()); } + + void setHeaderFlagBit(uint32_t flag) { + header_.set(header_.get() | uintptr_t(flag)); + } + void clearHeaderFlagBit(uint32_t flag) { + header_.set(header_.get() & ~uintptr_t(flag)); + } + void toggleHeaderFlagBit(uint32_t flag) { + header_.set(header_.get() ^ uintptr_t(flag)); + } + + void setHeaderLengthAndFlags(uint32_t len, uint32_t flags) { +#if JS_BITS_PER_WORD == 32 + header_.set(flags); + length_ = len; +#else + header_.set((uint64_t(len) << 32) | uint64_t(flags)); +#endif + } + + public: + // Returns the offset of header_. JIT code should use offsetOfFlags + // below. + static constexpr size_t offsetOfRawHeaderFlagsField() { + return offsetof(CellWithLengthAndFlags, header_); + } + + // Offsets for direct field from jit code. A number of places directly + // access 32-bit length and flags fields so do endian trickery here. +#if JS_BITS_PER_WORD == 32 + static constexpr size_t offsetOfHeaderFlags() { + return offsetof(CellWithLengthAndFlags, header_); + } + static constexpr size_t offsetOfHeaderLength() { + return offsetof(CellWithLengthAndFlags, length_); + } +#elif MOZ_LITTLE_ENDIAN() + static constexpr size_t offsetOfHeaderFlags() { + return offsetof(CellWithLengthAndFlags, header_); + } + static constexpr size_t offsetOfHeaderLength() { + return offsetof(CellWithLengthAndFlags, header_) + sizeof(uint32_t); + } +#else + static constexpr size_t offsetOfHeaderFlags() { + return offsetof(CellWithLengthAndFlags, header_) + sizeof(uint32_t); + } + static constexpr size_t offsetOfHeaderLength() { + return offsetof(CellWithLengthAndFlags, header_); + } +#endif +}; + +// Base class for non-nursery-allocatable GC things that allows storing a non-GC +// thing pointer in the first word. +// +// The low bits of the word (see CellFlagBitsReservedForGC) are reserved for GC. +template +class alignas(gc::CellAlignBytes) TenuredCellWithNonGCPointer + : public TenuredCell { + static_assert(!std::is_pointer_v, + "PtrT should be the type of the referent, not of the pointer"); + static_assert( + !std::is_base_of_v, + "Don't use TenuredCellWithNonGCPointer for pointers to GC things"); + + protected: + TenuredCellWithNonGCPointer() = default; + explicit TenuredCellWithNonGCPointer(PtrT* initial) { + uintptr_t data = uintptr_t(initial); + header_.set(data); + } + + PtrT* headerPtr() const { + MOZ_ASSERT(flags() == 0); + return reinterpret_cast(uintptr_t(header_.get())); + } + + void setHeaderPtr(PtrT* newValue) { + // As above, no flags are expected to be set here. + uintptr_t data = uintptr_t(newValue); + MOZ_ASSERT(flags() == 0); + header_.set(data); + } + + public: + static constexpr size_t offsetOfHeaderPtr() { + return offsetof(TenuredCellWithNonGCPointer, header_); + } +}; + +// Base class for non-nursery-allocatable GC things that allows storing flags +// in the first word. +// +// The low bits of the flags word (see CellFlagBitsReservedForGC) are reserved +// for GC. +class alignas(gc::CellAlignBytes) TenuredCellWithFlags : public TenuredCell { + protected: + TenuredCellWithFlags() { header_.set(0); } + explicit TenuredCellWithFlags(uintptr_t initial) { header_.set(initial); } + + uintptr_t headerFlagsField() const { + MOZ_ASSERT(flags() == 0); + return header_.get(); + } + + void setHeaderFlagBits(uintptr_t flags) { + header_.set(header_.get() | flags); + } + void clearHeaderFlagBits(uintptr_t flags) { + header_.set(header_.get() & ~flags); + } +}; + +// Base class for GC things that have a tenured GC pointer as their first word. +// +// The low bits of the first word (see CellFlagBitsReservedForGC) are reserved +// for GC. +// +// This includes a pre write barrier when the pointer is update. No post barrier +// is necessary as the pointer is always tenured. +template +class alignas(gc::CellAlignBytes) CellWithTenuredGCPointer : public BaseCell { + static void staticAsserts() { + // These static asserts are not in class scope because the PtrT may not be + // defined when this class template is instantiated. + static_assert( + std::is_same_v || std::is_same_v, + "BaseCell must be either Cell or TenuredCell"); + static_assert( + !std::is_pointer_v, + "PtrT should be the type of the referent, not of the pointer"); + static_assert( + std::is_base_of_v, + "Only use CellWithTenuredGCPointer for pointers to GC things"); + } + + protected: + CellWithTenuredGCPointer() = default; + explicit CellWithTenuredGCPointer(PtrT* initial) { initHeaderPtr(initial); } + + void initHeaderPtr(PtrT* initial) { + MOZ_ASSERT_IF(initial, !IsInsideNursery(initial)); + uintptr_t data = uintptr_t(initial); + this->header_.set(data); + } + + void setHeaderPtr(PtrT* newValue) { + // As above, no flags are expected to be set here. + MOZ_ASSERT_IF(newValue, !IsInsideNursery(newValue)); + PreWriteBarrier(headerPtr()); + unbarrieredSetHeaderPtr(newValue); + } + + public: + PtrT* headerPtr() const { + staticAsserts(); + MOZ_ASSERT(this->flags() == 0); + return reinterpret_cast(uintptr_t(this->header_.get())); + } + PtrT* headerPtrAtomic() const { + staticAsserts(); + MOZ_ASSERT(this->flags() == 0); + return reinterpret_cast(uintptr_t(this->header_.getAtomic())); + } + + void unbarrieredSetHeaderPtr(PtrT* newValue) { + uintptr_t data = uintptr_t(newValue); + MOZ_ASSERT(this->flags() == 0); + this->header_.set(data); + } + + static constexpr size_t offsetOfHeaderPtr() { + return offsetof(CellWithTenuredGCPointer, header_); + } +}; + +void CellHeaderPostWriteBarrier(JSObject** ptr, JSObject* prev, JSObject* next); + +template +constexpr inline bool GCTypeIsTenured() { + static_assert(std::is_base_of_v); + static_assert(!std::is_same_v && !std::is_same_v); + + return std::is_base_of_v || std::is_base_of_v; +} + +template +class alignas(gc::CellAlignBytes) TenuredCellWithGCPointer + : public TenuredCell { + static void staticAsserts() { + // These static asserts are not in class scope because the PtrT may not be + // defined when this class template is instantiated. + static_assert( + !std::is_pointer_v, + "PtrT should be the type of the referent, not of the pointer"); + static_assert( + std::is_base_of_v, + "Only use TenuredCellWithGCPointer for pointers to GC things"); + static_assert( + !GCTypeIsTenured, + "Don't use TenuredCellWithGCPointer for always-tenured GC things"); + } + + protected: + TenuredCellWithGCPointer() = default; + explicit TenuredCellWithGCPointer(PtrT* initial) { initHeaderPtr(initial); } + + void initHeaderPtr(PtrT* initial) { + uintptr_t data = uintptr_t(initial); + this->header_.set(data); + if (initial && IsInsideNursery(initial)) { + CellHeaderPostWriteBarrier(headerPtrAddress(), nullptr, initial); + } + } + + PtrT** headerPtrAddress() { + MOZ_ASSERT(this->flags() == 0); + return reinterpret_cast(&this->header_); + } + + public: + PtrT* headerPtr() const { + MOZ_ASSERT(this->flags() == 0); + return reinterpret_cast(uintptr_t(this->header_.get())); + } + + void unbarrieredSetHeaderPtr(PtrT* newValue) { + uintptr_t data = uintptr_t(newValue); + MOZ_ASSERT(this->flags() == 0); + this->header_.set(data); + } + + static constexpr size_t offsetOfHeaderPtr() { + return offsetof(TenuredCellWithGCPointer, header_); + } +}; + +// Check whether a typed GC thing is marked at all. Doesn't check gray bits for +// kinds that can't be marked gray. +template +static inline bool TenuredThingIsMarkedAny(T* thing) { + using BaseT = typename BaseGCType::type; + TenuredCell* cell = &thing->asTenured(); + if constexpr (TraceKindCanBeGray::value) { + return cell->isMarkedAny(); + } else { + MOZ_ASSERT(!cell->isMarkedGray()); + return cell->isMarkedBlack(); + } +} + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_Cell_h */ diff --git a/js/src/gc/ClearEdgesTracer.h b/js/src/gc/ClearEdgesTracer.h new file mode 100644 index 0000000000..3e8f0309a3 --- /dev/null +++ b/js/src/gc/ClearEdgesTracer.h @@ -0,0 +1,27 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_ClearEdgesTracer_h +#define gc_ClearEdgesTracer_h + +#include "js/TracingAPI.h" + +namespace js { +namespace gc { + +struct ClearEdgesTracer final : public GenericTracerImpl { + explicit ClearEdgesTracer(JSRuntime* rt); + + private: + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl; +}; + +} // namespace gc +} // namespace js + +#endif // gc_ClearEdgesTracer_h diff --git a/js/src/gc/Compacting.cpp b/js/src/gc/Compacting.cpp new file mode 100644 index 0000000000..6af0a98860 --- /dev/null +++ b/js/src/gc/Compacting.cpp @@ -0,0 +1,967 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Implementation of compacting GC. + */ + +#include "mozilla/Maybe.h" + +#include "debugger/DebugAPI.h" +#include "gc/ArenaList.h" +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/ParallelWork.h" +#include "gc/Zone.h" +#include "jit/JitCode.h" +#include "jit/JitRuntime.h" +#include "jit/JitZone.h" +#include "js/GCAPI.h" +#include "vm/HelperThreads.h" +#include "vm/Realm.h" +#include "wasm/WasmGcObject.h" + +#include "gc/Heap-inl.h" +#include "gc/Marking-inl.h" +#include "gc/PrivateIterators-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "gc/TraceMethods-inl.h" +#include "vm/GeckoProfiler-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::Maybe; + +bool GCRuntime::canRelocateZone(Zone* zone) const { + return !zone->isAtomsZone(); +} + +void GCRuntime::beginCompactPhase() { + MOZ_ASSERT(!isBackgroundSweeping()); + assertBackgroundSweepingFinished(); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT); + + MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty()); + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + if (canRelocateZone(zone)) { + zonesToMaybeCompact.ref().append(zone); + } + } + + startedCompacting = true; + zonesCompacted = 0; + +#ifdef DEBUG + AutoLockGC lock(this); + MOZ_ASSERT(!relocatedArenasToRelease); +#endif +} + +IncrementalProgress GCRuntime::compactPhase(JS::GCReason reason, + SliceBudget& sliceBudget, + AutoGCSession& session) { + assertBackgroundSweepingFinished(); + MOZ_ASSERT(startedCompacting); + + AutoMajorGCProfilerEntry s(this); + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT); + + // TODO: JSScripts can move. If the sampler interrupts the GC in the + // middle of relocating an arena, invalid JSScript pointers may be + // accessed. Suppress all sampling until a finer-grained solution can be + // found. See bug 1295775. + AutoSuppressProfilerSampling suppressSampling(rt->mainContextFromOwnThread()); + + ZoneList relocatedZones; + Arena* relocatedArenas = nullptr; + while (!zonesToMaybeCompact.ref().isEmpty()) { + Zone* zone = zonesToMaybeCompact.ref().front(); + zonesToMaybeCompact.ref().removeFront(); + + MOZ_ASSERT(nursery().isEmpty()); + zone->changeGCState(Zone::Finished, Zone::Compact); + + if (relocateArenas(zone, reason, relocatedArenas, sliceBudget)) { + updateZonePointersToRelocatedCells(zone); + relocatedZones.append(zone); + zonesCompacted++; + } else { + zone->changeGCState(Zone::Compact, Zone::Finished); + } + + if (sliceBudget.isOverBudget()) { + break; + } + } + + if (!relocatedZones.isEmpty()) { + updateRuntimePointersToRelocatedCells(session); + + do { + Zone* zone = relocatedZones.front(); + relocatedZones.removeFront(); + zone->changeGCState(Zone::Compact, Zone::Finished); + } while (!relocatedZones.isEmpty()); + } + + clearRelocatedArenas(relocatedArenas, reason); + +#ifdef DEBUG + protectOrReleaseRelocatedArenas(relocatedArenas, reason); +#else + releaseRelocatedArenas(relocatedArenas); +#endif + + // Clear caches that can contain cell pointers. + rt->caches().purgeForCompaction(); + +#ifdef DEBUG + checkHashTablesAfterMovingGC(); +#endif + + return zonesToMaybeCompact.ref().isEmpty() ? Finished : NotFinished; +} + +void GCRuntime::endCompactPhase() { startedCompacting = false; } + +static bool ShouldRelocateAllArenas(JS::GCReason reason) { + return reason == JS::GCReason::DEBUG_GC; +} + +/* + * Choose which arenas to relocate all cells from. Return an arena cursor that + * can be passed to removeRemainingArenas(). + */ +Arena** ArenaList::pickArenasToRelocate(size_t& arenaTotalOut, + size_t& relocTotalOut) { + // Relocate the greatest number of arenas such that the number of used cells + // in relocated arenas is less than or equal to the number of free cells in + // unrelocated arenas. In other words we only relocate cells we can move + // into existing arenas, and we choose the least full areans to relocate. + // + // This is made easier by the fact that the arena list has been sorted in + // descending order of number of used cells, so we will always relocate a + // tail of the arena list. All we need to do is find the point at which to + // start relocating. + + check(); + + if (isCursorAtEnd()) { + return nullptr; + } + + Arena** arenap = cursorp_; // Next arena to consider for relocation. + size_t previousFreeCells = 0; // Count of free cells before arenap. + size_t followingUsedCells = 0; // Count of used cells after arenap. + size_t fullArenaCount = 0; // Number of full arenas (not relocated). + size_t nonFullArenaCount = + 0; // Number of non-full arenas (considered for relocation). + size_t arenaIndex = 0; // Index of the next arena to consider. + + for (Arena* arena = head_; arena != *cursorp_; arena = arena->next) { + fullArenaCount++; + } + + for (Arena* arena = *cursorp_; arena; arena = arena->next) { + followingUsedCells += arena->countUsedCells(); + nonFullArenaCount++; + } + + mozilla::DebugOnly lastFreeCells(0); + size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getAllocKind()); + + while (*arenap) { + Arena* arena = *arenap; + if (followingUsedCells <= previousFreeCells) { + break; + } + + size_t freeCells = arena->countFreeCells(); + size_t usedCells = cellsPerArena - freeCells; + followingUsedCells -= usedCells; +#ifdef DEBUG + MOZ_ASSERT(freeCells >= lastFreeCells); + lastFreeCells = freeCells; +#endif + previousFreeCells += freeCells; + arenap = &arena->next; + arenaIndex++; + } + + size_t relocCount = nonFullArenaCount - arenaIndex; + MOZ_ASSERT(relocCount < nonFullArenaCount); + MOZ_ASSERT((relocCount == 0) == (!*arenap)); + arenaTotalOut += fullArenaCount + nonFullArenaCount; + relocTotalOut += relocCount; + + return arenap; +} + +#ifdef DEBUG +inline bool PtrIsInRange(const void* ptr, const void* start, size_t length) { + return uintptr_t(ptr) - uintptr_t(start) < length; +} +#endif + +static void RelocateCell(Zone* zone, TenuredCell* src, AllocKind thingKind, + size_t thingSize) { + JS::AutoSuppressGCAnalysis nogc; + + // Allocate a new cell. + MOZ_ASSERT(zone == src->zone()); + TenuredCell* dst = + reinterpret_cast(AllocateCellInGC(zone, thingKind)); + + // Copy source cell contents to destination. + memcpy(dst, src, thingSize); + + // Move any uid attached to the object. + gc::TransferUniqueId(dst, src); + + if (IsObjectAllocKind(thingKind)) { + auto* srcObj = static_cast(static_cast(src)); + auto* dstObj = static_cast(static_cast(dst)); + + if (srcObj->is()) { + NativeObject* srcNative = &srcObj->as(); + NativeObject* dstNative = &dstObj->as(); + + // Fixup the pointer to inline object elements if necessary. + if (srcNative->hasFixedElements()) { + uint32_t numShifted = + srcNative->getElementsHeader()->numShiftedElements(); + dstNative->setFixedElements(numShifted); + } + } else if (srcObj->is()) { + if (srcObj->as().usingInlineValueArray()) { + dstObj->as().setInlineValueArray(); + } + } + + // Call object moved hook if present. + if (JSObjectMovedOp op = srcObj->getClass()->extObjectMovedOp()) { + op(dstObj, srcObj); + } + + MOZ_ASSERT_IF( + dstObj->is(), + !PtrIsInRange( + (const Value*)dstObj->as().getDenseElements(), src, + thingSize)); + } + + // Copy the mark bits. + dst->copyMarkBitsFrom(src); + + // Poison the source cell contents except for the forwarding flag and pointer + // which will be stored in the first word. We can't do this for native object + // with fixed elements because this would overwrite the element flags and + // these are needed when updating COW elements referred to by other objects. +#ifdef DEBUG + JSObject* srcObj = IsObjectAllocKind(thingKind) + ? static_cast(static_cast(src)) + : nullptr; + if (!srcObj || !srcObj->is() || + !srcObj->as().hasFixedElements()) { + AlwaysPoison(reinterpret_cast(src) + sizeof(uintptr_t), + JS_MOVED_TENURED_PATTERN, thingSize - sizeof(uintptr_t), + MemCheckKind::MakeNoAccess); + } +#endif + + // Mark source cell as forwarded and leave a pointer to the destination. + RelocationOverlay::forwardCell(src, dst); +} + +static void RelocateArena(Arena* arena, SliceBudget& sliceBudget) { + MOZ_ASSERT(arena->allocated()); + MOZ_ASSERT(!arena->onDelayedMarkingList()); + MOZ_ASSERT(arena->bufferedCells()->isEmpty()); + + Zone* zone = arena->zone; + + AllocKind thingKind = arena->getAllocKind(); + size_t thingSize = arena->getThingSize(); + + for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) { + RelocateCell(zone, cell, thingKind, thingSize); + sliceBudget.step(); + } + +#ifdef DEBUG + for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) { + TenuredCell* src = cell; + MOZ_ASSERT(src->isForwarded()); + TenuredCell* dest = Forwarded(src); + MOZ_ASSERT(src->isMarkedBlack() == dest->isMarkedBlack()); + MOZ_ASSERT(src->isMarkedGray() == dest->isMarkedGray()); + } +#endif +} + +/* + * Relocate all arenas identified by pickArenasToRelocate: for each arena, + * relocate each cell within it, then add it to a list of relocated arenas. + */ +Arena* ArenaList::relocateArenas(Arena* toRelocate, Arena* relocated, + SliceBudget& sliceBudget, + gcstats::Statistics& stats) { + check(); + + while (Arena* arena = toRelocate) { + toRelocate = arena->next; + RelocateArena(arena, sliceBudget); + // Prepend to list of relocated arenas + arena->next = relocated; + relocated = arena; + stats.count(gcstats::COUNT_ARENA_RELOCATED); + } + + check(); + + return relocated; +} + +// Skip compacting zones unless we can free a certain proportion of their GC +// heap memory. +static const float MIN_ZONE_RECLAIM_PERCENT = 2.0; + +static bool ShouldRelocateZone(size_t arenaCount, size_t relocCount, + JS::GCReason reason) { + if (relocCount == 0) { + return false; + } + + if (IsOOMReason(reason)) { + return true; + } + + return (relocCount * 100.0f) / arenaCount >= MIN_ZONE_RECLAIM_PERCENT; +} + +static AllocKinds CompactingAllocKinds() { + AllocKinds result; + for (AllocKind kind : AllAllocKinds()) { + if (IsCompactingKind(kind)) { + result += kind; + } + } + return result; +} + +bool ArenaLists::relocateArenas(Arena*& relocatedListOut, JS::GCReason reason, + SliceBudget& sliceBudget, + gcstats::Statistics& stats) { + // This is only called from the main thread while we are doing a GC, so + // there is no need to lock. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + MOZ_ASSERT(runtime()->gc.isHeapCompacting()); + MOZ_ASSERT(!runtime()->gc.isBackgroundSweeping()); + + // Relocate all compatible kinds + AllocKinds allocKindsToRelocate = CompactingAllocKinds(); + + // Clear all the free lists. + clearFreeLists(); + + if (ShouldRelocateAllArenas(reason)) { + zone_->prepareForCompacting(); + for (auto kind : allocKindsToRelocate) { + ArenaList& al = arenaList(kind); + Arena* allArenas = al.head(); + al.clear(); + relocatedListOut = + al.relocateArenas(allArenas, relocatedListOut, sliceBudget, stats); + } + } else { + size_t arenaCount = 0; + size_t relocCount = 0; + AllAllocKindArray toRelocate; + + for (auto kind : allocKindsToRelocate) { + toRelocate[kind] = + arenaList(kind).pickArenasToRelocate(arenaCount, relocCount); + } + + if (!ShouldRelocateZone(arenaCount, relocCount, reason)) { + return false; + } + + zone_->prepareForCompacting(); + for (auto kind : allocKindsToRelocate) { + if (toRelocate[kind]) { + ArenaList& al = arenaList(kind); + Arena* arenas = al.removeRemainingArenas(toRelocate[kind]); + relocatedListOut = + al.relocateArenas(arenas, relocatedListOut, sliceBudget, stats); + } + } + } + + return true; +} + +bool GCRuntime::relocateArenas(Zone* zone, JS::GCReason reason, + Arena*& relocatedListOut, + SliceBudget& sliceBudget) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_MOVE); + + MOZ_ASSERT(!zone->isPreservingCode()); + MOZ_ASSERT(canRelocateZone(zone)); + + js::CancelOffThreadIonCompile(rt, JS::Zone::Compact); + + if (!zone->arenas.relocateArenas(relocatedListOut, reason, sliceBudget, + stats())) { + return false; + } + +#ifdef DEBUG + // Check that we did as much compaction as we should have. There + // should always be less than one arena's worth of free cells. + for (auto kind : CompactingAllocKinds()) { + ArenaList& al = zone->arenas.arenaList(kind); + size_t freeCells = 0; + for (Arena* arena = al.arenaAfterCursor(); arena; arena = arena->next) { + freeCells += arena->countFreeCells(); + } + MOZ_ASSERT(freeCells < Arena::thingsPerArena(kind)); + } +#endif + + return true; +} + +MovingTracer::MovingTracer(JSRuntime* rt) + : GenericTracerImpl(rt, JS::TracerKind::Moving, + JS::WeakMapTraceAction::TraceKeysAndValues) {} + +template +inline void MovingTracer::onEdge(T** thingp, const char* name) { + T* thing = *thingp; + if (thing->runtimeFromAnyThread() == runtime() && IsForwarded(thing)) { + *thingp = Forwarded(thing); + } +} + +void Zone::prepareForCompacting() { + JS::GCContext* gcx = runtimeFromMainThread()->gcContext(); + discardJitCode(gcx); +} + +void GCRuntime::sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone) { + MOZ_ASSERT(zone->isGCCompacting()); + + zone->traceWeakMaps(trc); + + traceWeakFinalizationObserverEdges(trc, zone); + + for (auto* cache : zone->weakCaches()) { + cache->traceWeak(trc, nullptr); + } + + if (jit::JitZone* jitZone = zone->jitZone()) { + jitZone->traceWeak(trc); + } + + for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) { + c->traceWeakNativeIterators(trc); + + for (RealmsInCompartmentIter r(c); !r.done(); r.next()) { + r->traceWeakRegExps(trc); + r->traceWeakSavedStacks(trc); + r->traceWeakGlobalEdge(trc); + r->traceWeakDebugEnvironmentEdges(trc); + r->traceWeakEdgesInJitRealm(trc); + } + } +} + +template +static inline void UpdateCellPointers(MovingTracer* trc, T* cell) { + // We only update unmoved GC things or the new copy of moved GC things, never + // the old copy. If this happened it could clear the forwarded flag which + // could lead to pointers to the old copy not being updated. + MOZ_ASSERT(!cell->isForwarded()); + + cell->fixupAfterMovingGC(); + cell->traceChildren(trc); +} + +template +static void UpdateArenaPointersTyped(MovingTracer* trc, Arena* arena) { + for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) { + UpdateCellPointers(trc, cell.as()); + } +} + +static bool CanUpdateKindInBackground(AllocKind kind) { + // We try to update as many GC things in parallel as we can, but there are + // kinds for which this might not be safe: + // - we assume JSObjects that are foreground finalized are not safe to + // update in parallel + // - updating a SharedPropMap touches child maps in + // SharedPropMap::fixupAfterMovingGC + return js::gc::IsBackgroundFinalized(kind) && !IsShapeAllocKind(kind) && + kind != AllocKind::BASE_SHAPE; +} + +/* + * Update the internal pointers for all cells in an arena. + */ +static void UpdateArenaPointers(MovingTracer* trc, Arena* arena) { + AllocKind kind = arena->getAllocKind(); + + MOZ_ASSERT_IF(!CanUpdateKindInBackground(kind), + CurrentThreadCanAccessRuntime(trc->runtime())); + + switch (kind) { +#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery, \ + compact) \ + case AllocKind::allocKind: \ + UpdateArenaPointersTyped(trc, arena); \ + return; + FOR_EACH_ALLOCKIND(EXPAND_CASE) +#undef EXPAND_CASE + + default: + MOZ_CRASH("Invalid alloc kind for UpdateArenaPointers"); + } +} + +struct ArenaListSegment { + Arena* begin; + Arena* end; +}; + +/* + * Update the internal pointers for all arenas in a segment of an arena list. + * + * Returns the number of steps to count against the slice budget. + */ +static size_t UpdateArenaListSegmentPointers(GCRuntime* gc, + const ArenaListSegment& arenas) { + MOZ_ASSERT(arenas.begin); + MovingTracer trc(gc->rt); + size_t count = 0; + for (Arena* arena = arenas.begin; arena != arenas.end; arena = arena->next) { + UpdateArenaPointers(&trc, arena); + count++; + } + return count * 256; +} + +class ArenasToUpdate { + // Maximum number of arenas to update in one block. +#ifdef DEBUG + static const unsigned MaxArenasToProcess = 16; +#else + static const unsigned MaxArenasToProcess = 256; +#endif + + public: + explicit ArenasToUpdate(Zone* zone); + ArenasToUpdate(Zone* zone, const AllocKinds& kinds); + + bool done() const { return !segmentBegin; } + + ArenaListSegment get() const { + MOZ_ASSERT(!done()); + return {segmentBegin, segmentEnd}; + } + + void next(); + + private: + Maybe kinds; // Selects which thing kinds to update. + Zone* zone; // Zone to process. + AllocKind kind = AllocKind::FIRST; // Current alloc kind to process. + Arena* segmentBegin = nullptr; + Arena* segmentEnd = nullptr; + + static AllocKind nextAllocKind(AllocKind i) { + return AllocKind(uint8_t(i) + 1); + } + + void settle(); + void findSegmentEnd(); +}; + +ArenasToUpdate::ArenasToUpdate(Zone* zone) : zone(zone) { settle(); } + +ArenasToUpdate::ArenasToUpdate(Zone* zone, const AllocKinds& kinds) + : kinds(Some(kinds)), zone(zone) { + settle(); +} + +void ArenasToUpdate::settle() { + // Called when we have set |kind| to a new kind. Sets |arena| to the next + // arena or null if there are no more arenas to update. + + MOZ_ASSERT(!segmentBegin); + + for (; kind < AllocKind::LIMIT; kind = nextAllocKind(kind)) { + if (kinds && !kinds.ref().contains(kind)) { + continue; + } + + Arena* arena = zone->arenas.getFirstArena(kind); + if (arena) { + segmentBegin = arena; + findSegmentEnd(); + break; + } + } +} + +void ArenasToUpdate::findSegmentEnd() { + // Take up to MaxArenasToProcess arenas from the list starting at + // |segmentBegin| and set |segmentEnd|. + Arena* arena = segmentBegin; + for (size_t i = 0; arena && i < MaxArenasToProcess; i++) { + arena = arena->next; + } + segmentEnd = arena; +} + +void ArenasToUpdate::next() { + MOZ_ASSERT(!done()); + + segmentBegin = segmentEnd; + if (segmentBegin) { + findSegmentEnd(); + return; + } + + kind = nextAllocKind(kind); + settle(); +} + +static AllocKinds ForegroundUpdateKinds(AllocKinds kinds) { + AllocKinds result; + for (AllocKind kind : kinds) { + if (!CanUpdateKindInBackground(kind)) { + result += kind; + } + } + return result; +} + +void GCRuntime::updateCellPointers(Zone* zone, AllocKinds kinds) { + AllocKinds fgKinds = ForegroundUpdateKinds(kinds); + AllocKinds bgKinds = kinds - fgKinds; + + ArenasToUpdate fgArenas(zone, fgKinds); + ArenasToUpdate bgArenas(zone, bgKinds); + + AutoLockHelperThreadState lock; + + AutoRunParallelWork bgTasks(this, UpdateArenaListSegmentPointers, + gcstats::PhaseKind::COMPACT_UPDATE_CELLS, + GCUse::Unspecified, bgArenas, + SliceBudget::unlimited(), lock); + + AutoUnlockHelperThreadState unlock(lock); + + for (; !fgArenas.done(); fgArenas.next()) { + UpdateArenaListSegmentPointers(this, fgArenas.get()); + } +} + +// After cells have been relocated any pointers to a cell's old locations must +// be updated to point to the new location. This happens by iterating through +// all cells in heap and tracing their children (non-recursively) to update +// them. +// +// This is complicated by the fact that updating a GC thing sometimes depends on +// making use of other GC things. After a moving GC these things may not be in +// a valid state since they may contain pointers which have not been updated +// yet. +// +// The main dependencies are: +// +// - Updating a JSObject makes use of its shape +// - Updating a typed object makes use of its type descriptor object +// +// This means we require at least three phases for update: +// +// 1) shapes +// 2) typed object type descriptor objects +// 3) all other objects +// +// Also, there can be data races calling IsForwarded() on the new location of a +// cell whose first word is being updated in parallel on another thread. This +// easiest way to avoid this is to not store a GC pointer in the first word of a +// cell. Otherwise this can be avoided by updating different kinds of cell in +// different phases. +// +// Since we want to minimize the number of phases, arrange kinds into three +// arbitrary phases. + +static constexpr AllocKinds UpdatePhaseOne{AllocKind::SCRIPT, + AllocKind::BASE_SHAPE, + AllocKind::SHAPE, + AllocKind::STRING, + AllocKind::JITCODE, + AllocKind::REGEXP_SHARED, + AllocKind::SCOPE, + AllocKind::GETTER_SETTER, + AllocKind::COMPACT_PROP_MAP, + AllocKind::NORMAL_PROP_MAP, + AllocKind::DICT_PROP_MAP}; + +// UpdatePhaseTwo is typed object descriptor objects. + +static constexpr AllocKinds UpdatePhaseThree{AllocKind::FUNCTION, + AllocKind::FUNCTION_EXTENDED, + AllocKind::OBJECT0, + AllocKind::OBJECT0_BACKGROUND, + AllocKind::OBJECT2, + AllocKind::OBJECT2_BACKGROUND, + AllocKind::ARRAYBUFFER4, + AllocKind::OBJECT4, + AllocKind::OBJECT4_BACKGROUND, + AllocKind::ARRAYBUFFER8, + AllocKind::OBJECT8, + AllocKind::OBJECT8_BACKGROUND, + AllocKind::ARRAYBUFFER12, + AllocKind::OBJECT12, + AllocKind::OBJECT12_BACKGROUND, + AllocKind::ARRAYBUFFER16, + AllocKind::OBJECT16, + AllocKind::OBJECT16_BACKGROUND}; + +void GCRuntime::updateAllCellPointers(MovingTracer* trc, Zone* zone) { + updateCellPointers(zone, UpdatePhaseOne); + + updateCellPointers(zone, UpdatePhaseThree); +} + +/* + * Update pointers to relocated cells in a single zone by doing a traversal of + * that zone's arenas and calling per-zone sweep hooks. + * + * The latter is necessary to update weak references which are not marked as + * part of the traversal. + */ +void GCRuntime::updateZonePointersToRelocatedCells(Zone* zone) { + MOZ_ASSERT(!rt->isBeingDestroyed()); + MOZ_ASSERT(zone->isGCCompacting()); + + AutoTouchingGrayThings tgt; + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_UPDATE); + MovingTracer trc(rt); + + zone->fixupAfterMovingGC(); + zone->fixupScriptMapsAfterMovingGC(&trc); + + // Fixup compartment global pointers as these get accessed during marking. + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + comp->fixupAfterMovingGC(&trc); + } + + zone->externalStringCache().purge(); + zone->functionToStringCache().purge(); + zone->shapeZone().purgeShapeCaches(rt->gcContext()); + rt->caches().stringToAtomCache.purge(); + + // Iterate through all cells that can contain relocatable pointers to update + // them. Since updating each cell is independent we try to parallelize this + // as much as possible. + updateAllCellPointers(&trc, zone); + + // Sweep everything to fix up weak pointers. + sweepZoneAfterCompacting(&trc, zone); + + // Call callbacks to get the rest of the system to fixup other untraced + // pointers. + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + callWeakPointerCompartmentCallbacks(&trc, comp); + } +} + +/* + * Update runtime-wide pointers to relocated cells. + */ +void GCRuntime::updateRuntimePointersToRelocatedCells(AutoGCSession& session) { + MOZ_ASSERT(!rt->isBeingDestroyed()); + + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::COMPACT_UPDATE); + MovingTracer trc(rt); + + Zone::fixupAllCrossCompartmentWrappersAfterMovingGC(&trc); + + rt->geckoProfiler().fixupStringsMapAfterMovingGC(); + + // Mark roots to update them. + + traceRuntimeForMajorGC(&trc, session); + + { + gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::MARK_ROOTS); + DebugAPI::traceAllForMovingGC(&trc); + DebugAPI::traceCrossCompartmentEdges(&trc); + + // Mark all gray roots. + traceEmbeddingGrayRoots(&trc); + Compartment::traceIncomingCrossCompartmentEdgesForZoneGC( + &trc, Compartment::GrayEdges); + } + + // Sweep everything to fix up weak pointers. + jit::JitRuntime::TraceWeakJitcodeGlobalTable(rt, &trc); + for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) { + cache->traceWeak(&trc, nullptr); + } + + if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) { + rt->jitRuntime()->getInterpreterEntryMap()->updateScriptsAfterMovingGC(); + } + + // Type inference may put more blocks here to free. + { + AutoLockHelperThreadState lock; + lifoBlocksToFree.ref().freeAll(); + } + + // Call callbacks to get the rest of the system to fixup other untraced + // pointers. + callWeakPointerZonesCallbacks(&trc); +} + +void GCRuntime::clearRelocatedArenas(Arena* arenaList, JS::GCReason reason) { + AutoLockGC lock(this); + clearRelocatedArenasWithoutUnlocking(arenaList, reason, lock); +} + +void GCRuntime::clearRelocatedArenasWithoutUnlocking(Arena* arenaList, + JS::GCReason reason, + const AutoLockGC& lock) { + // Clear the relocated arenas, now containing only forwarding pointers + while (arenaList) { + Arena* arena = arenaList; + arenaList = arenaList->next; + + // Clear the mark bits + arena->unmarkAll(); + + // Mark arena as empty + arena->setAsFullyUnused(); + +#ifdef DEBUG + // The cell contents have been partially marked no access in RelocateCell, + // so we need to mark the region as undefined again so we can poison it. + SetMemCheckKind(reinterpret_cast(arena->thingsStart()), + arena->getThingsSpan(), MemCheckKind::MakeUndefined); +#endif + + AlwaysPoison(reinterpret_cast(arena->thingsStart()), + JS_MOVED_TENURED_PATTERN, arena->getThingsSpan(), + MemCheckKind::MakeNoAccess); + + // Don't count emptied arenas as being freed by the current GC: + // - if we purposely moved everything to new arenas, as that will already + // have allocated a similar number of arenas. (This only happens for + // collections triggered by GC zeal.) + // - if they were allocated since the start of the GC. + bool allArenasRelocated = ShouldRelocateAllArenas(reason); + bool updateRetainedSize = !allArenasRelocated && !arena->isNewlyCreated(); + arena->zone->gcHeapSize.removeBytes(ArenaSize, updateRetainedSize, + heapSize); + + // Release the arena but don't return it to the chunk yet. + arena->release(lock); + } +} + +#ifdef DEBUG + +// In debug mode we don't always release relocated arenas straight away. +// Sometimes protect them instead and hold onto them until the next GC sweep +// phase to catch any pointers to them that didn't get forwarded. + +static inline bool CanProtectArenas() { + // On some systems the page size is larger than the size of an arena so we + // can't change the mapping permissions per arena. + return SystemPageSize() <= ArenaSize; +} + +static inline bool ShouldProtectRelocatedArenas(JS::GCReason reason) { + // For zeal mode collections we don't release the relocated arenas + // immediately. Instead we protect them and keep them around until the next + // collection so we can catch any stray accesses to them. + return reason == JS::GCReason::DEBUG_GC && CanProtectArenas(); +} + +void GCRuntime::protectOrReleaseRelocatedArenas(Arena* arenaList, + JS::GCReason reason) { + if (ShouldProtectRelocatedArenas(reason)) { + protectAndHoldArenas(arenaList); + return; + } + + releaseRelocatedArenas(arenaList); +} + +void GCRuntime::protectAndHoldArenas(Arena* arenaList) { + for (Arena* arena = arenaList; arena;) { + MOZ_ASSERT(!arena->allocated()); + Arena* next = arena->next; + if (!next) { + // Prepend to hold list before we protect the memory. + AutoLockGC lock(this); + arena->next = relocatedArenasToRelease; + relocatedArenasToRelease = arenaList; + } + ProtectPages(arena, ArenaSize); + arena = next; + } +} + +void GCRuntime::unprotectHeldRelocatedArenas(const AutoLockGC& lock) { + for (Arena* arena = relocatedArenasToRelease; arena; arena = arena->next) { + UnprotectPages(arena, ArenaSize); + MOZ_ASSERT(!arena->allocated()); + } +} + +void GCRuntime::releaseHeldRelocatedArenas() { + AutoLockGC lock(this); + unprotectHeldRelocatedArenas(lock); + Arena* arenas = relocatedArenasToRelease; + relocatedArenasToRelease = nullptr; + releaseRelocatedArenasWithoutUnlocking(arenas, lock); +} + +void GCRuntime::releaseHeldRelocatedArenasWithoutUnlocking( + const AutoLockGC& lock) { + unprotectHeldRelocatedArenas(lock); + releaseRelocatedArenasWithoutUnlocking(relocatedArenasToRelease, lock); + relocatedArenasToRelease = nullptr; +} + +#endif + +void GCRuntime::releaseRelocatedArenas(Arena* arenaList) { + AutoLockGC lock(this); + releaseRelocatedArenasWithoutUnlocking(arenaList, lock); +} + +void GCRuntime::releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, + const AutoLockGC& lock) { + // Release relocated arenas previously cleared with clearRelocatedArenas(). + while (arenaList) { + Arena* arena = arenaList; + arenaList = arenaList->next; + + // We already updated the memory accounting so just call + // Chunk::releaseArena. + arena->chunk()->releaseArena(this, arena, lock); + } +} diff --git a/js/src/gc/FinalizationObservers.cpp b/js/src/gc/FinalizationObservers.cpp new file mode 100644 index 0000000000..3a7a114645 --- /dev/null +++ b/js/src/gc/FinalizationObservers.cpp @@ -0,0 +1,509 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC support for FinalizationRegistry and WeakRef objects. + */ + +#include "gc/FinalizationObservers.h" + +#include "mozilla/ScopeExit.h" + +#include "builtin/FinalizationRegistryObject.h" +#include "builtin/WeakRefObject.h" +#include "gc/GCRuntime.h" +#include "gc/Zone.h" +#include "vm/JSContext.h" + +#include "gc/WeakMap-inl.h" +#include "vm/JSObject-inl.h" +#include "vm/NativeObject-inl.h" + +using namespace js; +using namespace js::gc; + +FinalizationObservers::FinalizationObservers(Zone* zone) + : zone(zone), + registries(zone), + recordMap(zone), + crossZoneRecords(zone), + weakRefMap(zone), + crossZoneWeakRefs(zone) {} + +FinalizationObservers::~FinalizationObservers() { + MOZ_ASSERT(registries.empty()); + MOZ_ASSERT(recordMap.empty()); + MOZ_ASSERT(crossZoneRecords.empty()); + MOZ_ASSERT(crossZoneWeakRefs.empty()); +} + +bool GCRuntime::addFinalizationRegistry( + JSContext* cx, Handle registry) { + if (!cx->zone()->ensureFinalizationObservers() || + !cx->zone()->finalizationObservers()->addRegistry(registry)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool FinalizationObservers::addRegistry( + Handle registry) { + return registries.put(registry); +} + +bool GCRuntime::registerWithFinalizationRegistry(JSContext* cx, + HandleObject target, + HandleObject record) { + MOZ_ASSERT(!IsCrossCompartmentWrapper(target)); + MOZ_ASSERT( + UncheckedUnwrapWithoutExpose(record)->is()); + MOZ_ASSERT(target->compartment() == record->compartment()); + + Zone* zone = cx->zone(); + if (!zone->ensureFinalizationObservers() || + !zone->finalizationObservers()->addRecord(target, record)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool FinalizationObservers::addRecord(HandleObject target, + HandleObject record) { + // Add a record to the record map and clean up on failure. + // + // The following must be updated and kept in sync: + // - the zone's recordMap (to observe the target) + // - the registry's global objects's recordSet (to trace the record) + // - the count of cross zone records (to calculate sweep groups) + + MOZ_ASSERT(target->zone() == zone); + + FinalizationRecordObject* unwrappedRecord = + &UncheckedUnwrapWithoutExpose(record)->as(); + + Zone* registryZone = unwrappedRecord->zone(); + bool crossZone = registryZone != zone; + if (crossZone && !addCrossZoneWrapper(crossZoneRecords, record)) { + return false; + } + auto wrapperGuard = mozilla::MakeScopeExit([&] { + if (crossZone) { + removeCrossZoneWrapper(crossZoneRecords, record); + } + }); + + GlobalObject* registryGlobal = &unwrappedRecord->global(); + auto* globalData = registryGlobal->getOrCreateFinalizationRegistryData(); + if (!globalData || !globalData->addRecord(unwrappedRecord)) { + return false; + } + auto globalDataGuard = mozilla::MakeScopeExit( + [&] { globalData->removeRecord(unwrappedRecord); }); + + auto ptr = recordMap.lookupForAdd(target); + if (!ptr && !recordMap.add(ptr, target, RecordVector(zone))) { + return false; + } + + if (!ptr->value().append(record)) { + return false; + } + + unwrappedRecord->setInRecordMap(true); + + globalDataGuard.release(); + wrapperGuard.release(); + return true; +} + +bool FinalizationObservers::addCrossZoneWrapper(WrapperWeakSet& weakSet, + JSObject* wrapper) { + MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper)); + MOZ_ASSERT(UncheckedUnwrapWithoutExpose(wrapper)->zone() != zone); + + auto ptr = weakSet.lookupForAdd(wrapper); + MOZ_ASSERT(!ptr); + return weakSet.add(ptr, wrapper, UndefinedValue()); +} + +void FinalizationObservers::removeCrossZoneWrapper(WrapperWeakSet& weakSet, + JSObject* wrapper) { + MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper)); + MOZ_ASSERT(UncheckedUnwrapWithoutExpose(wrapper)->zone() != zone); + + auto ptr = weakSet.lookupForAdd(wrapper); + MOZ_ASSERT(ptr); + weakSet.remove(ptr); +} + +static FinalizationRecordObject* UnwrapFinalizationRecord(JSObject* obj) { + obj = UncheckedUnwrapWithoutExpose(obj); + if (!obj->is()) { + MOZ_ASSERT(JS_IsDeadWrapper(obj)); + // CCWs between the compartments have been nuked. The + // FinalizationRegistry's callback doesn't run in this case. + return nullptr; + } + return &obj->as(); +} + +void FinalizationObservers::clearRecords() { + // Clear table entries related to FinalizationRecordObjects, which are not + // processed after the start of shutdown. + // + // WeakRefs are still updated during shutdown to avoid the possibility of + // stale or dangling pointers. + +#ifdef DEBUG + checkTables(); +#endif + + recordMap.clear(); + crossZoneRecords.clear(); +} + +void GCRuntime::traceWeakFinalizationObserverEdges(JSTracer* trc, Zone* zone) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(trc->runtime())); + FinalizationObservers* observers = zone->finalizationObservers(); + if (observers) { + observers->traceWeakEdges(trc); + } +} + +void FinalizationObservers::traceRoots(JSTracer* trc) { + // The cross-zone wrapper weak maps are traced as roots; this does not keep + // any of their entries alive by itself. + crossZoneRecords.trace(trc); + crossZoneWeakRefs.trace(trc); +} + +void FinalizationObservers::traceWeakEdges(JSTracer* trc) { + // Removing dead pointers from vectors may reorder live pointers to gray + // things in the vector. This is OK. + AutoTouchingGrayThings atgt; + + traceWeakWeakRefEdges(trc); + traceWeakFinalizationRegistryEdges(trc); +} + +void FinalizationObservers::traceWeakFinalizationRegistryEdges(JSTracer* trc) { + // Sweep finalization registry data and queue finalization records for cleanup + // for any entries whose target is dying and remove them from the map. + + GCRuntime* gc = &trc->runtime()->gc; + + for (RegistrySet::Enum e(registries); !e.empty(); e.popFront()) { + auto result = TraceWeakEdge(trc, &e.mutableFront(), "FinalizationRegistry"); + if (result.isDead()) { + auto* registry = + &result.initialTarget()->as(); + registry->queue()->setHasRegistry(false); + e.removeFront(); + } else { + result.finalTarget()->as().traceWeak(trc); + } + } + + for (RecordMap::Enum e(recordMap); !e.empty(); e.popFront()) { + RecordVector& records = e.front().value(); + + // Sweep finalization records, updating any pointers moved by the GC and + // remove if necessary. + records.mutableEraseIf([&](HeapPtr& heapPtr) { + auto result = TraceWeakEdge(trc, &heapPtr, "FinalizationRecord"); + JSObject* obj = + result.isLive() ? result.finalTarget() : result.initialTarget(); + FinalizationRecordObject* record = UnwrapFinalizationRecord(obj); + MOZ_ASSERT_IF(record, record->isInRecordMap()); + + bool shouldRemove = !result.isLive() || shouldRemoveRecord(record); + if (shouldRemove && record && record->isInRecordMap()) { + updateForRemovedRecord(obj, record); + } + + return shouldRemove; + }); + +#ifdef DEBUG + for (JSObject* obj : records) { + MOZ_ASSERT(UnwrapFinalizationRecord(obj)->isInRecordMap()); + } +#endif + + // Queue finalization records for targets that are dying. + if (!TraceWeakEdge(trc, &e.front().mutableKey(), + "FinalizationRecord target")) { + for (JSObject* obj : records) { + FinalizationRecordObject* record = UnwrapFinalizationRecord(obj); + FinalizationQueueObject* queue = record->queue(); + updateForRemovedRecord(obj, record); + queue->queueRecordToBeCleanedUp(record); + gc->queueFinalizationRegistryForCleanup(queue); + } + e.removeFront(); + } + } +} + +// static +bool FinalizationObservers::shouldRemoveRecord( + FinalizationRecordObject* record) { + // Records are removed from the target's vector for the following reasons: + return !record || // Nuked CCW to record. + !record->isRegistered() || // Unregistered record. + !record->queue()->hasRegistry(); // Dead finalization registry. +} + +void FinalizationObservers::updateForRemovedRecord( + JSObject* wrapper, FinalizationRecordObject* record) { + // Remove other references to a record when it has been removed from the + // zone's record map. See addRecord(). + MOZ_ASSERT(record->isInRecordMap()); + + Zone* registryZone = record->zone(); + if (registryZone != zone) { + removeCrossZoneWrapper(crossZoneRecords, wrapper); + } + + GlobalObject* registryGlobal = &record->global(); + auto* globalData = registryGlobal->maybeFinalizationRegistryData(); + globalData->removeRecord(record); + + // The removed record may be gray, and that's OK. + AutoTouchingGrayThings atgt; + + record->setInRecordMap(false); +} + +void GCRuntime::nukeFinalizationRecordWrapper( + JSObject* wrapper, FinalizationRecordObject* record) { + if (record->isInRecordMap()) { + FinalizationRegistryObject::unregisterRecord(record); + FinalizationObservers* observers = wrapper->zone()->finalizationObservers(); + observers->updateForRemovedRecord(wrapper, record); + } +} + +void GCRuntime::queueFinalizationRegistryForCleanup( + FinalizationQueueObject* queue) { + // Prod the embedding to call us back later to run the finalization callbacks, + // if necessary. + + if (queue->isQueuedForCleanup()) { + return; + } + + // Derive the incumbent global by unwrapping the incumbent global object and + // then getting its global. + JSObject* object = UncheckedUnwrapWithoutExpose(queue->incumbentObject()); + MOZ_ASSERT(object); + GlobalObject* incumbentGlobal = &object->nonCCWGlobal(); + + callHostCleanupFinalizationRegistryCallback(queue->doCleanupFunction(), + incumbentGlobal); + + // The queue object may be gray, and that's OK. + AutoTouchingGrayThings atgt; + + queue->setQueuedForCleanup(true); +} + +// Insert a target -> weakRef mapping in the target's Zone so that a dying +// target will clear out the weakRef's target. If the weakRef is in a different +// Zone, then the crossZoneWeakRefs table will keep the weakRef alive. If the +// weakRef is in the same Zone, then it must be the actual WeakRefObject and +// not a cross-compartment wrapper, since nothing would keep that alive. +bool GCRuntime::registerWeakRef(HandleObject target, HandleObject weakRef) { + MOZ_ASSERT(!IsCrossCompartmentWrapper(target)); + MOZ_ASSERT(UncheckedUnwrap(weakRef)->is()); + MOZ_ASSERT_IF(target->zone() != weakRef->zone(), + target->compartment() == weakRef->compartment()); + + Zone* zone = target->zone(); + return zone->ensureFinalizationObservers() && + zone->finalizationObservers()->addWeakRefTarget(target, weakRef); +} + +bool FinalizationObservers::addWeakRefTarget(HandleObject target, + HandleObject weakRef) { + WeakRefObject* unwrappedWeakRef = + &UncheckedUnwrapWithoutExpose(weakRef)->as(); + + Zone* weakRefZone = unwrappedWeakRef->zone(); + bool crossZone = weakRefZone != zone; + if (crossZone && !addCrossZoneWrapper(crossZoneWeakRefs, weakRef)) { + return false; + } + auto wrapperGuard = mozilla::MakeScopeExit([&] { + if (crossZone) { + removeCrossZoneWrapper(crossZoneWeakRefs, weakRef); + } + }); + + auto ptr = weakRefMap.lookupForAdd(target); + if (!ptr && !weakRefMap.add(ptr, target, WeakRefHeapPtrVector(zone))) { + return false; + } + + if (!ptr->value().emplaceBack(weakRef)) { + return false; + } + + wrapperGuard.release(); + return true; +} + +static WeakRefObject* UnwrapWeakRef(JSObject* obj) { + MOZ_ASSERT(!JS_IsDeadWrapper(obj)); + obj = UncheckedUnwrapWithoutExpose(obj); + return &obj->as(); +} + +void FinalizationObservers::removeWeakRefTarget( + Handle target, Handle weakRef) { + MOZ_ASSERT(target); + + WeakRefHeapPtrVector& weakRefs = weakRefMap.lookup(target)->value(); + JSObject* wrapper = nullptr; + weakRefs.eraseIf([weakRef, &wrapper](JSObject* obj) { + if (UnwrapWeakRef(obj) == weakRef) { + wrapper = obj; + return true; + } + return false; + }); + + MOZ_ASSERT(wrapper); + updateForRemovedWeakRef(wrapper, weakRef); +} + +void GCRuntime::nukeWeakRefWrapper(JSObject* wrapper, WeakRefObject* weakRef) { + // WeakRef wrappers can exist independently of the ones we create for the + // weakRefMap so don't assume |wrapper| is in the same zone as the WeakRef + // target. + JSObject* target = weakRef->target(); + if (!target) { + return; + } + + FinalizationObservers* observers = target->zone()->finalizationObservers(); + if (observers) { + observers->unregisterWeakRefWrapper(wrapper, weakRef); + } +} + +void FinalizationObservers::unregisterWeakRefWrapper(JSObject* wrapper, + WeakRefObject* weakRef) { + JSObject* target = weakRef->target(); + MOZ_ASSERT(target); + + bool removed = false; + WeakRefHeapPtrVector& weakRefs = weakRefMap.lookup(target)->value(); + weakRefs.eraseIf([wrapper, &removed](JSObject* obj) { + bool remove = obj == wrapper; + if (remove) { + removed = true; + } + return remove; + }); + + if (removed) { + updateForRemovedWeakRef(wrapper, weakRef); + } +} + +void FinalizationObservers::updateForRemovedWeakRef(JSObject* wrapper, + WeakRefObject* weakRef) { + weakRef->clearTarget(); + + Zone* weakRefZone = weakRef->zone(); + if (weakRefZone != zone) { + removeCrossZoneWrapper(crossZoneWeakRefs, wrapper); + } +} + +void FinalizationObservers::traceWeakWeakRefEdges(JSTracer* trc) { + for (WeakRefMap::Enum e(weakRefMap); !e.empty(); e.popFront()) { + // If target is dying, clear the target field of all weakRefs, and remove + // the entry from the map. + auto result = TraceWeakEdge(trc, &e.front().mutableKey(), "WeakRef target"); + if (result.isDead()) { + for (JSObject* obj : e.front().value()) { + updateForRemovedWeakRef(obj, UnwrapWeakRef(obj)); + } + e.removeFront(); + } else { + // Update the target field after compacting. + traceWeakWeakRefVector(trc, e.front().value(), result.finalTarget()); + } + } +} + +void FinalizationObservers::traceWeakWeakRefVector( + JSTracer* trc, WeakRefHeapPtrVector& weakRefs, JSObject* target) { + weakRefs.mutableEraseIf([&](HeapPtr& obj) -> bool { + auto result = TraceWeakEdge(trc, &obj, "WeakRef"); + if (result.isDead()) { + JSObject* wrapper = result.initialTarget(); + updateForRemovedWeakRef(wrapper, UnwrapWeakRef(wrapper)); + } else { + UnwrapWeakRef(result.finalTarget())->setTargetUnbarriered(target); + } + return result.isDead(); + }); +} + +#ifdef DEBUG +void FinalizationObservers::checkTables() const { + // Check all cross-zone wrappers are present in the appropriate table. + size_t recordCount = 0; + for (auto r = recordMap.all(); !r.empty(); r.popFront()) { + for (JSObject* object : r.front().value()) { + FinalizationRecordObject* record = UnwrapFinalizationRecord(object); + if (record && record->isInRecordMap() && record->zone() != zone) { + MOZ_ASSERT(crossZoneRecords.has(object)); + recordCount++; + } + } + } + MOZ_ASSERT(crossZoneRecords.count() == recordCount); + + size_t weakRefCount = 0; + for (auto r = weakRefMap.all(); !r.empty(); r.popFront()) { + for (JSObject* object : r.front().value()) { + WeakRefObject* weakRef = UnwrapWeakRef(object); + if (weakRef && weakRef->zone() != zone) { + MOZ_ASSERT(crossZoneWeakRefs.has(object)); + weakRefCount++; + } + } + } + MOZ_ASSERT(crossZoneWeakRefs.count() == weakRefCount); +} +#endif + +FinalizationRegistryGlobalData::FinalizationRegistryGlobalData(Zone* zone) + : recordSet(zone) {} + +bool FinalizationRegistryGlobalData::addRecord( + FinalizationRecordObject* record) { + return recordSet.putNew(record); +} + +void FinalizationRegistryGlobalData::removeRecord( + FinalizationRecordObject* record) { + MOZ_ASSERT_IF(!record->runtimeFromMainThread()->gc.isShuttingDown(), + recordSet.has(record)); + recordSet.remove(record); +} + +void FinalizationRegistryGlobalData::trace(JSTracer* trc) { + recordSet.trace(trc); +} diff --git a/js/src/gc/FinalizationObservers.h b/js/src/gc/FinalizationObservers.h new file mode 100644 index 0000000000..eee8ef14e0 --- /dev/null +++ b/js/src/gc/FinalizationObservers.h @@ -0,0 +1,129 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_FinalizationObservers_h +#define gc_FinalizationObservers_h + +#include "gc/Barrier.h" +#include "gc/WeakMap.h" +#include "gc/ZoneAllocator.h" +#include "js/GCHashTable.h" +#include "js/GCVector.h" + +namespace js { + +class FinalizationRegistryObject; +class FinalizationRecordObject; +class WeakRefObject; + +namespace gc { + +// Per-zone data structures to support FinalizationRegistry and WeakRef. +class FinalizationObservers { + Zone* const zone; + + // The set of all finalization registries in the associated zone. These are + // direct pointers and are not wrapped. + using RegistrySet = + GCHashSet, StableCellHasher>, + ZoneAllocPolicy>; + RegistrySet registries; + + // A vector of FinalizationRecord objects, or CCWs to them. + using RecordVector = GCVector, 1, ZoneAllocPolicy>; + + // A map from finalization registry targets to a vector of finalization + // records representing registries that the target is registered with and + // their associated held values. The records may be in other zones and are + // wrapped appropriately. + using RecordMap = + GCHashMap, RecordVector, + StableCellHasher>, ZoneAllocPolicy>; + RecordMap recordMap; + + // A weak map used as a set of cross-zone wrappers. This is used for both + // finalization registries and weak refs. For the former it has wrappers to + // finalization record objects and for the latter wrappers to weak refs. + // + // The weak map marking rules keep the wrappers alive while their targets are + // alive and ensure that they are both swept in the same sweep group. + using WrapperWeakSet = ObjectValueWeakMap; + WrapperWeakSet crossZoneRecords; + + // A map of weak ref targets to a vector of weak refs that are observing the + // target. The weak refs may be in other zones and are wrapped appropriately. + using WeakRefHeapPtrVector = + GCVector, 1, js::ZoneAllocPolicy>; + using WeakRefMap = + GCHashMap, WeakRefHeapPtrVector, + StableCellHasher>, ZoneAllocPolicy>; + WeakRefMap weakRefMap; + + // A weak map used as a set of cross-zone weak refs wrappers. + WrapperWeakSet crossZoneWeakRefs; + + public: + explicit FinalizationObservers(Zone* zone); + ~FinalizationObservers(); + + // FinalizationRegistry support: + bool addRegistry(Handle registry); + bool addRecord(HandleObject target, HandleObject record); + void clearRecords(); + + void updateForRemovedRecord(JSObject* wrapper, + FinalizationRecordObject* record); + + // WeakRef support: + bool addWeakRefTarget(Handle target, Handle weakRef); + void removeWeakRefTarget(Handle target, + Handle weakRef); + + void unregisterWeakRefWrapper(JSObject* wrapper, WeakRefObject* weakRef); + + void traceRoots(JSTracer* trc); + void traceWeakEdges(JSTracer* trc); + +#ifdef DEBUG + void checkTables() const; +#endif + + private: + bool addCrossZoneWrapper(WrapperWeakSet& weakSet, JSObject* wrapper); + void removeCrossZoneWrapper(WrapperWeakSet& weakSet, JSObject* wrapper); + + void updateForRemovedWeakRef(JSObject* wrapper, WeakRefObject* weakRef); + + void traceWeakFinalizationRegistryEdges(JSTracer* trc); + void traceWeakWeakRefEdges(JSTracer* trc); + void traceWeakWeakRefVector(JSTracer* trc, WeakRefHeapPtrVector& weakRefs, + JSObject* target); + + static bool shouldRemoveRecord(FinalizationRecordObject* record); +}; + +// Per-global data structures to support FinalizationRegistry. +class FinalizationRegistryGlobalData { + // Set of finalization records for finalization registries in this + // realm. These are traced as part of the realm's global. + using RecordSet = + GCHashSet, StableCellHasher>, + ZoneAllocPolicy>; + RecordSet recordSet; + + public: + explicit FinalizationRegistryGlobalData(Zone* zone); + + bool addRecord(FinalizationRecordObject* record); + void removeRecord(FinalizationRecordObject* record); + + void trace(JSTracer* trc); +}; + +} // namespace gc +} // namespace js + +#endif // gc_FinalizationObservers_h diff --git a/js/src/gc/FindSCCs.h b/js/src/gc/FindSCCs.h new file mode 100644 index 0000000000..97069bcd09 --- /dev/null +++ b/js/src/gc/FindSCCs.h @@ -0,0 +1,207 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_FindSCCs_h +#define gc_FindSCCs_h + +#include "mozilla/Assertions.h" // MOZ_ASSERT + +#include // std::min + +#include "js/AllocPolicy.h" // js::SystemAllocPolicy +#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit +#include "js/HashTable.h" // js::HashSet, js::DefaultHasher + +namespace js { +namespace gc { + +template +struct GraphNodeBase { + using NodeSet = + js::HashSet, js::SystemAllocPolicy>; + + NodeSet gcGraphEdges; + Node* gcNextGraphNode = nullptr; + Node* gcNextGraphComponent = nullptr; + unsigned gcDiscoveryTime = 0; + unsigned gcLowLink = 0; + + Node* nextNodeInGroup() const { + if (gcNextGraphNode && + gcNextGraphNode->gcNextGraphComponent == gcNextGraphComponent) { + return gcNextGraphNode; + } + return nullptr; + } + + Node* nextGroup() const { return gcNextGraphComponent; } +}; + +/* + * Find the strongly connected components of a graph using Tarjan's algorithm, + * and return them in topological order. + * + * Nodes derive from GraphNodeBase and add target edge pointers to + * sourceNode.gcGraphEdges to describe the graph: + * + * struct MyGraphNode : public GraphNodeBase + * { + * ... + * } + * + * MyGraphNode node1, node2, node3; + * node1.gcGraphEdges.put(node2); // Error checking elided. + * node2.gcGraphEdges.put(node3); + * node3.gcGraphEdges.put(node2); + * + * ComponentFinder finder; + * finder.addNode(node1); + * finder.addNode(node2); + * finder.addNode(node3); + * MyGraphNode* result = finder.getResultsList(); + */ + +template +class ComponentFinder { + public: + explicit ComponentFinder(JSContext* cx) : cx(cx) {} + + ~ComponentFinder() { + MOZ_ASSERT(!stack); + MOZ_ASSERT(!firstComponent); + } + + /* Forces all nodes to be added to a single component. */ + void useOneComponent() { stackFull = true; } + + void addNode(Node* v) { + if (v->gcDiscoveryTime == Undefined) { + MOZ_ASSERT(v->gcLowLink == Undefined); + processNode(v); + } + } + + Node* getResultsList() { + if (stackFull) { + /* + * All nodes after the stack overflow are in |stack|. Put them all in + * one big component of their own. + */ + Node* firstGoodComponent = firstComponent; + for (Node* v = stack; v; v = stack) { + stack = v->gcNextGraphNode; + v->gcNextGraphComponent = firstGoodComponent; + v->gcNextGraphNode = firstComponent; + firstComponent = v; + } + stackFull = false; + } + + MOZ_ASSERT(!stack); + + Node* result = firstComponent; + firstComponent = nullptr; + + for (Node* v = result; v; v = v->gcNextGraphNode) { + v->gcDiscoveryTime = Undefined; + v->gcLowLink = Undefined; + } + + return result; + } + + static void mergeGroups(Node* first) { + for (Node* v = first; v; v = v->gcNextGraphNode) { + v->gcNextGraphComponent = nullptr; + } + } + + private: + // Constant used to indicate an unprocessed vertex. + static const unsigned Undefined = 0; + + // Constant used to indicate a processed vertex that is no longer on the + // stack. + static const unsigned Finished = (unsigned)-1; + + void addEdgeTo(Node* w) { + if (w->gcDiscoveryTime == Undefined) { + processNode(w); + cur->gcLowLink = std::min(cur->gcLowLink, w->gcLowLink); + } else if (w->gcDiscoveryTime != Finished) { + cur->gcLowLink = std::min(cur->gcLowLink, w->gcDiscoveryTime); + } + } + + void processNode(Node* v) { + v->gcDiscoveryTime = clock; + v->gcLowLink = clock; + ++clock; + + v->gcNextGraphNode = stack; + stack = v; + + if (stackFull) { + return; + } + + AutoCheckRecursionLimit recursion(cx); + if (!recursion.checkSystemDontReport(cx)) { + stackFull = true; + return; + } + + Node* old = cur; + cur = v; + for (auto r = cur->gcGraphEdges.all(); !r.empty(); r.popFront()) { + addEdgeTo(r.front()); + } + cur = old; + + if (stackFull) { + return; + } + + if (v->gcLowLink == v->gcDiscoveryTime) { + Node* nextComponent = firstComponent; + Node* w; + do { + MOZ_ASSERT(stack); + w = stack; + stack = w->gcNextGraphNode; + + /* + * Record that the element is no longer on the stack by setting the + * discovery time to a special value that's not Undefined. + */ + w->gcDiscoveryTime = Finished; + + /* Figure out which group we're in. */ + w->gcNextGraphComponent = nextComponent; + + /* + * Prepend the component to the beginning of the output list to + * reverse the list and achieve the desired order. + */ + w->gcNextGraphNode = firstComponent; + firstComponent = w; + } while (w != v); + } + } + + private: + unsigned clock = 1; + Node* stack = nullptr; + Node* firstComponent = nullptr; + Node* cur = nullptr; + JSContext* cx; + bool stackFull = false; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_FindSCCs_h */ diff --git a/js/src/gc/GC-inl.h b/js/src/gc/GC-inl.h new file mode 100644 index 0000000000..1c177f51ab --- /dev/null +++ b/js/src/gc/GC-inl.h @@ -0,0 +1,344 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GC_inl_h +#define gc_GC_inl_h + +#include "gc/GC.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include "gc/IteratorUtils.h" +#include "gc/Marking.h" +#include "gc/Zone.h" +#include "vm/Runtime.h" + +#include "gc/ArenaList-inl.h" + +namespace js { +namespace gc { + +class AutoAssertEmptyNursery; + +class ArenaListIter { + Arena* arena; + + public: + explicit ArenaListIter(Arena* head) : arena(head) {} + bool done() const { return !arena; } + Arena* get() const { + MOZ_ASSERT(!done()); + return arena; + } + void next() { + MOZ_ASSERT(!done()); + arena = arena->next; + } + + operator Arena*() const { return get(); } + Arena* operator->() const { return get(); } +}; + +class ArenaIter : public ChainedIterator { + public: + ArenaIter(JS::Zone* zone, AllocKind kind) + : ChainedIterator(zone->arenas.getFirstArena(kind), + zone->arenas.getFirstCollectingArena(kind), + zone->arenas.getFirstSweptArena(kind)) {} +}; + +class ArenaCellIter { + size_t firstThingOffset; + size_t thingSize; + Arena* arenaAddr; + FreeSpan span; + uint_fast16_t thing; + mozilla::DebugOnly traceKind; + + // Upon entry, |thing| points to any thing (free or used) and finds the + // first used thing, which may be |thing|. + void settle() { + MOZ_ASSERT(!done()); + MOZ_ASSERT(thing); + // Note: if |span| is empty, this test will fail, which is what we want + // -- |span| being empty means that we're past the end of the last free + // thing, all the remaining things in the arena are used, and we'll + // never need to move forward. + if (thing == span.first) { + thing = span.last + thingSize; + span = *span.nextSpan(arenaAddr); + } + } + + public: + explicit ArenaCellIter(Arena* arena) { + MOZ_ASSERT(arena); + AllocKind kind = arena->getAllocKind(); + firstThingOffset = Arena::firstThingOffset(kind); + thingSize = Arena::thingSize(kind); + traceKind = MapAllocToTraceKind(kind); + arenaAddr = arena; + span = *arena->getFirstFreeSpan(); + thing = firstThingOffset; + settle(); + } + + bool done() const { + MOZ_ASSERT(thing <= ArenaSize); + return thing == ArenaSize; + } + + TenuredCell* get() const { + MOZ_ASSERT(!done()); + return reinterpret_cast(uintptr_t(arenaAddr) + thing); + } + + template + T* as() const { + MOZ_ASSERT(!done()); + MOZ_ASSERT(JS::MapTypeToTraceKind::kind == traceKind); + return reinterpret_cast(get()); + } + + void next() { + MOZ_ASSERT(!done()); + thing += thingSize; + if (thing < ArenaSize) { + settle(); + } + } + + operator TenuredCell*() const { return get(); } + TenuredCell* operator->() const { return get(); } +}; + +template +class ZoneAllCellIter; + +template <> +class ZoneAllCellIter { + mozilla::Maybe> iter; + mozilla::Maybe nogc; + + protected: + // For use when a subclass wants to insert some setup before init(). + ZoneAllCellIter() = default; + + void init(JS::Zone* zone, AllocKind kind) { + MOZ_ASSERT_IF(IsNurseryAllocable(kind), + (zone->isAtomsZone() || + zone->runtimeFromMainThread()->gc.nursery().isEmpty())); + initForTenuredIteration(zone, kind); + } + + void initForTenuredIteration(JS::Zone* zone, AllocKind kind) { + JSRuntime* rt = zone->runtimeFromAnyThread(); + + // If called from outside a GC, ensure that the heap is in a state + // that allows us to iterate. + if (!JS::RuntimeHeapIsBusy()) { + // Assert that no GCs can occur while a ZoneAllCellIter is live. + nogc.emplace(); + } + + // We have a single-threaded runtime, so there's no need to protect + // against other threads iterating or allocating. However, we do have + // background finalization; we may have to wait for this to finish if + // it's currently active. + if (IsBackgroundFinalized(kind) && + zone->arenas.needBackgroundFinalizeWait(kind)) { + rt->gc.waitBackgroundSweepEnd(); + } + iter.emplace(zone, kind); + } + + public: + ZoneAllCellIter(JS::Zone* zone, AllocKind kind) { + // If we are iterating a nursery-allocated kind then we need to + // evict first so that we can see all things. + if (IsNurseryAllocable(kind)) { + zone->runtimeFromMainThread()->gc.evictNursery(); + } + + init(zone, kind); + } + + ZoneAllCellIter(JS::Zone* zone, AllocKind kind, + const js::gc::AutoAssertEmptyNursery&) { + // No need to evict the nursery. (This constructor is known statically + // to not GC.) + init(zone, kind); + } + + bool done() const { return iter->done(); } + + template + T* get() const { + return iter->ref().as(); + } + + TenuredCell* getCell() const { return iter->get(); } + + void next() { iter->next(); } +}; + +/* clang-format off */ +// +// Iterator over the cells in a Zone, where the GC type (JSString, JSObject) is +// known, for a single AllocKind. Example usages: +// +// for (auto obj = zone->cellIter(AllocKind::OBJECT0); !obj.done(); obj.next()) { +// ... +// } +// +// for (auto script = zone->cellIter(); !script.done(); script.next()) { +// f(script->code()); +// } +// +// As this code demonstrates, you can use 'script' as if it were a JSScript*. +// Its actual type is ZoneAllCellIter, but for most purposes it will +// autoconvert to JSScript*. +// +// Note that in the JSScript case, ZoneAllCellIter is able to infer the AllocKind +// from the type 'JSScript', whereas in the JSObject case, the kind must be +// given (because there are multiple AllocKinds for objects). +// +// Also, the static rooting hazard analysis knows that the JSScript case will +// not GC during construction. The JSObject case needs to GC, or more precisely +// to empty the nursery and clear out the store buffer, so that it can see all +// objects to iterate over (the nursery is not iterable) and remove the +// possibility of having pointers from the store buffer to data hanging off +// stuff we're iterating over that we are going to delete. (The latter should +// not be a problem, since such instances should be using RelocatablePtr do +// remove themselves from the store buffer on deletion, but currently for +// subtle reasons that isn't good enough.) +// +// If the iterator is used within a GC, then there is no need to evict the +// nursery (again). You may select a variant that will skip the eviction either +// by specializing on a GCType that is never allocated in the nursery, or +// explicitly by passing in a trailing AutoAssertEmptyNursery argument. +// +// NOTE: This class can return items that are about to be swept/finalized. +// You must not keep pointers to such items across GCs. Use +// ZoneCellIter below to filter these out. +// +// NOTE: This class also does not read barrier returned items, so may return +// gray cells. You must not store such items anywhere on the heap without +// gray-unmarking them. Use ZoneCellIter to automatically unmark them. +// +/* clang-format on */ +template +class ZoneAllCellIter : public ZoneAllCellIter { + public: + // Non-nursery allocated (equivalent to having an entry in + // MapTypeToAllocKind). The template declaration here is to discard this + // constructor overload if MapTypeToAllocKind::kind does not + // exist. Note that there will be no remaining overloads that will work, which + // makes sense given that you haven't specified which of the AllocKinds to use + // for GCType. + // + // If we later add a nursery allocable GCType with a single AllocKind, we will + // want to add an overload of this constructor that does the right thing (ie, + // it empties the nursery before iterating.) + explicit ZoneAllCellIter(JS::Zone* zone) : ZoneAllCellIter() { + init(zone, MapTypeToAllocKind::kind); + } + + // Non-nursery allocated, nursery is known to be empty: same behavior as + // above. + ZoneAllCellIter(JS::Zone* zone, const js::gc::AutoAssertEmptyNursery&) + : ZoneAllCellIter(zone) {} + + // Arbitrary kind, which will be assumed to be nursery allocable (and + // therefore the nursery will be emptied before iterating.) + ZoneAllCellIter(JS::Zone* zone, AllocKind kind) + : ZoneAllCellIter(zone, kind) {} + + // Arbitrary kind, which will be assumed to be nursery allocable, but the + // nursery is known to be empty already: same behavior as non-nursery types. + ZoneAllCellIter(JS::Zone* zone, AllocKind kind, + const js::gc::AutoAssertEmptyNursery& empty) + : ZoneAllCellIter(zone, kind, empty) {} + + GCType* get() const { return ZoneAllCellIter::get(); } + operator GCType*() const { return get(); } + GCType* operator->() const { return get(); } +}; + +// Like the above class but filter out cells that are about to be finalized. +// Also, read barrier all cells returned (unless the Unbarriered variants are +// used) to prevent gray cells from escaping. +template +class ZoneCellIter : protected ZoneAllCellIter { + using Base = ZoneAllCellIter; + + public: + /* + * The same constructors as above. + */ + explicit ZoneCellIter(JS::Zone* zone) : ZoneAllCellIter(zone) { + skipDying(); + } + ZoneCellIter(JS::Zone* zone, const js::gc::AutoAssertEmptyNursery& empty) + : ZoneAllCellIter(zone, empty) { + skipDying(); + } + ZoneCellIter(JS::Zone* zone, AllocKind kind) + : ZoneAllCellIter(zone, kind) { + skipDying(); + } + ZoneCellIter(JS::Zone* zone, AllocKind kind, + const js::gc::AutoAssertEmptyNursery& empty) + : ZoneAllCellIter(zone, kind, empty) { + skipDying(); + } + + using Base::done; + + void next() { + ZoneAllCellIter::next(); + skipDying(); + } + + TenuredCell* getCell() const { + TenuredCell* cell = Base::getCell(); + + // This can result in a new reference being created to an object that an + // ongoing incremental GC may find to be unreachable, so we may need a + // barrier here. + JSRuntime* rt = cell->runtimeFromAnyThread(); + if (!JS::RuntimeHeapIsCollecting(rt->heapState())) { + JS::TraceKind traceKind = JS::MapTypeToTraceKind::kind; + ExposeGCThingToActiveJS(JS::GCCellPtr(cell, traceKind)); + } + + return cell; + } + + T* get() const { return reinterpret_cast(getCell()); } + + TenuredCell* unbarrieredGetCell() const { return Base::getCell(); } + T* unbarrieredGet() const { return Base::get(); } + operator T*() const { return get(); } + T* operator->() const { return get(); } + + private: + void skipDying() { + while (!ZoneAllCellIter::done()) { + T* current = ZoneAllCellIter::get(); + if (!IsAboutToBeFinalizedUnbarriered(current)) { + return; + } + ZoneAllCellIter::next(); + } + } +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_GC_inl_h */ diff --git a/js/src/gc/GC.cpp b/js/src/gc/GC.cpp new file mode 100644 index 0000000000..b8e1d21f2a --- /dev/null +++ b/js/src/gc/GC.cpp @@ -0,0 +1,5101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * [SMDOC] Garbage Collector + * + * This code implements an incremental mark-and-sweep garbage collector, with + * most sweeping carried out in the background on a parallel thread. + * + * Full vs. zone GC + * ---------------- + * + * The collector can collect all zones at once, or a subset. These types of + * collection are referred to as a full GC and a zone GC respectively. + * + * It is possible for an incremental collection that started out as a full GC to + * become a zone GC if new zones are created during the course of the + * collection. + * + * Incremental collection + * ---------------------- + * + * For a collection to be carried out incrementally the following conditions + * must be met: + * - the collection must be run by calling js::GCSlice() rather than js::GC() + * - the GC parameter JSGC_INCREMENTAL_GC_ENABLED must be true. + * + * The last condition is an engine-internal mechanism to ensure that incremental + * collection is not carried out without the correct barriers being implemented. + * For more information see 'Incremental marking' below. + * + * If the collection is not incremental, all foreground activity happens inside + * a single call to GC() or GCSlice(). However the collection is not complete + * until the background sweeping activity has finished. + * + * An incremental collection proceeds as a series of slices, interleaved with + * mutator activity, i.e. running JavaScript code. Slices are limited by a time + * budget. The slice finishes as soon as possible after the requested time has + * passed. + * + * Collector states + * ---------------- + * + * The collector proceeds through the following states, the current state being + * held in JSRuntime::gcIncrementalState: + * + * - Prepare - unmarks GC things, discards JIT code and other setup + * - MarkRoots - marks the stack and other roots + * - Mark - incrementally marks reachable things + * - Sweep - sweeps zones in groups and continues marking unswept zones + * - Finalize - performs background finalization, concurrent with mutator + * - Compact - incrementally compacts by zone + * - Decommit - performs background decommit and chunk removal + * + * Roots are marked in the first MarkRoots slice; this is the start of the GC + * proper. The following states can take place over one or more slices. + * + * In other words an incremental collection proceeds like this: + * + * Slice 1: Prepare: Starts background task to unmark GC things + * + * ... JS code runs, background unmarking finishes ... + * + * Slice 2: MarkRoots: Roots are pushed onto the mark stack. + * Mark: The mark stack is processed by popping an element, + * marking it, and pushing its children. + * + * ... JS code runs ... + * + * Slice 3: Mark: More mark stack processing. + * + * ... JS code runs ... + * + * Slice n-1: Mark: More mark stack processing. + * + * ... JS code runs ... + * + * Slice n: Mark: Mark stack is completely drained. + * Sweep: Select first group of zones to sweep and sweep them. + * + * ... JS code runs ... + * + * Slice n+1: Sweep: Mark objects in unswept zones that were newly + * identified as alive (see below). Then sweep more zone + * sweep groups. + * + * ... JS code runs ... + * + * Slice n+2: Sweep: Mark objects in unswept zones that were newly + * identified as alive. Then sweep more zones. + * + * ... JS code runs ... + * + * Slice m: Sweep: Sweeping is finished, and background sweeping + * started on the helper thread. + * + * ... JS code runs, remaining sweeping done on background thread ... + * + * When background sweeping finishes the GC is complete. + * + * Incremental marking + * ------------------- + * + * Incremental collection requires close collaboration with the mutator (i.e., + * JS code) to guarantee correctness. + * + * - During an incremental GC, if a memory location (except a root) is written + * to, then the value it previously held must be marked. Write barriers + * ensure this. + * + * - Any object that is allocated during incremental GC must start out marked. + * + * - Roots are marked in the first slice and hence don't need write barriers. + * Roots are things like the C stack and the VM stack. + * + * The problem that write barriers solve is that between slices the mutator can + * change the object graph. We must ensure that it cannot do this in such a way + * that makes us fail to mark a reachable object (marking an unreachable object + * is tolerable). + * + * We use a snapshot-at-the-beginning algorithm to do this. This means that we + * promise to mark at least everything that is reachable at the beginning of + * collection. To implement it we mark the old contents of every non-root memory + * location written to by the mutator while the collection is in progress, using + * write barriers. This is described in gc/Barrier.h. + * + * Incremental sweeping + * -------------------- + * + * Sweeping is difficult to do incrementally because object finalizers must be + * run at the start of sweeping, before any mutator code runs. The reason is + * that some objects use their finalizers to remove themselves from caches. If + * mutator code was allowed to run after the start of sweeping, it could observe + * the state of the cache and create a new reference to an object that was just + * about to be destroyed. + * + * Sweeping all finalizable objects in one go would introduce long pauses, so + * instead sweeping broken up into groups of zones. Zones which are not yet + * being swept are still marked, so the issue above does not apply. + * + * The order of sweeping is restricted by cross compartment pointers - for + * example say that object |a| from zone A points to object |b| in zone B and + * neither object was marked when we transitioned to the Sweep phase. Imagine we + * sweep B first and then return to the mutator. It's possible that the mutator + * could cause |a| to become alive through a read barrier (perhaps it was a + * shape that was accessed via a shape table). Then we would need to mark |b|, + * which |a| points to, but |b| has already been swept. + * + * So if there is such a pointer then marking of zone B must not finish before + * marking of zone A. Pointers which form a cycle between zones therefore + * restrict those zones to being swept at the same time, and these are found + * using Tarjan's algorithm for finding the strongly connected components of a + * graph. + * + * GC things without finalizers, and things with finalizers that are able to run + * in the background, are swept on the background thread. This accounts for most + * of the sweeping work. + * + * Reset + * ----- + * + * During incremental collection it is possible, although unlikely, for + * conditions to change such that incremental collection is no longer safe. In + * this case, the collection is 'reset' by resetIncrementalGC(). If we are in + * the mark state, this just stops marking, but if we have started sweeping + * already, we continue non-incrementally until we have swept the current sweep + * group. Following a reset, a new collection is started. + * + * Compacting GC + * ------------- + * + * Compacting GC happens at the end of a major GC as part of the last slice. + * There are three parts: + * + * - Arenas are selected for compaction. + * - The contents of those arenas are moved to new arenas. + * - All references to moved things are updated. + * + * Collecting Atoms + * ---------------- + * + * Atoms are collected differently from other GC things. They are contained in + * a special zone and things in other zones may have pointers to them that are + * not recorded in the cross compartment pointer map. Each zone holds a bitmap + * with the atoms it might be keeping alive, and atoms are only collected if + * they are not included in any zone's atom bitmap. See AtomMarking.cpp for how + * this bitmap is managed. + */ + +#include "gc/GC-inl.h" + +#include "mozilla/Range.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/TextUtils.h" +#include "mozilla/TimeStamp.h" + +#include +#include +#include +#include +#include +#include + +#include "jsapi.h" // JS_AbortIfWrongThread +#include "jstypes.h" + +#include "debugger/DebugAPI.h" +#include "gc/ClearEdgesTracer.h" +#include "gc/GCContext.h" +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/GCProbes.h" +#include "gc/Memory.h" +#include "gc/ParallelMarking.h" +#include "gc/ParallelWork.h" +#include "gc/WeakMap.h" +#include "jit/ExecutableAllocator.h" +#include "jit/JitCode.h" +#include "jit/JitRealm.h" +#include "jit/JitRuntime.h" +#include "jit/ProcessExecutableMemory.h" +#include "js/HeapAPI.h" // JS::GCCellPtr +#include "js/Printer.h" +#include "js/SliceBudget.h" +#include "util/DifferentialTesting.h" +#include "vm/BigIntType.h" +#include "vm/EnvironmentObject.h" +#include "vm/GetterSetter.h" +#include "vm/HelperThreadState.h" +#include "vm/JitActivation.h" +#include "vm/JSObject.h" +#include "vm/JSScript.h" +#include "vm/PropMap.h" +#include "vm/Realm.h" +#include "vm/Shape.h" +#include "vm/StringType.h" +#include "vm/SymbolType.h" +#include "vm/Time.h" + +#include "gc/Heap-inl.h" +#include "gc/Nursery-inl.h" +#include "gc/ObjectKind-inl.h" +#include "gc/PrivateIterators-inl.h" +#include "vm/GeckoProfiler-inl.h" +#include "vm/JSContext-inl.h" +#include "vm/Realm-inl.h" +#include "vm/Stack-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::MakeScopeExit; +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Some; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +using JS::AutoGCRooter; + +const AllocKind gc::slotsToThingKind[] = { + // clang-format off + /* 0 */ AllocKind::OBJECT0, AllocKind::OBJECT2, AllocKind::OBJECT2, AllocKind::OBJECT4, + /* 4 */ AllocKind::OBJECT4, AllocKind::OBJECT8, AllocKind::OBJECT8, AllocKind::OBJECT8, + /* 8 */ AllocKind::OBJECT8, AllocKind::OBJECT12, AllocKind::OBJECT12, AllocKind::OBJECT12, + /* 12 */ AllocKind::OBJECT12, AllocKind::OBJECT16, AllocKind::OBJECT16, AllocKind::OBJECT16, + /* 16 */ AllocKind::OBJECT16 + // clang-format on +}; + +static_assert(std::size(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT, + "We have defined a slot count for each kind."); + +MOZ_THREAD_LOCAL(JS::GCContext*) js::TlsGCContext; + +JS::GCContext::GCContext(JSRuntime* runtime) : runtime_(runtime) {} + +JS::GCContext::~GCContext() { + MOZ_ASSERT(!hasJitCodeToPoison()); + MOZ_ASSERT(!isCollecting()); + MOZ_ASSERT(gcUse() == GCUse::None); + MOZ_ASSERT(!gcSweepZone()); + MOZ_ASSERT(!isTouchingGrayThings()); +} + +void JS::GCContext::poisonJitCode() { + if (hasJitCodeToPoison()) { + jit::ExecutableAllocator::poisonCode(runtime(), jitPoisonRanges); + jitPoisonRanges.clearAndFree(); + } +} + +#ifdef DEBUG +void GCRuntime::verifyAllChunks() { + AutoLockGC lock(this); + fullChunks(lock).verifyChunks(); + availableChunks(lock).verifyChunks(); + emptyChunks(lock).verifyChunks(); +} +#endif + +void GCRuntime::setMinEmptyChunkCount(uint32_t value, const AutoLockGC& lock) { + minEmptyChunkCount_ = value; + if (minEmptyChunkCount_ > maxEmptyChunkCount_) { + maxEmptyChunkCount_ = minEmptyChunkCount_; + } + MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_); +} + +void GCRuntime::setMaxEmptyChunkCount(uint32_t value, const AutoLockGC& lock) { + maxEmptyChunkCount_ = value; + if (minEmptyChunkCount_ > maxEmptyChunkCount_) { + minEmptyChunkCount_ = maxEmptyChunkCount_; + } + MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_); +} + +inline bool GCRuntime::tooManyEmptyChunks(const AutoLockGC& lock) { + return emptyChunks(lock).count() > minEmptyChunkCount(lock); +} + +ChunkPool GCRuntime::expireEmptyChunkPool(const AutoLockGC& lock) { + MOZ_ASSERT(emptyChunks(lock).verify()); + MOZ_ASSERT(minEmptyChunkCount(lock) <= maxEmptyChunkCount(lock)); + + ChunkPool expired; + while (tooManyEmptyChunks(lock)) { + TenuredChunk* chunk = emptyChunks(lock).pop(); + prepareToFreeChunk(chunk->info); + expired.push(chunk); + } + + MOZ_ASSERT(expired.verify()); + MOZ_ASSERT(emptyChunks(lock).verify()); + MOZ_ASSERT(emptyChunks(lock).count() <= maxEmptyChunkCount(lock)); + MOZ_ASSERT(emptyChunks(lock).count() <= minEmptyChunkCount(lock)); + return expired; +} + +static void FreeChunkPool(ChunkPool& pool) { + for (ChunkPool::Iter iter(pool); !iter.done();) { + TenuredChunk* chunk = iter.get(); + iter.next(); + pool.remove(chunk); + MOZ_ASSERT(chunk->unused()); + UnmapPages(static_cast(chunk), ChunkSize); + } + MOZ_ASSERT(pool.count() == 0); +} + +void GCRuntime::freeEmptyChunks(const AutoLockGC& lock) { + FreeChunkPool(emptyChunks(lock)); +} + +inline void GCRuntime::prepareToFreeChunk(TenuredChunkInfo& info) { + MOZ_ASSERT(numArenasFreeCommitted >= info.numArenasFreeCommitted); + numArenasFreeCommitted -= info.numArenasFreeCommitted; + stats().count(gcstats::COUNT_DESTROY_CHUNK); +#ifdef DEBUG + /* + * Let FreeChunkPool detect a missing prepareToFreeChunk call before it + * frees chunk. + */ + info.numArenasFreeCommitted = 0; +#endif +} + +void GCRuntime::releaseArena(Arena* arena, const AutoLockGC& lock) { + MOZ_ASSERT(arena->allocated()); + MOZ_ASSERT(!arena->onDelayedMarkingList()); + MOZ_ASSERT(TlsGCContext.get()->isFinalizing()); + + arena->zone->gcHeapSize.removeGCArena(heapSize); + arena->release(lock); + arena->chunk()->releaseArena(this, arena, lock); +} + +GCRuntime::GCRuntime(JSRuntime* rt) + : rt(rt), + systemZone(nullptr), + mainThreadContext(rt), + heapState_(JS::HeapState::Idle), + stats_(this), + sweepingTracer(rt), + fullGCRequested(false), + helperThreadRatio(TuningDefaults::HelperThreadRatio), + maxHelperThreads(TuningDefaults::MaxHelperThreads), + helperThreadCount(1), + createBudgetCallback(nullptr), + minEmptyChunkCount_(TuningDefaults::MinEmptyChunkCount), + maxEmptyChunkCount_(TuningDefaults::MaxEmptyChunkCount), + rootsHash(256), + nextCellUniqueId_(LargestTaggedNullCellPointer + + 1), // Ensure disjoint from null tagged pointers. + numArenasFreeCommitted(0), + verifyPreData(nullptr), + lastGCStartTime_(TimeStamp::Now()), + lastGCEndTime_(TimeStamp::Now()), + incrementalGCEnabled(TuningDefaults::IncrementalGCEnabled), + perZoneGCEnabled(TuningDefaults::PerZoneGCEnabled), + numActiveZoneIters(0), + cleanUpEverything(false), + grayBitsValid(true), + majorGCTriggerReason(JS::GCReason::NO_REASON), + minorGCNumber(0), + majorGCNumber(0), + number(0), + sliceNumber(0), + isFull(false), + incrementalState(gc::State::NotActive), + initialState(gc::State::NotActive), + useZeal(false), + lastMarkSlice(false), + safeToYield(true), + markOnBackgroundThreadDuringSweeping(false), + useBackgroundThreads(false), +#ifdef DEBUG + hadShutdownGC(false), +#endif + requestSliceAfterBackgroundTask(false), + lifoBlocksToFree((size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + lifoBlocksToFreeAfterMinorGC( + (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + sweepGroupIndex(0), + sweepGroups(nullptr), + currentSweepGroup(nullptr), + sweepZone(nullptr), + abortSweepAfterCurrentGroup(false), + sweepMarkResult(IncrementalProgress::NotFinished), +#ifdef DEBUG + testMarkQueue(rt), +#endif + startedCompacting(false), + zonesCompacted(0), +#ifdef DEBUG + relocatedArenasToRelease(nullptr), +#endif +#ifdef JS_GC_ZEAL + markingValidator(nullptr), +#endif + defaultTimeBudgetMS_(TuningDefaults::DefaultTimeBudgetMS), + incrementalAllowed(true), + compactingEnabled(TuningDefaults::CompactingEnabled), + parallelMarkingEnabled(TuningDefaults::ParallelMarkingEnabled), + rootsRemoved(false), +#ifdef JS_GC_ZEAL + zealModeBits(0), + zealFrequency(0), + nextScheduled(0), + deterministicOnly(false), + zealSliceBudget(0), + selectedForMarking(rt), +#endif + fullCompartmentChecks(false), + gcCallbackDepth(0), + alwaysPreserveCode(false), + lowMemoryState(false), + lock(mutexid::GCLock), + delayedMarkingLock(mutexid::GCDelayedMarkingLock), + allocTask(this, emptyChunks_.ref()), + unmarkTask(this), + markTask(this), + sweepTask(this), + freeTask(this), + decommitTask(this), + nursery_(this), + storeBuffer_(rt, nursery()), + lastAllocRateUpdateTime(TimeStamp::Now()) { +} + +using CharRange = mozilla::Range; +using CharRangeVector = Vector; + +static bool SplitStringBy(CharRange text, char delimiter, + CharRangeVector* result) { + auto start = text.begin(); + for (auto ptr = start; ptr != text.end(); ptr++) { + if (*ptr == delimiter) { + if (!result->emplaceBack(start, ptr)) { + return false; + } + start = ptr + 1; + } + } + + return result->emplaceBack(start, text.end()); +} + +static bool ParseTimeDuration(CharRange text, TimeDuration* durationOut) { + const char* str = text.begin().get(); + char* end; + *durationOut = TimeDuration::FromMilliseconds(strtol(str, &end, 10)); + return str != end && end == text.end().get(); +} + +static void PrintProfileHelpAndExit(const char* envName, const char* helpText) { + fprintf(stderr, "%s=N[,(main|all)]\n", envName); + fprintf(stderr, "%s", helpText); + exit(0); +} + +void js::gc::ReadProfileEnv(const char* envName, const char* helpText, + bool* enableOut, bool* workersOut, + TimeDuration* thresholdOut) { + *enableOut = false; + *workersOut = false; + *thresholdOut = TimeDuration(); + + const char* env = getenv(envName); + if (!env) { + return; + } + + if (strcmp(env, "help") == 0) { + PrintProfileHelpAndExit(envName, helpText); + } + + CharRangeVector parts; + auto text = CharRange(env, strlen(env)); + if (!SplitStringBy(text, ',', &parts)) { + MOZ_CRASH("OOM parsing environment variable"); + } + + if (parts.length() == 0 || parts.length() > 2) { + PrintProfileHelpAndExit(envName, helpText); + } + + *enableOut = true; + + if (!ParseTimeDuration(parts[0], thresholdOut)) { + PrintProfileHelpAndExit(envName, helpText); + } + + if (parts.length() == 2) { + const char* threads = parts[1].begin().get(); + if (strcmp(threads, "all") == 0) { + *workersOut = true; + } else if (strcmp(threads, "main") != 0) { + PrintProfileHelpAndExit(envName, helpText); + } + } +} + +bool js::gc::ShouldPrintProfile(JSRuntime* runtime, bool enable, + bool profileWorkers, TimeDuration threshold, + TimeDuration duration) { + return enable && (runtime->isMainRuntime() || profileWorkers) && + duration >= threshold; +} + +#ifdef JS_GC_ZEAL + +void GCRuntime::getZealBits(uint32_t* zealBits, uint32_t* frequency, + uint32_t* scheduled) { + *zealBits = zealModeBits; + *frequency = zealFrequency; + *scheduled = nextScheduled; +} + +const char gc::ZealModeHelpText[] = + " Specifies how zealous the garbage collector should be. Some of these " + "modes can\n" + " be set simultaneously, by passing multiple level options, e.g. \"2;4\" " + "will activate\n" + " both modes 2 and 4. Modes can be specified by name or number.\n" + " \n" + " Values:\n" + " 0: (None) Normal amount of collection (resets all modes)\n" + " 1: (RootsChange) Collect when roots are added or removed\n" + " 2: (Alloc) Collect when every N allocations (default: 100)\n" + " 4: (VerifierPre) Verify pre write barriers between instructions\n" + " 6: (YieldBeforeRootMarking) Incremental GC in two slices that yields " + "before root marking\n" + " 7: (GenerationalGC) Collect the nursery every N nursery allocations\n" + " 8: (YieldBeforeMarking) Incremental GC in two slices that yields " + "between\n" + " the root marking and marking phases\n" + " 9: (YieldBeforeSweeping) Incremental GC in two slices that yields " + "between\n" + " the marking and sweeping phases\n" + " 10: (IncrementalMultipleSlices) Incremental GC in many slices\n" + " 11: (IncrementalMarkingValidator) Verify incremental marking\n" + " 12: (ElementsBarrier) Use the individual element post-write barrier\n" + " regardless of elements size\n" + " 13: (CheckHashTablesOnMinorGC) Check internal hashtables on minor GC\n" + " 14: (Compact) Perform a shrinking collection every N allocations\n" + " 15: (CheckHeapAfterGC) Walk the heap to check its integrity after " + "every GC\n" + " 17: (YieldBeforeSweepingAtoms) Incremental GC in two slices that " + "yields\n" + " before sweeping the atoms table\n" + " 18: (CheckGrayMarking) Check gray marking invariants after every GC\n" + " 19: (YieldBeforeSweepingCaches) Incremental GC in two slices that " + "yields\n" + " before sweeping weak caches\n" + " 21: (YieldBeforeSweepingObjects) Incremental GC in two slices that " + "yields\n" + " before sweeping foreground finalized objects\n" + " 22: (YieldBeforeSweepingNonObjects) Incremental GC in two slices that " + "yields\n" + " before sweeping non-object GC things\n" + " 23: (YieldBeforeSweepingPropMapTrees) Incremental GC in two slices " + "that " + "yields\n" + " before sweeping shape trees\n" + " 24: (CheckWeakMapMarking) Check weak map marking invariants after " + "every GC\n" + " 25: (YieldWhileGrayMarking) Incremental GC in two slices that yields\n" + " during gray marking\n"; + +// The set of zeal modes that control incremental slices. These modes are +// mutually exclusive. +static const mozilla::EnumSet IncrementalSliceZealModes = { + ZealMode::YieldBeforeRootMarking, + ZealMode::YieldBeforeMarking, + ZealMode::YieldBeforeSweeping, + ZealMode::IncrementalMultipleSlices, + ZealMode::YieldBeforeSweepingAtoms, + ZealMode::YieldBeforeSweepingCaches, + ZealMode::YieldBeforeSweepingObjects, + ZealMode::YieldBeforeSweepingNonObjects, + ZealMode::YieldBeforeSweepingPropMapTrees}; + +void GCRuntime::setZeal(uint8_t zeal, uint32_t frequency) { + MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit)); + + if (verifyPreData) { + VerifyBarriers(rt, PreBarrierVerifier); + } + + if (zeal == 0) { + if (hasZealMode(ZealMode::GenerationalGC)) { + evictNursery(JS::GCReason::DEBUG_GC); + nursery().leaveZealMode(); + } + + if (isIncrementalGCInProgress()) { + finishGC(JS::GCReason::DEBUG_GC); + } + } + + ZealMode zealMode = ZealMode(zeal); + if (zealMode == ZealMode::GenerationalGC) { + evictNursery(JS::GCReason::DEBUG_GC); + nursery().enterZealMode(); + } + + // Some modes are mutually exclusive. If we're setting one of those, we + // first reset all of them. + if (IncrementalSliceZealModes.contains(zealMode)) { + for (auto mode : IncrementalSliceZealModes) { + clearZealMode(mode); + } + } + + bool schedule = zealMode >= ZealMode::Alloc; + if (zeal != 0) { + zealModeBits |= 1 << unsigned(zeal); + } else { + zealModeBits = 0; + } + zealFrequency = frequency; + nextScheduled = schedule ? frequency : 0; +} + +void GCRuntime::unsetZeal(uint8_t zeal) { + MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit)); + ZealMode zealMode = ZealMode(zeal); + + if (!hasZealMode(zealMode)) { + return; + } + + if (verifyPreData) { + VerifyBarriers(rt, PreBarrierVerifier); + } + + if (zealMode == ZealMode::GenerationalGC) { + evictNursery(JS::GCReason::DEBUG_GC); + nursery().leaveZealMode(); + } + + clearZealMode(zealMode); + + if (zealModeBits == 0) { + if (isIncrementalGCInProgress()) { + finishGC(JS::GCReason::DEBUG_GC); + } + + zealFrequency = 0; + nextScheduled = 0; + } +} + +void GCRuntime::setNextScheduled(uint32_t count) { nextScheduled = count; } + +static bool ParseZealModeName(CharRange text, uint32_t* modeOut) { + struct ModeInfo { + const char* name; + size_t length; + uint32_t value; + }; + + static const ModeInfo zealModes[] = {{"None", 0}, +# define ZEAL_MODE(name, value) {#name, strlen(#name), value}, + JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE) +# undef ZEAL_MODE + }; + + for (auto mode : zealModes) { + if (text.length() == mode.length && + memcmp(text.begin().get(), mode.name, mode.length) == 0) { + *modeOut = mode.value; + return true; + } + } + + return false; +} + +static bool ParseZealModeNumericParam(CharRange text, uint32_t* paramOut) { + if (text.length() == 0) { + return false; + } + + for (auto c : text) { + if (!mozilla::IsAsciiDigit(c)) { + return false; + } + } + + *paramOut = atoi(text.begin().get()); + return true; +} + +static bool PrintZealHelpAndFail() { + fprintf(stderr, "Format: JS_GC_ZEAL=level(;level)*[,N]\n"); + fputs(ZealModeHelpText, stderr); + return false; +} + +bool GCRuntime::parseAndSetZeal(const char* str) { + // Set the zeal mode from a string consisting of one or more mode specifiers + // separated by ';', optionally followed by a ',' and the trigger frequency. + // The mode specifiers can by a mode name or its number. + + auto text = CharRange(str, strlen(str)); + + CharRangeVector parts; + if (!SplitStringBy(text, ',', &parts)) { + return false; + } + + if (parts.length() == 0 || parts.length() > 2) { + return PrintZealHelpAndFail(); + } + + uint32_t frequency = JS_DEFAULT_ZEAL_FREQ; + if (parts.length() == 2 && !ParseZealModeNumericParam(parts[1], &frequency)) { + return PrintZealHelpAndFail(); + } + + CharRangeVector modes; + if (!SplitStringBy(parts[0], ';', &modes)) { + return false; + } + + for (const auto& descr : modes) { + uint32_t mode; + if (!ParseZealModeName(descr, &mode) && + !(ParseZealModeNumericParam(descr, &mode) && + mode <= unsigned(ZealMode::Limit))) { + return PrintZealHelpAndFail(); + } + + setZeal(mode, frequency); + } + + return true; +} + +const char* js::gc::AllocKindName(AllocKind kind) { + static const char* const names[] = { +# define EXPAND_THING_NAME(allocKind, _1, _2, _3, _4, _5, _6) #allocKind, + FOR_EACH_ALLOCKIND(EXPAND_THING_NAME) +# undef EXPAND_THING_NAME + }; + static_assert(std::size(names) == AllocKindCount, + "names array should have an entry for every AllocKind"); + + size_t i = size_t(kind); + MOZ_ASSERT(i < std::size(names)); + return names[i]; +} + +void js::gc::DumpArenaInfo() { + fprintf(stderr, "Arena header size: %zu\n\n", ArenaHeaderSize); + + fprintf(stderr, "GC thing kinds:\n"); + fprintf(stderr, "%25s %8s %8s %8s\n", + "AllocKind:", "Size:", "Count:", "Padding:"); + for (auto kind : AllAllocKinds()) { + fprintf(stderr, "%25s %8zu %8zu %8zu\n", AllocKindName(kind), + Arena::thingSize(kind), Arena::thingsPerArena(kind), + Arena::firstThingOffset(kind) - ArenaHeaderSize); + } +} + +#endif // JS_GC_ZEAL + +bool GCRuntime::init(uint32_t maxbytes) { + MOZ_ASSERT(!wasInitialized()); + + MOZ_ASSERT(SystemPageSize()); + Arena::checkLookupTables(); + + if (!TlsGCContext.init()) { + return false; + } + TlsGCContext.set(&mainThreadContext.ref()); + + updateHelperThreadCount(); + +#ifdef JS_GC_ZEAL + const char* size = getenv("JSGC_MARK_STACK_LIMIT"); + if (size) { + maybeMarkStackLimit = atoi(size); + } +#endif + + if (!updateMarkersVector()) { + return false; + } + + { + AutoLockGCBgAlloc lock(this); + + MOZ_ALWAYS_TRUE(tunables.setParameter(JSGC_MAX_BYTES, maxbytes)); + + if (!nursery().init(lock)) { + return false; + } + + const char* pretenureThresholdStr = getenv("JSGC_PRETENURE_THRESHOLD"); + if (pretenureThresholdStr && pretenureThresholdStr[0]) { + char* last; + long pretenureThreshold = strtol(pretenureThresholdStr, &last, 10); + if (last[0] || !tunables.setParameter(JSGC_PRETENURE_THRESHOLD, + pretenureThreshold)) { + fprintf(stderr, "Invalid value for JSGC_PRETENURE_THRESHOLD: %s\n", + pretenureThresholdStr); + } + } + } + +#ifdef JS_GC_ZEAL + const char* zealSpec = getenv("JS_GC_ZEAL"); + if (zealSpec && zealSpec[0] && !parseAndSetZeal(zealSpec)) { + return false; + } +#endif + + for (auto& marker : markers) { + if (!marker->init()) { + return false; + } + } + + if (!initSweepActions()) { + return false; + } + + UniquePtr zone = MakeUnique(rt, Zone::AtomsZone); + if (!zone || !zone->init()) { + return false; + } + + // The atoms zone is stored as the first element of the zones vector. + MOZ_ASSERT(zone->isAtomsZone()); + MOZ_ASSERT(zones().empty()); + MOZ_ALWAYS_TRUE(zones().reserve(1)); // ZonesVector has inline capacity 4. + zones().infallibleAppend(zone.release()); + + gcprobes::Init(this); + + initialized = true; + return true; +} + +void GCRuntime::finish() { + MOZ_ASSERT(inPageLoadCount == 0); + MOZ_ASSERT(!sharedAtomsZone_); + + // Wait for nursery background free to end and disable it to release memory. + if (nursery().isEnabled()) { + nursery().disable(); + } + + // Wait until the background finalization and allocation stops and the + // helper thread shuts down before we forcefully release any remaining GC + // memory. + sweepTask.join(); + markTask.join(); + freeTask.join(); + allocTask.cancelAndWait(); + decommitTask.cancelAndWait(); + +#ifdef JS_GC_ZEAL + // Free memory associated with GC verification. + finishVerifier(); +#endif + + // Delete all remaining zones. + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + AutoSetThreadIsSweeping threadIsSweeping(rt->gcContext(), zone); + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + for (RealmsInCompartmentIter realm(comp); !realm.done(); realm.next()) { + js_delete(realm.get()); + } + comp->realms().clear(); + js_delete(comp.get()); + } + zone->compartments().clear(); + js_delete(zone.get()); + } + + zones().clear(); + + FreeChunkPool(fullChunks_.ref()); + FreeChunkPool(availableChunks_.ref()); + FreeChunkPool(emptyChunks_.ref()); + + TlsGCContext.set(nullptr); + + gcprobes::Finish(this); + + nursery().printTotalProfileTimes(); + stats().printTotalProfileTimes(); +} + +bool GCRuntime::freezeSharedAtomsZone() { + // This is called just after permanent atoms and well-known symbols have been + // created. At this point all existing atoms and symbols are permanent. + // + // This method makes the current atoms zone into a shared atoms zone and + // removes it from the zones list. Everything in it is marked black. A new + // empty atoms zone is created, where all atoms local to this runtime will + // live. + // + // The shared atoms zone will not be collected until shutdown when it is + // returned to the zone list by restoreSharedAtomsZone(). + + MOZ_ASSERT(rt->isMainRuntime()); + MOZ_ASSERT(!sharedAtomsZone_); + MOZ_ASSERT(zones().length() == 1); + MOZ_ASSERT(atomsZone()); + MOZ_ASSERT(!atomsZone()->wasGCStarted()); + MOZ_ASSERT(!atomsZone()->needsIncrementalBarrier()); + + AutoAssertEmptyNursery nurseryIsEmpty(rt->mainContextFromOwnThread()); + + atomsZone()->arenas.clearFreeLists(); + + for (auto kind : AllAllocKinds()) { + for (auto thing = + atomsZone()->cellIterUnsafe(kind, nurseryIsEmpty); + !thing.done(); thing.next()) { + TenuredCell* cell = thing.getCell(); + MOZ_ASSERT((cell->is() && + cell->as()->isPermanentAndMayBeShared()) || + (cell->is() && + cell->as()->isPermanentAndMayBeShared())); + cell->markBlack(); + } + } + + sharedAtomsZone_ = atomsZone(); + zones().clear(); + + UniquePtr zone = MakeUnique(rt, Zone::AtomsZone); + if (!zone || !zone->init()) { + return false; + } + + MOZ_ASSERT(zone->isAtomsZone()); + zones().infallibleAppend(zone.release()); + + return true; +} + +void GCRuntime::restoreSharedAtomsZone() { + // Return the shared atoms zone to the zone list. This allows the contents of + // the shared atoms zone to be collected when the parent runtime is shut down. + + if (!sharedAtomsZone_) { + return; + } + + MOZ_ASSERT(rt->isMainRuntime()); + MOZ_ASSERT(rt->childRuntimeCount == 0); + + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zones().append(sharedAtomsZone_)) { + oomUnsafe.crash("restoreSharedAtomsZone"); + } + + sharedAtomsZone_ = nullptr; +} + +bool GCRuntime::setParameter(JSContext* cx, JSGCParamKey key, uint32_t value) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + AutoStopVerifyingBarriers pauseVerification(rt, false); + FinishGC(cx); + waitBackgroundSweepEnd(); + + AutoLockGC lock(this); + return setParameter(key, value, lock); +} + +static bool IsGCThreadParameter(JSGCParamKey key) { + return key == JSGC_HELPER_THREAD_RATIO || key == JSGC_MAX_HELPER_THREADS || + key == JSGC_MARKING_THREAD_COUNT; +} + +bool GCRuntime::setParameter(JSGCParamKey key, uint32_t value, + AutoLockGC& lock) { + switch (key) { + case JSGC_SLICE_TIME_BUDGET_MS: + defaultTimeBudgetMS_ = value; + break; + case JSGC_INCREMENTAL_GC_ENABLED: + setIncrementalGCEnabled(value != 0); + break; + case JSGC_PER_ZONE_GC_ENABLED: + perZoneGCEnabled = value != 0; + break; + case JSGC_COMPACTING_ENABLED: + compactingEnabled = value != 0; + break; + case JSGC_PARALLEL_MARKING_ENABLED: + // Not supported on workers. + parallelMarkingEnabled = rt->isMainRuntime() && value != 0; + updateMarkersVector(); + break; + case JSGC_INCREMENTAL_WEAKMAP_ENABLED: + for (auto& marker : markers) { + marker->incrementalWeakMapMarkingEnabled = value != 0; + } + break; + case JSGC_MIN_EMPTY_CHUNK_COUNT: + setMinEmptyChunkCount(value, lock); + break; + case JSGC_MAX_EMPTY_CHUNK_COUNT: + setMaxEmptyChunkCount(value, lock); + break; + default: + if (IsGCThreadParameter(key)) { + return setThreadParameter(key, value, lock); + } + + if (!tunables.setParameter(key, value)) { + return false; + } + updateAllGCStartThresholds(); + } + + return true; +} + +bool GCRuntime::setThreadParameter(JSGCParamKey key, uint32_t value, + AutoLockGC& lock) { + if (rt->parentRuntime) { + // Don't allow these to be set for worker runtimes. + return false; + } + + switch (key) { + case JSGC_HELPER_THREAD_RATIO: + if (value == 0) { + return false; + } + helperThreadRatio = double(value) / 100.0; + break; + case JSGC_MAX_HELPER_THREADS: + if (value == 0) { + return false; + } + maxHelperThreads = value; + break; + case JSGC_MARKING_THREAD_COUNT: + markingThreadCount = std::min(size_t(value), MaxParallelWorkers); + break; + default: + MOZ_CRASH("Unexpected parameter key"); + } + + updateHelperThreadCount(); + updateMarkersVector(); + + return true; +} + +void GCRuntime::resetParameter(JSContext* cx, JSGCParamKey key) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + AutoStopVerifyingBarriers pauseVerification(rt, false); + FinishGC(cx); + waitBackgroundSweepEnd(); + + AutoLockGC lock(this); + resetParameter(key, lock); +} + +void GCRuntime::resetParameter(JSGCParamKey key, AutoLockGC& lock) { + switch (key) { + case JSGC_SLICE_TIME_BUDGET_MS: + defaultTimeBudgetMS_ = TuningDefaults::DefaultTimeBudgetMS; + break; + case JSGC_INCREMENTAL_GC_ENABLED: + setIncrementalGCEnabled(TuningDefaults::IncrementalGCEnabled); + break; + case JSGC_PER_ZONE_GC_ENABLED: + perZoneGCEnabled = TuningDefaults::PerZoneGCEnabled; + break; + case JSGC_COMPACTING_ENABLED: + compactingEnabled = TuningDefaults::CompactingEnabled; + break; + case JSGC_PARALLEL_MARKING_ENABLED: + parallelMarkingEnabled = TuningDefaults::ParallelMarkingEnabled; + updateMarkersVector(); + break; + case JSGC_INCREMENTAL_WEAKMAP_ENABLED: + for (auto& marker : markers) { + marker->incrementalWeakMapMarkingEnabled = + TuningDefaults::IncrementalWeakMapMarkingEnabled; + } + break; + case JSGC_MIN_EMPTY_CHUNK_COUNT: + setMinEmptyChunkCount(TuningDefaults::MinEmptyChunkCount, lock); + break; + case JSGC_MAX_EMPTY_CHUNK_COUNT: + setMaxEmptyChunkCount(TuningDefaults::MaxEmptyChunkCount, lock); + break; + default: + if (IsGCThreadParameter(key)) { + resetThreadParameter(key, lock); + return; + } + + tunables.resetParameter(key); + updateAllGCStartThresholds(); + } +} + +void GCRuntime::resetThreadParameter(JSGCParamKey key, AutoLockGC& lock) { + if (rt->parentRuntime) { + return; + } + + switch (key) { + case JSGC_HELPER_THREAD_RATIO: + helperThreadRatio = TuningDefaults::HelperThreadRatio; + break; + case JSGC_MAX_HELPER_THREADS: + maxHelperThreads = TuningDefaults::MaxHelperThreads; + break; + case JSGC_MARKING_THREAD_COUNT: + markingThreadCount = 0; + break; + default: + MOZ_CRASH("Unexpected parameter key"); + } + + updateHelperThreadCount(); + updateMarkersVector(); +} + +uint32_t GCRuntime::getParameter(JSGCParamKey key) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + AutoLockGC lock(this); + return getParameter(key, lock); +} + +uint32_t GCRuntime::getParameter(JSGCParamKey key, const AutoLockGC& lock) { + switch (key) { + case JSGC_BYTES: + return uint32_t(heapSize.bytes()); + case JSGC_NURSERY_BYTES: + return nursery().capacity(); + case JSGC_NUMBER: + return uint32_t(number); + case JSGC_MAJOR_GC_NUMBER: + return uint32_t(majorGCNumber); + case JSGC_MINOR_GC_NUMBER: + return uint32_t(minorGCNumber); + case JSGC_INCREMENTAL_GC_ENABLED: + return incrementalGCEnabled; + case JSGC_PER_ZONE_GC_ENABLED: + return perZoneGCEnabled; + case JSGC_UNUSED_CHUNKS: + return uint32_t(emptyChunks(lock).count()); + case JSGC_TOTAL_CHUNKS: + return uint32_t(fullChunks(lock).count() + availableChunks(lock).count() + + emptyChunks(lock).count()); + case JSGC_SLICE_TIME_BUDGET_MS: + MOZ_RELEASE_ASSERT(defaultTimeBudgetMS_ >= 0); + MOZ_RELEASE_ASSERT(defaultTimeBudgetMS_ <= UINT32_MAX); + return uint32_t(defaultTimeBudgetMS_); + case JSGC_MIN_EMPTY_CHUNK_COUNT: + return minEmptyChunkCount(lock); + case JSGC_MAX_EMPTY_CHUNK_COUNT: + return maxEmptyChunkCount(lock); + case JSGC_COMPACTING_ENABLED: + return compactingEnabled; + case JSGC_PARALLEL_MARKING_ENABLED: + return parallelMarkingEnabled; + case JSGC_INCREMENTAL_WEAKMAP_ENABLED: + return marker().incrementalWeakMapMarkingEnabled; + case JSGC_CHUNK_BYTES: + return ChunkSize; + case JSGC_HELPER_THREAD_RATIO: + MOZ_ASSERT(helperThreadRatio > 0.0); + return uint32_t(helperThreadRatio * 100.0); + case JSGC_MAX_HELPER_THREADS: + MOZ_ASSERT(maxHelperThreads <= UINT32_MAX); + return maxHelperThreads; + case JSGC_HELPER_THREAD_COUNT: + return helperThreadCount; + case JSGC_MARKING_THREAD_COUNT: + return markingThreadCount; + case JSGC_SYSTEM_PAGE_SIZE_KB: + return SystemPageSize() / 1024; + default: + return tunables.getParameter(key); + } +} + +#ifdef JS_GC_ZEAL +void GCRuntime::setMarkStackLimit(size_t limit, AutoLockGC& lock) { + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + + maybeMarkStackLimit = limit; + + AutoUnlockGC unlock(lock); + AutoStopVerifyingBarriers pauseVerification(rt, false); + for (auto& marker : markers) { + marker->setMaxCapacity(limit); + } +} +#endif + +void GCRuntime::setIncrementalGCEnabled(bool enabled) { + incrementalGCEnabled = enabled; +} + +void GCRuntime::updateHelperThreadCount() { + if (!CanUseExtraThreads()) { + // startTask will run the work on the main thread if the count is 1. + MOZ_ASSERT(helperThreadCount == 1); + return; + } + + // Number of extra threads required during parallel marking to ensure we can + // start the necessary marking tasks. Background free and background + // allocation may already be running and we want to avoid these tasks blocking + // marking. In real configurations there will be enough threads that this + // won't affect anything. + static constexpr size_t SpareThreadsDuringParallelMarking = 2; + + // The count of helper threads used for GC tasks is process wide. Don't set it + // for worker JS runtimes. + if (rt->parentRuntime) { + helperThreadCount = rt->parentRuntime->gc.helperThreadCount; + return; + } + + // Calculate the target thread count for GC parallel tasks. + double cpuCount = GetHelperThreadCPUCount(); + helperThreadCount = std::clamp(size_t(cpuCount * helperThreadRatio.ref()), + size_t(1), maxHelperThreads.ref()); + + // Calculate the overall target thread count taking into account the separate + // parameter for parallel marking threads. Add spare threads to avoid blocking + // parallel marking when there is other GC work happening. + size_t targetCount = + std::max(helperThreadCount.ref(), + markingThreadCount.ref() + SpareThreadsDuringParallelMarking); + + // Attempt to create extra threads if possible. This is not supported when + // using an external thread pool. + AutoLockHelperThreadState lock; + (void)HelperThreadState().ensureThreadCount(targetCount, lock); + + // Limit all thread counts based on the number of threads available, which may + // be fewer than requested. + size_t availableThreadCount = GetHelperThreadCount(); + MOZ_ASSERT(availableThreadCount != 0); + targetCount = std::min(targetCount, availableThreadCount); + helperThreadCount = std::min(helperThreadCount.ref(), availableThreadCount); + markingThreadCount = + std::min(markingThreadCount.ref(), + availableThreadCount - SpareThreadsDuringParallelMarking); + + // Update the maximum number of threads that will be used for GC work. + HelperThreadState().setGCParallelThreadCount(targetCount, lock); +} + +size_t GCRuntime::markingWorkerCount() const { + if (!CanUseExtraThreads() || !parallelMarkingEnabled) { + return 1; + } + + if (markingThreadCount) { + return markingThreadCount; + } + + // Limit parallel marking to use at most two threads initially. + return 2; +} + +#ifdef DEBUG +void GCRuntime::assertNoMarkingWork() const { + for (auto& marker : markers) { + MOZ_ASSERT(marker->isDrained()); + } + MOZ_ASSERT(!hasDelayedMarking()); +} +#endif + +static size_t GetGCParallelThreadCount() { + AutoLockHelperThreadState lock; + return HelperThreadState().getGCParallelThreadCount(lock); +} + +bool GCRuntime::updateMarkersVector() { + MOZ_ASSERT(helperThreadCount >= 1, + "There must always be at least one mark task"); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + assertNoMarkingWork(); + + // Limit worker count to number of GC parallel tasks that can run + // concurrently, otherwise one thread can deadlock waiting on another. + size_t targetCount = + std::min(markingWorkerCount(), GetGCParallelThreadCount()); + + if (markers.length() > targetCount) { + return markers.resize(targetCount); + } + + while (markers.length() < targetCount) { + auto marker = MakeUnique(rt); + if (!marker) { + return false; + } + +#ifdef JS_GC_ZEAL + if (maybeMarkStackLimit) { + marker->setMaxCapacity(maybeMarkStackLimit); + } +#endif + + if (!marker->init()) { + return false; + } + + if (!markers.emplaceBack(std::move(marker))) { + return false; + } + } + + return true; +} + +bool GCRuntime::addBlackRootsTracer(JSTraceDataOp traceOp, void* data) { + AssertHeapIsIdle(); + return !!blackRootTracers.ref().append( + Callback(traceOp, data)); +} + +void GCRuntime::removeBlackRootsTracer(JSTraceDataOp traceOp, void* data) { + // Can be called from finalizers + for (size_t i = 0; i < blackRootTracers.ref().length(); i++) { + Callback* e = &blackRootTracers.ref()[i]; + if (e->op == traceOp && e->data == data) { + blackRootTracers.ref().erase(e); + break; + } + } +} + +void GCRuntime::setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data) { + AssertHeapIsIdle(); + grayRootTracer.ref() = {traceOp, data}; +} + +void GCRuntime::clearBlackAndGrayRootTracers() { + MOZ_ASSERT(rt->isBeingDestroyed()); + blackRootTracers.ref().clear(); + setGrayRootsTracer(nullptr, nullptr); +} + +void GCRuntime::setGCCallback(JSGCCallback callback, void* data) { + gcCallback.ref() = {callback, data}; +} + +void GCRuntime::callGCCallback(JSGCStatus status, JS::GCReason reason) const { + const auto& callback = gcCallback.ref(); + MOZ_ASSERT(callback.op); + callback.op(rt->mainContextFromOwnThread(), status, reason, callback.data); +} + +void GCRuntime::setObjectsTenuredCallback(JSObjectsTenuredCallback callback, + void* data) { + tenuredCallback.ref() = {callback, data}; +} + +void GCRuntime::callObjectsTenuredCallback() { + JS::AutoSuppressGCAnalysis nogc; + const auto& callback = tenuredCallback.ref(); + if (callback.op) { + callback.op(rt->mainContextFromOwnThread(), callback.data); + } +} + +bool GCRuntime::addFinalizeCallback(JSFinalizeCallback callback, void* data) { + return finalizeCallbacks.ref().append( + Callback(callback, data)); +} + +template +static void EraseCallback(CallbackVector& vector, F callback) { + for (Callback* p = vector.begin(); p != vector.end(); p++) { + if (p->op == callback) { + vector.erase(p); + return; + } + } +} + +void GCRuntime::removeFinalizeCallback(JSFinalizeCallback callback) { + EraseCallback(finalizeCallbacks.ref(), callback); +} + +void GCRuntime::callFinalizeCallbacks(JS::GCContext* gcx, + JSFinalizeStatus status) const { + for (auto& p : finalizeCallbacks.ref()) { + p.op(gcx, status, p.data); + } +} + +void GCRuntime::setHostCleanupFinalizationRegistryCallback( + JSHostCleanupFinalizationRegistryCallback callback, void* data) { + hostCleanupFinalizationRegistryCallback.ref() = {callback, data}; +} + +void GCRuntime::callHostCleanupFinalizationRegistryCallback( + JSFunction* doCleanup, GlobalObject* incumbentGlobal) { + JS::AutoSuppressGCAnalysis nogc; + const auto& callback = hostCleanupFinalizationRegistryCallback.ref(); + if (callback.op) { + callback.op(doCleanup, incumbentGlobal, callback.data); + } +} + +bool GCRuntime::addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback, + void* data) { + return updateWeakPointerZonesCallbacks.ref().append( + Callback(callback, data)); +} + +void GCRuntime::removeWeakPointerZonesCallback( + JSWeakPointerZonesCallback callback) { + EraseCallback(updateWeakPointerZonesCallbacks.ref(), callback); +} + +void GCRuntime::callWeakPointerZonesCallbacks(JSTracer* trc) const { + for (auto const& p : updateWeakPointerZonesCallbacks.ref()) { + p.op(trc, p.data); + } +} + +bool GCRuntime::addWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback, void* data) { + return updateWeakPointerCompartmentCallbacks.ref().append( + Callback(callback, data)); +} + +void GCRuntime::removeWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback) { + EraseCallback(updateWeakPointerCompartmentCallbacks.ref(), callback); +} + +void GCRuntime::callWeakPointerCompartmentCallbacks( + JSTracer* trc, JS::Compartment* comp) const { + for (auto const& p : updateWeakPointerCompartmentCallbacks.ref()) { + p.op(trc, comp, p.data); + } +} + +JS::GCSliceCallback GCRuntime::setSliceCallback(JS::GCSliceCallback callback) { + return stats().setSliceCallback(callback); +} + +JS::GCNurseryCollectionCallback GCRuntime::setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback callback) { + return stats().setNurseryCollectionCallback(callback); +} + +JS::DoCycleCollectionCallback GCRuntime::setDoCycleCollectionCallback( + JS::DoCycleCollectionCallback callback) { + const auto prior = gcDoCycleCollectionCallback.ref(); + gcDoCycleCollectionCallback.ref() = {callback, nullptr}; + return prior.op; +} + +void GCRuntime::callDoCycleCollectionCallback(JSContext* cx) { + const auto& callback = gcDoCycleCollectionCallback.ref(); + if (callback.op) { + callback.op(cx); + } +} + +bool GCRuntime::addRoot(Value* vp, const char* name) { + /* + * Sometimes Firefox will hold weak references to objects and then convert + * them to strong references by calling AddRoot (e.g., via PreserveWrapper, + * or ModifyBusyCount in workers). We need a read barrier to cover these + * cases. + */ + MOZ_ASSERT(vp); + Value value = *vp; + if (value.isGCThing()) { + ValuePreWriteBarrier(value); + } + + return rootsHash.ref().put(vp, name); +} + +void GCRuntime::removeRoot(Value* vp) { + rootsHash.ref().remove(vp); + notifyRootsRemoved(); +} + +/* Compacting GC */ + +bool js::gc::IsCurrentlyAnimating(const TimeStamp& lastAnimationTime, + const TimeStamp& currentTime) { + // Assume that we're currently animating if js::NotifyAnimationActivity has + // been called in the last second. + static const auto oneSecond = TimeDuration::FromSeconds(1); + return !lastAnimationTime.IsNull() && + currentTime < (lastAnimationTime + oneSecond); +} + +static bool DiscardedCodeRecently(Zone* zone, const TimeStamp& currentTime) { + static const auto thirtySeconds = TimeDuration::FromSeconds(30); + return !zone->lastDiscardedCodeTime().IsNull() && + currentTime < (zone->lastDiscardedCodeTime() + thirtySeconds); +} + +bool GCRuntime::shouldCompact() { + // Compact on shrinking GC if enabled. Skip compacting in incremental GCs + // if we are currently animating, unless the user is inactive or we're + // responding to memory pressure. + + if (!isShrinkingGC() || !isCompactingGCEnabled()) { + return false; + } + + if (initialReason == JS::GCReason::USER_INACTIVE || + initialReason == JS::GCReason::MEM_PRESSURE) { + return true; + } + + return !isIncremental || + !IsCurrentlyAnimating(rt->lastAnimationTime, TimeStamp::Now()); +} + +bool GCRuntime::isCompactingGCEnabled() const { + return compactingEnabled && + rt->mainContextFromOwnThread()->compactingDisabledCount == 0; +} + +JS_PUBLIC_API void JS::SetCreateGCSliceBudgetCallback( + JSContext* cx, JS::CreateSliceBudgetCallback cb) { + cx->runtime()->gc.createBudgetCallback = cb; +} + +void TimeBudget::setDeadlineFromNow() { deadline = TimeStamp::Now() + budget; } + +SliceBudget::SliceBudget(TimeBudget time, InterruptRequestFlag* interrupt) + : budget(TimeBudget(time)), + interruptRequested(interrupt), + counter(StepsPerExpensiveCheck) { + budget.as().setDeadlineFromNow(); +} + +SliceBudget::SliceBudget(WorkBudget work) + : budget(work), interruptRequested(nullptr), counter(work.budget) {} + +int SliceBudget::describe(char* buffer, size_t maxlen) const { + if (isUnlimited()) { + return snprintf(buffer, maxlen, "unlimited"); + } else if (isWorkBudget()) { + return snprintf(buffer, maxlen, "work(%" PRId64 ")", workBudget()); + } else { + const char* interruptStr = ""; + if (interruptRequested) { + interruptStr = interrupted ? "INTERRUPTED " : "interruptible "; + } + const char* extra = ""; + if (idle) { + extra = extended ? " (started idle but extended)" : " (idle)"; + } + return snprintf(buffer, maxlen, "%s%" PRId64 "ms%s", interruptStr, + timeBudget(), extra); + } +} + +bool SliceBudget::checkOverBudget() { + MOZ_ASSERT(counter <= 0); + MOZ_ASSERT(!isUnlimited()); + + if (isWorkBudget()) { + return true; + } + + if (interruptRequested && *interruptRequested) { + *interruptRequested = false; + interrupted = true; + } + + if (interrupted) { + return true; + } + + if (TimeStamp::Now() >= budget.as().deadline) { + return true; + } + + counter = StepsPerExpensiveCheck; + return false; +} + +void GCRuntime::requestMajorGC(JS::GCReason reason) { + MOZ_ASSERT_IF(reason != JS::GCReason::BG_TASK_FINISHED, + !CurrentThreadIsPerformingGC()); + + if (majorGCRequested()) { + return; + } + + majorGCTriggerReason = reason; + rt->mainContextFromAnyThread()->requestInterrupt(InterruptReason::GC); +} + +void Nursery::requestMinorGC(JS::GCReason reason) const { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + + if (minorGCRequested()) { + return; + } + + minorGCTriggerReason_ = reason; + runtime()->mainContextFromOwnThread()->requestInterrupt(InterruptReason::GC); +} + +bool GCRuntime::triggerGC(JS::GCReason reason) { + /* + * Don't trigger GCs if this is being called off the main thread from + * onTooMuchMalloc(). + */ + if (!CurrentThreadCanAccessRuntime(rt)) { + return false; + } + + /* GC is already running. */ + if (JS::RuntimeHeapIsCollecting()) { + return false; + } + + JS::PrepareForFullGC(rt->mainContextFromOwnThread()); + requestMajorGC(reason); + return true; +} + +void GCRuntime::maybeTriggerGCAfterAlloc(Zone* zone) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + + TriggerResult trigger = + checkHeapThreshold(zone, zone->gcHeapSize, zone->gcHeapThreshold); + + if (trigger.shouldTrigger) { + // Start or continue an in progress incremental GC. We do this to try to + // avoid performing non-incremental GCs on zones which allocate a lot of + // data, even when incremental slices can't be triggered via scheduling in + // the event loop. + triggerZoneGC(zone, JS::GCReason::ALLOC_TRIGGER, trigger.usedBytes, + trigger.thresholdBytes); + } +} + +void js::gc::MaybeMallocTriggerZoneGC(JSRuntime* rt, ZoneAllocator* zoneAlloc, + const HeapSize& heap, + const HeapThreshold& threshold, + JS::GCReason reason) { + rt->gc.maybeTriggerGCAfterMalloc(Zone::from(zoneAlloc), heap, threshold, + reason); +} + +void GCRuntime::maybeTriggerGCAfterMalloc(Zone* zone) { + if (maybeTriggerGCAfterMalloc(zone, zone->mallocHeapSize, + zone->mallocHeapThreshold, + JS::GCReason::TOO_MUCH_MALLOC)) { + return; + } + + maybeTriggerGCAfterMalloc(zone, zone->jitHeapSize, zone->jitHeapThreshold, + JS::GCReason::TOO_MUCH_JIT_CODE); +} + +bool GCRuntime::maybeTriggerGCAfterMalloc(Zone* zone, const HeapSize& heap, + const HeapThreshold& threshold, + JS::GCReason reason) { + // Ignore malloc during sweeping, for example when we resize hash tables. + if (heapState() != JS::HeapState::Idle) { + return false; + } + + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + TriggerResult trigger = checkHeapThreshold(zone, heap, threshold); + if (!trigger.shouldTrigger) { + return false; + } + + // Trigger a zone GC. budgetIncrementalGC() will work out whether to do an + // incremental or non-incremental collection. + triggerZoneGC(zone, reason, trigger.usedBytes, trigger.thresholdBytes); + return true; +} + +TriggerResult GCRuntime::checkHeapThreshold( + Zone* zone, const HeapSize& heapSize, const HeapThreshold& heapThreshold) { + MOZ_ASSERT_IF(heapThreshold.hasSliceThreshold(), zone->wasGCStarted()); + + size_t usedBytes = heapSize.bytes(); + size_t thresholdBytes = heapThreshold.hasSliceThreshold() + ? heapThreshold.sliceBytes() + : heapThreshold.startBytes(); + + // The incremental limit will be checked if we trigger a GC slice. + MOZ_ASSERT(thresholdBytes <= heapThreshold.incrementalLimitBytes()); + + return TriggerResult{usedBytes >= thresholdBytes, usedBytes, thresholdBytes}; +} + +bool GCRuntime::triggerZoneGC(Zone* zone, JS::GCReason reason, size_t used, + size_t threshold) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + /* GC is already running. */ + if (JS::RuntimeHeapIsBusy()) { + return false; + } + +#ifdef JS_GC_ZEAL + if (hasZealMode(ZealMode::Alloc)) { + MOZ_RELEASE_ASSERT(triggerGC(reason)); + return true; + } +#endif + + if (zone->isAtomsZone()) { + stats().recordTrigger(used, threshold); + MOZ_RELEASE_ASSERT(triggerGC(reason)); + return true; + } + + stats().recordTrigger(used, threshold); + zone->scheduleGC(); + requestMajorGC(reason); + return true; +} + +void GCRuntime::maybeGC() { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + +#ifdef JS_GC_ZEAL + if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::RootsChange)) { + JS::PrepareForFullGC(rt->mainContextFromOwnThread()); + gc(JS::GCOptions::Normal, JS::GCReason::DEBUG_GC); + return; + } +#endif + + (void)gcIfRequestedImpl(/* eagerOk = */ true); +} + +JS::GCReason GCRuntime::wantMajorGC(bool eagerOk) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + if (majorGCRequested()) { + return majorGCTriggerReason; + } + + if (isIncrementalGCInProgress() || !eagerOk) { + return JS::GCReason::NO_REASON; + } + + JS::GCReason reason = JS::GCReason::NO_REASON; + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + if (checkEagerAllocTrigger(zone->gcHeapSize, zone->gcHeapThreshold) || + checkEagerAllocTrigger(zone->mallocHeapSize, + zone->mallocHeapThreshold)) { + zone->scheduleGC(); + reason = JS::GCReason::EAGER_ALLOC_TRIGGER; + } + } + + return reason; +} + +bool GCRuntime::checkEagerAllocTrigger(const HeapSize& size, + const HeapThreshold& threshold) { + double thresholdBytes = + threshold.eagerAllocTrigger(schedulingState.inHighFrequencyGCMode()); + double usedBytes = size.bytes(); + if (usedBytes <= 1024 * 1024 || usedBytes < thresholdBytes) { + return false; + } + + stats().recordTrigger(usedBytes, thresholdBytes); + return true; +} + +bool GCRuntime::shouldDecommit() const { + // If we're doing a shrinking GC we always decommit to release as much memory + // as possible. + if (cleanUpEverything) { + return true; + } + + // If we are allocating heavily enough to trigger "high frequency" GC then + // skip decommit so that we do not compete with the mutator. + return !schedulingState.inHighFrequencyGCMode(); +} + +void GCRuntime::startDecommit() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::DECOMMIT); + +#ifdef DEBUG + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + MOZ_ASSERT(decommitTask.isIdle()); + + { + AutoLockGC lock(this); + MOZ_ASSERT(fullChunks(lock).verify()); + MOZ_ASSERT(availableChunks(lock).verify()); + MOZ_ASSERT(emptyChunks(lock).verify()); + + // Verify that all entries in the empty chunks pool are unused. + for (ChunkPool::Iter chunk(emptyChunks(lock)); !chunk.done(); + chunk.next()) { + MOZ_ASSERT(chunk->unused()); + } + } +#endif + + if (!shouldDecommit()) { + return; + } + + { + AutoLockGC lock(this); + if (availableChunks(lock).empty() && !tooManyEmptyChunks(lock) && + emptyChunks(lock).empty()) { + return; // Nothing to do. + } + } + +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(!requestSliceAfterBackgroundTask); + } +#endif + + if (useBackgroundThreads) { + decommitTask.start(); + return; + } + + decommitTask.runFromMainThread(); +} + +BackgroundDecommitTask::BackgroundDecommitTask(GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::DECOMMIT) {} + +void js::gc::BackgroundDecommitTask::run(AutoLockHelperThreadState& lock) { + { + AutoUnlockHelperThreadState unlock(lock); + + ChunkPool emptyChunksToFree; + { + AutoLockGC gcLock(gc); + emptyChunksToFree = gc->expireEmptyChunkPool(gcLock); + } + + FreeChunkPool(emptyChunksToFree); + + { + AutoLockGC gcLock(gc); + + // To help minimize the total number of chunks needed over time, sort the + // available chunks list so that we allocate into more-used chunks first. + gc->availableChunks(gcLock).sort(); + + if (DecommitEnabled()) { + gc->decommitEmptyChunks(cancel_, gcLock); + gc->decommitFreeArenas(cancel_, gcLock); + } + } + } + + gc->maybeRequestGCAfterBackgroundTask(lock); +} + +static inline bool CanDecommitWholeChunk(TenuredChunk* chunk) { + return chunk->unused() && chunk->info.numArenasFreeCommitted != 0; +} + +// Called from a background thread to decommit free arenas. Releases the GC +// lock. +void GCRuntime::decommitEmptyChunks(const bool& cancel, AutoLockGC& lock) { + Vector chunksToDecommit; + for (ChunkPool::Iter chunk(emptyChunks(lock)); !chunk.done(); chunk.next()) { + if (CanDecommitWholeChunk(chunk) && !chunksToDecommit.append(chunk)) { + onOutOfMallocMemory(lock); + return; + } + } + + for (TenuredChunk* chunk : chunksToDecommit) { + if (cancel) { + break; + } + + // Check whether something used the chunk while lock was released. + if (!CanDecommitWholeChunk(chunk)) { + continue; + } + + // Temporarily remove the chunk while decommitting its memory so that the + // mutator doesn't start allocating from it when we drop the lock. + emptyChunks(lock).remove(chunk); + + { + AutoUnlockGC unlock(lock); + chunk->decommitAllArenas(); + MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0); + } + + emptyChunks(lock).push(chunk); + } +} + +// Called from a background thread to decommit free arenas. Releases the GC +// lock. +void GCRuntime::decommitFreeArenas(const bool& cancel, AutoLockGC& lock) { + MOZ_ASSERT(DecommitEnabled()); + + // Since we release the GC lock while doing the decommit syscall below, + // it is dangerous to iterate the available list directly, as the active + // thread could modify it concurrently. Instead, we build and pass an + // explicit Vector containing the Chunks we want to visit. + Vector chunksToDecommit; + for (ChunkPool::Iter chunk(availableChunks(lock)); !chunk.done(); + chunk.next()) { + if (chunk->info.numArenasFreeCommitted != 0 && + !chunksToDecommit.append(chunk)) { + onOutOfMallocMemory(lock); + return; + } + } + + for (TenuredChunk* chunk : chunksToDecommit) { + chunk->decommitFreeArenas(this, cancel, lock); + } +} + +// Do all possible decommit immediately from the current thread without +// releasing the GC lock or allocating any memory. +void GCRuntime::decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock) { + MOZ_ASSERT(DecommitEnabled()); + for (ChunkPool::Iter chunk(availableChunks(lock)); !chunk.done(); + chunk.next()) { + chunk->decommitFreeArenasWithoutUnlocking(lock); + } + MOZ_ASSERT(availableChunks(lock).verify()); +} + +void GCRuntime::maybeRequestGCAfterBackgroundTask( + const AutoLockHelperThreadState& lock) { + if (requestSliceAfterBackgroundTask) { + // Trigger a slice so the main thread can continue the collection + // immediately. + requestSliceAfterBackgroundTask = false; + requestMajorGC(JS::GCReason::BG_TASK_FINISHED); + } +} + +void GCRuntime::cancelRequestedGCAfterBackgroundTask() { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(!requestSliceAfterBackgroundTask); + } +#endif + + majorGCTriggerReason.compareExchange(JS::GCReason::BG_TASK_FINISHED, + JS::GCReason::NO_REASON); +} + +bool GCRuntime::isWaitingOnBackgroundTask() const { + AutoLockHelperThreadState lock; + return requestSliceAfterBackgroundTask; +} + +void GCRuntime::queueUnusedLifoBlocksForFree(LifoAlloc* lifo) { + MOZ_ASSERT(JS::RuntimeHeapIsBusy()); + AutoLockHelperThreadState lock; + lifoBlocksToFree.ref().transferUnusedFrom(lifo); +} + +void GCRuntime::queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo) { + lifoBlocksToFreeAfterMinorGC.ref().transferFrom(lifo); +} + +void GCRuntime::queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers) { + AutoLockHelperThreadState lock; + + if (!buffersToFreeAfterMinorGC.ref().empty()) { + // In the rare case that this hasn't processed the buffers from a previous + // minor GC we have to wait here. + MOZ_ASSERT(!freeTask.isIdle(lock)); + freeTask.joinWithLockHeld(lock); + } + + MOZ_ASSERT(buffersToFreeAfterMinorGC.ref().empty()); + std::swap(buffersToFreeAfterMinorGC.ref(), buffers); +} + +void Realm::destroy(JS::GCContext* gcx) { + JSRuntime* rt = gcx->runtime(); + if (auto callback = rt->destroyRealmCallback) { + callback(gcx, this); + } + if (principals()) { + JS_DropPrincipals(rt->mainContextFromOwnThread(), principals()); + } + // Bug 1560019: Malloc memory associated with a zone but not with a specific + // GC thing is not currently tracked. + gcx->deleteUntracked(this); +} + +void Compartment::destroy(JS::GCContext* gcx) { + JSRuntime* rt = gcx->runtime(); + if (auto callback = rt->destroyCompartmentCallback) { + callback(gcx, this); + } + // Bug 1560019: Malloc memory associated with a zone but not with a specific + // GC thing is not currently tracked. + gcx->deleteUntracked(this); + rt->gc.stats().sweptCompartment(); +} + +void Zone::destroy(JS::GCContext* gcx) { + MOZ_ASSERT(compartments().empty()); + JSRuntime* rt = gcx->runtime(); + if (auto callback = rt->destroyZoneCallback) { + callback(gcx, this); + } + // Bug 1560019: Malloc memory associated with a zone but not with a specific + // GC thing is not currently tracked. + gcx->deleteUntracked(this); + gcx->runtime()->gc.stats().sweptZone(); +} + +/* + * It's simpler if we preserve the invariant that every zone (except atoms + * zones) has at least one compartment, and every compartment has at least one + * realm. If we know we're deleting the entire zone, then sweepCompartments is + * allowed to delete all compartments. In this case, |keepAtleastOne| is false. + * If any cells remain alive in the zone, set |keepAtleastOne| true to prohibit + * sweepCompartments from deleting every compartment. Instead, it preserves an + * arbitrary compartment in the zone. + */ +void Zone::sweepCompartments(JS::GCContext* gcx, bool keepAtleastOne, + bool destroyingRuntime) { + MOZ_ASSERT_IF(!isAtomsZone(), !compartments().empty()); + MOZ_ASSERT_IF(destroyingRuntime, !keepAtleastOne); + + Compartment** read = compartments().begin(); + Compartment** end = compartments().end(); + Compartment** write = read; + while (read < end) { + Compartment* comp = *read++; + + /* + * Don't delete the last compartment and realm if keepAtleastOne is + * still true, meaning all the other compartments were deleted. + */ + bool keepAtleastOneRealm = read == end && keepAtleastOne; + comp->sweepRealms(gcx, keepAtleastOneRealm, destroyingRuntime); + + if (!comp->realms().empty()) { + *write++ = comp; + keepAtleastOne = false; + } else { + comp->destroy(gcx); + } + } + compartments().shrinkTo(write - compartments().begin()); + MOZ_ASSERT_IF(keepAtleastOne, !compartments().empty()); + MOZ_ASSERT_IF(destroyingRuntime, compartments().empty()); +} + +void Compartment::sweepRealms(JS::GCContext* gcx, bool keepAtleastOne, + bool destroyingRuntime) { + MOZ_ASSERT(!realms().empty()); + MOZ_ASSERT_IF(destroyingRuntime, !keepAtleastOne); + + Realm** read = realms().begin(); + Realm** end = realms().end(); + Realm** write = read; + while (read < end) { + Realm* realm = *read++; + + /* + * Don't delete the last realm if keepAtleastOne is still true, meaning + * all the other realms were deleted. + */ + bool dontDelete = read == end && keepAtleastOne; + if ((realm->marked() || dontDelete) && !destroyingRuntime) { + *write++ = realm; + keepAtleastOne = false; + } else { + realm->destroy(gcx); + } + } + realms().shrinkTo(write - realms().begin()); + MOZ_ASSERT_IF(keepAtleastOne, !realms().empty()); + MOZ_ASSERT_IF(destroyingRuntime, realms().empty()); +} + +void GCRuntime::sweepZones(JS::GCContext* gcx, bool destroyingRuntime) { + MOZ_ASSERT_IF(destroyingRuntime, numActiveZoneIters == 0); + + if (numActiveZoneIters) { + return; + } + + assertBackgroundSweepingFinished(); + + // Sweep zones following the atoms zone. + MOZ_ASSERT(zones()[0]->isAtomsZone()); + Zone** read = zones().begin() + 1; + Zone** end = zones().end(); + Zone** write = read; + + while (read < end) { + Zone* zone = *read++; + + if (zone->wasGCStarted()) { + MOZ_ASSERT(!zone->isQueuedForBackgroundSweep()); + AutoSetThreadIsSweeping threadIsSweeping(zone); + const bool zoneIsDead = + zone->arenas.arenaListsAreEmpty() && !zone->hasMarkedRealms(); + MOZ_ASSERT_IF(destroyingRuntime, zoneIsDead); + if (zoneIsDead) { + zone->arenas.checkEmptyFreeLists(); + zone->sweepCompartments(gcx, false, destroyingRuntime); + MOZ_ASSERT(zone->compartments().empty()); + zone->destroy(gcx); + continue; + } + zone->sweepCompartments(gcx, true, destroyingRuntime); + } + *write++ = zone; + } + zones().shrinkTo(write - zones().begin()); +} + +void ArenaLists::checkEmptyArenaList(AllocKind kind) { + MOZ_ASSERT(arenaList(kind).isEmpty()); +} + +void GCRuntime::purgeRuntimeForMinorGC() { + // If external strings become nursery allocable, remember to call + // zone->externalStringCache().purge() (and delete this assert.) + MOZ_ASSERT(!IsNurseryAllocable(AllocKind::EXTERNAL_STRING)); + + for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) { + zone->functionToStringCache().purge(); + } +} + +void GCRuntime::purgeRuntime() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE); + + for (GCRealmsIter realm(rt); !realm.done(); realm.next()) { + realm->purge(); + } + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->purgeAtomCache(); + zone->externalStringCache().purge(); + zone->functionToStringCache().purge(); + zone->boundPrefixCache().clearAndCompact(); + zone->shapeZone().purgeShapeCaches(rt->gcContext()); + } + + JSContext* cx = rt->mainContextFromOwnThread(); + queueUnusedLifoBlocksForFree(&cx->tempLifoAlloc()); + cx->interpreterStack().purge(rt); + cx->frontendCollectionPool().purge(); + + rt->caches().purge(); + + if (rt->isMainRuntime()) { + SharedImmutableStringsCache::getSingleton().purge(); + } + + MOZ_ASSERT(marker().unmarkGrayStack.empty()); + marker().unmarkGrayStack.clearAndFree(); + + // If we're the main runtime, tell helper threads to free their unused + // memory when they are next idle. + if (!rt->parentRuntime) { + HelperThreadState().triggerFreeUnusedMemory(); + } +} + +bool GCRuntime::shouldPreserveJITCode(Realm* realm, + const TimeStamp& currentTime, + JS::GCReason reason, + bool canAllocateMoreCode, + bool isActiveCompartment) { + if (cleanUpEverything) { + return false; + } + if (!canAllocateMoreCode) { + return false; + } + + if (isActiveCompartment) { + return true; + } + if (alwaysPreserveCode) { + return true; + } + if (realm->preserveJitCode()) { + return true; + } + if (IsCurrentlyAnimating(realm->lastAnimationTime, currentTime) && + DiscardedCodeRecently(realm->zone(), currentTime)) { + return true; + } + if (reason == JS::GCReason::DEBUG_GC) { + return true; + } + + return false; +} + +#ifdef DEBUG +class CompartmentCheckTracer final : public JS::CallbackTracer { + void onChild(JS::GCCellPtr thing, const char* name) override; + bool edgeIsInCrossCompartmentMap(JS::GCCellPtr dst); + + public: + explicit CompartmentCheckTracer(JSRuntime* rt) + : JS::CallbackTracer(rt, JS::TracerKind::CompartmentCheck, + JS::WeakEdgeTraceAction::Skip), + src(nullptr), + zone(nullptr), + compartment(nullptr) {} + + Cell* src; + JS::TraceKind srcKind; + Zone* zone; + Compartment* compartment; +}; + +static bool InCrossCompartmentMap(JSRuntime* rt, JSObject* src, + JS::GCCellPtr dst) { + // Cross compartment edges are either in the cross compartment map or in a + // debugger weakmap. + + Compartment* srccomp = src->compartment(); + + if (dst.is()) { + if (ObjectWrapperMap::Ptr p = srccomp->lookupWrapper(&dst.as())) { + if (*p->value().unsafeGet() == src) { + return true; + } + } + } + + if (DebugAPI::edgeIsInDebuggerWeakmap(rt, src, dst)) { + return true; + } + + return false; +} + +void CompartmentCheckTracer::onChild(JS::GCCellPtr thing, const char* name) { + Compartment* comp = + MapGCThingTyped(thing, [](auto t) { return t->maybeCompartment(); }); + if (comp && compartment) { + MOZ_ASSERT(comp == compartment || edgeIsInCrossCompartmentMap(thing)); + } else { + TenuredCell* tenured = &thing.asCell()->asTenured(); + Zone* thingZone = tenured->zoneFromAnyThread(); + MOZ_ASSERT(thingZone == zone || thingZone->isAtomsZone()); + } +} + +bool CompartmentCheckTracer::edgeIsInCrossCompartmentMap(JS::GCCellPtr dst) { + return srcKind == JS::TraceKind::Object && + InCrossCompartmentMap(runtime(), static_cast(src), dst); +} + +static bool IsPartiallyInitializedObject(Cell* cell) { + if (!cell->is()) { + return false; + } + + JSObject* obj = cell->as(); + if (!obj->is()) { + return false; + } + + NativeObject* nobj = &obj->as(); + + // Check for failed allocation of dynamic slots in + // NativeObject::allocateInitialSlots. + size_t nDynamicSlots = NativeObject::calculateDynamicSlots( + nobj->numFixedSlots(), nobj->slotSpan(), nobj->getClass()); + return nDynamicSlots != 0 && !nobj->hasDynamicSlots(); +} + +void GCRuntime::checkForCompartmentMismatches() { + JSContext* cx = rt->mainContextFromOwnThread(); + if (cx->disableStrictProxyCheckingCount) { + return; + } + + CompartmentCheckTracer trc(rt); + AutoAssertEmptyNursery empty(cx); + for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) { + trc.zone = zone; + for (auto thingKind : AllAllocKinds()) { + for (auto i = zone->cellIterUnsafe(thingKind, empty); + !i.done(); i.next()) { + // We may encounter partially initialized objects. These are unreachable + // and it's safe to ignore them. + if (IsPartiallyInitializedObject(i.getCell())) { + continue; + } + + trc.src = i.getCell(); + trc.srcKind = MapAllocToTraceKind(thingKind); + trc.compartment = MapGCThingTyped( + trc.src, trc.srcKind, [](auto t) { return t->maybeCompartment(); }); + JS::TraceChildren(&trc, JS::GCCellPtr(trc.src, trc.srcKind)); + } + } + } +} +#endif + +static bool ShouldCleanUpEverything(JS::GCOptions options) { + // During shutdown, we must clean everything up, for the sake of leak + // detection. When a runtime has no contexts, or we're doing a GC before a + // shutdown CC, those are strong indications that we're shutting down. + return options == JS::GCOptions::Shutdown || options == JS::GCOptions::Shrink; +} + +static bool ShouldUseBackgroundThreads(bool isIncremental, + JS::GCReason reason) { + bool shouldUse = isIncremental && CanUseExtraThreads(); + MOZ_ASSERT_IF(reason == JS::GCReason::DESTROY_RUNTIME, !shouldUse); + return shouldUse; +} + +void GCRuntime::startCollection(JS::GCReason reason) { + checkGCStateNotInUse(); + MOZ_ASSERT_IF( + isShuttingDown(), + isShutdownGC() || + reason == JS::GCReason::XPCONNECT_SHUTDOWN /* Bug 1650075 */); + + initialReason = reason; + cleanUpEverything = ShouldCleanUpEverything(gcOptions()); + isCompacting = shouldCompact(); + rootsRemoved = false; + sweepGroupIndex = 0; + lastGCStartTime_ = TimeStamp::Now(); + +#ifdef DEBUG + if (isShutdownGC()) { + hadShutdownGC = true; + } + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->gcSweepGroupIndex = 0; + } +#endif +} + +static void RelazifyFunctions(Zone* zone, AllocKind kind) { + MOZ_ASSERT(kind == AllocKind::FUNCTION || + kind == AllocKind::FUNCTION_EXTENDED); + + JSRuntime* rt = zone->runtimeFromMainThread(); + AutoAssertEmptyNursery empty(rt->mainContextFromOwnThread()); + + for (auto i = zone->cellIterUnsafe(kind, empty); !i.done(); + i.next()) { + JSFunction* fun = &i->as(); + // When iterating over the GC-heap, we may encounter function objects that + // are incomplete (missing a BaseScript when we expect one). We must check + // for this case before we can call JSFunction::hasBytecode(). + if (fun->isIncomplete()) { + continue; + } + if (fun->hasBytecode()) { + fun->maybeRelazify(rt); + } + } +} + +static bool ShouldCollectZone(Zone* zone, JS::GCReason reason) { + // If we are repeating a GC because we noticed dead compartments haven't + // been collected, then only collect zones containing those compartments. + if (reason == JS::GCReason::COMPARTMENT_REVIVED) { + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + if (comp->gcState.scheduledForDestruction) { + return true; + } + } + + return false; + } + + // Otherwise we only collect scheduled zones. + return zone->isGCScheduled(); +} + +bool GCRuntime::prepareZonesForCollection(JS::GCReason reason, + bool* isFullOut) { +#ifdef DEBUG + /* Assert that zone state is as we expect */ + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + MOZ_ASSERT(!zone->isCollecting()); + MOZ_ASSERT_IF(!zone->isAtomsZone(), !zone->compartments().empty()); + for (auto i : AllAllocKinds()) { + MOZ_ASSERT(zone->arenas.collectingArenaList(i).isEmpty()); + } + } +#endif + + *isFullOut = true; + bool any = false; + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + /* Set up which zones will be collected. */ + bool shouldCollect = ShouldCollectZone(zone, reason); + if (shouldCollect) { + any = true; + zone->changeGCState(Zone::NoGC, Zone::Prepare); + } else { + *isFullOut = false; + } + + zone->setWasCollected(shouldCollect); + } + + /* Check that at least one zone is scheduled for collection. */ + return any; +} + +void GCRuntime::discardJITCodeForGC() { + size_t nurserySiteResetCount = 0; + size_t pretenuredSiteResetCount = 0; + + js::CancelOffThreadIonCompile(rt, JS::Zone::Prepare); + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_DISCARD_CODE); + + // We may need to reset allocation sites and discard JIT code to recover if + // we find object lifetimes have changed. + PretenuringZone& pz = zone->pretenuring; + bool resetNurserySites = pz.shouldResetNurseryAllocSites(); + bool resetPretenuredSites = pz.shouldResetPretenuredAllocSites(); + + if (!zone->isPreservingCode()) { + Zone::DiscardOptions options; + options.discardBaselineCode = true; + options.discardJitScripts = true; + options.resetNurseryAllocSites = resetNurserySites; + options.resetPretenuredAllocSites = resetPretenuredSites; + zone->discardJitCode(rt->gcContext(), options); + } else if (resetNurserySites || resetPretenuredSites) { + zone->resetAllocSitesAndInvalidate(resetNurserySites, + resetPretenuredSites); + } + + if (resetNurserySites) { + nurserySiteResetCount++; + } + if (resetPretenuredSites) { + pretenuredSiteResetCount++; + } + } + + if (nursery().reportPretenuring()) { + if (nurserySiteResetCount) { + fprintf( + stderr, + "GC reset nursery alloc sites and invalidated code in %zu zones\n", + nurserySiteResetCount); + } + if (pretenuredSiteResetCount) { + fprintf( + stderr, + "GC reset pretenured alloc sites and invalidated code in %zu zones\n", + pretenuredSiteResetCount); + } + } +} + +void GCRuntime::relazifyFunctionsForShrinkingGC() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::RELAZIFY_FUNCTIONS); + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + RelazifyFunctions(zone, AllocKind::FUNCTION); + RelazifyFunctions(zone, AllocKind::FUNCTION_EXTENDED); + } +} + +void GCRuntime::purgePropMapTablesForShrinkingGC() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE_PROP_MAP_TABLES); + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + if (!canRelocateZone(zone) || zone->keepPropMapTables()) { + continue; + } + + // Note: CompactPropMaps never have a table. + for (auto map = zone->cellIterUnsafe(); !map.done(); + map.next()) { + if (map->asLinked()->hasTable()) { + map->asLinked()->purgeTable(rt->gcContext()); + } + } + for (auto map = zone->cellIterUnsafe(); !map.done(); + map.next()) { + if (map->asLinked()->hasTable()) { + map->asLinked()->purgeTable(rt->gcContext()); + } + } + } +} + +// The debugger keeps track of the URLs for the sources of each realm's scripts. +// These URLs are purged on shrinking GCs. +void GCRuntime::purgeSourceURLsForShrinkingGC() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE_SOURCE_URLS); + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + // URLs are not tracked for realms in the system zone. + if (!canRelocateZone(zone) || zone->isSystemZone()) { + continue; + } + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + for (RealmsInCompartmentIter realm(comp); !realm.done(); realm.next()) { + GlobalObject* global = realm.get()->unsafeUnbarrieredMaybeGlobal(); + if (global) { + global->clearSourceURLSHolder(); + } + } + } + } +} + +void GCRuntime::unmarkWeakMaps() { + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + /* Unmark all weak maps in the zones being collected. */ + WeakMapBase::unmarkZone(zone); + } +} + +bool GCRuntime::beginPreparePhase(JS::GCReason reason, AutoGCSession& session) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PREPARE); + + if (!prepareZonesForCollection(reason, &isFull.ref())) { + return false; + } + + /* + * Start a parallel task to clear all mark state for the zones we are + * collecting. This is linear in the size of the heap we are collecting and so + * can be slow. This usually happens concurrently with the mutator and GC + * proper does not start until this is complete. + */ + unmarkTask.initZones(); + if (useBackgroundThreads) { + unmarkTask.start(); + } else { + unmarkTask.runFromMainThread(); + } + + /* + * Process any queued source compressions during the start of a major + * GC. + * + * Bug 1650075: When we start passing GCOptions::Shutdown for + * GCReason::XPCONNECT_SHUTDOWN GCs we can remove the extra check. + */ + if (!isShutdownGC() && reason != JS::GCReason::XPCONNECT_SHUTDOWN) { + StartHandlingCompressionsOnGC(rt); + } + + return true; +} + +BackgroundUnmarkTask::BackgroundUnmarkTask(GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::UNMARK) {} + +void BackgroundUnmarkTask::initZones() { + MOZ_ASSERT(isIdle()); + MOZ_ASSERT(zones.empty()); + MOZ_ASSERT(!isCancelled()); + + // We can't safely iterate the zones vector from another thread so we copy the + // zones to be collected into another vector. + AutoEnterOOMUnsafeRegion oomUnsafe; + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + if (!zones.append(zone.get())) { + oomUnsafe.crash("BackgroundUnmarkTask::initZones"); + } + + zone->arenas.clearFreeLists(); + zone->arenas.moveArenasToCollectingLists(); + } +} + +void BackgroundUnmarkTask::run(AutoLockHelperThreadState& helperTheadLock) { + AutoUnlockHelperThreadState unlock(helperTheadLock); + + for (Zone* zone : zones) { + for (auto kind : AllAllocKinds()) { + ArenaList& arenas = zone->arenas.collectingArenaList(kind); + for (ArenaListIter arena(arenas.head()); !arena.done(); arena.next()) { + arena->unmarkAll(); + if (isCancelled()) { + break; + } + } + } + } + + zones.clear(); +} + +void GCRuntime::endPreparePhase(JS::GCReason reason) { + MOZ_ASSERT(unmarkTask.isIdle()); + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + /* + * In an incremental GC, clear the area free lists to ensure that subsequent + * allocations refill them and end up marking new cells back. See + * arenaAllocatedDuringGC(). + */ + zone->arenas.clearFreeLists(); + + zone->markedStrings = 0; + zone->finalizedStrings = 0; + + zone->setPreservingCode(false); + +#ifdef JS_GC_ZEAL + if (hasZealMode(ZealMode::YieldBeforeRootMarking)) { + for (auto kind : AllAllocKinds()) { + for (ArenaIter arena(zone, kind); !arena.done(); arena.next()) { + arena->checkNoMarkedCells(); + } + } + } +#endif + } + + // Discard JIT code more aggressively if the process is approaching its + // executable code limit. + bool canAllocateMoreCode = jit::CanLikelyAllocateMoreExecutableMemory(); + auto currentTime = TimeStamp::Now(); + + Compartment* activeCompartment = nullptr; + jit::JitActivationIterator activation(rt->mainContextFromOwnThread()); + if (!activation.done()) { + activeCompartment = activation->compartment(); + } + + for (CompartmentsIter c(rt); !c.done(); c.next()) { + c->gcState.scheduledForDestruction = false; + c->gcState.maybeAlive = false; + c->gcState.hasEnteredRealm = false; + bool isActiveCompartment = c == activeCompartment; + for (RealmsInCompartmentIter r(c); !r.done(); r.next()) { + if (r->shouldTraceGlobal() || !r->zone()->isGCScheduled()) { + c->gcState.maybeAlive = true; + } + if (shouldPreserveJITCode(r, currentTime, reason, canAllocateMoreCode, + isActiveCompartment)) { + r->zone()->setPreservingCode(true); + } + if (r->hasBeenEnteredIgnoringJit()) { + c->gcState.hasEnteredRealm = true; + } + } + } + + /* + * Perform remaining preparation work that must take place in the first true + * GC slice. + */ + + { + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::PREPARE); + + AutoLockHelperThreadState helperLock; + + /* Clear mark state for WeakMaps in parallel with other work. */ + AutoRunParallelTask unmarkWeakMaps(this, &GCRuntime::unmarkWeakMaps, + gcstats::PhaseKind::UNMARK_WEAKMAPS, + GCUse::Unspecified, helperLock); + + AutoUnlockHelperThreadState unlock(helperLock); + + // Discard JIT code. For incremental collections, the sweep phase will + // also discard JIT code. + discardJITCodeForGC(); + + /* + * Relazify functions after discarding JIT code (we can't relazify + * functions with JIT code) and before the actual mark phase, so that + * the current GC can collect the JSScripts we're unlinking here. We do + * this only when we're performing a shrinking GC, as too much + * relazification can cause performance issues when we have to reparse + * the same functions over and over. + */ + if (isShrinkingGC()) { + relazifyFunctionsForShrinkingGC(); + purgePropMapTablesForShrinkingGC(); + purgeSourceURLsForShrinkingGC(); + } + + /* + * We must purge the runtime at the beginning of an incremental GC. The + * danger if we purge later is that the snapshot invariant of + * incremental GC will be broken, as follows. If some object is + * reachable only through some cache (say the dtoaCache) then it will + * not be part of the snapshot. If we purge after root marking, then + * the mutator could obtain a pointer to the object and start using + * it. This object might never be marked, so a GC hazard would exist. + */ + purgeRuntime(); + + startBackgroundFreeAfterMinorGC(); + + if (isShutdownGC()) { + /* Clear any engine roots that may hold external data live. */ + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->clearRootsForShutdownGC(); + } + +#ifdef DEBUG + testMarkQueue.clear(); + queuePos = 0; +#endif + } + } + +#ifdef DEBUG + if (fullCompartmentChecks) { + checkForCompartmentMismatches(); + } +#endif +} + +AutoUpdateLiveCompartments::AutoUpdateLiveCompartments(GCRuntime* gc) : gc(gc) { + for (GCCompartmentsIter c(gc->rt); !c.done(); c.next()) { + c->gcState.hasMarkedCells = false; + } +} + +AutoUpdateLiveCompartments::~AutoUpdateLiveCompartments() { + for (GCCompartmentsIter c(gc->rt); !c.done(); c.next()) { + if (c->gcState.hasMarkedCells) { + c->gcState.maybeAlive = true; + } + } +} + +Zone::GCState Zone::initialMarkingState() const { + if (isAtomsZone()) { + // Don't delay gray marking in the atoms zone like we do in other zones. + return MarkBlackAndGray; + } + + return MarkBlackOnly; +} + +void GCRuntime::beginMarkPhase(AutoGCSession& session) { + /* + * Mark phase. + */ + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + + // This is the slice we actually start collecting. The number can be used to + // check whether a major GC has started so we must not increment it until we + // get here. + incMajorGcNumber(); + + MOZ_ASSERT(!hasDelayedMarking()); + for (auto& marker : markers) { + marker->start(); + } + +#ifdef DEBUG + queuePos = 0; + queueMarkColor.reset(); +#endif + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + // Incremental marking barriers are enabled at this point. + zone->changeGCState(Zone::Prepare, zone->initialMarkingState()); + + // Merge arenas allocated during the prepare phase, then move all arenas to + // the collecting arena lists. + zone->arenas.mergeArenasFromCollectingLists(); + zone->arenas.moveArenasToCollectingLists(); + + for (RealmsInZoneIter realm(zone); !realm.done(); realm.next()) { + realm->clearAllocatedDuringGC(); + } + } + + if (rt->isBeingDestroyed()) { + checkNoRuntimeRoots(session); + } else { + AutoUpdateLiveCompartments updateLive(this); + marker().setRootMarkingMode(true); + traceRuntimeForMajorGC(marker().tracer(), session); + marker().setRootMarkingMode(false); + } + + updateSchedulingStateOnGCStart(); + stats().measureInitialHeapSize(); +} + +void GCRuntime::findDeadCompartments() { + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::FIND_DEAD_COMPARTMENTS); + + /* + * This code ensures that if a compartment is "dead", then it will be + * collected in this GC. A compartment is considered dead if its maybeAlive + * flag is false. The maybeAlive flag is set if: + * + * (1) the compartment has been entered (set in beginMarkPhase() above) + * (2) the compartment's zone is not being collected (set in + * beginMarkPhase() above) + * (3) an object in the compartment was marked during root marking, either + * as a black root or a gray root. This is arranged by + * SetCompartmentHasMarkedCells and AutoUpdateLiveCompartments. + * (4) the compartment has incoming cross-compartment edges from another + * compartment that has maybeAlive set (set by this method). + * + * If the maybeAlive is false, then we set the scheduledForDestruction flag. + * At the end of the GC, we look for compartments where + * scheduledForDestruction is true. These are compartments that were somehow + * "revived" during the incremental GC. If any are found, we do a special, + * non-incremental GC of those compartments to try to collect them. + * + * Compartments can be revived for a variety of reasons. On reason is bug + * 811587, where a reflector that was dead can be revived by DOM code that + * still refers to the underlying DOM node. + * + * Read barriers and allocations can also cause revival. This might happen + * during a function like JS_TransplantObject, which iterates over all + * compartments, live or dead, and operates on their objects. See bug 803376 + * for details on this problem. To avoid the problem, we try to avoid + * allocation and read barriers during JS_TransplantObject and the like. + */ + + // Propagate the maybeAlive flag via cross-compartment edges. + + Vector workList; + + for (CompartmentsIter comp(rt); !comp.done(); comp.next()) { + if (comp->gcState.maybeAlive) { + if (!workList.append(comp)) { + return; + } + } + } + + while (!workList.empty()) { + Compartment* comp = workList.popCopy(); + for (Compartment::WrappedObjectCompartmentEnum e(comp); !e.empty(); + e.popFront()) { + Compartment* dest = e.front(); + if (!dest->gcState.maybeAlive) { + dest->gcState.maybeAlive = true; + if (!workList.append(dest)) { + return; + } + } + } + } + + // Set scheduledForDestruction based on maybeAlive. + + for (GCCompartmentsIter comp(rt); !comp.done(); comp.next()) { + MOZ_ASSERT(!comp->gcState.scheduledForDestruction); + if (!comp->gcState.maybeAlive) { + comp->gcState.scheduledForDestruction = true; + } + } +} + +void GCRuntime::updateSchedulingStateOnGCStart() { + heapSize.updateOnGCStart(); + + // Update memory counters for the zones we are collecting. + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->updateSchedulingStateOnGCStart(); + } +} + +inline bool GCRuntime::canMarkInParallel() const { +#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT) + // OOM testing limits the engine to using a single helper thread. + if (oom::simulator.targetThread() == THREAD_TYPE_GCPARALLEL) { + return false; + } +#endif + + return markers.length() > 1 && stats().initialCollectedBytes() >= + tunables.parallelMarkingThresholdBytes(); +} + +IncrementalProgress GCRuntime::markUntilBudgetExhausted( + SliceBudget& sliceBudget, ParallelMarking allowParallelMarking, + ShouldReportMarkTime reportTime) { + // Run a marking slice and return whether the stack is now empty. + + AutoMajorGCProfilerEntry s(this); + + if (processTestMarkQueue() == QueueYielded) { + return NotFinished; + } + + if (allowParallelMarking && canMarkInParallel()) { + MOZ_ASSERT(parallelMarkingEnabled); + MOZ_ASSERT(reportTime); + MOZ_ASSERT(!isBackgroundMarking()); + + ParallelMarker pm(this); + if (!pm.mark(sliceBudget)) { + return NotFinished; + } + + assertNoMarkingWork(); + return Finished; + } + +#ifdef DEBUG + AutoSetThreadIsMarking threadIsMarking; +#endif // DEBUG + + return marker().markUntilBudgetExhausted(sliceBudget, reportTime) + ? Finished + : NotFinished; +} + +void GCRuntime::drainMarkStack() { + auto unlimited = SliceBudget::unlimited(); + MOZ_RELEASE_ASSERT(marker().markUntilBudgetExhausted(unlimited)); +} + +#ifdef DEBUG + +const GCVector, 0, SystemAllocPolicy>& +GCRuntime::getTestMarkQueue() const { + return testMarkQueue.get(); +} + +bool GCRuntime::appendTestMarkQueue(const JS::Value& value) { + return testMarkQueue.append(value); +} + +void GCRuntime::clearTestMarkQueue() { + testMarkQueue.clear(); + queuePos = 0; +} + +size_t GCRuntime::testMarkQueuePos() const { return queuePos; } + +#endif + +GCRuntime::MarkQueueProgress GCRuntime::processTestMarkQueue() { +#ifdef DEBUG + if (testMarkQueue.empty()) { + return QueueComplete; + } + + if (queueMarkColor == mozilla::Some(MarkColor::Gray) && + state() != State::Sweep) { + return QueueSuspended; + } + + // If the queue wants to be gray marking, but we've pushed a black object + // since set-color-gray was processed, then we can't switch to gray and must + // again wait until gray marking is possible. + // + // Remove this code if the restriction against marking gray during black is + // relaxed. + if (queueMarkColor == mozilla::Some(MarkColor::Gray) && + marker().hasBlackEntries()) { + return QueueSuspended; + } + + // If the queue wants to be marking a particular color, switch to that color. + // In any case, restore the mark color to whatever it was when we entered + // this function. + bool willRevertToGray = marker().markColor() == MarkColor::Gray; + AutoSetMarkColor autoRevertColor( + marker(), queueMarkColor.valueOr(marker().markColor())); + + // Process the mark queue by taking each object in turn, pushing it onto the + // mark stack, and processing just the top element with processMarkStackTop + // without recursing into reachable objects. + while (queuePos < testMarkQueue.length()) { + Value val = testMarkQueue[queuePos++].get(); + if (val.isObject()) { + JSObject* obj = &val.toObject(); + JS::Zone* zone = obj->zone(); + if (!zone->isGCMarking() || obj->isMarkedAtLeast(marker().markColor())) { + continue; + } + + // If we have started sweeping, obey sweep group ordering. But note that + // we will first be called during the initial sweep slice, when the sweep + // group indexes have not yet been computed. In that case, we can mark + // freely. + if (state() == State::Sweep && initialState != State::Sweep) { + if (zone->gcSweepGroupIndex < getCurrentSweepGroupIndex()) { + // Too late. This must have been added after we started collecting, + // and we've already processed its sweep group. Skip it. + continue; + } + if (zone->gcSweepGroupIndex > getCurrentSweepGroupIndex()) { + // Not ready yet. Wait until we reach the object's sweep group. + queuePos--; + return QueueSuspended; + } + } + + if (marker().markColor() == MarkColor::Gray && + zone->isGCMarkingBlackOnly()) { + // Have not yet reached the point where we can mark this object, so + // continue with the GC. + queuePos--; + return QueueSuspended; + } + + if (marker().markColor() == MarkColor::Black && willRevertToGray) { + // If we put any black objects on the stack, we wouldn't be able to + // return to gray marking. So delay the marking until we're back to + // black marking. + queuePos--; + return QueueSuspended; + } + + // Mark the object and push it onto the stack. + size_t oldPosition = marker().stack.position(); + marker().markAndTraverse(obj); + + // If we overflow the stack here and delay marking, then we won't be + // testing what we think we're testing. + if (marker().stack.position() == oldPosition) { + MOZ_ASSERT(obj->asTenured().arena()->onDelayedMarkingList()); + AutoEnterOOMUnsafeRegion oomUnsafe; + oomUnsafe.crash("Overflowed stack while marking test queue"); + } + + SliceBudget unlimited = SliceBudget::unlimited(); + marker().processMarkStackTop(unlimited); + } else if (val.isString()) { + JSLinearString* str = &val.toString()->asLinear(); + if (js::StringEqualsLiteral(str, "yield") && isIncrementalGc()) { + return QueueYielded; + } else if (js::StringEqualsLiteral(str, "enter-weak-marking-mode") || + js::StringEqualsLiteral(str, "abort-weak-marking-mode")) { + if (marker().isRegularMarking()) { + // We can't enter weak marking mode at just any time, so instead + // we'll stop processing the queue and continue on with the GC. Once + // we enter weak marking mode, we can continue to the rest of the + // queue. Note that we will also suspend for aborting, and then abort + // the earliest following weak marking mode. + queuePos--; + return QueueSuspended; + } + if (js::StringEqualsLiteral(str, "abort-weak-marking-mode")) { + marker().abortLinearWeakMarking(); + } + } else if (js::StringEqualsLiteral(str, "drain")) { + auto unlimited = SliceBudget::unlimited(); + MOZ_RELEASE_ASSERT( + marker().markUntilBudgetExhausted(unlimited, DontReportMarkTime)); + } else if (js::StringEqualsLiteral(str, "set-color-gray")) { + queueMarkColor = mozilla::Some(MarkColor::Gray); + if (state() != State::Sweep || marker().hasBlackEntries()) { + // Cannot mark gray yet, so continue with the GC. + queuePos--; + return QueueSuspended; + } + marker().setMarkColor(MarkColor::Gray); + } else if (js::StringEqualsLiteral(str, "set-color-black")) { + queueMarkColor = mozilla::Some(MarkColor::Black); + marker().setMarkColor(MarkColor::Black); + } else if (js::StringEqualsLiteral(str, "unset-color")) { + queueMarkColor.reset(); + } + } + } +#endif + + return QueueComplete; +} + +static bool IsEmergencyGC(JS::GCReason reason) { + return reason == JS::GCReason::LAST_DITCH || + reason == JS::GCReason::MEM_PRESSURE; +} + +void GCRuntime::finishCollection(JS::GCReason reason) { + assertBackgroundSweepingFinished(); + + MOZ_ASSERT(!hasDelayedMarking()); + for (auto& marker : markers) { + marker->stop(); + } + + maybeStopPretenuring(); + + if (IsEmergencyGC(reason)) { + waitBackgroundFreeEnd(); + } + + TimeStamp currentTime = TimeStamp::Now(); + + updateSchedulingStateAfterCollection(currentTime); + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->changeGCState(Zone::Finished, Zone::NoGC); + zone->notifyObservingDebuggers(); + } + +#ifdef JS_GC_ZEAL + clearSelectedForMarking(); +#endif + + schedulingState.updateHighFrequencyMode(lastGCEndTime_, currentTime, + tunables); + lastGCEndTime_ = currentTime; + + checkGCStateNotInUse(); +} + +void GCRuntime::checkGCStateNotInUse() { +#ifdef DEBUG + for (auto& marker : markers) { + MOZ_ASSERT(!marker->isActive()); + MOZ_ASSERT(marker->isDrained()); + } + MOZ_ASSERT(!hasDelayedMarking()); + + MOZ_ASSERT(!lastMarkSlice); + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + if (zone->wasCollected()) { + zone->arenas.checkGCStateNotInUse(); + } + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + MOZ_ASSERT(!zone->isOnList()); + } + + MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty()); + MOZ_ASSERT(cellsToAssertNotGray.ref().empty()); + + AutoLockHelperThreadState lock; + MOZ_ASSERT(!requestSliceAfterBackgroundTask); + MOZ_ASSERT(unmarkTask.isIdle(lock)); + MOZ_ASSERT(markTask.isIdle(lock)); + MOZ_ASSERT(sweepTask.isIdle(lock)); + MOZ_ASSERT(decommitTask.isIdle(lock)); +#endif +} + +void GCRuntime::maybeStopPretenuring() { + nursery().maybeStopPretenuring(this); + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + if (!zone->nurseryStringsDisabled) { + continue; + } + + // Count the number of strings before the major GC. + size_t numStrings = zone->markedStrings + zone->finalizedStrings; + double rate = double(zone->finalizedStrings) / double(numStrings); + if (rate > tunables.stopPretenureStringThreshold()) { + zone->markedStrings = 0; + zone->finalizedStrings = 0; + zone->nurseryStringsDisabled = false; + nursery().updateAllocFlagsForZone(zone); + } + } +} + +void GCRuntime::updateSchedulingStateAfterCollection(TimeStamp currentTime) { + TimeDuration totalGCTime = stats().totalGCTime(); + size_t totalInitialBytes = stats().initialCollectedBytes(); + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + if (tunables.balancedHeapLimitsEnabled() && totalInitialBytes != 0) { + zone->updateCollectionRate(totalGCTime, totalInitialBytes); + } + zone->clearGCSliceThresholds(); + zone->updateGCStartThresholds(*this); + } +} + +void GCRuntime::updateAllGCStartThresholds() { + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->updateGCStartThresholds(*this); + } +} + +void GCRuntime::updateAllocationRates() { + // Calculate mutator time since the last update. This ignores the fact that + // the zone could have been created since the last update. + + TimeStamp currentTime = TimeStamp::Now(); + TimeDuration totalTime = currentTime - lastAllocRateUpdateTime; + if (collectorTimeSinceAllocRateUpdate >= totalTime) { + // It shouldn't happen but occasionally we see collector time being larger + // than total time. Skip the update in that case. + return; + } + + TimeDuration mutatorTime = totalTime - collectorTimeSinceAllocRateUpdate; + + for (AllZonesIter zone(this); !zone.done(); zone.next()) { + zone->updateAllocationRate(mutatorTime); + zone->updateGCStartThresholds(*this); + } + + lastAllocRateUpdateTime = currentTime; + collectorTimeSinceAllocRateUpdate = TimeDuration(); +} + +static const char* GCHeapStateToLabel(JS::HeapState heapState) { + switch (heapState) { + case JS::HeapState::MinorCollecting: + return "js::Nursery::collect"; + case JS::HeapState::MajorCollecting: + return "js::GCRuntime::collect"; + default: + MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame"); + } + MOZ_ASSERT_UNREACHABLE("Should have exhausted every JS::HeapState variant!"); + return nullptr; +} + +static JS::ProfilingCategoryPair GCHeapStateToProfilingCategory( + JS::HeapState heapState) { + return heapState == JS::HeapState::MinorCollecting + ? JS::ProfilingCategoryPair::GCCC_MinorGC + : JS::ProfilingCategoryPair::GCCC_MajorGC; +} + +/* Start a new heap session. */ +AutoHeapSession::AutoHeapSession(GCRuntime* gc, JS::HeapState heapState) + : gc(gc), prevState(gc->heapState_) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt)); + MOZ_ASSERT(prevState == JS::HeapState::Idle || + (prevState == JS::HeapState::MajorCollecting && + heapState == JS::HeapState::MinorCollecting)); + MOZ_ASSERT(heapState != JS::HeapState::Idle); + + gc->heapState_ = heapState; + + if (heapState == JS::HeapState::MinorCollecting || + heapState == JS::HeapState::MajorCollecting) { + profilingStackFrame.emplace(gc->rt->mainContextFromOwnThread(), + GCHeapStateToLabel(heapState), + GCHeapStateToProfilingCategory(heapState)); + } +} + +AutoHeapSession::~AutoHeapSession() { + MOZ_ASSERT(JS::RuntimeHeapIsBusy()); + gc->heapState_ = prevState; +} + +static const char* MajorGCStateToLabel(State state) { + switch (state) { + case State::Mark: + return "js::GCRuntime::markUntilBudgetExhausted"; + case State::Sweep: + return "js::GCRuntime::performSweepActions"; + case State::Compact: + return "js::GCRuntime::compactPhase"; + default: + MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame"); + } + + MOZ_ASSERT_UNREACHABLE("Should have exhausted every State variant!"); + return nullptr; +} + +static JS::ProfilingCategoryPair MajorGCStateToProfilingCategory(State state) { + switch (state) { + case State::Mark: + return JS::ProfilingCategoryPair::GCCC_MajorGC_Mark; + case State::Sweep: + return JS::ProfilingCategoryPair::GCCC_MajorGC_Sweep; + case State::Compact: + return JS::ProfilingCategoryPair::GCCC_MajorGC_Compact; + default: + MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame"); + } +} + +AutoMajorGCProfilerEntry::AutoMajorGCProfilerEntry(GCRuntime* gc) + : AutoGeckoProfilerEntry(gc->rt->mainContextFromAnyThread(), + MajorGCStateToLabel(gc->state()), + MajorGCStateToProfilingCategory(gc->state())) { + MOZ_ASSERT(gc->heapState() == JS::HeapState::MajorCollecting); +} + +GCRuntime::IncrementalResult GCRuntime::resetIncrementalGC( + GCAbortReason reason) { + MOZ_ASSERT(reason != GCAbortReason::None); + + // Drop as much work as possible from an ongoing incremental GC so + // we can start a new GC after it has finished. + if (incrementalState == State::NotActive) { + return IncrementalResult::Ok; + } + + AutoGCSession session(this, JS::HeapState::MajorCollecting); + + switch (incrementalState) { + case State::NotActive: + case State::MarkRoots: + case State::Finish: + MOZ_CRASH("Unexpected GC state in resetIncrementalGC"); + break; + + case State::Prepare: + unmarkTask.cancelAndWait(); + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->changeGCState(Zone::Prepare, Zone::NoGC); + zone->clearGCSliceThresholds(); + zone->arenas.clearFreeLists(); + zone->arenas.mergeArenasFromCollectingLists(); + } + + incrementalState = State::NotActive; + checkGCStateNotInUse(); + break; + + case State::Mark: { + // Cancel any ongoing marking. + for (auto& marker : markers) { + marker->reset(); + } + resetDelayedMarking(); + + for (GCCompartmentsIter c(rt); !c.done(); c.next()) { + resetGrayList(c); + } + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->changeGCState(zone->initialMarkingState(), Zone::NoGC); + zone->clearGCSliceThresholds(); + zone->arenas.unmarkPreMarkedFreeCells(); + zone->arenas.mergeArenasFromCollectingLists(); + } + + { + AutoLockHelperThreadState lock; + lifoBlocksToFree.ref().freeAll(); + } + + lastMarkSlice = false; + incrementalState = State::Finish; + +#ifdef DEBUG + for (auto& marker : markers) { + MOZ_ASSERT(!marker->shouldCheckCompartments()); + } +#endif + + break; + } + + case State::Sweep: { + // Finish sweeping the current sweep group, then abort. + for (CompartmentsIter c(rt); !c.done(); c.next()) { + c->gcState.scheduledForDestruction = false; + } + + abortSweepAfterCurrentGroup = true; + isCompacting = false; + + break; + } + + case State::Finalize: { + isCompacting = false; + break; + } + + case State::Compact: { + // Skip any remaining zones that would have been compacted. + MOZ_ASSERT(isCompacting); + startedCompacting = true; + zonesToMaybeCompact.ref().clear(); + break; + } + + case State::Decommit: { + break; + } + } + + stats().reset(reason); + + return IncrementalResult::ResetIncremental; +} + +AutoDisableBarriers::AutoDisableBarriers(GCRuntime* gc) : gc(gc) { + /* + * Clear needsIncrementalBarrier early so we don't do any write barriers + * during sweeping. + */ + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + if (zone->isGCMarking()) { + MOZ_ASSERT(zone->needsIncrementalBarrier()); + zone->setNeedsIncrementalBarrier(false); + } + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + } +} + +AutoDisableBarriers::~AutoDisableBarriers() { + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + if (zone->isGCMarking()) { + zone->setNeedsIncrementalBarrier(true); + } + } +} + +static bool NeedToCollectNursery(GCRuntime* gc) { + return !gc->nursery().isEmpty() || !gc->storeBuffer().isEmpty(); +} + +#ifdef DEBUG +static const char* DescribeBudget(const SliceBudget& budget) { + MOZ_ASSERT(TlsContext.get()->isMainThreadContext()); + constexpr size_t length = 32; + static char buffer[length]; + budget.describe(buffer, length); + return buffer; +} +#endif + +static bool ShouldPauseMutatorWhileWaiting(const SliceBudget& budget, + JS::GCReason reason, + bool budgetWasIncreased) { + // When we're nearing the incremental limit at which we will finish the + // collection synchronously, pause the main thread if there is only background + // GC work happening. This allows the GC to catch up and avoid hitting the + // limit. + return budget.isTimeBudget() && + (reason == JS::GCReason::ALLOC_TRIGGER || + reason == JS::GCReason::TOO_MUCH_MALLOC) && + budgetWasIncreased; +} + +void GCRuntime::incrementalSlice(SliceBudget& budget, JS::GCReason reason, + bool budgetWasIncreased) { + MOZ_ASSERT_IF(isIncrementalGCInProgress(), isIncremental); + + AutoSetThreadIsPerformingGC performingGC(rt->gcContext()); + + AutoGCSession session(this, JS::HeapState::MajorCollecting); + + bool destroyingRuntime = (reason == JS::GCReason::DESTROY_RUNTIME); + + initialState = incrementalState; + isIncremental = !budget.isUnlimited(); + useBackgroundThreads = ShouldUseBackgroundThreads(isIncremental, reason); + +#ifdef JS_GC_ZEAL + // Do the incremental collection type specified by zeal mode if the collection + // was triggered by runDebugGC() and incremental GC has not been cancelled by + // resetIncrementalGC(). + useZeal = isIncremental && reason == JS::GCReason::DEBUG_GC; +#endif + +#ifdef DEBUG + stats().log( + "Incremental: %d, lastMarkSlice: %d, useZeal: %d, budget: %s, " + "budgetWasIncreased: %d", + bool(isIncremental), bool(lastMarkSlice), bool(useZeal), + DescribeBudget(budget), budgetWasIncreased); +#endif + + if (useZeal && hasIncrementalTwoSliceZealMode()) { + // Yields between slices occurs at predetermined points in these modes; the + // budget is not used. |isIncremental| is still true. + stats().log("Using unlimited budget for two-slice zeal mode"); + budget = SliceBudget::unlimited(); + } + + bool shouldPauseMutator = + ShouldPauseMutatorWhileWaiting(budget, reason, budgetWasIncreased); + + switch (incrementalState) { + case State::NotActive: + startCollection(reason); + + incrementalState = State::Prepare; + if (!beginPreparePhase(reason, session)) { + incrementalState = State::NotActive; + break; + } + + if (useZeal && hasZealMode(ZealMode::YieldBeforeRootMarking)) { + break; + } + + [[fallthrough]]; + + case State::Prepare: + if (waitForBackgroundTask(unmarkTask, budget, shouldPauseMutator, + DontTriggerSliceWhenFinished) == NotFinished) { + break; + } + + incrementalState = State::MarkRoots; + [[fallthrough]]; + + case State::MarkRoots: + if (NeedToCollectNursery(this)) { + collectNurseryFromMajorGC(reason); + } + + endPreparePhase(reason); + beginMarkPhase(session); + incrementalState = State::Mark; + + if (useZeal && hasZealMode(ZealMode::YieldBeforeMarking) && + isIncremental) { + break; + } + + [[fallthrough]]; + + case State::Mark: + if (mightSweepInThisSlice(budget.isUnlimited())) { + // Trace wrapper rooters before marking if we might start sweeping in + // this slice. + rt->mainContextFromOwnThread()->traceWrapperGCRooters( + marker().tracer()); + } + + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + if (markUntilBudgetExhausted(budget, AllowParallelMarking) == + NotFinished) { + break; + } + } + + assertNoMarkingWork(); + + /* + * There are a number of reasons why we break out of collection here, + * either ending the slice or to run a new interation of the loop in + * GCRuntime::collect() + */ + + /* + * In incremental GCs where we have already performed more than one + * slice we yield after marking with the aim of starting the sweep in + * the next slice, since the first slice of sweeping can be expensive. + * + * This is modified by the various zeal modes. We don't yield in + * YieldBeforeMarking mode and we always yield in YieldBeforeSweeping + * mode. + * + * We will need to mark anything new on the stack when we resume, so + * we stay in Mark state. + */ + if (isIncremental && !lastMarkSlice) { + if ((initialState == State::Mark && + !(useZeal && hasZealMode(ZealMode::YieldBeforeMarking))) || + (useZeal && hasZealMode(ZealMode::YieldBeforeSweeping))) { + lastMarkSlice = true; + stats().log("Yielding before starting sweeping"); + break; + } + } + + incrementalState = State::Sweep; + lastMarkSlice = false; + + beginSweepPhase(reason, session); + + [[fallthrough]]; + + case State::Sweep: + if (storeBuffer().mayHavePointersToDeadCells()) { + collectNurseryFromMajorGC(reason); + } + + if (initialState == State::Sweep) { + rt->mainContextFromOwnThread()->traceWrapperGCRooters( + marker().tracer()); + } + + if (performSweepActions(budget) == NotFinished) { + break; + } + + endSweepPhase(destroyingRuntime); + + incrementalState = State::Finalize; + + [[fallthrough]]; + + case State::Finalize: + if (waitForBackgroundTask(sweepTask, budget, shouldPauseMutator, + TriggerSliceWhenFinished) == NotFinished) { + break; + } + + assertBackgroundSweepingFinished(); + + { + // Sweep the zones list now that background finalization is finished to + // remove and free dead zones, compartments and realms. + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::SWEEP); + gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::DESTROY); + sweepZones(rt->gcContext(), destroyingRuntime); + } + + MOZ_ASSERT(!startedCompacting); + incrementalState = State::Compact; + + // Always yield before compacting since it is not incremental. + if (isCompacting && !budget.isUnlimited()) { + break; + } + + [[fallthrough]]; + + case State::Compact: + if (isCompacting) { + if (NeedToCollectNursery(this)) { + collectNurseryFromMajorGC(reason); + } + + storeBuffer().checkEmpty(); + if (!startedCompacting) { + beginCompactPhase(); + } + + if (compactPhase(reason, budget, session) == NotFinished) { + break; + } + + endCompactPhase(); + } + + startDecommit(); + incrementalState = State::Decommit; + + [[fallthrough]]; + + case State::Decommit: + if (waitForBackgroundTask(decommitTask, budget, shouldPauseMutator, + TriggerSliceWhenFinished) == NotFinished) { + break; + } + + incrementalState = State::Finish; + + [[fallthrough]]; + + case State::Finish: + finishCollection(reason); + incrementalState = State::NotActive; + break; + } + +#ifdef DEBUG + MOZ_ASSERT(safeToYield); + for (auto& marker : markers) { + MOZ_ASSERT(marker->markColor() == MarkColor::Black); + } + MOZ_ASSERT(!rt->gcContext()->hasJitCodeToPoison()); +#endif +} + +void GCRuntime::collectNurseryFromMajorGC(JS::GCReason reason) { + collectNursery(gcOptions(), reason, + gcstats::PhaseKind::EVICT_NURSERY_FOR_MAJOR_GC); +} + +bool GCRuntime::hasForegroundWork() const { + switch (incrementalState) { + case State::NotActive: + // Incremental GC is not running and no work is pending. + return false; + case State::Prepare: + // We yield in the Prepare state after starting unmarking. + return !unmarkTask.wasStarted(); + case State::Finalize: + // We yield in the Finalize state to wait for background sweeping. + return !isBackgroundSweeping(); + case State::Decommit: + // We yield in the Decommit state to wait for background decommit. + return !decommitTask.wasStarted(); + default: + // In all other states there is still work to do. + return true; + } +} + +IncrementalProgress GCRuntime::waitForBackgroundTask( + GCParallelTask& task, const SliceBudget& budget, bool shouldPauseMutator, + ShouldTriggerSliceWhenFinished triggerSlice) { + // Wait here in non-incremental collections, or if we want to pause the + // mutator to let the GC catch up. + if (budget.isUnlimited() || shouldPauseMutator) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::WAIT_BACKGROUND_THREAD); + Maybe deadline; + if (budget.isTimeBudget()) { + deadline.emplace(budget.deadline()); + } + task.join(deadline); + } + + // In incremental collections, yield if the task has not finished and + // optionally request a slice to notify us when this happens. + if (!budget.isUnlimited()) { + AutoLockHelperThreadState lock; + if (task.wasStarted(lock)) { + if (triggerSlice) { + requestSliceAfterBackgroundTask = true; + } + return NotFinished; + } + + task.joinWithLockHeld(lock); + } + + MOZ_ASSERT(task.isIdle()); + + if (triggerSlice) { + cancelRequestedGCAfterBackgroundTask(); + } + + return Finished; +} + +GCAbortReason gc::IsIncrementalGCUnsafe(JSRuntime* rt) { + MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC); + + if (!rt->gc.isIncrementalGCAllowed()) { + return GCAbortReason::IncrementalDisabled; + } + + return GCAbortReason::None; +} + +inline void GCRuntime::checkZoneIsScheduled(Zone* zone, JS::GCReason reason, + const char* trigger) { +#ifdef DEBUG + if (zone->isGCScheduled()) { + return; + } + + fprintf(stderr, + "checkZoneIsScheduled: Zone %p not scheduled as expected in %s GC " + "for %s trigger\n", + zone, JS::ExplainGCReason(reason), trigger); + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + fprintf(stderr, " Zone %p:%s%s\n", zone.get(), + zone->isAtomsZone() ? " atoms" : "", + zone->isGCScheduled() ? " scheduled" : ""); + } + fflush(stderr); + MOZ_CRASH("Zone not scheduled"); +#endif +} + +GCRuntime::IncrementalResult GCRuntime::budgetIncrementalGC( + bool nonincrementalByAPI, JS::GCReason reason, SliceBudget& budget) { + if (nonincrementalByAPI) { + stats().nonincremental(GCAbortReason::NonIncrementalRequested); + budget = SliceBudget::unlimited(); + + // Reset any in progress incremental GC if this was triggered via the + // API. This isn't required for correctness, but sometimes during tests + // the caller expects this GC to collect certain objects, and we need + // to make sure to collect everything possible. + if (reason != JS::GCReason::ALLOC_TRIGGER) { + return resetIncrementalGC(GCAbortReason::NonIncrementalRequested); + } + + return IncrementalResult::Ok; + } + + if (reason == JS::GCReason::ABORT_GC) { + budget = SliceBudget::unlimited(); + stats().nonincremental(GCAbortReason::AbortRequested); + return resetIncrementalGC(GCAbortReason::AbortRequested); + } + + if (!budget.isUnlimited()) { + GCAbortReason unsafeReason = IsIncrementalGCUnsafe(rt); + if (unsafeReason == GCAbortReason::None) { + if (reason == JS::GCReason::COMPARTMENT_REVIVED) { + unsafeReason = GCAbortReason::CompartmentRevived; + } else if (!incrementalGCEnabled) { + unsafeReason = GCAbortReason::ModeChange; + } + } + + if (unsafeReason != GCAbortReason::None) { + budget = SliceBudget::unlimited(); + stats().nonincremental(unsafeReason); + return resetIncrementalGC(unsafeReason); + } + } + + GCAbortReason resetReason = GCAbortReason::None; + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + if (zone->gcHeapSize.bytes() >= + zone->gcHeapThreshold.incrementalLimitBytes()) { + checkZoneIsScheduled(zone, reason, "GC bytes"); + budget = SliceBudget::unlimited(); + stats().nonincremental(GCAbortReason::GCBytesTrigger); + if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) { + resetReason = GCAbortReason::GCBytesTrigger; + } + } + + if (zone->mallocHeapSize.bytes() >= + zone->mallocHeapThreshold.incrementalLimitBytes()) { + checkZoneIsScheduled(zone, reason, "malloc bytes"); + budget = SliceBudget::unlimited(); + stats().nonincremental(GCAbortReason::MallocBytesTrigger); + if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) { + resetReason = GCAbortReason::MallocBytesTrigger; + } + } + + if (zone->jitHeapSize.bytes() >= + zone->jitHeapThreshold.incrementalLimitBytes()) { + checkZoneIsScheduled(zone, reason, "JIT code bytes"); + budget = SliceBudget::unlimited(); + stats().nonincremental(GCAbortReason::JitCodeBytesTrigger); + if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) { + resetReason = GCAbortReason::JitCodeBytesTrigger; + } + } + + if (isIncrementalGCInProgress() && + zone->isGCScheduled() != zone->wasGCStarted()) { + budget = SliceBudget::unlimited(); + resetReason = GCAbortReason::ZoneChange; + } + } + + if (resetReason != GCAbortReason::None) { + return resetIncrementalGC(resetReason); + } + + return IncrementalResult::Ok; +} + +bool GCRuntime::maybeIncreaseSliceBudget(SliceBudget& budget) { + if (js::SupportDifferentialTesting()) { + return false; + } + + if (!budget.isTimeBudget() || !isIncrementalGCInProgress()) { + return false; + } + + bool wasIncreasedForLongCollections = + maybeIncreaseSliceBudgetForLongCollections(budget); + bool wasIncreasedForUgentCollections = + maybeIncreaseSliceBudgetForUrgentCollections(budget); + + return wasIncreasedForLongCollections || wasIncreasedForUgentCollections; +} + +// Return true if the budget is actually extended after rounding. +static bool ExtendBudget(SliceBudget& budget, double newDuration) { + long newDurationMS = lround(newDuration); + if (newDurationMS <= budget.timeBudget()) { + return false; + } + + bool idleTriggered = budget.idle; + budget = SliceBudget(TimeBudget(newDuration), nullptr); // Uninterruptible. + budget.idle = idleTriggered; + budget.extended = true; + return true; +} + +bool GCRuntime::maybeIncreaseSliceBudgetForLongCollections( + SliceBudget& budget) { + // For long-running collections, enforce a minimum time budget that increases + // linearly with time up to a maximum. + + // All times are in milliseconds. + struct BudgetAtTime { + double time; + double budget; + }; + const BudgetAtTime MinBudgetStart{1500, 0.0}; + const BudgetAtTime MinBudgetEnd{2500, 100.0}; + + double totalTime = (TimeStamp::Now() - lastGCStartTime()).ToMilliseconds(); + + double minBudget = + LinearInterpolate(totalTime, MinBudgetStart.time, MinBudgetStart.budget, + MinBudgetEnd.time, MinBudgetEnd.budget); + + return ExtendBudget(budget, minBudget); +} + +bool GCRuntime::maybeIncreaseSliceBudgetForUrgentCollections( + SliceBudget& budget) { + // Enforce a minimum time budget based on how close we are to the incremental + // limit. + + size_t minBytesRemaining = SIZE_MAX; + for (AllZonesIter zone(this); !zone.done(); zone.next()) { + if (!zone->wasGCStarted()) { + continue; + } + size_t gcBytesRemaining = + zone->gcHeapThreshold.incrementalBytesRemaining(zone->gcHeapSize); + minBytesRemaining = std::min(minBytesRemaining, gcBytesRemaining); + size_t mallocBytesRemaining = + zone->mallocHeapThreshold.incrementalBytesRemaining( + zone->mallocHeapSize); + minBytesRemaining = std::min(minBytesRemaining, mallocBytesRemaining); + } + + if (minBytesRemaining < tunables.urgentThresholdBytes() && + minBytesRemaining != 0) { + // Increase budget based on the reciprocal of the fraction remaining. + double fractionRemaining = + double(minBytesRemaining) / double(tunables.urgentThresholdBytes()); + double minBudget = double(defaultSliceBudgetMS()) / fractionRemaining; + return ExtendBudget(budget, minBudget); + } + + return false; +} + +static void ScheduleZones(GCRuntime* gc, JS::GCReason reason) { + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + // Re-check heap threshold for alloc-triggered zones that were not + // previously collected. Now we have allocation rate data, the heap limit + // may have been increased beyond the current size. + if (gc->tunables.balancedHeapLimitsEnabled() && zone->isGCScheduled() && + zone->smoothedCollectionRate.ref().isNothing() && + reason == JS::GCReason::ALLOC_TRIGGER && + zone->gcHeapSize.bytes() < zone->gcHeapThreshold.startBytes()) { + zone->unscheduleGC(); // May still be re-scheduled below. + } + + if (gc->isShutdownGC()) { + zone->scheduleGC(); + } + + if (!gc->isPerZoneGCEnabled()) { + zone->scheduleGC(); + } + + // To avoid resets, continue to collect any zones that were being + // collected in a previous slice. + if (gc->isIncrementalGCInProgress() && zone->wasGCStarted()) { + zone->scheduleGC(); + } + + // This is a heuristic to reduce the total number of collections. + bool inHighFrequencyMode = gc->schedulingState.inHighFrequencyGCMode(); + if (zone->gcHeapSize.bytes() >= + zone->gcHeapThreshold.eagerAllocTrigger(inHighFrequencyMode) || + zone->mallocHeapSize.bytes() >= + zone->mallocHeapThreshold.eagerAllocTrigger(inHighFrequencyMode) || + zone->jitHeapSize.bytes() >= zone->jitHeapThreshold.startBytes()) { + zone->scheduleGC(); + } + } +} + +static void UnscheduleZones(GCRuntime* gc) { + for (ZonesIter zone(gc->rt, WithAtoms); !zone.done(); zone.next()) { + zone->unscheduleGC(); + } +} + +class js::gc::AutoCallGCCallbacks { + GCRuntime& gc_; + JS::GCReason reason_; + + public: + explicit AutoCallGCCallbacks(GCRuntime& gc, JS::GCReason reason) + : gc_(gc), reason_(reason) { + gc_.maybeCallGCCallback(JSGC_BEGIN, reason); + } + ~AutoCallGCCallbacks() { gc_.maybeCallGCCallback(JSGC_END, reason_); } +}; + +void GCRuntime::maybeCallGCCallback(JSGCStatus status, JS::GCReason reason) { + if (!gcCallback.ref().op) { + return; + } + + if (isIncrementalGCInProgress()) { + return; + } + + if (gcCallbackDepth == 0) { + // Save scheduled zone information in case the callback clears it. + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->gcScheduledSaved_ = zone->gcScheduled_; + } + } + + // Save and clear GC options and state in case the callback reenters GC. + JS::GCOptions options = gcOptions(); + maybeGcOptions = Nothing(); + bool savedFullGCRequested = fullGCRequested; + fullGCRequested = false; + + gcCallbackDepth++; + + callGCCallback(status, reason); + + MOZ_ASSERT(gcCallbackDepth != 0); + gcCallbackDepth--; + + // Restore the original GC options. + maybeGcOptions = Some(options); + + // At the end of a GC, clear out the fullGCRequested state. At the start, + // restore the previous setting. + fullGCRequested = (status == JSGC_END) ? false : savedFullGCRequested; + + if (gcCallbackDepth == 0) { + // Ensure any zone that was originally scheduled stays scheduled. + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->gcScheduled_ = zone->gcScheduled_ || zone->gcScheduledSaved_; + } + } +} + +/* + * We disable inlining to ensure that the bottom of the stack with possible GC + * roots recorded in MarkRuntime excludes any pointers we use during the marking + * implementation. + */ +MOZ_NEVER_INLINE GCRuntime::IncrementalResult GCRuntime::gcCycle( + bool nonincrementalByAPI, const SliceBudget& budgetArg, + JS::GCReason reason) { + // Assert if this is a GC unsafe region. + rt->mainContextFromOwnThread()->verifyIsSafeToGC(); + + // It's ok if threads other than the main thread have suppressGC set, as + // they are operating on zones which will not be collected from here. + MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC); + + // This reason is used internally. See below. + MOZ_ASSERT(reason != JS::GCReason::RESET); + + // Background finalization and decommit are finished by definition before we + // can start a new major GC. Background allocation may still be running, but + // that's OK because chunk pools are protected by the GC lock. + if (!isIncrementalGCInProgress()) { + assertBackgroundSweepingFinished(); + MOZ_ASSERT(decommitTask.isIdle()); + } + + // Note that GC callbacks are allowed to re-enter GC. + AutoCallGCCallbacks callCallbacks(*this, reason); + + // Increase slice budget for long running collections before it is recorded by + // AutoGCSlice. + SliceBudget budget(budgetArg); + bool budgetWasIncreased = maybeIncreaseSliceBudget(budget); + + ScheduleZones(this, reason); + + auto updateCollectorTime = MakeScopeExit([&] { + if (const gcstats::Statistics::SliceData* slice = stats().lastSlice()) { + collectorTimeSinceAllocRateUpdate += slice->duration(); + } + }); + + gcstats::AutoGCSlice agc(stats(), scanZonesBeforeGC(), gcOptions(), budget, + reason, budgetWasIncreased); + + IncrementalResult result = + budgetIncrementalGC(nonincrementalByAPI, reason, budget); + if (result == IncrementalResult::ResetIncremental) { + if (incrementalState == State::NotActive) { + // The collection was reset and has finished. + return result; + } + + // The collection was reset but we must finish up some remaining work. + reason = JS::GCReason::RESET; + } + + majorGCTriggerReason = JS::GCReason::NO_REASON; + MOZ_ASSERT(!stats().hasTrigger()); + + incGcNumber(); + incGcSliceNumber(); + + gcprobes::MajorGCStart(); + incrementalSlice(budget, reason, budgetWasIncreased); + gcprobes::MajorGCEnd(); + + MOZ_ASSERT_IF(result == IncrementalResult::ResetIncremental, + !isIncrementalGCInProgress()); + return result; +} + +inline bool GCRuntime::mightSweepInThisSlice(bool nonIncremental) { + MOZ_ASSERT(incrementalState < State::Sweep); + return nonIncremental || lastMarkSlice || hasIncrementalTwoSliceZealMode(); +} + +#ifdef JS_GC_ZEAL +static bool IsDeterministicGCReason(JS::GCReason reason) { + switch (reason) { + case JS::GCReason::API: + case JS::GCReason::DESTROY_RUNTIME: + case JS::GCReason::LAST_DITCH: + case JS::GCReason::TOO_MUCH_MALLOC: + case JS::GCReason::TOO_MUCH_WASM_MEMORY: + case JS::GCReason::TOO_MUCH_JIT_CODE: + case JS::GCReason::ALLOC_TRIGGER: + case JS::GCReason::DEBUG_GC: + case JS::GCReason::CC_FORCED: + case JS::GCReason::SHUTDOWN_CC: + case JS::GCReason::ABORT_GC: + case JS::GCReason::DISABLE_GENERATIONAL_GC: + case JS::GCReason::FINISH_GC: + case JS::GCReason::PREPARE_FOR_TRACING: + return true; + + default: + return false; + } +} +#endif + +gcstats::ZoneGCStats GCRuntime::scanZonesBeforeGC() { + gcstats::ZoneGCStats zoneStats; + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zoneStats.zoneCount++; + zoneStats.compartmentCount += zone->compartments().length(); + if (zone->isGCScheduled()) { + zoneStats.collectedZoneCount++; + zoneStats.collectedCompartmentCount += zone->compartments().length(); + } + } + + return zoneStats; +} + +// The GC can only clean up scheduledForDestruction realms that were marked live +// by a barrier (e.g. by RemapWrappers from a navigation event). It is also +// common to have realms held live because they are part of a cycle in gecko, +// e.g. involving the HTMLDocument wrapper. In this case, we need to run the +// CycleCollector in order to remove these edges before the realm can be freed. +void GCRuntime::maybeDoCycleCollection() { + const static float ExcessiveGrayRealms = 0.8f; + const static size_t LimitGrayRealms = 200; + + size_t realmsTotal = 0; + size_t realmsGray = 0; + for (RealmsIter realm(rt); !realm.done(); realm.next()) { + ++realmsTotal; + GlobalObject* global = realm->unsafeUnbarrieredMaybeGlobal(); + if (global && global->isMarkedGray()) { + ++realmsGray; + } + } + float grayFraction = float(realmsGray) / float(realmsTotal); + if (grayFraction > ExcessiveGrayRealms || realmsGray > LimitGrayRealms) { + callDoCycleCollectionCallback(rt->mainContextFromOwnThread()); + } +} + +void GCRuntime::checkCanCallAPI() { + MOZ_RELEASE_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + /* If we attempt to invoke the GC while we are running in the GC, assert. */ + MOZ_RELEASE_ASSERT(!JS::RuntimeHeapIsBusy()); +} + +bool GCRuntime::checkIfGCAllowedInCurrentState(JS::GCReason reason) { + if (rt->mainContextFromOwnThread()->suppressGC) { + return false; + } + + // Only allow shutdown GCs when we're destroying the runtime. This keeps + // the GC callback from triggering a nested GC and resetting global state. + if (rt->isBeingDestroyed() && !isShutdownGC()) { + return false; + } + +#ifdef JS_GC_ZEAL + if (deterministicOnly && !IsDeterministicGCReason(reason)) { + return false; + } +#endif + + return true; +} + +bool GCRuntime::shouldRepeatForDeadZone(JS::GCReason reason) { + MOZ_ASSERT_IF(reason == JS::GCReason::COMPARTMENT_REVIVED, !isIncremental); + MOZ_ASSERT(!isIncrementalGCInProgress()); + + if (!isIncremental) { + return false; + } + + for (CompartmentsIter c(rt); !c.done(); c.next()) { + if (c->gcState.scheduledForDestruction) { + return true; + } + } + + return false; +} + +struct MOZ_RAII AutoSetZoneSliceThresholds { + explicit AutoSetZoneSliceThresholds(GCRuntime* gc) : gc(gc) { + // On entry, zones that are already collecting should have a slice threshold + // set. + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->wasGCStarted() == + zone->gcHeapThreshold.hasSliceThreshold()); + MOZ_ASSERT(zone->wasGCStarted() == + zone->mallocHeapThreshold.hasSliceThreshold()); + } + } + + ~AutoSetZoneSliceThresholds() { + // On exit, update the thresholds for all collecting zones. + bool waitingOnBGTask = gc->isWaitingOnBackgroundTask(); + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + if (zone->wasGCStarted()) { + zone->setGCSliceThresholds(*gc, waitingOnBGTask); + } else { + MOZ_ASSERT(!zone->gcHeapThreshold.hasSliceThreshold()); + MOZ_ASSERT(!zone->mallocHeapThreshold.hasSliceThreshold()); + } + } + } + + GCRuntime* gc; +}; + +void GCRuntime::collect(bool nonincrementalByAPI, const SliceBudget& budget, + JS::GCReason reason) { + TimeStamp startTime = TimeStamp::Now(); + auto timer = MakeScopeExit([&] { + if (Realm* realm = rt->mainContextFromOwnThread()->realm()) { + realm->timers.gcTime += TimeStamp::Now() - startTime; + } + }); + + auto clearGCOptions = MakeScopeExit([&] { + if (!isIncrementalGCInProgress()) { + maybeGcOptions = Nothing(); + } + }); + + MOZ_ASSERT(reason != JS::GCReason::NO_REASON); + + // Checks run for each request, even if we do not actually GC. + checkCanCallAPI(); + + // Check if we are allowed to GC at this time before proceeding. + if (!checkIfGCAllowedInCurrentState(reason)) { + return; + } + + stats().log("GC slice starting in state %s", StateName(incrementalState)); + + AutoStopVerifyingBarriers av(rt, isShutdownGC()); + AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread()); + AutoSetZoneSliceThresholds sliceThresholds(this); + + schedulingState.updateHighFrequencyModeForReason(reason); + + if (!isIncrementalGCInProgress() && tunables.balancedHeapLimitsEnabled()) { + updateAllocationRates(); + } + + bool repeat; + do { + IncrementalResult cycleResult = + gcCycle(nonincrementalByAPI, budget, reason); + + if (reason == JS::GCReason::ABORT_GC) { + MOZ_ASSERT(!isIncrementalGCInProgress()); + stats().log("GC aborted by request"); + break; + } + + /* + * Sometimes when we finish a GC we need to immediately start a new one. + * This happens in the following cases: + * - when we reset the current GC + * - when finalizers drop roots during shutdown + * - when zones that we thought were dead at the start of GC are + * not collected (see the large comment in beginMarkPhase) + */ + repeat = false; + if (!isIncrementalGCInProgress()) { + if (cycleResult == ResetIncremental) { + repeat = true; + } else if (rootsRemoved && isShutdownGC()) { + /* Need to re-schedule all zones for GC. */ + JS::PrepareForFullGC(rt->mainContextFromOwnThread()); + repeat = true; + reason = JS::GCReason::ROOTS_REMOVED; + } else if (shouldRepeatForDeadZone(reason)) { + repeat = true; + reason = JS::GCReason::COMPARTMENT_REVIVED; + } + } + } while (repeat); + + if (reason == JS::GCReason::COMPARTMENT_REVIVED) { + maybeDoCycleCollection(); + } + +#ifdef JS_GC_ZEAL + if (hasZealMode(ZealMode::CheckHeapAfterGC)) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::TRACE_HEAP); + CheckHeapAfterGC(rt); + } + if (hasZealMode(ZealMode::CheckGrayMarking) && !isIncrementalGCInProgress()) { + MOZ_RELEASE_ASSERT(CheckGrayMarkingState(rt)); + } +#endif + stats().log("GC slice ending in state %s", StateName(incrementalState)); + + UnscheduleZones(this); +} + +SliceBudget GCRuntime::defaultBudget(JS::GCReason reason, int64_t millis) { + // millis == 0 means use internal GC scheduling logic to come up with + // a duration for the slice budget. This may end up still being zero + // based on preferences. + if (millis == 0) { + millis = defaultSliceBudgetMS(); + } + + // If the embedding has registered a callback for creating SliceBudgets, + // then use it. + if (createBudgetCallback) { + return createBudgetCallback(reason, millis); + } + + // Otherwise, the preference can request an unlimited duration slice. + if (millis == 0) { + return SliceBudget::unlimited(); + } + + return SliceBudget(TimeBudget(millis)); +} + +void GCRuntime::gc(JS::GCOptions options, JS::GCReason reason) { + if (!isIncrementalGCInProgress()) { + setGCOptions(options); + } + + collect(true, SliceBudget::unlimited(), reason); +} + +void GCRuntime::startGC(JS::GCOptions options, JS::GCReason reason, + const js::SliceBudget& budget) { + MOZ_ASSERT(!isIncrementalGCInProgress()); + setGCOptions(options); + + if (!JS::IsIncrementalGCEnabled(rt->mainContextFromOwnThread())) { + collect(true, SliceBudget::unlimited(), reason); + return; + } + + collect(false, budget, reason); +} + +void GCRuntime::setGCOptions(JS::GCOptions options) { + MOZ_ASSERT(maybeGcOptions == Nothing()); + maybeGcOptions = Some(options); +} + +void GCRuntime::gcSlice(JS::GCReason reason, const js::SliceBudget& budget) { + MOZ_ASSERT(isIncrementalGCInProgress()); + collect(false, budget, reason); +} + +void GCRuntime::finishGC(JS::GCReason reason) { + MOZ_ASSERT(isIncrementalGCInProgress()); + + // If we're not collecting because we're out of memory then skip the + // compacting phase if we need to finish an ongoing incremental GC + // non-incrementally to avoid janking the browser. + if (!IsOOMReason(initialReason)) { + if (incrementalState == State::Compact) { + abortGC(); + return; + } + + isCompacting = false; + } + + collect(false, SliceBudget::unlimited(), reason); +} + +void GCRuntime::abortGC() { + MOZ_ASSERT(isIncrementalGCInProgress()); + checkCanCallAPI(); + MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC); + + collect(false, SliceBudget::unlimited(), JS::GCReason::ABORT_GC); +} + +static bool ZonesSelected(GCRuntime* gc) { + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + if (zone->isGCScheduled()) { + return true; + } + } + return false; +} + +void GCRuntime::startDebugGC(JS::GCOptions options, const SliceBudget& budget) { + MOZ_ASSERT(!isIncrementalGCInProgress()); + setGCOptions(options); + + if (!ZonesSelected(this)) { + JS::PrepareForFullGC(rt->mainContextFromOwnThread()); + } + + collect(false, budget, JS::GCReason::DEBUG_GC); +} + +void GCRuntime::debugGCSlice(const SliceBudget& budget) { + MOZ_ASSERT(isIncrementalGCInProgress()); + + if (!ZonesSelected(this)) { + JS::PrepareForIncrementalGC(rt->mainContextFromOwnThread()); + } + + collect(false, budget, JS::GCReason::DEBUG_GC); +} + +/* Schedule a full GC unless a zone will already be collected. */ +void js::PrepareForDebugGC(JSRuntime* rt) { + if (!ZonesSelected(&rt->gc)) { + JS::PrepareForFullGC(rt->mainContextFromOwnThread()); + } +} + +void GCRuntime::onOutOfMallocMemory() { + // Stop allocating new chunks. + allocTask.cancelAndWait(); + + // Make sure we release anything queued for release. + decommitTask.join(); + nursery().joinDecommitTask(); + + // Wait for background free of nursery huge slots to finish. + sweepTask.join(); + + AutoLockGC lock(this); + onOutOfMallocMemory(lock); +} + +void GCRuntime::onOutOfMallocMemory(const AutoLockGC& lock) { +#ifdef DEBUG + // Release any relocated arenas we may be holding on to, without releasing + // the GC lock. + releaseHeldRelocatedArenasWithoutUnlocking(lock); +#endif + + // Throw away any excess chunks we have lying around. + freeEmptyChunks(lock); + + // Immediately decommit as many arenas as possible in the hopes that this + // might let the OS scrape together enough pages to satisfy the failing + // malloc request. + if (DecommitEnabled()) { + decommitFreeArenasWithoutUnlocking(lock); + } +} + +void GCRuntime::minorGC(JS::GCReason reason, gcstats::PhaseKind phase) { + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + + MOZ_ASSERT_IF(reason == JS::GCReason::EVICT_NURSERY, + !rt->mainContextFromOwnThread()->suppressGC); + if (rt->mainContextFromOwnThread()->suppressGC) { + return; + } + + incGcNumber(); + + collectNursery(JS::GCOptions::Normal, reason, phase); + +#ifdef JS_GC_ZEAL + if (hasZealMode(ZealMode::CheckHeapAfterGC)) { + gcstats::AutoPhase ap(stats(), phase); + CheckHeapAfterGC(rt); + } +#endif + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + maybeTriggerGCAfterAlloc(zone); + maybeTriggerGCAfterMalloc(zone); + } +} + +void GCRuntime::collectNursery(JS::GCOptions options, JS::GCReason reason, + gcstats::PhaseKind phase) { + AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread()); + + uint32_t numAllocs = 0; + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + numAllocs += zone->getAndResetTenuredAllocsSinceMinorGC(); + } + stats().setAllocsSinceMinorGCTenured(numAllocs); + + gcstats::AutoPhase ap(stats(), phase); + + nursery().clearMinorGCRequest(); + nursery().collect(options, reason); + MOZ_ASSERT(nursery().isEmpty()); + + startBackgroundFreeAfterMinorGC(); +} + +void GCRuntime::startBackgroundFreeAfterMinorGC() { + MOZ_ASSERT(nursery().isEmpty()); + + { + AutoLockHelperThreadState lock; + + lifoBlocksToFree.ref().transferFrom(&lifoBlocksToFreeAfterMinorGC.ref()); + + if (lifoBlocksToFree.ref().isEmpty() && + buffersToFreeAfterMinorGC.ref().empty()) { + return; + } + } + + startBackgroundFree(); +} + +bool GCRuntime::gcIfRequestedImpl(bool eagerOk) { + // This method returns whether a major GC was performed. + + if (nursery().minorGCRequested()) { + minorGC(nursery().minorGCTriggerReason()); + } + + JS::GCReason reason = wantMajorGC(eagerOk); + if (reason == JS::GCReason::NO_REASON) { + return false; + } + + SliceBudget budget = defaultBudget(reason, 0); + if (!isIncrementalGCInProgress()) { + startGC(JS::GCOptions::Normal, reason, budget); + } else { + gcSlice(reason, budget); + } + return true; +} + +void js::gc::FinishGC(JSContext* cx, JS::GCReason reason) { + // Calling this when GC is suppressed won't have any effect. + MOZ_ASSERT(!cx->suppressGC); + + // GC callbacks may run arbitrary code, including JS. Check this regardless of + // whether we GC for this invocation. + MOZ_ASSERT(cx->isNurseryAllocAllowed()); + + if (JS::IsIncrementalGCInProgress(cx)) { + JS::PrepareForIncrementalGC(cx); + JS::FinishIncrementalGC(cx, reason); + } +} + +void js::gc::WaitForBackgroundTasks(JSContext* cx) { + cx->runtime()->gc.waitForBackgroundTasks(); +} + +void GCRuntime::waitForBackgroundTasks() { + MOZ_ASSERT(!isIncrementalGCInProgress()); + MOZ_ASSERT(sweepTask.isIdle()); + MOZ_ASSERT(decommitTask.isIdle()); + MOZ_ASSERT(markTask.isIdle()); + + allocTask.join(); + freeTask.join(); + nursery().joinDecommitTask(); +} + +Realm* js::NewRealm(JSContext* cx, JSPrincipals* principals, + const JS::RealmOptions& options) { + JSRuntime* rt = cx->runtime(); + JS_AbortIfWrongThread(cx); + + UniquePtr zoneHolder; + UniquePtr compHolder; + + Compartment* comp = nullptr; + Zone* zone = nullptr; + JS::CompartmentSpecifier compSpec = + options.creationOptions().compartmentSpecifier(); + switch (compSpec) { + case JS::CompartmentSpecifier::NewCompartmentInSystemZone: + // systemZone might be null here, in which case we'll make a zone and + // set this field below. + zone = rt->gc.systemZone; + break; + case JS::CompartmentSpecifier::NewCompartmentInExistingZone: + zone = options.creationOptions().zone(); + MOZ_ASSERT(zone); + break; + case JS::CompartmentSpecifier::ExistingCompartment: + comp = options.creationOptions().compartment(); + zone = comp->zone(); + break; + case JS::CompartmentSpecifier::NewCompartmentAndZone: + break; + } + + if (!zone) { + Zone::Kind kind = Zone::NormalZone; + const JSPrincipals* trusted = rt->trustedPrincipals(); + if (compSpec == JS::CompartmentSpecifier::NewCompartmentInSystemZone || + (principals && principals == trusted)) { + kind = Zone::SystemZone; + } + + zoneHolder = MakeUnique(cx->runtime(), kind); + if (!zoneHolder || !zoneHolder->init()) { + ReportOutOfMemory(cx); + return nullptr; + } + + zone = zoneHolder.get(); + } + + bool invisibleToDebugger = options.creationOptions().invisibleToDebugger(); + if (comp) { + // Debugger visibility is per-compartment, not per-realm, so make sure the + // new realm's visibility matches its compartment's. + MOZ_ASSERT(comp->invisibleToDebugger() == invisibleToDebugger); + } else { + compHolder = cx->make_unique(zone, invisibleToDebugger); + if (!compHolder) { + return nullptr; + } + + comp = compHolder.get(); + } + + UniquePtr realm(cx->new_(comp, options)); + if (!realm) { + return nullptr; + } + realm->init(cx, principals); + + // Make sure we don't put system and non-system realms in the same + // compartment. + if (!compHolder) { + MOZ_RELEASE_ASSERT(realm->isSystem() == IsSystemCompartment(comp)); + } + + AutoLockGC lock(rt); + + // Reserve space in the Vectors before we start mutating them. + if (!comp->realms().reserve(comp->realms().length() + 1) || + (compHolder && + !zone->compartments().reserve(zone->compartments().length() + 1)) || + (zoneHolder && !rt->gc.zones().reserve(rt->gc.zones().length() + 1))) { + ReportOutOfMemory(cx); + return nullptr; + } + + // After this everything must be infallible. + + comp->realms().infallibleAppend(realm.get()); + + if (compHolder) { + zone->compartments().infallibleAppend(compHolder.release()); + } + + if (zoneHolder) { + rt->gc.zones().infallibleAppend(zoneHolder.release()); + + // Lazily set the runtime's system zone. + if (compSpec == JS::CompartmentSpecifier::NewCompartmentInSystemZone) { + MOZ_RELEASE_ASSERT(!rt->gc.systemZone); + MOZ_ASSERT(zone->isSystemZone()); + rt->gc.systemZone = zone; + } + } + + return realm.release(); +} + +void GCRuntime::runDebugGC() { +#ifdef JS_GC_ZEAL + if (rt->mainContextFromOwnThread()->suppressGC) { + return; + } + + if (hasZealMode(ZealMode::GenerationalGC)) { + return minorGC(JS::GCReason::DEBUG_GC); + } + + PrepareForDebugGC(rt); + + auto budget = SliceBudget::unlimited(); + if (hasZealMode(ZealMode::IncrementalMultipleSlices)) { + /* + * Start with a small slice limit and double it every slice. This + * ensure that we get multiple slices, and collection runs to + * completion. + */ + if (!isIncrementalGCInProgress()) { + zealSliceBudget = zealFrequency / 2; + } else { + zealSliceBudget *= 2; + } + budget = SliceBudget(WorkBudget(zealSliceBudget)); + + js::gc::State initialState = incrementalState; + if (!isIncrementalGCInProgress()) { + setGCOptions(JS::GCOptions::Shrink); + } + collect(false, budget, JS::GCReason::DEBUG_GC); + + /* Reset the slice size when we get to the sweep or compact phases. */ + if ((initialState == State::Mark && incrementalState == State::Sweep) || + (initialState == State::Sweep && incrementalState == State::Compact)) { + zealSliceBudget = zealFrequency / 2; + } + } else if (hasIncrementalTwoSliceZealMode()) { + // These modes trigger incremental GC that happens in two slices and the + // supplied budget is ignored by incrementalSlice. + budget = SliceBudget(WorkBudget(1)); + + if (!isIncrementalGCInProgress()) { + setGCOptions(JS::GCOptions::Normal); + } + collect(false, budget, JS::GCReason::DEBUG_GC); + } else if (hasZealMode(ZealMode::Compact)) { + gc(JS::GCOptions::Shrink, JS::GCReason::DEBUG_GC); + } else { + gc(JS::GCOptions::Normal, JS::GCReason::DEBUG_GC); + } + +#endif +} + +void GCRuntime::setFullCompartmentChecks(bool enabled) { + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + fullCompartmentChecks = enabled; +} + +void GCRuntime::notifyRootsRemoved() { + rootsRemoved = true; + +#ifdef JS_GC_ZEAL + /* Schedule a GC to happen "soon". */ + if (hasZealMode(ZealMode::RootsChange)) { + nextScheduled = 1; + } +#endif +} + +#ifdef JS_GC_ZEAL +bool GCRuntime::selectForMarking(JSObject* object) { + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + return selectedForMarking.ref().get().append(object); +} + +void GCRuntime::clearSelectedForMarking() { + selectedForMarking.ref().get().clearAndFree(); +} + +void GCRuntime::setDeterministic(bool enabled) { + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + deterministicOnly = enabled; +} +#endif + +#ifdef DEBUG + +AutoAssertNoNurseryAlloc::AutoAssertNoNurseryAlloc() { + TlsContext.get()->disallowNurseryAlloc(); +} + +AutoAssertNoNurseryAlloc::~AutoAssertNoNurseryAlloc() { + TlsContext.get()->allowNurseryAlloc(); +} + +#endif // DEBUG + +#ifdef JSGC_HASH_TABLE_CHECKS +void GCRuntime::checkHashTablesAfterMovingGC() { + /* + * Check that internal hash tables no longer have any pointers to things + * that have been moved. + */ + rt->geckoProfiler().checkStringsMapAfterMovingGC(); + if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) { + rt->jitRuntime()->getInterpreterEntryMap()->checkScriptsAfterMovingGC(); + } + for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) { + zone->checkUniqueIdTableAfterMovingGC(); + zone->shapeZone().checkTablesAfterMovingGC(); + zone->checkAllCrossCompartmentWrappersAfterMovingGC(); + zone->checkScriptMapsAfterMovingGC(); + + // Note: CompactPropMaps never have a table. + JS::AutoCheckCannotGC nogc; + for (auto map = zone->cellIterUnsafe(); !map.done(); + map.next()) { + if (PropMapTable* table = map->asLinked()->maybeTable(nogc)) { + table->checkAfterMovingGC(); + } + } + for (auto map = zone->cellIterUnsafe(); !map.done(); + map.next()) { + if (PropMapTable* table = map->asLinked()->maybeTable(nogc)) { + table->checkAfterMovingGC(); + } + } + } + + for (CompartmentsIter c(this); !c.done(); c.next()) { + for (RealmsInCompartmentIter r(c); !r.done(); r.next()) { + r->dtoaCache.checkCacheAfterMovingGC(); + if (r->debugEnvs()) { + r->debugEnvs()->checkHashTablesAfterMovingGC(); + } + } + } +} +#endif + +#ifdef DEBUG +bool GCRuntime::hasZone(Zone* target) { + for (AllZonesIter zone(this); !zone.done(); zone.next()) { + if (zone == target) { + return true; + } + } + return false; +} +#endif + +void AutoAssertEmptyNursery::checkCondition(JSContext* cx) { + if (!noAlloc) { + noAlloc.emplace(); + } + this->cx = cx; + MOZ_ASSERT(cx->nursery().isEmpty()); +} + +AutoEmptyNursery::AutoEmptyNursery(JSContext* cx) : AutoAssertEmptyNursery() { + MOZ_ASSERT(!cx->suppressGC); + cx->runtime()->gc.stats().suspendPhases(); + cx->runtime()->gc.evictNursery(JS::GCReason::EVICT_NURSERY); + cx->runtime()->gc.stats().resumePhases(); + checkCondition(cx); +} + +#ifdef DEBUG + +namespace js { + +// We don't want jsfriendapi.h to depend on GenericPrinter, +// so these functions are declared directly in the cpp. + +extern JS_PUBLIC_API void DumpString(JSString* str, js::GenericPrinter& out); + +} // namespace js + +void js::gc::Cell::dump(js::GenericPrinter& out) const { + switch (getTraceKind()) { + case JS::TraceKind::Object: + reinterpret_cast(this)->dump(out); + break; + + case JS::TraceKind::String: + js::DumpString(reinterpret_cast(const_cast(this)), out); + break; + + case JS::TraceKind::Shape: + reinterpret_cast(this)->dump(out); + break; + + default: + out.printf("%s(%p)\n", JS::GCTraceKindToAscii(getTraceKind()), + (void*)this); + } +} + +// For use in a debugger. +void js::gc::Cell::dump() const { + js::Fprinter out(stderr); + dump(out); +} +#endif + +JS_PUBLIC_API bool js::gc::detail::CanCheckGrayBits(const TenuredCell* cell) { + // We do not check the gray marking state of cells in the following cases: + // + // 1) When OOM has caused us to clear the gcGrayBitsValid_ flag. + // + // 2) When we are in an incremental GC and examine a cell that is in a zone + // that is not being collected. Gray targets of CCWs that are marked black + // by a barrier will eventually be marked black in a later GC slice. + // + // 3) When mark bits are being cleared concurrently by a helper thread. + + MOZ_ASSERT(cell); + + auto runtime = cell->runtimeFromAnyThread(); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime)); + + if (!runtime->gc.areGrayBitsValid()) { + return false; + } + + JS::Zone* zone = cell->zone(); + + if (runtime->gc.isIncrementalGCInProgress() && !zone->wasGCStarted()) { + return false; + } + + return !zone->isGCPreparing(); +} + +JS_PUBLIC_API bool js::gc::detail::CellIsMarkedGrayIfKnown( + const TenuredCell* cell) { + MOZ_ASSERT_IF(cell->isPermanentAndMayBeShared(), cell->isMarkedBlack()); + if (!cell->isMarkedGray()) { + return false; + } + + return CanCheckGrayBits(cell); +} + +#ifdef DEBUG + +JS_PUBLIC_API void js::gc::detail::AssertCellIsNotGray(const Cell* cell) { + if (!cell->isTenured()) { + return; + } + + // Check that a cell is not marked gray. + // + // Since this is a debug-only check, take account of the eventual mark state + // of cells that will be marked black by the next GC slice in an incremental + // GC. For performance reasons we don't do this in CellIsMarkedGrayIfKnown. + + auto tc = &cell->asTenured(); + if (!tc->isMarkedGray() || !CanCheckGrayBits(tc)) { + return; + } + + // TODO: I'd like to AssertHeapIsIdle() here, but this ends up getting + // called during GC and while iterating the heap for memory reporting. + MOZ_ASSERT(!JS::RuntimeHeapIsCycleCollecting()); + + if (tc->zone()->isGCMarkingBlackAndGray()) { + // We are doing gray marking in the cell's zone. Even if the cell is + // currently marked gray it may eventually be marked black. Delay checking + // non-black cells until we finish gray marking. + + if (!tc->isMarkedBlack()) { + JSRuntime* rt = tc->zone()->runtimeFromMainThread(); + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!rt->gc.cellsToAssertNotGray.ref().append(cell)) { + oomUnsafe.crash("Can't append to delayed gray checks list"); + } + } + return; + } + + MOZ_ASSERT(!tc->isMarkedGray()); +} + +extern JS_PUBLIC_API bool js::gc::detail::ObjectIsMarkedBlack( + const JSObject* obj) { + return obj->isMarkedBlack(); +} + +#endif + +js::gc::ClearEdgesTracer::ClearEdgesTracer(JSRuntime* rt) + : GenericTracerImpl(rt, JS::TracerKind::ClearEdges, + JS::WeakMapTraceAction::TraceKeysAndValues) {} + +template +void js::gc::ClearEdgesTracer::onEdge(T** thingp, const char* name) { + // We don't handle removing pointers to nursery edges from the store buffer + // with this tracer. Check that this doesn't happen. + T* thing = *thingp; + MOZ_ASSERT(!IsInsideNursery(thing)); + + // Fire the pre-barrier since we're removing an edge from the graph. + InternalBarrierMethods::preBarrier(thing); + + *thingp = nullptr; +} + +void GCRuntime::setPerformanceHint(PerformanceHint hint) { + if (hint == PerformanceHint::InPageLoad) { + inPageLoadCount++; + } else { + MOZ_ASSERT(inPageLoadCount); + inPageLoadCount--; + } +} diff --git a/js/src/gc/GC.h b/js/src/gc/GC.h new file mode 100644 index 0000000000..58475bc74a --- /dev/null +++ b/js/src/gc/GC.h @@ -0,0 +1,243 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * JS engine garbage collector API. + */ + +#ifndef gc_GC_h +#define gc_GC_h + +#include "gc/GCEnum.h" +#include "js/GCAPI.h" +#include "js/HeapAPI.h" +#include "js/RealmIterators.h" +#include "js/TraceKind.h" + +class JSTracer; + +namespace JS { +class RealmOptions; +} + +namespace js { + +class Nursery; + +namespace gc { + +class Arena; +class TenuredChunk; + +} /* namespace gc */ + +// Define name, key and writability for the GC parameters. +#define FOR_EACH_GC_PARAM(_) \ + _("maxBytes", JSGC_MAX_BYTES, true) \ + _("minNurseryBytes", JSGC_MIN_NURSERY_BYTES, true) \ + _("maxNurseryBytes", JSGC_MAX_NURSERY_BYTES, true) \ + _("gcBytes", JSGC_BYTES, false) \ + _("nurseryBytes", JSGC_NURSERY_BYTES, false) \ + _("gcNumber", JSGC_NUMBER, false) \ + _("majorGCNumber", JSGC_MAJOR_GC_NUMBER, false) \ + _("minorGCNumber", JSGC_MINOR_GC_NUMBER, false) \ + _("incrementalGCEnabled", JSGC_INCREMENTAL_GC_ENABLED, true) \ + _("perZoneGCEnabled", JSGC_PER_ZONE_GC_ENABLED, true) \ + _("unusedChunks", JSGC_UNUSED_CHUNKS, false) \ + _("totalChunks", JSGC_TOTAL_CHUNKS, false) \ + _("sliceTimeBudgetMS", JSGC_SLICE_TIME_BUDGET_MS, true) \ + _("highFrequencyTimeLimit", JSGC_HIGH_FREQUENCY_TIME_LIMIT, true) \ + _("smallHeapSizeMax", JSGC_SMALL_HEAP_SIZE_MAX, true) \ + _("largeHeapSizeMin", JSGC_LARGE_HEAP_SIZE_MIN, true) \ + _("highFrequencySmallHeapGrowth", JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH, \ + true) \ + _("highFrequencyLargeHeapGrowth", JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH, \ + true) \ + _("lowFrequencyHeapGrowth", JSGC_LOW_FREQUENCY_HEAP_GROWTH, true) \ + _("balancedHeapLimitsEnabled", JSGC_BALANCED_HEAP_LIMITS_ENABLED, true) \ + _("heapGrowthFactor", JSGC_HEAP_GROWTH_FACTOR, true) \ + _("allocationThreshold", JSGC_ALLOCATION_THRESHOLD, true) \ + _("smallHeapIncrementalLimit", JSGC_SMALL_HEAP_INCREMENTAL_LIMIT, true) \ + _("largeHeapIncrementalLimit", JSGC_LARGE_HEAP_INCREMENTAL_LIMIT, true) \ + _("minEmptyChunkCount", JSGC_MIN_EMPTY_CHUNK_COUNT, true) \ + _("maxEmptyChunkCount", JSGC_MAX_EMPTY_CHUNK_COUNT, true) \ + _("compactingEnabled", JSGC_COMPACTING_ENABLED, true) \ + _("parallelMarkingEnabled", JSGC_PARALLEL_MARKING_ENABLED, true) \ + _("parallelMarkingThresholdKB", JSGC_PARALLEL_MARKING_THRESHOLD_KB, true) \ + _("minLastDitchGCPeriod", JSGC_MIN_LAST_DITCH_GC_PERIOD, true) \ + _("nurseryFreeThresholdForIdleCollection", \ + JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION, true) \ + _("nurseryFreeThresholdForIdleCollectionPercent", \ + JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_PERCENT, true) \ + _("nurseryTimeoutForIdleCollectionMS", \ + JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS, true) \ + _("pretenureThreshold", JSGC_PRETENURE_THRESHOLD, true) \ + _("zoneAllocDelayKB", JSGC_ZONE_ALLOC_DELAY_KB, true) \ + _("mallocThresholdBase", JSGC_MALLOC_THRESHOLD_BASE, true) \ + _("urgentThreshold", JSGC_URGENT_THRESHOLD_MB, true) \ + _("chunkBytes", JSGC_CHUNK_BYTES, false) \ + _("helperThreadRatio", JSGC_HELPER_THREAD_RATIO, true) \ + _("maxHelperThreads", JSGC_MAX_HELPER_THREADS, true) \ + _("helperThreadCount", JSGC_HELPER_THREAD_COUNT, false) \ + _("markingThreadCount", JSGC_MARKING_THREAD_COUNT, true) \ + _("systemPageSizeKB", JSGC_SYSTEM_PAGE_SIZE_KB, false) + +// Get the key and writability give a GC parameter name. +extern bool GetGCParameterInfo(const char* name, JSGCParamKey* keyOut, + bool* writableOut); + +extern void TraceRuntime(JSTracer* trc); + +// Trace roots but don't evict the nursery first; used from DumpHeap. +extern void TraceRuntimeWithoutEviction(JSTracer* trc); + +extern void ReleaseAllJITCode(JS::GCContext* gcx); + +extern void PrepareForDebugGC(JSRuntime* rt); + +/* Functions for managing cross compartment gray pointers. */ + +extern void NotifyGCNukeWrapper(JSContext* cx, JSObject* wrapper); + +extern unsigned NotifyGCPreSwap(JSObject* a, JSObject* b); + +extern void NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned preResult); + +using IterateChunkCallback = void (*)(JSRuntime*, void*, gc::TenuredChunk*, + const JS::AutoRequireNoGC&); +using IterateZoneCallback = void (*)(JSRuntime*, void*, JS::Zone*, + const JS::AutoRequireNoGC&); +using IterateArenaCallback = void (*)(JSRuntime*, void*, gc::Arena*, + JS::TraceKind, size_t, + const JS::AutoRequireNoGC&); +using IterateCellCallback = void (*)(JSRuntime*, void*, JS::GCCellPtr, size_t, + const JS::AutoRequireNoGC&); + +/* + * This function calls |zoneCallback| on every zone, |realmCallback| on + * every realm, |arenaCallback| on every in-use arena, and |cellCallback| + * on every in-use cell in the GC heap. + * + * Note that no read barrier is triggered on the cells passed to cellCallback, + * so no these pointers must not escape the callback. + */ +extern void IterateHeapUnbarriered(JSContext* cx, void* data, + IterateZoneCallback zoneCallback, + JS::IterateRealmCallback realmCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback); + +/* + * This function is like IterateHeapUnbarriered, but does it for a single zone. + */ +extern void IterateHeapUnbarrieredForZone( + JSContext* cx, JS::Zone* zone, void* data, IterateZoneCallback zoneCallback, + JS::IterateRealmCallback realmCallback, IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback); + +/* + * Invoke chunkCallback on every in-use chunk. + */ +extern void IterateChunks(JSContext* cx, void* data, + IterateChunkCallback chunkCallback); + +using IterateScriptCallback = void (*)(JSRuntime*, void*, BaseScript*, + const JS::AutoRequireNoGC&); + +/* + * Invoke scriptCallback on every in-use script for the given realm or for all + * realms if it is null. The scripts may or may not have bytecode. + */ +extern void IterateScripts(JSContext* cx, JS::Realm* realm, void* data, + IterateScriptCallback scriptCallback); + +JS::Realm* NewRealm(JSContext* cx, JSPrincipals* principals, + const JS::RealmOptions& options); + +namespace gc { + +void FinishGC(JSContext* cx, JS::GCReason = JS::GCReason::FINISH_GC); + +void WaitForBackgroundTasks(JSContext* cx); + +enum VerifierType { PreBarrierVerifier }; + +#ifdef JS_GC_ZEAL + +extern const char ZealModeHelpText[]; + +/* Check that write barriers have been used correctly. See gc/Verifier.cpp. */ +void VerifyBarriers(JSRuntime* rt, VerifierType type); + +void MaybeVerifyBarriers(JSContext* cx, bool always = false); + +void DumpArenaInfo(); + +#else + +static inline void VerifyBarriers(JSRuntime* rt, VerifierType type) {} + +static inline void MaybeVerifyBarriers(JSContext* cx, bool always = false) {} + +#endif + +/* + * Instances of this class prevent GC from happening while they are live. If an + * allocation causes a heap threshold to be exceeded, no GC will be performed + * and the allocation will succeed. Allocation may still fail for other reasons. + * + * Use of this class is highly discouraged, since without GC system memory can + * become exhausted and this can cause crashes at places where we can't handle + * allocation failure. + * + * Use of this is permissible in situations where it would be impossible (or at + * least very difficult) to tolerate GC and where only a fixed number of objects + * are allocated, such as: + * + * - error reporting + * - JIT bailout handling + * - brain transplants (JSObject::swap) + * - debugging utilities not exposed to the browser + * + * This works by updating the |JSContext::suppressGC| counter which is checked + * at the start of GC. + */ +class MOZ_RAII JS_HAZ_GC_SUPPRESSED AutoSuppressGC + : public JS::AutoRequireNoGC { + int32_t& suppressGC_; + + public: + explicit AutoSuppressGC(JSContext* cx); + + ~AutoSuppressGC() { suppressGC_--; } +}; + +const char* StateName(State state); + +} /* namespace gc */ + +/* Use this to avoid assertions when manipulating the wrapper map. */ +class MOZ_RAII AutoDisableProxyCheck { + public: +#ifdef DEBUG + AutoDisableProxyCheck(); + ~AutoDisableProxyCheck(); +#else + AutoDisableProxyCheck() {} +#endif +}; + +struct MOZ_RAII AutoDisableCompactingGC { + explicit AutoDisableCompactingGC(JSContext* cx); + ~AutoDisableCompactingGC(); + + private: + JSContext* cx; +}; + +} /* namespace js */ + +#endif /* gc_GC_h */ diff --git a/js/src/gc/GCAPI.cpp b/js/src/gc/GCAPI.cpp new file mode 100644 index 0000000000..7911baaf93 --- /dev/null +++ b/js/src/gc/GCAPI.cpp @@ -0,0 +1,798 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * API functions and methods used by the rest of SpiderMonkey and by embeddings. + */ + +#include "mozilla/TimeStamp.h" + +#include "jsapi.h" +#include "jsfriendapi.h" + +#include "gc/GC.h" +#include "gc/PublicIterators.h" +#include "jit/JitRealm.h" +#include "js/HeapAPI.h" +#include "js/Value.h" +#include "util/DifferentialTesting.h" +#include "vm/HelperThreads.h" +#include "vm/Realm.h" +#include "vm/Scope.h" + +#include "gc/Marking-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "vm/GeckoProfiler-inl.h" +#include "vm/JSContext-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::TimeStamp; + +extern JS_PUBLIC_API bool js::AddRawValueRoot(JSContext* cx, Value* vp, + const char* name) { + MOZ_ASSERT(vp); + MOZ_ASSERT(name); + bool ok = cx->runtime()->gc.addRoot(vp, name); + if (!ok) { + JS_ReportOutOfMemory(cx); + } + return ok; +} + +extern JS_PUBLIC_API void js::RemoveRawValueRoot(JSContext* cx, Value* vp) { + cx->runtime()->gc.removeRoot(vp); +} + +JS_PUBLIC_API JS::HeapState JS::RuntimeHeapState() { + return TlsContext.get()->runtime()->gc.heapState(); +} + +JS::AutoDisableGenerationalGC::AutoDisableGenerationalGC(JSContext* cx) + : cx(cx) { + if (!cx->generationalDisabled) { + cx->runtime()->gc.evictNursery(JS::GCReason::DISABLE_GENERATIONAL_GC); + cx->nursery().disable(); + } + ++cx->generationalDisabled; + MOZ_ASSERT(cx->nursery().isEmpty()); +} + +JS::AutoDisableGenerationalGC::~AutoDisableGenerationalGC() { + if (--cx->generationalDisabled == 0 && + cx->runtime()->gc.tunables.gcMaxNurseryBytes() > 0) { + cx->nursery().enable(); + } +} + +JS_PUBLIC_API bool JS::IsGenerationalGCEnabled(JSRuntime* rt) { + return !rt->mainContextFromOwnThread()->generationalDisabled; +} + +AutoDisableCompactingGC::AutoDisableCompactingGC(JSContext* cx) : cx(cx) { + ++cx->compactingDisabledCount; + if (cx->runtime()->gc.isIncrementalGCInProgress() && + cx->runtime()->gc.isCompactingGc()) { + FinishGC(cx); + } +} + +AutoDisableCompactingGC::~AutoDisableCompactingGC() { + MOZ_ASSERT(cx->compactingDisabledCount > 0); + --cx->compactingDisabledCount; +} + +#ifdef DEBUG + +/* Should only be called manually under gdb */ +void PreventGCDuringInteractiveDebug() { TlsContext.get()->suppressGC++; } + +#endif + +void js::ReleaseAllJITCode(JS::GCContext* gcx) { + js::CancelOffThreadIonCompile(gcx->runtime()); + + for (ZonesIter zone(gcx->runtime(), SkipAtoms); !zone.done(); zone.next()) { + zone->forceDiscardJitCode(gcx); + } + + for (RealmsIter realm(gcx->runtime()); !realm.done(); realm.next()) { + if (jit::JitRealm* jitRealm = realm->jitRealm()) { + jitRealm->discardStubs(); + } + } +} + +AutoSuppressGC::AutoSuppressGC(JSContext* cx) + : suppressGC_(cx->suppressGC.ref()) { + suppressGC_++; +} + +#ifdef DEBUG +AutoDisableProxyCheck::AutoDisableProxyCheck() { + TlsContext.get()->disableStrictProxyChecking(); +} + +AutoDisableProxyCheck::~AutoDisableProxyCheck() { + TlsContext.get()->enableStrictProxyChecking(); +} + +JS_PUBLIC_API void JS::AssertGCThingMustBeTenured(JSObject* obj) { + MOZ_ASSERT(obj->isTenured()); + MOZ_ASSERT(obj->getClass()->hasFinalize() && + !(obj->getClass()->flags & JSCLASS_SKIP_NURSERY_FINALIZE)); +} + +JS_PUBLIC_API void JS::AssertGCThingIsNotNurseryAllocable(Cell* cell) { + MOZ_ASSERT(cell); + MOZ_ASSERT(!cell->is() && !cell->is() && + !cell->is()); +} + +JS_PUBLIC_API void js::gc::AssertGCThingHasType(js::gc::Cell* cell, + JS::TraceKind kind) { + if (!cell) { + MOZ_ASSERT(kind == JS::TraceKind::Null); + return; + } + + MOZ_ASSERT(IsCellPointerValid(cell)); + MOZ_ASSERT(cell->getTraceKind() == kind); +} +#endif + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + +JS::AutoAssertNoGC::AutoAssertNoGC(JSContext* maybecx) { + if (maybecx) { + cx_ = maybecx; + } else if (TlsContext.initialized()) { + cx_ = TlsContext.get(); + } else { + cx_ = nullptr; + } + if (cx_) { + cx_->inUnsafeRegion++; + } +} + +JS::AutoAssertNoGC::~AutoAssertNoGC() { + if (cx_) { + MOZ_ASSERT(cx_->inUnsafeRegion > 0); + cx_->inUnsafeRegion--; + } +} + +#endif // MOZ_DIAGNOSTIC_ASSERT_ENABLED + +#ifdef DEBUG + +JS::AutoEnterCycleCollection::AutoEnterCycleCollection(JSRuntime* rt) + : runtime_(rt) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + runtime_->gc.heapState_ = HeapState::CycleCollecting; +} + +JS::AutoEnterCycleCollection::~AutoEnterCycleCollection() { + MOZ_ASSERT(JS::RuntimeHeapIsCycleCollecting()); + runtime_->gc.heapState_ = HeapState::Idle; +} + +JS::AutoAssertGCCallback::AutoAssertGCCallback() : AutoSuppressGCAnalysis() { + MOZ_ASSERT(JS::RuntimeHeapIsCollecting()); +} + +#endif // DEBUG + +JS_PUBLIC_API const char* JS::GCTraceKindToAscii(JS::TraceKind kind) { + switch (kind) { +#define MAP_NAME(name, _0, _1, _2) \ + case JS::TraceKind::name: \ + return "JS " #name; + JS_FOR_EACH_TRACEKIND(MAP_NAME); +#undef MAP_NAME + default: + return "Invalid"; + } +} + +JS_PUBLIC_API size_t JS::GCTraceKindSize(JS::TraceKind kind) { + switch (kind) { +#define MAP_SIZE(name, type, _0, _1) \ + case JS::TraceKind::name: \ + return sizeof(type); + JS_FOR_EACH_TRACEKIND(MAP_SIZE); +#undef MAP_SIZE + default: + return 0; + } +} + +JS::GCCellPtr::GCCellPtr(const Value& v) + : GCCellPtr(v.toGCThing(), v.traceKind()) {} + +JS::TraceKind JS::GCCellPtr::outOfLineKind() const { + MOZ_ASSERT((ptr & OutOfLineTraceKindMask) == OutOfLineTraceKindMask); + MOZ_ASSERT(asCell()->isTenured()); + return MapAllocToTraceKind(asCell()->asTenured().getAllocKind()); +} + +JS_PUBLIC_API void JS::PrepareZoneForGC(JSContext* cx, Zone* zone) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + // If we got the zone from a shared atom, we may have the wrong atoms zone + // here. + if (zone->isAtomsZone()) { + zone = cx->runtime()->atomsZone(); + } + + MOZ_ASSERT(cx->runtime()->gc.hasZone(zone)); + zone->scheduleGC(); +} + +JS_PUBLIC_API void JS::PrepareForFullGC(JSContext* cx) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + cx->runtime()->gc.fullGCRequested = true; + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + zone->scheduleGC(); + } +} + +JS_PUBLIC_API void JS::PrepareForIncrementalGC(JSContext* cx) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + if (!JS::IsIncrementalGCInProgress(cx)) { + return; + } + + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + if (zone->wasGCStarted()) { + zone->scheduleGC(); + } + } +} + +JS_PUBLIC_API bool JS::IsGCScheduled(JSContext* cx) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + if (zone->isGCScheduled()) { + return true; + } + } + + return false; +} + +JS_PUBLIC_API void JS::SkipZoneForGC(JSContext* cx, Zone* zone) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + MOZ_ASSERT(cx->runtime()->gc.hasZone(zone)); + + cx->runtime()->gc.fullGCRequested = false; + zone->unscheduleGC(); +} + +static inline void CheckGCOptions(JS::GCOptions options) { + MOZ_ASSERT(options == JS::GCOptions::Normal || + options == JS::GCOptions::Shrink || + options == JS::GCOptions::Shutdown); +} + +JS_PUBLIC_API void JS::NonIncrementalGC(JSContext* cx, JS::GCOptions options, + GCReason reason) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + CheckGCOptions(options); + + cx->runtime()->gc.gc(options, reason); + + MOZ_ASSERT(!IsIncrementalGCInProgress(cx)); +} + +JS_PUBLIC_API void JS::StartIncrementalGC(JSContext* cx, JS::GCOptions options, + GCReason reason, + const js::SliceBudget& budget) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + CheckGCOptions(options); + + cx->runtime()->gc.startGC(options, reason, budget); +} + +JS_PUBLIC_API void JS::IncrementalGCSlice(JSContext* cx, GCReason reason, + const js::SliceBudget& budget) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + cx->runtime()->gc.gcSlice(reason, budget); +} + +JS_PUBLIC_API bool JS::IncrementalGCHasForegroundWork(JSContext* cx) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + return cx->runtime()->gc.hasForegroundWork(); +} + +JS_PUBLIC_API void JS::FinishIncrementalGC(JSContext* cx, GCReason reason) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + cx->runtime()->gc.finishGC(reason); +} + +JS_PUBLIC_API void JS::AbortIncrementalGC(JSContext* cx) { + AssertHeapIsIdle(); + CHECK_THREAD(cx); + + if (IsIncrementalGCInProgress(cx)) { + cx->runtime()->gc.abortGC(); + } +} + +char16_t* JS::GCDescription::formatSliceMessage(JSContext* cx) const { + UniqueChars cstr = cx->runtime()->gc.stats().formatCompactSliceMessage(); + + size_t nchars = strlen(cstr.get()); + UniqueTwoByteChars out(js_pod_malloc(nchars + 1)); + if (!out) { + return nullptr; + } + out.get()[nchars] = 0; + + CopyAndInflateChars(out.get(), cstr.get(), nchars); + return out.release(); +} + +char16_t* JS::GCDescription::formatSummaryMessage(JSContext* cx) const { + UniqueChars cstr = cx->runtime()->gc.stats().formatCompactSummaryMessage(); + + size_t nchars = strlen(cstr.get()); + UniqueTwoByteChars out(js_pod_malloc(nchars + 1)); + if (!out) { + return nullptr; + } + out.get()[nchars] = 0; + + CopyAndInflateChars(out.get(), cstr.get(), nchars); + return out.release(); +} + +JS::dbg::GarbageCollectionEvent::Ptr JS::GCDescription::toGCEvent( + JSContext* cx) const { + return JS::dbg::GarbageCollectionEvent::Create( + cx->runtime(), cx->runtime()->gc.stats(), + cx->runtime()->gc.majorGCCount()); +} + +TimeStamp JS::GCDescription::startTime(JSContext* cx) const { + return cx->runtime()->gc.stats().start(); +} + +TimeStamp JS::GCDescription::endTime(JSContext* cx) const { + return cx->runtime()->gc.stats().end(); +} + +TimeStamp JS::GCDescription::lastSliceStart(JSContext* cx) const { + return cx->runtime()->gc.stats().slices().back().start; +} + +TimeStamp JS::GCDescription::lastSliceEnd(JSContext* cx) const { + return cx->runtime()->gc.stats().slices().back().end; +} + +JS::UniqueChars JS::GCDescription::sliceToJSONProfiler(JSContext* cx) const { + size_t slices = cx->runtime()->gc.stats().slices().length(); + MOZ_ASSERT(slices > 0); + return cx->runtime()->gc.stats().renderJsonSlice(slices - 1); +} + +JS::UniqueChars JS::GCDescription::formatJSONProfiler(JSContext* cx) const { + return cx->runtime()->gc.stats().renderJsonMessage(); +} + +JS_PUBLIC_API JS::UniqueChars JS::MinorGcToJSON(JSContext* cx) { + JSRuntime* rt = cx->runtime(); + return rt->gc.stats().renderNurseryJson(); +} + +JS_PUBLIC_API JS::GCSliceCallback JS::SetGCSliceCallback( + JSContext* cx, GCSliceCallback callback) { + return cx->runtime()->gc.setSliceCallback(callback); +} + +JS_PUBLIC_API JS::DoCycleCollectionCallback JS::SetDoCycleCollectionCallback( + JSContext* cx, JS::DoCycleCollectionCallback callback) { + return cx->runtime()->gc.setDoCycleCollectionCallback(callback); +} + +JS_PUBLIC_API JS::GCNurseryCollectionCallback +JS::SetGCNurseryCollectionCallback(JSContext* cx, + GCNurseryCollectionCallback callback) { + return cx->runtime()->gc.setNurseryCollectionCallback(callback); +} + +JS_PUBLIC_API void JS::SetLowMemoryState(JSContext* cx, bool newState) { + return cx->runtime()->gc.setLowMemoryState(newState); +} + +JS_PUBLIC_API void JS::DisableIncrementalGC(JSContext* cx) { + cx->runtime()->gc.disallowIncrementalGC(); +} + +JS_PUBLIC_API bool JS::IsIncrementalGCEnabled(JSContext* cx) { + GCRuntime& gc = cx->runtime()->gc; + return gc.isIncrementalGCEnabled() && gc.isIncrementalGCAllowed(); +} + +JS_PUBLIC_API bool JS::IsIncrementalGCInProgress(JSContext* cx) { + return cx->runtime()->gc.isIncrementalGCInProgress(); +} + +JS_PUBLIC_API bool JS::IsIncrementalGCInProgress(JSRuntime* rt) { + return rt->gc.isIncrementalGCInProgress() && + !rt->gc.isVerifyPreBarriersEnabled(); +} + +JS_PUBLIC_API bool JS::IsIncrementalBarrierNeeded(JSContext* cx) { + if (JS::RuntimeHeapIsBusy()) { + return false; + } + + auto state = cx->runtime()->gc.state(); + return state != gc::State::NotActive && state <= gc::State::Sweep; +} + +JS_PUBLIC_API void JS::IncrementalPreWriteBarrier(JSObject* obj) { + if (!obj) { + return; + } + + AutoGeckoProfilerEntry profilingStackFrame( + TlsContext.get(), "IncrementalPreWriteBarrier(JSObject*)", + JS::ProfilingCategoryPair::GCCC_Barrier); + PreWriteBarrier(obj); +} + +JS_PUBLIC_API void JS::IncrementalPreWriteBarrier(GCCellPtr thing) { + if (!thing) { + return; + } + + AutoGeckoProfilerEntry profilingStackFrame( + TlsContext.get(), "IncrementalPreWriteBarrier(GCCellPtr)", + JS::ProfilingCategoryPair::GCCC_Barrier); + CellPtrPreWriteBarrier(thing); +} + +JS_PUBLIC_API bool JS::WasIncrementalGC(JSRuntime* rt) { + return rt->gc.isIncrementalGc(); +} + +bool js::gc::CreateUniqueIdForNativeObject(NativeObject* nobj, uint64_t* uidp) { + JSRuntime* runtime = nobj->runtimeFromMainThread(); + *uidp = NextCellUniqueId(runtime); + JSContext* cx = runtime->mainContextFromOwnThread(); + return nobj->setUniqueId(cx, *uidp); +} + +bool js::gc::CreateUniqueIdForNonNativeObject(Cell* cell, + UniqueIdMap::AddPtr ptr, + uint64_t* uidp) { + // If the cell is in the nursery, hopefully unlikely, then we need to tell the + // nursery about it so that it can sweep the uid if the thing does not get + // tenured. + JSRuntime* runtime = cell->runtimeFromMainThread(); + if (IsInsideNursery(cell) && + !runtime->gc.nursery().addedUniqueIdToCell(cell)) { + return false; + } + + // Set a new uid on the cell. + *uidp = NextCellUniqueId(runtime); + return cell->zone()->uniqueIds().add(ptr, cell, *uidp); +} + +uint64_t js::gc::NextCellUniqueId(JSRuntime* rt) { + return rt->gc.nextCellUniqueId(); +} + +namespace js { + +static const struct GCParamInfo { + const char* name; + JSGCParamKey key; + bool writable; +} GCParameters[] = { +#define DEFINE_PARAM_INFO(name, key, writable) {name, key, writable}, + FOR_EACH_GC_PARAM(DEFINE_PARAM_INFO) +#undef DEFINE_PARAM_INFO +}; + +bool GetGCParameterInfo(const char* name, JSGCParamKey* keyOut, + bool* writableOut) { + MOZ_ASSERT(keyOut); + MOZ_ASSERT(writableOut); + + for (const GCParamInfo& info : GCParameters) { + if (strcmp(name, info.name) == 0) { + *keyOut = info.key; + *writableOut = info.writable; + return true; + } + } + + return false; +} + +namespace gc { +namespace MemInfo { + +static bool GCBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.heapSize.bytes())); + return true; +} + +static bool MallocBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + double bytes = 0; + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + bytes += zone->mallocHeapSize.bytes(); + } + args.rval().setNumber(bytes); + return true; +} + +static bool GCMaxBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.tunables.gcMaxBytes())); + return true; +} + +static bool GCHighFreqGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setBoolean( + cx->runtime()->gc.schedulingState.inHighFrequencyGCMode()); + return true; +} + +static bool GCNumberGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.gcNumber())); + return true; +} + +static bool MajorGCCountGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.majorGCCount())); + return true; +} + +static bool MinorGCCountGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.minorGCCount())); + return true; +} + +static bool GCSliceCountGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.gcSliceCount())); + return true; +} + +static bool GCCompartmentCount(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + size_t count = 0; + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + count += zone->compartments().length(); + } + + args.rval().setNumber(double(count)); + return true; +} + +static bool GCLastStartReason(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + const char* reason = ExplainGCReason(cx->runtime()->gc.lastStartReason()); + RootedString str(cx, JS_NewStringCopyZ(cx, reason)); + if (!str) { + return false; + } + + args.rval().setString(str); + return true; +} + +static bool ZoneGCBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->zone()->gcHeapSize.bytes())); + return true; +} + +static bool ZoneGCTriggerBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->zone()->gcHeapThreshold.startBytes())); + return true; +} + +static bool ZoneGCAllocTriggerGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + bool highFrequency = + cx->runtime()->gc.schedulingState.inHighFrequencyGCMode(); + args.rval().setNumber( + double(cx->zone()->gcHeapThreshold.eagerAllocTrigger(highFrequency))); + return true; +} + +static bool ZoneMallocBytesGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->zone()->mallocHeapSize.bytes())); + return true; +} + +static bool ZoneMallocTriggerBytesGetter(JSContext* cx, unsigned argc, + Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->zone()->mallocHeapThreshold.startBytes())); + return true; +} + +static bool ZoneGCNumberGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setNumber(double(cx->runtime()->gc.gcNumber())); + return true; +} + +#ifdef DEBUG +static bool DummyGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setUndefined(); + return true; +} +#endif + +} /* namespace MemInfo */ + +JSObject* NewMemoryInfoObject(JSContext* cx) { + RootedObject obj(cx, JS_NewObject(cx, nullptr)); + if (!obj) { + return nullptr; + } + + using namespace MemInfo; + struct NamedGetter { + const char* name; + JSNative getter; + } getters[] = {{"gcBytes", GCBytesGetter}, + {"gcMaxBytes", GCMaxBytesGetter}, + {"mallocBytes", MallocBytesGetter}, + {"gcIsHighFrequencyMode", GCHighFreqGetter}, + {"gcNumber", GCNumberGetter}, + {"majorGCCount", MajorGCCountGetter}, + {"minorGCCount", MinorGCCountGetter}, + {"sliceCount", GCSliceCountGetter}, + {"compartmentCount", GCCompartmentCount}, + {"lastStartReason", GCLastStartReason}}; + + for (auto pair : getters) { + JSNative getter = pair.getter; + +#ifdef DEBUG + if (js::SupportDifferentialTesting()) { + getter = DummyGetter; + } +#endif + + if (!JS_DefineProperty(cx, obj, pair.name, getter, nullptr, + JSPROP_ENUMERATE)) { + return nullptr; + } + } + + RootedObject zoneObj(cx, JS_NewObject(cx, nullptr)); + if (!zoneObj) { + return nullptr; + } + + if (!JS_DefineProperty(cx, obj, "zone", zoneObj, JSPROP_ENUMERATE)) { + return nullptr; + } + + struct NamedZoneGetter { + const char* name; + JSNative getter; + } zoneGetters[] = {{"gcBytes", ZoneGCBytesGetter}, + {"gcTriggerBytes", ZoneGCTriggerBytesGetter}, + {"gcAllocTrigger", ZoneGCAllocTriggerGetter}, + {"mallocBytes", ZoneMallocBytesGetter}, + {"mallocTriggerBytes", ZoneMallocTriggerBytesGetter}, + {"gcNumber", ZoneGCNumberGetter}}; + + for (auto pair : zoneGetters) { + JSNative getter = pair.getter; + +#ifdef DEBUG + if (js::SupportDifferentialTesting()) { + getter = DummyGetter; + } +#endif + + if (!JS_DefineProperty(cx, zoneObj, pair.name, getter, nullptr, + JSPROP_ENUMERATE)) { + return nullptr; + } + } + + return obj; +} + +const char* StateName(State state) { + switch (state) { +#define MAKE_CASE(name) \ + case State::name: \ + return #name; + GCSTATES(MAKE_CASE) +#undef MAKE_CASE + } + MOZ_CRASH("Invalid gc::State enum value"); +} + +const char* StateName(JS::Zone::GCState state) { + switch (state) { + case JS::Zone::NoGC: + return "NoGC"; + case JS::Zone::Prepare: + return "Prepare"; + case JS::Zone::MarkBlackOnly: + return "MarkBlackOnly"; + case JS::Zone::MarkBlackAndGray: + return "MarkBlackAndGray"; + case JS::Zone::Sweep: + return "Sweep"; + case JS::Zone::Finished: + return "Finished"; + case JS::Zone::Compact: + return "Compact"; + case JS::Zone::VerifyPreBarriers: + return "VerifyPreBarriers"; + case JS::Zone::Limit: + break; + } + MOZ_CRASH("Invalid Zone::GCState enum value"); +} + +} /* namespace gc */ +} /* namespace js */ + +JS_PUBLIC_API void js::gc::FinalizeDeadNurseryObject(JSContext* cx, + JSObject* obj) { + CHECK_THREAD(cx); + MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting()); + + MOZ_ASSERT(obj); + MOZ_ASSERT(IsInsideNursery(obj)); + MOZ_ASSERT(!IsForwarded(obj)); + + const JSClass* jsClass = JS::GetClass(obj); + jsClass->doFinalize(cx->gcContext(), obj); +} + +JS_PUBLIC_API void js::gc::SetPerformanceHint(JSContext* cx, + PerformanceHint hint) { + CHECK_THREAD(cx); + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + + cx->runtime()->gc.setPerformanceHint(hint); +} diff --git a/js/src/gc/GCContext-inl.h b/js/src/gc/GCContext-inl.h new file mode 100644 index 0000000000..7b53d50b77 --- /dev/null +++ b/js/src/gc/GCContext-inl.h @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCContext_inl_h +#define gc_GCContext_inl_h + +#include "gc/GCContext.h" + +#include "gc/ZoneAllocator.h" + +inline void JS::GCContext::free_(Cell* cell, void* p, size_t nbytes, + MemoryUse use) { + if (p) { + removeCellMemory(cell, nbytes, use); + js_free(p); + } +} + +template +inline void JS::GCContext::release(Cell* cell, T* p, size_t nbytes, + MemoryUse use) { + if (p) { + removeCellMemory(cell, nbytes, use); + p->Release(); + } +} + +inline void JS::GCContext::removeCellMemory(Cell* cell, size_t nbytes, + MemoryUse use) { + // This may or may not be called as part of GC. + if (nbytes && cell->isTenured()) { + auto zone = js::ZoneAllocator::from(cell->asTenured().zoneFromAnyThread()); + zone->removeCellMemory(cell, nbytes, use, isFinalizing()); + } +} + +#endif // gc_GCContext_inl_h diff --git a/js/src/gc/GCContext.h b/js/src/gc/GCContext.h new file mode 100644 index 0000000000..8c74a81e0c --- /dev/null +++ b/js/src/gc/GCContext.h @@ -0,0 +1,257 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCContext_h +#define gc_GCContext_h + +#include "mozilla/Assertions.h" // MOZ_ASSERT +#include "mozilla/ThreadLocal.h" + +#include "jspubtd.h" +#include "jstypes.h" // JS_PUBLIC_API +#include "gc/GCEnum.h" // js::MemoryUse +#include "jit/ExecutableAllocator.h" // jit::JitPoisonRangeVector +#include "js/Utility.h" // js_free + +struct JS_PUBLIC_API JSRuntime; + +namespace js { + +class AutoTouchingGrayThings; + +namespace gc { + +class AutoSetThreadGCUse; +class AutoSetThreadIsSweeping; + +enum class GCUse { + // This thread is not running in the garbage collector. + None, + + // This thread is currently collecting. Used when no finer detail is known. + Unspecified, + + // This thread is currently marking GC things. This thread could be the main + // thread or a helper thread doing sweep-marking. + Marking, + + // This thread is currently sweeping GC things. This thread could be the + // main thread or a helper thread while the main thread is running the + // mutator. + Sweeping, + + // Whether this thread is currently finalizing GC things. This thread could + // be the main thread or a helper thread doing finalization while the main + // thread is running the mutator. + Finalizing +}; + +} // namespace gc +} // namespace js + +namespace JS { + +/* + * GCContext is by GC operations that can run on or off the main thread. + * + * Its main function is to provide methods to free memory and update memory + * accounting. For convenience, it also has delete_ convenience methods that + * also call destructors. + * + * It is passed to finalizers and other sweep-phase hooks as JSContext is not + * available off the main thread. + */ +class GCContext { + using Cell = js::gc::Cell; + using MemoryUse = js::MemoryUse; + + JSRuntime* const runtime_; + + js::jit::JitPoisonRangeVector jitPoisonRanges; + + // Which part of the garbage collector this context is running at the moment. + js::gc::GCUse gcUse_ = js::gc::GCUse::None; + friend class js::gc::AutoSetThreadGCUse; + friend class js::gc::AutoSetThreadIsSweeping; + +#ifdef DEBUG + // The specific zone currently being swept, if any. + Zone* gcSweepZone_ = nullptr; + + // Whether this thread is currently manipulating possibly-gray GC things. + size_t isTouchingGrayThings_ = false; + friend class js::AutoTouchingGrayThings; +#endif + + public: + explicit GCContext(JSRuntime* maybeRuntime); + ~GCContext(); + + JSRuntime* runtime() const { + MOZ_ASSERT(onMainThread()); + return runtimeFromAnyThread(); + } + JSRuntime* runtimeFromAnyThread() const { + MOZ_ASSERT(runtime_); + return runtime_; + } + + js::gc::GCUse gcUse() const { return gcUse_; } + bool isCollecting() const { return gcUse() != js::gc::GCUse::None; } + bool isFinalizing() const { return gcUse_ == js::gc::GCUse::Finalizing; } + +#ifdef DEBUG + bool onMainThread() const { + return js::CurrentThreadCanAccessRuntime(runtime_); + } + + Zone* gcSweepZone() const { return gcSweepZone_; } + bool isTouchingGrayThings() const { return isTouchingGrayThings_; } +#endif + + // Deprecated. Where possible, memory should be tracked against the owning GC + // thing by calling js::AddCellMemory and the memory freed with free_() below. + void freeUntracked(void* p) { js_free(p); } + + // Free memory associated with a GC thing and update the memory accounting. + // + // The memory should have been associated with the GC thing using + // js::InitReservedSlot or js::InitObjectPrivate, or possibly + // js::AddCellMemory. + void free_(Cell* cell, void* p, size_t nbytes, MemoryUse use); + + bool appendJitPoisonRange(const js::jit::JitPoisonRange& range) { + return jitPoisonRanges.append(range); + } + bool hasJitCodeToPoison() const { return !jitPoisonRanges.empty(); } + void poisonJitCode(); + + // Deprecated. Where possible, memory should be tracked against the owning GC + // thing by calling js::AddCellMemory and the memory freed with delete_() + // below. + template + void deleteUntracked(T* p) { + if (p) { + p->~T(); + js_free(p); + } + } + + // Delete a C++ object that was associated with a GC thing and update the + // memory accounting. The size is determined by the type T. + // + // The memory should have been associated with the GC thing using + // js::InitReservedSlot or js::InitObjectPrivate, or possibly + // js::AddCellMemory. + template + void delete_(Cell* cell, T* p, MemoryUse use) { + delete_(cell, p, sizeof(T), use); + } + + // Delete a C++ object that was associated with a GC thing and update the + // memory accounting. + // + // The memory should have been associated with the GC thing using + // js::InitReservedSlot or js::InitObjectPrivate, or possibly + // js::AddCellMemory. + template + void delete_(Cell* cell, T* p, size_t nbytes, MemoryUse use) { + if (p) { + p->~T(); + free_(cell, p, nbytes, use); + } + } + + // Release a RefCounted object that was associated with a GC thing and update + // the memory accounting. + // + // The memory should have been associated with the GC thing using + // js::InitReservedSlot or js::InitObjectPrivate, or possibly + // js::AddCellMemory. + // + // This counts the memory once per association with a GC thing. It's not + // expected that the same object is associated with more than one GC thing in + // each zone. If this is the case then some other form of accounting would be + // more appropriate. + template + void release(Cell* cell, T* p, MemoryUse use) { + release(cell, p, sizeof(T), use); + } + + // Release a RefCounted object and that was associated with a GC thing and + // update the memory accounting. + // + // The memory should have been associated with the GC thing using + // js::InitReservedSlot or js::InitObjectPrivate, or possibly + // js::AddCellMemory. + template + void release(Cell* cell, T* p, size_t nbytes, MemoryUse use); + + // Update the memory accounting for a GC for memory freed by some other + // method. + void removeCellMemory(Cell* cell, size_t nbytes, MemoryUse use); +}; + +} // namespace JS + +namespace js { + +/* Thread Local Storage for storing the GCContext for a thread. */ +extern MOZ_THREAD_LOCAL(JS::GCContext*) TlsGCContext; + +inline JS::GCContext* MaybeGetGCContext() { + if (!TlsGCContext.init()) { + return nullptr; + } + return TlsGCContext.get(); +} + +class MOZ_RAII AutoTouchingGrayThings { + public: +#ifdef DEBUG + AutoTouchingGrayThings() { TlsGCContext.get()->isTouchingGrayThings_++; } + ~AutoTouchingGrayThings() { + JS::GCContext* gcx = TlsGCContext.get(); + MOZ_ASSERT(gcx->isTouchingGrayThings_); + gcx->isTouchingGrayThings_--; + } +#else + AutoTouchingGrayThings() {} +#endif +}; + +#ifdef DEBUG + +inline bool CurrentThreadIsGCMarking() { + JS::GCContext* gcx = MaybeGetGCContext(); + return gcx && gcx->gcUse() == gc::GCUse::Marking; +} + +inline bool CurrentThreadIsGCSweeping() { + JS::GCContext* gcx = MaybeGetGCContext(); + return gcx && gcx->gcUse() == gc::GCUse::Sweeping; +} + +inline bool CurrentThreadIsGCFinalizing() { + JS::GCContext* gcx = MaybeGetGCContext(); + return gcx && gcx->gcUse() == gc::GCUse::Finalizing; +} + +inline bool CurrentThreadIsTouchingGrayThings() { + JS::GCContext* gcx = MaybeGetGCContext(); + return gcx && gcx->isTouchingGrayThings(); +} + +inline bool CurrentThreadIsPerformingGC() { + JS::GCContext* gcx = MaybeGetGCContext(); + return gcx && gcx->isCollecting(); +} + +#endif + +} // namespace js + +#endif // gc_GCContext_h diff --git a/js/src/gc/GCEnum.h b/js/src/gc/GCEnum.h new file mode 100644 index 0000000000..7434e34b1c --- /dev/null +++ b/js/src/gc/GCEnum.h @@ -0,0 +1,160 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal enum definitions. + */ + +#ifndef gc_GCEnum_h +#define gc_GCEnum_h + +#include + +#include "js/MemoryFunctions.h" // JS_FOR_EACH_PUBLIC_MEMORY_USE + +namespace js { +namespace gc { + +// The phases of an incremental GC. +#define GCSTATES(D) \ + D(NotActive) \ + D(Prepare) \ + D(MarkRoots) \ + D(Mark) \ + D(Sweep) \ + D(Finalize) \ + D(Compact) \ + D(Decommit) \ + D(Finish) +enum class State { +#define MAKE_STATE(name) name, + GCSTATES(MAKE_STATE) +#undef MAKE_STATE +}; + +#define JS_FOR_EACH_ZEAL_MODE(D) \ + D(RootsChange, 1) \ + D(Alloc, 2) \ + D(VerifierPre, 4) \ + D(YieldBeforeRootMarking, 6) \ + D(GenerationalGC, 7) \ + D(YieldBeforeMarking, 8) \ + D(YieldBeforeSweeping, 9) \ + D(IncrementalMultipleSlices, 10) \ + D(IncrementalMarkingValidator, 11) \ + D(ElementsBarrier, 12) \ + D(CheckHashTablesOnMinorGC, 13) \ + D(Compact, 14) \ + D(CheckHeapAfterGC, 15) \ + D(YieldBeforeSweepingAtoms, 17) \ + D(CheckGrayMarking, 18) \ + D(YieldBeforeSweepingCaches, 19) \ + D(YieldBeforeSweepingObjects, 21) \ + D(YieldBeforeSweepingNonObjects, 22) \ + D(YieldBeforeSweepingPropMapTrees, 23) \ + D(CheckWeakMapMarking, 24) \ + D(YieldWhileGrayMarking, 25) + +enum class ZealMode { +#define ZEAL_MODE(name, value) name = value, + JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE) +#undef ZEAL_MODE + Count, + Limit = Count - 1 +}; + +} /* namespace gc */ + +// Reasons we reset an ongoing incremental GC or perform a non-incremental GC. +#define GC_ABORT_REASONS(D) \ + D(None, 0) \ + D(NonIncrementalRequested, 1) \ + D(AbortRequested, 2) \ + D(Unused1, 3) \ + D(IncrementalDisabled, 4) \ + D(ModeChange, 5) \ + D(MallocBytesTrigger, 6) \ + D(GCBytesTrigger, 7) \ + D(ZoneChange, 8) \ + D(CompartmentRevived, 9) \ + D(GrayRootBufferingFailed, 10) \ + D(JitCodeBytesTrigger, 11) +enum class GCAbortReason { +#define MAKE_REASON(name, num) name = num, + GC_ABORT_REASONS(MAKE_REASON) +#undef MAKE_REASON +}; + +#define JS_FOR_EACH_INTERNAL_MEMORY_USE(_) \ + _(ArrayBufferContents) \ + _(StringContents) \ + _(ObjectElements) \ + _(ObjectSlots) \ + _(ScriptPrivateData) \ + _(MapObjectTable) \ + _(BigIntDigits) \ + _(ScopeData) \ + _(WeakMapObject) \ + _(ShapeSetForAdd) \ + _(PropMapChildren) \ + _(PropMapTable) \ + _(ModuleBindingMap) \ + _(ModuleCyclicFields) \ + _(ModuleExports) \ + _(BaselineScript) \ + _(IonScript) \ + _(ArgumentsData) \ + _(RareArgumentsData) \ + _(RegExpSharedBytecode) \ + _(RegExpSharedNamedCaptureData) \ + _(TypedArrayElements) \ + _(NativeIterator) \ + _(JitScript) \ + _(ScriptDebugScript) \ + _(BreakpointSite) \ + _(Breakpoint) \ + _(ForOfPIC) \ + _(ForOfPICStub) \ + _(WasmInstanceExports) \ + _(WasmInstanceScopes) \ + _(WasmInstanceGlobals) \ + _(WasmInstanceInstance) \ + _(WasmMemoryObservers) \ + _(WasmGlobalCell) \ + _(WasmResolveResponseClosure) \ + _(WasmModule) \ + _(WasmTableTable) \ + _(WasmExceptionData) \ + _(WasmTagType) \ + _(FileObjectFile) \ + _(Debugger) \ + _(DebuggerFrameGeneratorInfo) \ + _(DebuggerFrameIterData) \ + _(DebuggerOnStepHandler) \ + _(DebuggerOnPopHandler) \ + _(ICUObject) \ + _(FinalizationRegistryRecordVector) \ + _(FinalizationRegistryRegistrations) \ + _(FinalizationRecordVector) \ + _(TrackedAllocPolicy) \ + _(SharedArrayRawBuffer) \ + _(XDRBufferElements) \ + _(GlobalObjectData) \ + _(ProxyExternalValueArray) + +#define JS_FOR_EACH_MEMORY_USE(_) \ + JS_FOR_EACH_PUBLIC_MEMORY_USE(_) \ + JS_FOR_EACH_INTERNAL_MEMORY_USE(_) + +enum class MemoryUse : uint8_t { +#define DEFINE_MEMORY_USE(Name) Name, + JS_FOR_EACH_MEMORY_USE(DEFINE_MEMORY_USE) +#undef DEFINE_MEMORY_USE +}; + +} /* namespace js */ + +#endif /* gc_GCEnum_h */ diff --git a/js/src/gc/GCInternals.h b/js/src/gc/GCInternals.h new file mode 100644 index 0000000000..c234ad4b2b --- /dev/null +++ b/js/src/gc/GCInternals.h @@ -0,0 +1,344 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definitions. + */ + +#ifndef gc_GCInternals_h +#define gc_GCInternals_h + +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "gc/Cell.h" +#include "gc/GC.h" +#include "gc/GCContext.h" +#include "vm/GeckoProfiler.h" +#include "vm/HelperThreads.h" +#include "vm/JSContext.h" + +namespace js { +namespace gc { + +/* + * There are a couple of classes here that serve mostly as "tokens" indicating + * that a precondition holds. Some functions force the caller to possess such a + * token because they require the precondition to hold, and it is better to make + * the precondition explicit at the API entry point than to crash in an + * assertion later on when it is relied upon. + */ + +struct MOZ_RAII AutoAssertNoNurseryAlloc { +#ifdef DEBUG + AutoAssertNoNurseryAlloc(); + ~AutoAssertNoNurseryAlloc(); +#else + AutoAssertNoNurseryAlloc() {} +#endif +}; + +/* + * A class that serves as a token that the nursery in the current thread's zone + * group is empty. + */ +class MOZ_RAII AutoAssertEmptyNursery { + protected: + JSContext* cx; + + mozilla::Maybe noAlloc; + + // Check that the nursery is empty. + void checkCondition(JSContext* cx); + + // For subclasses that need to empty the nursery in their constructors. + AutoAssertEmptyNursery() : cx(nullptr) {} + + public: + explicit AutoAssertEmptyNursery(JSContext* cx) : cx(nullptr) { + checkCondition(cx); + } + + AutoAssertEmptyNursery(const AutoAssertEmptyNursery& other) + : AutoAssertEmptyNursery(other.cx) {} +}; + +/* + * Evict the nursery upon construction. Serves as a token indicating that the + * nursery is empty. (See AutoAssertEmptyNursery, above.) + */ +class MOZ_RAII AutoEmptyNursery : public AutoAssertEmptyNursery { + public: + explicit AutoEmptyNursery(JSContext* cx); +}; + +// Abstract base class for exclusive heap access for tracing or GC. +class MOZ_RAII AutoHeapSession { + public: + ~AutoHeapSession(); + + protected: + AutoHeapSession(GCRuntime* gc, JS::HeapState state); + + private: + AutoHeapSession(const AutoHeapSession&) = delete; + void operator=(const AutoHeapSession&) = delete; + + GCRuntime* gc; + JS::HeapState prevState; + mozilla::Maybe profilingStackFrame; +}; + +class MOZ_RAII AutoGCSession : public AutoHeapSession { + public: + explicit AutoGCSession(GCRuntime* gc, JS::HeapState state) + : AutoHeapSession(gc, state) {} +}; + +class MOZ_RAII AutoMajorGCProfilerEntry : public AutoGeckoProfilerEntry { + public: + explicit AutoMajorGCProfilerEntry(GCRuntime* gc); +}; + +class MOZ_RAII AutoTraceSession : public AutoHeapSession { + public: + explicit AutoTraceSession(JSRuntime* rt) + : AutoHeapSession(&rt->gc, JS::HeapState::Tracing) {} +}; + +struct MOZ_RAII AutoFinishGC { + explicit AutoFinishGC(JSContext* cx, JS::GCReason reason) { + FinishGC(cx, reason); + } +}; + +// This class should be used by any code that needs exclusive access to the heap +// in order to trace through it. +class MOZ_RAII AutoPrepareForTracing : private AutoFinishGC, + public AutoTraceSession { + public: + explicit AutoPrepareForTracing(JSContext* cx) + : AutoFinishGC(cx, JS::GCReason::PREPARE_FOR_TRACING), + AutoTraceSession(cx->runtime()) {} +}; + +// This class should be used by any code that needs exclusive access to the heap +// in order to trace through it. +// +// This version also empties the nursery after finishing any ongoing GC. +class MOZ_RAII AutoEmptyNurseryAndPrepareForTracing : private AutoFinishGC, + public AutoEmptyNursery, + public AutoTraceSession { + public: + explicit AutoEmptyNurseryAndPrepareForTracing(JSContext* cx) + : AutoFinishGC(cx, JS::GCReason::PREPARE_FOR_TRACING), + AutoEmptyNursery(cx), + AutoTraceSession(cx->runtime()) {} +}; + +/* + * Temporarily disable incremental barriers. + */ +class AutoDisableBarriers { + public: + explicit AutoDisableBarriers(GCRuntime* gc); + ~AutoDisableBarriers(); + + private: + GCRuntime* gc; +}; + +// Set compartments' maybeAlive flags if anything is marked while this class is +// live. This is used while marking roots. +class AutoUpdateLiveCompartments { + GCRuntime* gc; + + public: + explicit AutoUpdateLiveCompartments(GCRuntime* gc); + ~AutoUpdateLiveCompartments(); +}; + +class MOZ_RAII AutoRunParallelTask : public GCParallelTask { + // This class takes a pointer to a member function of GCRuntime. + using TaskFunc = JS_MEMBER_FN_PTR_TYPE(GCRuntime, void); + + TaskFunc func_; + AutoLockHelperThreadState& lock_; + + public: + AutoRunParallelTask(GCRuntime* gc, TaskFunc func, gcstats::PhaseKind phase, + GCUse use, AutoLockHelperThreadState& lock) + : GCParallelTask(gc, phase, use), func_(func), lock_(lock) { + gc->startTask(*this, lock_); + } + + ~AutoRunParallelTask() { gc->joinTask(*this, lock_); } + + void run(AutoLockHelperThreadState& lock) override { + AutoUnlockHelperThreadState unlock(lock); + + // The hazard analysis can't tell what the call to func_ will do but it's + // not allowed to GC. + JS::AutoSuppressGCAnalysis nogc; + + // Call pointer to member function on |gc|. + JS_CALL_MEMBER_FN_PTR(gc, func_); + } +}; + +GCAbortReason IsIncrementalGCUnsafe(JSRuntime* rt); + +#ifdef JS_GC_ZEAL + +class MOZ_RAII AutoStopVerifyingBarriers { + GCRuntime* gc; + bool restartPreVerifier; + + public: + AutoStopVerifyingBarriers(JSRuntime* rt, bool isShutdown) : gc(&rt->gc) { + if (gc->isVerifyPreBarriersEnabled()) { + gc->endVerifyPreBarriers(); + restartPreVerifier = !isShutdown; + } else { + restartPreVerifier = false; + } + } + + ~AutoStopVerifyingBarriers() { + // Nasty special case: verification runs a minor GC, which *may* nest + // inside of an outer minor GC. This is not allowed by the + // gc::Statistics phase tree. So we pause the "real" GC, if in fact one + // is in progress. + gcstats::PhaseKind outer = gc->stats().currentPhaseKind(); + if (outer != gcstats::PhaseKind::NONE) { + gc->stats().endPhase(outer); + } + MOZ_ASSERT(gc->stats().currentPhaseKind() == gcstats::PhaseKind::NONE); + + if (restartPreVerifier) { + gc->startVerifyPreBarriers(); + } + + if (outer != gcstats::PhaseKind::NONE) { + gc->stats().beginPhase(outer); + } + } +}; +#else +struct MOZ_RAII AutoStopVerifyingBarriers { + AutoStopVerifyingBarriers(JSRuntime*, bool) {} +}; +#endif /* JS_GC_ZEAL */ + +class MOZ_RAII AutoPoisonFreedJitCode { + JS::GCContext* const gcx; + + public: + explicit AutoPoisonFreedJitCode(JS::GCContext* gcx) : gcx(gcx) {} + ~AutoPoisonFreedJitCode() { gcx->poisonJitCode(); } +}; + +// Set/restore the GCContext GC use flag for the current thread. + +class MOZ_RAII AutoSetThreadGCUse { + public: + AutoSetThreadGCUse(JS::GCContext* gcx, GCUse use) + : gcx(gcx), prevUse(gcx->gcUse_) { + gcx->gcUse_ = use; + } + explicit AutoSetThreadGCUse(GCUse use) + : AutoSetThreadGCUse(TlsGCContext.get(), use) {} + + ~AutoSetThreadGCUse() { gcx->gcUse_ = prevUse; } + + protected: + JS::GCContext* gcx; + GCUse prevUse; +}; + +template +class AutoSetThreadGCUseT : public AutoSetThreadGCUse { + public: + explicit AutoSetThreadGCUseT(JS::GCContext* gcx) + : AutoSetThreadGCUse(gcx, Use) {} + AutoSetThreadGCUseT() : AutoSetThreadGCUseT(TlsGCContext.get()) {} +}; + +using AutoSetThreadIsPerformingGC = AutoSetThreadGCUseT; +using AutoSetThreadIsMarking = AutoSetThreadGCUseT; +using AutoSetThreadIsFinalizing = AutoSetThreadGCUseT; + +class AutoSetThreadIsSweeping : public AutoSetThreadGCUseT { + public: + explicit AutoSetThreadIsSweeping(JS::GCContext* gcx, + JS::Zone* sweepZone = nullptr) + : AutoSetThreadGCUseT(gcx) { +#ifdef DEBUG + prevZone = gcx->gcSweepZone_; + gcx->gcSweepZone_ = sweepZone; +#endif + } + explicit AutoSetThreadIsSweeping(JS::Zone* sweepZone = nullptr) + : AutoSetThreadIsSweeping(TlsGCContext.get(), sweepZone) {} + + ~AutoSetThreadIsSweeping() { +#ifdef DEBUG + MOZ_ASSERT_IF(prevUse == GCUse::None, !prevZone); + gcx->gcSweepZone_ = prevZone; +#endif + } + + private: +#ifdef DEBUG + JS::Zone* prevZone; +#endif +}; + +#ifdef JSGC_HASH_TABLE_CHECKS +void CheckHashTablesAfterMovingGC(JSRuntime* rt); +void CheckHeapAfterGC(JSRuntime* rt); +#endif + +struct MovingTracer final : public GenericTracerImpl { + explicit MovingTracer(JSRuntime* rt); + + private: + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl; +}; + +struct MinorSweepingTracer final + : public GenericTracerImpl { + explicit MinorSweepingTracer(JSRuntime* rt); + + private: + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl; +}; + +extern void DelayCrossCompartmentGrayMarking(GCMarker* maybeMarker, + JSObject* src); + +inline bool IsOOMReason(JS::GCReason reason) { + return reason == JS::GCReason::LAST_DITCH || + reason == JS::GCReason::MEM_PRESSURE; +} + +void* AllocateCellInGC(JS::Zone* zone, AllocKind thingKind); + +void ReadProfileEnv(const char* envName, const char* helpText, bool* enableOut, + bool* workersOut, mozilla::TimeDuration* thresholdOut); + +bool ShouldPrintProfile(JSRuntime* runtime, bool enable, bool workers, + mozilla::TimeDuration threshold, + mozilla::TimeDuration duration); + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_GCInternals_h */ diff --git a/js/src/gc/GCLock.h b/js/src/gc/GCLock.h new file mode 100644 index 0000000000..64c28ac544 --- /dev/null +++ b/js/src/gc/GCLock.h @@ -0,0 +1,110 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal classes for acquiring and releasing the GC lock. + */ + +#ifndef gc_GCLock_h +#define gc_GCLock_h + +#include "vm/Runtime.h" + +namespace js { + +class AutoUnlockGC; + +/* + * RAII class that takes the GC lock while it is live. + * + * Usually functions will pass const references of this class. However + * non-const references can be used to either temporarily release the lock by + * use of AutoUnlockGC or to start background allocation when the lock is + * released. + */ +class MOZ_RAII AutoLockGC { + public: + explicit AutoLockGC(gc::GCRuntime* gc) : gc(gc) { lock(); } + explicit AutoLockGC(JSRuntime* rt) : AutoLockGC(&rt->gc) {} + + ~AutoLockGC() { lockGuard_.reset(); } + + js::LockGuard& guard() { return lockGuard_.ref(); } + + protected: + void lock() { + MOZ_ASSERT(lockGuard_.isNothing()); + lockGuard_.emplace(gc->lock); + } + + void unlock() { + MOZ_ASSERT(lockGuard_.isSome()); + lockGuard_.reset(); + } + + gc::GCRuntime* const gc; + + private: + mozilla::Maybe> lockGuard_; + + AutoLockGC(const AutoLockGC&) = delete; + AutoLockGC& operator=(const AutoLockGC&) = delete; + + friend class AutoUnlockGC; // For lock/unlock. +}; + +/* + * Same as AutoLockGC except it can optionally start a background chunk + * allocation task when the lock is released. + */ +class MOZ_RAII AutoLockGCBgAlloc : public AutoLockGC { + public: + explicit AutoLockGCBgAlloc(gc::GCRuntime* gc) : AutoLockGC(gc) {} + explicit AutoLockGCBgAlloc(JSRuntime* rt) : AutoLockGCBgAlloc(&rt->gc) {} + + ~AutoLockGCBgAlloc() { + unlock(); + + /* + * We have to do this after releasing the lock because it may acquire + * the helper lock which could cause lock inversion if we still held + * the GC lock. + */ + if (startBgAlloc) { + gc->startBackgroundAllocTaskIfIdle(); + } + } + + /* + * This can be used to start a background allocation task (if one isn't + * already running) that allocates chunks and makes them available in the + * free chunks list. This happens after the lock is released in order to + * avoid lock inversion. + */ + void tryToStartBackgroundAllocation() { startBgAlloc = true; } + + private: + // true if we should start a background chunk allocation task after the + // lock is released. + bool startBgAlloc = false; +}; + +class MOZ_RAII AutoUnlockGC { + public: + explicit AutoUnlockGC(AutoLockGC& lock) : lock(lock) { lock.unlock(); } + + ~AutoUnlockGC() { lock.lock(); } + + private: + AutoLockGC& lock; + + AutoUnlockGC(const AutoUnlockGC&) = delete; + AutoUnlockGC& operator=(const AutoUnlockGC&) = delete; +}; + +} // namespace js + +#endif /* gc_GCLock_h */ diff --git a/js/src/gc/GCMarker.h b/js/src/gc/GCMarker.h new file mode 100644 index 0000000000..053ba90e18 --- /dev/null +++ b/js/src/gc/GCMarker.h @@ -0,0 +1,598 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCMarker_h +#define gc_GCMarker_h + +#include "mozilla/Maybe.h" +#include "mozilla/Variant.h" + +#include "ds/OrderedHashTable.h" +#include "gc/Barrier.h" +#include "js/TracingAPI.h" +#include "js/TypeDecls.h" +#include "threading/ProtectedData.h" + +class JSRope; + +namespace js { + +class GCMarker; +class SliceBudget; +class WeakMapBase; + +static const size_t MARK_STACK_BASE_CAPACITY = 4096; + +enum class SlotsOrElementsKind { + Unused = 0, // Must match SlotsOrElementsRangeTag + Elements, + FixedSlots, + DynamicSlots +}; + +namespace gc { + +enum IncrementalProgress { NotFinished = 0, Finished }; + +class AutoSetMarkColor; +struct Cell; +class ParallelMarker; +class UnmarkGrayTracer; + +struct EphemeronEdgeTableHashPolicy { + using Lookup = Cell*; + static HashNumber hash(const Lookup& v, + const mozilla::HashCodeScrambler& hcs) { + return hcs.scramble(mozilla::HashGeneric(v)); + } + static bool match(Cell* const& k, const Lookup& l) { return k == l; } + static bool isEmpty(Cell* const& v) { return !v; } + static void makeEmpty(Cell** vp) { *vp = nullptr; } +}; + +// Ephemeron edges have two source nodes and one target, and mark the target +// with the minimum (least-marked) color of the sources. Currently, one of +// those sources will always be a WeakMapBase, so this will refer to its color +// at the time the edge is traced through. The other source's color will be +// given by the current mark color of the GCMarker. +struct EphemeronEdge { + CellColor color; + Cell* target; + + EphemeronEdge(CellColor color_, Cell* cell) : color(color_), target(cell) {} +}; + +using EphemeronEdgeVector = Vector; + +using EphemeronEdgeTable = + OrderedHashMap; + +/* + * The mark stack. Pointers in this stack are "gray" in the GC sense, but + * their references may be marked either black or gray (in the CC sense). + * + * When the mark stack is full, the GC does not call js::TraceChildren to mark + * the reachable "children" of the thing. Rather the thing is put aside and + * js::TraceChildren is called later when the mark stack is empty. + * + * To implement such delayed marking of the children with minimal overhead for + * the normal case of sufficient stack, we link arenas into a list using + * Arena::setNextDelayedMarkingArena(). The head of the list is stored in + * GCMarker::delayedMarkingList. GCMarker::delayMarkingChildren() adds arenas + * to the list as necessary while markAllDelayedChildren() pops the arenas from + * the stack until it is empty. + */ +class MarkStack { + public: + /* + * We use a common mark stack to mark GC things of different types and use + * the explicit tags to distinguish them when it cannot be deduced from + * the context of push or pop operation. + */ + enum Tag { + SlotsOrElementsRangeTag = 0, // Must match SlotsOrElementsKind::Unused. + ObjectTag, + JitCodeTag, + ScriptTag, + TempRopeTag, + + LastTag = TempRopeTag + }; + + static const uintptr_t TagMask = 7; + static_assert(TagMask >= uintptr_t(LastTag), + "The tag mask must subsume the tags."); + static_assert(TagMask <= gc::CellAlignMask, + "The tag mask must be embeddable in a Cell*."); + + class TaggedPtr { + uintptr_t bits; + + Cell* ptr() const; + + public: + TaggedPtr() = default; + TaggedPtr(Tag tag, Cell* ptr); + Tag tag() const; + uintptr_t tagUnchecked() const; + template + T* as() const; + + JSObject* asRangeObject() const; + JSRope* asTempRope() const; + + void assertValid() const; + }; + + struct SlotsOrElementsRange { + SlotsOrElementsRange(SlotsOrElementsKind kind, JSObject* obj, size_t start); + void assertValid() const; + + SlotsOrElementsKind kind() const; + size_t start() const; + TaggedPtr ptr() const; + + static constexpr size_t StartShift = 2; + static constexpr size_t KindMask = (1 << StartShift) - 1; + + private: + uintptr_t startAndKind_; + TaggedPtr ptr_; + }; + + MarkStack(); + ~MarkStack(); + + explicit MarkStack(const MarkStack& other); + MarkStack& operator=(const MarkStack& other); + + MarkStack(MarkStack&& other); + MarkStack& operator=(MarkStack&& other); + + // The unit for MarkStack::capacity() is mark stack words. + size_t capacity() { return stack().length(); } + + size_t position() const { return topIndex_; } + + [[nodiscard]] bool init(); + [[nodiscard]] bool resetStackCapacity(); + +#ifdef JS_GC_ZEAL + void setMaxCapacity(size_t maxCapacity); +#endif + + template + [[nodiscard]] bool push(T* ptr); + + [[nodiscard]] bool push(JSObject* obj, SlotsOrElementsKind kind, + size_t start); + [[nodiscard]] bool push(const TaggedPtr& ptr); + [[nodiscard]] bool push(const SlotsOrElementsRange& array); + void infalliblePush(const TaggedPtr& ptr); + void infalliblePush(const SlotsOrElementsRange& array); + + // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary + // storage to hold rope pointers. + [[nodiscard]] bool pushTempRope(JSRope* ptr); + + bool isEmpty() const { return position() == 0; } + bool hasEntries() const { return !isEmpty(); } + + Tag peekTag() const; + TaggedPtr popPtr(); + SlotsOrElementsRange popSlotsOrElementsRange(); + + void clearAndResetCapacity(); + void clearAndFreeStack(); + + void poisonUnused(); + + [[nodiscard]] bool ensureSpace(size_t count); + + static void moveWork(MarkStack& dst, MarkStack& src); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + + private: + using StackVector = Vector; + const StackVector& stack() const { return stack_.ref(); } + StackVector& stack() { return stack_.ref(); } + + /* Grow the stack, ensuring there is space for at least count elements. */ + [[nodiscard]] bool enlarge(size_t count); + + [[nodiscard]] bool resize(size_t newCapacity); + + TaggedPtr* topPtr(); + + const TaggedPtr& peekPtr() const; + [[nodiscard]] bool pushTaggedPtr(Tag tag, Cell* ptr); + + bool indexIsEntryBase(size_t index) const; + + // Vector containing allocated stack memory. Unused beyond topIndex_. + MainThreadOrGCTaskData stack_; + + // Index of the top of the stack. + MainThreadOrGCTaskData topIndex_; + +#ifdef JS_GC_ZEAL + // The maximum stack capacity to grow to. + MainThreadOrGCTaskData maxCapacity_{SIZE_MAX}; +#endif +}; + +static_assert(unsigned(SlotsOrElementsKind::Unused) == + unsigned(MarkStack::SlotsOrElementsRangeTag), + "To split the mark stack we depend on being able to tell the " + "difference between SlotsOrElementsRange::startAndKind_ and a " + "tagged SlotsOrElementsRange"); + +// Bitmask of options to parameterize MarkingTracerT. +namespace MarkingOptions { +enum : uint32_t { + // Set the compartment's hasMarkedCells flag for roots. + MarkRootCompartments = 1, + + // The marking tracer is operating in parallel. Use appropriate atomic + // accesses to update the mark bits correctly. + ParallelMarking = 2, + + // Mark any implicit edges if we are in weak marking mode. + MarkImplicitEdges = 4, +}; +} // namespace MarkingOptions + +constexpr uint32_t NormalMarkingOptions = MarkingOptions::MarkImplicitEdges; + +template +class MarkingTracerT + : public GenericTracerImpl> { + public: + MarkingTracerT(JSRuntime* runtime, GCMarker* marker); + virtual ~MarkingTracerT() = default; + + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl>; + + GCMarker* getMarker(); +}; + +using MarkingTracer = MarkingTracerT; +using RootMarkingTracer = MarkingTracerT; +using ParallelMarkingTracer = MarkingTracerT; + +enum ShouldReportMarkTime : bool { + ReportMarkTime = true, + DontReportMarkTime = false +}; + +} /* namespace gc */ + +class GCMarker { + enum MarkingState : uint8_t { + // Have not yet started marking. + NotActive, + + // Root marking mode. This sets the hasMarkedCells flag on compartments + // containing objects and scripts, which is used to make sure we clean up + // dead compartments. + RootMarking, + + // Main marking mode. Weakmap marking will be populating the + // gcEphemeronEdges tables but not consulting them. The state will + // transition to WeakMarking until it is done, then back to RegularMarking. + RegularMarking, + + // Like RegularMarking but with multiple threads running in parallel. + ParallelMarking, + + // Same as RegularMarking except now every marked obj/script is immediately + // looked up in the gcEphemeronEdges table to find edges generated by + // weakmap keys, and traversing them to their values. Transitions back to + // RegularMarking when done. + WeakMarking, + }; + + public: + explicit GCMarker(JSRuntime* rt); + [[nodiscard]] bool init(); + + JSRuntime* runtime() { return runtime_; } + JSTracer* tracer() { + return tracer_.match([](auto& t) -> JSTracer* { return &t; }); + } + +#ifdef JS_GC_ZEAL + void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); } +#endif + + bool isActive() const { return state != NotActive; } + bool isRegularMarking() const { return state == RegularMarking; } + bool isParallelMarking() const { return state == ParallelMarking; } + bool isWeakMarking() const { return state == WeakMarking; } + + gc::MarkColor markColor() const { return markColor_; } + + bool isDrained() const { return stack.isEmpty() && otherStack.isEmpty(); } + + bool hasEntriesForCurrentColor() { return stack.hasEntries(); } + bool hasBlackEntries() const { return hasEntries(gc::MarkColor::Black); } + bool hasGrayEntries() const { return hasEntries(gc::MarkColor::Gray); } + bool hasEntries(gc::MarkColor color) const; + + bool canDonateWork() const; + + void start(); + void stop(); + void reset(); + + [[nodiscard]] bool markUntilBudgetExhausted( + SliceBudget& budget, + gc::ShouldReportMarkTime reportTime = gc::ReportMarkTime); + + void setRootMarkingMode(bool newState); + + bool enterWeakMarkingMode(); + void leaveWeakMarkingMode(); + + void enterParallelMarkingMode(gc::ParallelMarker* pm); + void leaveParallelMarkingMode(); + + // Do not use linear-time weak marking for the rest of this collection. + // Currently, this will only be triggered by an OOM when updating needed data + // structures. + void abortLinearWeakMarking(); + + // 'delegate' is no longer the delegate of 'key'. + void severWeakDelegate(JSObject* key, JSObject* delegate); + + // 'delegate' is now the delegate of 'key'. Update weakmap marking state. + void restoreWeakDelegate(JSObject* key, JSObject* delegate); + +#ifdef DEBUG + // We can't check atom marking if the helper thread lock is already held by + // the current thread. This allows us to disable the check. + void setCheckAtomMarking(bool check); + + bool shouldCheckCompartments() { return strictCompartmentChecking; } +#endif + + bool markCurrentColorInParallel(SliceBudget& budget); + + template + bool markOneColor(SliceBudget& budget); + + static void moveWork(GCMarker* dst, GCMarker* src); + + size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + + static GCMarker* fromTracer(JSTracer* trc) { + MOZ_ASSERT(trc->isMarkingTracer()); + auto* marker = reinterpret_cast(uintptr_t(trc) - + offsetof(GCMarker, tracer_)); + MOZ_ASSERT(marker->tracer() == trc); + return marker; + } + + // Internal public methods, for ease of use by the rest of the GC: + + // If |thing| is unmarked, mark it and then traverse its children. + template + void markAndTraverse(T* thing); + + template + void markImplicitEdges(T* oldThing); + + private: + /* + * Care must be taken changing the mark color from gray to black. The cycle + * collector depends on the invariant that there are no black to gray edges + * in the GC heap. This invariant lets the CC not trace through black + * objects. If this invariant is violated, the cycle collector may free + * objects that are still reachable. + */ + void setMarkColor(gc::MarkColor newColor); + friend class js::gc::AutoSetMarkColor; + + template + void setMarkingStateAndTracer(MarkingState prev, MarkingState next); + + template + bool processMarkStackTop(SliceBudget& budget); + friend class gc::GCRuntime; + + // Helper methods that coerce their second argument to the base pointer + // type. + template + void markAndTraverseObjectEdge(S source, JSObject* target) { + markAndTraverseEdge(source, target); + } + template + void markAndTraverseStringEdge(S source, JSString* target) { + markAndTraverseEdge(source, target); + } + + template + void markAndTraverseEdge(S source, T* target); + template + void markAndTraverseEdge(S source, const T& target); + + template + void checkTraversedEdge(S source, T* target); + + // Mark the given GC thing, but do not trace its children. Return true + // if the thing became marked. + template + [[nodiscard]] bool mark(T* thing); + + // Traverse a GC thing's children, using a strategy depending on the type. + // This can either processing them immediately or push them onto the mark + // stack for later. +#define DEFINE_TRAVERSE_METHOD(_1, Type, _2, _3) \ + template \ + void traverse(Type* thing); + JS_FOR_EACH_TRACEKIND(DEFINE_TRAVERSE_METHOD) +#undef DEFINE_TRAVERSE_METHOD + + // Process a marked thing's children by calling T::traceChildren(). + template + void traceChildren(T* thing); + + // Process a marked thing's children recursively using an iterative loop and + // manual dispatch, for kinds where this is possible. + template + void scanChildren(T* thing); + + // Push a marked thing onto the mark stack. Its children will be marked later. + template + void pushThing(T* thing); + + template + void eagerlyMarkChildren(JSLinearString* str); + template + void eagerlyMarkChildren(JSRope* rope); + template + void eagerlyMarkChildren(JSString* str); + template + void eagerlyMarkChildren(Shape* shape); + template + void eagerlyMarkChildren(PropMap* map); + template + void eagerlyMarkChildren(Scope* scope); + + template + inline void pushTaggedPtr(T* ptr); + + inline void pushValueRange(JSObject* obj, SlotsOrElementsKind kind, + size_t start, size_t end); + + // Push an object onto the stack for later tracing and assert that it has + // already been marked. + inline void repush(JSObject* obj); + + template + void markImplicitEdgesHelper(T oldThing); + + // Mark through edges whose target color depends on the colors of two source + // entities (eg a WeakMap and one of its keys), and push the target onto the + // mark stack. + void markEphemeronEdges(gc::EphemeronEdgeVector& edges, + gc::CellColor srcColor); + friend class JS::Zone; + +#ifdef DEBUG + void checkZone(void* p); +#else + void checkZone(void* p) {} +#endif + + template + bool doMarking(SliceBudget& budget, gc::ShouldReportMarkTime reportTime); + + void delayMarkingChildrenOnOOM(gc::Cell* cell); + + /* + * The JSTracer used for marking. This can change depending on the current + * state. + */ + mozilla::Variant + tracer_; + + JSRuntime* const runtime_; + + // The main mark stack, holding entries of color |markColor_|. + gc::MarkStack stack; + + // The auxiliary mark stack, which may contain entries of the other color. + gc::MarkStack otherStack; + + // Track whether we're using the main or auxiliary stack. + MainThreadOrGCTaskData haveSwappedStacks; + + // The current mark stack color. + MainThreadOrGCTaskData markColor_; + + MainThreadOrGCTaskData parallelMarker_; + + Vector unmarkGrayStack; + friend class gc::UnmarkGrayTracer; + + /* Track the state of marking. */ + MainThreadOrGCTaskData state; + + /* Whether we successfully added all edges to the implicit edges table. */ + MainThreadOrGCTaskData haveAllImplicitEdges; + + public: + /* + * Whether weakmaps can be marked incrementally. + * + * JSGC_INCREMENTAL_WEAKMAP_ENABLED + * pref: javascript.options.mem.incremental_weakmap + */ + MainThreadOrGCTaskData incrementalWeakMapMarkingEnabled; + +#ifdef DEBUG + private: + /* Assert that start and stop are called with correct ordering. */ + MainThreadOrGCTaskData started; + + /* + * Whether to check that atoms traversed are present in atom marking + * bitmap. + */ + MainThreadOrGCTaskData checkAtomMarking; + + /* + * If this is true, all marked objects must belong to a compartment being + * GCed. This is used to look for compartment bugs. + */ + MainThreadOrGCTaskData strictCompartmentChecking; + + public: + /* + * The compartment and zone of the object whose trace hook is currently being + * called, if any. Used to catch cross-compartment edges traced without use of + * TraceCrossCompartmentEdge. + */ + MainThreadOrGCTaskData tracingCompartment; + MainThreadOrGCTaskData tracingZone; +#endif // DEBUG +}; + +namespace gc { + +/* + * Temporarily change the mark color while this class is on the stack. + * + * During incremental sweeping this also transitions zones in the + * current sweep group into the Mark or MarkGray state as appropriate. + */ +class MOZ_RAII AutoSetMarkColor { + GCMarker& marker_; + MarkColor initialColor_; + + public: + AutoSetMarkColor(GCMarker& marker, MarkColor newColor) + : marker_(marker), initialColor_(marker.markColor()) { + marker_.setMarkColor(newColor); + } + + AutoSetMarkColor(GCMarker& marker, CellColor newColor) + : AutoSetMarkColor(marker, newColor.asMarkColor()) {} + + ~AutoSetMarkColor() { marker_.setMarkColor(initialColor_); } +}; + +} /* namespace gc */ + +} /* namespace js */ + +#endif /* gc_GCMarker_h */ diff --git a/js/src/gc/GCParallelTask.cpp b/js/src/gc/GCParallelTask.cpp new file mode 100644 index 0000000000..029315b119 --- /dev/null +++ b/js/src/gc/GCParallelTask.cpp @@ -0,0 +1,231 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/GCParallelTask.h" + +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "gc/GCContext.h" +#include "gc/GCInternals.h" +#include "gc/ParallelWork.h" +#include "vm/HelperThreadState.h" +#include "vm/Runtime.h" +#include "vm/Time.h" + +using namespace js; +using namespace js::gc; + +using mozilla::Maybe; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +js::GCParallelTask::~GCParallelTask() { + // The LinkedListElement destructor will remove us from any list we are part + // of without synchronization, so ensure that doesn't happen. + MOZ_DIAGNOSTIC_ASSERT(!isInList()); + + // Only most-derived classes' destructors may do the join: base class + // destructors run after those for derived classes' members, so a join in a + // base class can't ensure that the task is done using the members. All we + // can do now is check that someone has previously stopped the task. + assertIdle(); +} + +static bool ShouldMeasureTaskStartDelay() { + // We use many tasks during GC so randomly sample a small fraction for the + // purposes of recording telemetry. + return (rand() % 100) == 0; +} + +void js::GCParallelTask::startWithLockHeld(AutoLockHelperThreadState& lock) { + MOZ_ASSERT(CanUseExtraThreads()); + MOZ_ASSERT(HelperThreadState().isInitialized(lock)); + assertIdle(); + + maybeQueueTime_ = TimeStamp(); + if (ShouldMeasureTaskStartDelay()) { + maybeQueueTime_ = TimeStamp::Now(); + } + + setDispatched(lock); + HelperThreadState().submitTask(this, lock); +} + +void js::GCParallelTask::start() { + if (!CanUseExtraThreads()) { + runFromMainThread(); + return; + } + + AutoLockHelperThreadState lock; + startWithLockHeld(lock); +} + +void js::GCParallelTask::startOrRunIfIdle(AutoLockHelperThreadState& lock) { + if (wasStarted(lock)) { + return; + } + + // Join the previous invocation of the task. This will return immediately + // if the thread has never been started. + joinWithLockHeld(lock); + + if (!CanUseExtraThreads()) { + runFromMainThread(lock); + return; + } + + startWithLockHeld(lock); +} + +void js::GCParallelTask::cancelAndWait() { + MOZ_ASSERT(!isCancelled()); + cancel_ = true; + join(); + cancel_ = false; +} + +void js::GCParallelTask::join(Maybe deadline) { + AutoLockHelperThreadState lock; + joinWithLockHeld(lock, deadline); +} + +void js::GCParallelTask::joinWithLockHeld(AutoLockHelperThreadState& lock, + Maybe deadline) { + // Task has not been started; there's nothing to do. + if (isIdle(lock)) { + return; + } + + if (isDispatched(lock) && deadline.isNothing()) { + // If the task was dispatched but has not yet started then cancel the task + // and run it from the main thread. This stops us from blocking here when + // the helper threads are busy with other tasks. + cancelDispatchedTask(lock); + runFromMainThread(lock); + } else { + // Otherwise wait for the task to complete. + joinNonIdleTask(deadline, lock); + } + + if (isIdle(lock)) { + recordDuration(); + } +} + +void GCParallelTask::recordDuration() { + if (phaseKind != gcstats::PhaseKind::NONE) { + gc->stats().recordParallelPhase(phaseKind, duration_); + } +} + +void js::GCParallelTask::joinNonIdleTask(Maybe deadline, + AutoLockHelperThreadState& lock) { + MOZ_ASSERT(!isIdle(lock)); + + while (!isFinished(lock)) { + TimeDuration timeout = TimeDuration::Forever(); + if (deadline) { + TimeStamp now = TimeStamp::Now(); + if (*deadline <= now) { + break; + } + timeout = *deadline - now; + } + + HelperThreadState().wait(lock, timeout); + } + + if (isFinished(lock)) { + setIdle(lock); + } +} + +void js::GCParallelTask::cancelDispatchedTask(AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isDispatched(lock)); + MOZ_ASSERT(isInList()); + remove(); + setIdle(lock); +} + +void js::GCParallelTask::runFromMainThread(AutoLockHelperThreadState& lock) { + assertIdle(); + MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(gc->rt)); + state_ = State::Running; + runTask(gc->rt->gcContext(), lock); + state_ = State::Idle; +} + +void js::GCParallelTask::runFromMainThread() { + AutoLockHelperThreadState lock; + runFromMainThread(lock); +} + +class MOZ_RAII AutoGCContext { + JS::GCContext context; + + public: + explicit AutoGCContext(JSRuntime* runtime) : context(runtime) { + MOZ_RELEASE_ASSERT(TlsGCContext.init(), + "Failed to initialize TLS for GC context"); + + MOZ_ASSERT(!TlsGCContext.get()); + TlsGCContext.set(&context); + } + + ~AutoGCContext() { + MOZ_ASSERT(TlsGCContext.get() == &context); + TlsGCContext.set(nullptr); + } + + JS::GCContext* get() { return &context; } +}; + +void js::GCParallelTask::runHelperThreadTask(AutoLockHelperThreadState& lock) { + setRunning(lock); + + AutoGCContext gcContext(gc->rt); + + runTask(gcContext.get(), lock); + + setFinished(lock); +} + +void GCParallelTask::runTask(JS::GCContext* gcx, + AutoLockHelperThreadState& lock) { + // Run the task from either the main thread or a helper thread. + + AutoSetThreadGCUse setUse(gcx, use); + + // The hazard analysis can't tell what the call to func_ will do but it's not + // allowed to GC. + JS::AutoSuppressGCAnalysis nogc; + + TimeStamp timeStart = TimeStamp::Now(); + run(lock); + duration_ = TimeSince(timeStart); + + if (maybeQueueTime_) { + TimeDuration delay = timeStart - maybeQueueTime_; + gc->rt->metrics().GC_TASK_START_DELAY_US(delay); + } +} + +bool js::GCParallelTask::isIdle() const { + AutoLockHelperThreadState lock; + return isIdle(lock); +} + +bool js::GCParallelTask::wasStarted() const { + AutoLockHelperThreadState lock; + return wasStarted(lock); +} + +/* static */ +size_t js::gc::GCRuntime::parallelWorkerCount() const { + return std::min(helperThreadCount.ref(), MaxParallelWorkers); +} diff --git a/js/src/gc/GCParallelTask.h b/js/src/gc/GCParallelTask.h new file mode 100644 index 0000000000..4784382ab5 --- /dev/null +++ b/js/src/gc/GCParallelTask.h @@ -0,0 +1,246 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCParallelTask_h +#define gc_GCParallelTask_h + +#include "mozilla/LinkedList.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include + +#include "gc/GCContext.h" +#include "js/Utility.h" +#include "threading/ProtectedData.h" +#include "vm/HelperThreads.h" +#include "vm/HelperThreadTask.h" + +#define JS_MEMBER_FN_PTR_TYPE(ClassT, ReturnT, /* ArgTs */...) \ + ReturnT (ClassT::*)(__VA_ARGS__) + +#define JS_CALL_MEMBER_FN_PTR(Receiver, Ptr, /* Args */...) \ + ((Receiver)->*(Ptr))(__VA_ARGS__) + +namespace js { + +namespace gcstats { +enum class PhaseKind : uint8_t; +} + +namespace gc { + +class GCRuntime; + +static inline mozilla::TimeDuration TimeSince(mozilla::TimeStamp prev) { + mozilla::TimeStamp now = mozilla::TimeStamp::Now(); + // Sadly this happens sometimes. + MOZ_ASSERT(now >= prev); + if (now < prev) { + now = prev; + } + return now - prev; +} + +} // namespace gc + +class AutoLockHelperThreadState; +class GCParallelTask; +class HelperThread; + +// A wrapper around a linked list to enforce synchronization. +class GCParallelTaskList { + mozilla::LinkedList tasks; + + public: + bool isEmpty(const AutoLockHelperThreadState& lock) { + gHelperThreadLock.assertOwnedByCurrentThread(); + return tasks.isEmpty(); + } + + void insertBack(GCParallelTask* task, const AutoLockHelperThreadState& lock) { + gHelperThreadLock.assertOwnedByCurrentThread(); + tasks.insertBack(task); + } + + GCParallelTask* popFirst(const AutoLockHelperThreadState& lock) { + gHelperThreadLock.assertOwnedByCurrentThread(); + return tasks.popFirst(); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf, + const AutoLockHelperThreadState& lock) const { + gHelperThreadLock.assertOwnedByCurrentThread(); + return tasks.sizeOfExcludingThis(aMallocSizeOf); + } +}; + +// A generic task used to dispatch work to the helper thread system. +// Users override the pure-virtual run() method. +class GCParallelTask : private mozilla::LinkedListElement, + public HelperThreadTask { + friend class mozilla::LinkedList; + friend class mozilla::LinkedListElement; + + public: + gc::GCRuntime* const gc; + + // This can be PhaseKind::NONE for tasks that take place outside a GC. + const gcstats::PhaseKind phaseKind; + + gc::GCUse use; + + private: + // The state of the parallel computation. + enum class State { + // The task is idle. Either start() has not been called or join() has + // returned. + Idle, + + // The task has been started but has not yet begun running on a helper + // thread. + Dispatched, + + // The task is currently running on a helper thread. + Running, + + // The task has finished running but has not yet been joined by the main + // thread. + Finished + }; + + UnprotectedData state_; + + // May be set to the time this task was queued to collect telemetry. + mozilla::TimeStamp maybeQueueTime_; + + // Amount of time this task took to execute. + MainThreadOrGCTaskData duration_; + + explicit GCParallelTask(const GCParallelTask&) = delete; + + protected: + // A flag to signal a request for early completion of the off-thread task. + mozilla::Atomic cancel_; + + public: + explicit GCParallelTask(gc::GCRuntime* gc, gcstats::PhaseKind phaseKind, + gc::GCUse use = gc::GCUse::Unspecified) + : gc(gc), + phaseKind(phaseKind), + use(use), + state_(State::Idle), + cancel_(false) {} + GCParallelTask(GCParallelTask&& other) + : gc(other.gc), + phaseKind(other.phaseKind), + use(other.use), + state_(other.state_), + cancel_(false) {} + + // Derived classes must override this to ensure that join() gets called + // before members get destructed. + virtual ~GCParallelTask(); + + // Time spent in the most recent invocation of this task. + mozilla::TimeDuration duration() const { return duration_; } + + // The simple interface to a parallel task works exactly like pthreads. + void start(); + void join(mozilla::Maybe deadline = mozilla::Nothing()); + + // If multiple tasks are to be started or joined at once, it is more + // efficient to take the helper thread lock once and use these methods. + void startWithLockHeld(AutoLockHelperThreadState& lock); + void joinWithLockHeld( + AutoLockHelperThreadState& lock, + mozilla::Maybe deadline = mozilla::Nothing()); + void joinNonIdleTask(mozilla::Maybe deadline, + AutoLockHelperThreadState& lock); + + // Instead of dispatching to a helper, run the task on the current thread. + void runFromMainThread(); + void runFromMainThread(AutoLockHelperThreadState& lock); + + // If the task is not already running, either start it or run it on the main + // thread if that fails. + void startOrRunIfIdle(AutoLockHelperThreadState& lock); + + // Cancel a dispatched task before it started executing. + void cancelDispatchedTask(AutoLockHelperThreadState& lock); + + // Set the cancel flag and wait for the task to finish. + void cancelAndWait(); + + // Report whether the task is idle. This means either before start() has been + // called or after join() has been called. + bool isIdle() const; + bool isIdle(const AutoLockHelperThreadState& lock) const { + return state_ == State::Idle; + } + + // Report whether the task has been started. This means after start() has been + // called but before the task has run to completion. The task may not yet have + // started running. + bool wasStarted() const; + bool wasStarted(const AutoLockHelperThreadState& lock) const { + return isDispatched(lock) || isRunning(lock); + } + + bool isDispatched(const AutoLockHelperThreadState& lock) const { + return state_ == State::Dispatched; + } + + protected: + // Override this method to provide the task's functionality. + virtual void run(AutoLockHelperThreadState& lock) = 0; + + virtual void recordDuration(); + + bool isCancelled() const { return cancel_; } + + private: + void assertIdle() const { + // Don't lock here because that adds extra synchronization in debug + // builds that may hide bugs. There's no race if the assertion passes. + MOZ_ASSERT(state_ == State::Idle); + } + + bool isRunning(const AutoLockHelperThreadState& lock) const { + return state_ == State::Running; + } + bool isFinished(const AutoLockHelperThreadState& lock) const { + return state_ == State::Finished; + } + + void setDispatched(const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isIdle(lock)); + state_ = State::Dispatched; + } + void setRunning(const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isDispatched(lock)); + state_ = State::Running; + } + void setFinished(const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isRunning(lock)); + state_ = State::Finished; + } + void setIdle(const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isDispatched(lock) || isFinished(lock)); + state_ = State::Idle; + } + + void runTask(JS::GCContext* gcx, AutoLockHelperThreadState& lock); + + // Implement the HelperThreadTask interface. + ThreadType threadType() override { + return ThreadType::THREAD_TYPE_GCPARALLEL; + } + void runHelperThreadTask(AutoLockHelperThreadState& locked) override; +}; + +} /* namespace js */ +#endif /* gc_GCParallelTask_h */ diff --git a/js/src/gc/GCProbes.h b/js/src/gc/GCProbes.h new file mode 100644 index 0000000000..6942610493 --- /dev/null +++ b/js/src/gc/GCProbes.h @@ -0,0 +1,49 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCProbes_h +#define gc_GCProbes_h + +/* + * This interface can be used to insert probes for GC related events. + * + * The code must be built with JS_GC_PROBES for these probes to be called + * from JIT code. + */ + +#include + +#include "gc/AllocKind.h" +#include "js/TraceKind.h" + +class JSObject; + +namespace js { +namespace gc { + +class GCRuntime; +struct Cell; + +namespace gcprobes { + +inline void Init(gc::GCRuntime* gc) {} +inline void Finish(gc::GCRuntime* gc) {} +inline void NurseryAlloc(void* ptr, JS::TraceKind kind) {} +inline void TenuredAlloc(void* ptr, gc::AllocKind kind) {} +inline void CreateObject(JSObject* object) {} +inline void MinorGCStart() {} +inline void PromoteToTenured(gc::Cell* src, gc::Cell* dst) {} +inline void MinorGCEnd() {} +inline void MajorGCStart() {} +inline void TenuredFinalize(gc::Cell* thing) { +} // May be called off main thread. +inline void MajorGCEnd() {} + +} // namespace gcprobes +} // namespace gc +} // namespace js + +#endif // gc_GCProbes_h diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h new file mode 100644 index 0000000000..5336114020 --- /dev/null +++ b/js/src/gc/GCRuntime.h @@ -0,0 +1,1444 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCRuntime_h +#define gc_GCRuntime_h + +#include "mozilla/Atomics.h" +#include "mozilla/DoublyLinkedList.h" +#include "mozilla/EnumSet.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "gc/ArenaList.h" +#include "gc/AtomMarking.h" +#include "gc/GCContext.h" +#include "gc/GCMarker.h" +#include "gc/GCParallelTask.h" +#include "gc/IteratorUtils.h" +#include "gc/Nursery.h" +#include "gc/Scheduling.h" +#include "gc/Statistics.h" +#include "gc/StoreBuffer.h" +#include "js/friend/PerformanceHint.h" +#include "js/GCAnnotations.h" +#include "js/UniquePtr.h" +#include "vm/AtomsTable.h" + +namespace js { + +class AutoLockGC; +class AutoLockGCBgAlloc; +class AutoLockHelperThreadState; +class FinalizationRegistryObject; +class FinalizationRecordObject; +class FinalizationQueueObject; +class GlobalObject; +class VerifyPreTracer; +class WeakRefObject; + +namespace gc { + +using BlackGrayEdgeVector = Vector; +using ZoneVector = Vector; + +class AutoCallGCCallbacks; +class AutoGCSession; +class AutoHeapSession; +class AutoTraceSession; +struct FinalizePhase; +class MarkingValidator; +struct MovingTracer; +class ParallelMarkTask; +enum class ShouldCheckThresholds; +class SweepGroupsIter; + +// Interface to a sweep action. +struct SweepAction { + // The arguments passed to each action. + struct Args { + GCRuntime* gc; + JS::GCContext* gcx; + SliceBudget& budget; + }; + + virtual ~SweepAction() = default; + virtual IncrementalProgress run(Args& state) = 0; + virtual void assertFinished() const = 0; + virtual bool shouldSkip() { return false; } +}; + +class ChunkPool { + TenuredChunk* head_; + size_t count_; + + public: + ChunkPool() : head_(nullptr), count_(0) {} + ChunkPool(const ChunkPool& other) = delete; + ChunkPool(ChunkPool&& other) { *this = std::move(other); } + + ~ChunkPool() { + MOZ_ASSERT(!head_); + MOZ_ASSERT(count_ == 0); + } + + ChunkPool& operator=(const ChunkPool& other) = delete; + ChunkPool& operator=(ChunkPool&& other) { + head_ = other.head_; + other.head_ = nullptr; + count_ = other.count_; + other.count_ = 0; + return *this; + } + + bool empty() const { return !head_; } + size_t count() const { return count_; } + + TenuredChunk* head() { + MOZ_ASSERT(head_); + return head_; + } + TenuredChunk* pop(); + void push(TenuredChunk* chunk); + TenuredChunk* remove(TenuredChunk* chunk); + + void sort(); + + private: + TenuredChunk* mergeSort(TenuredChunk* list, size_t count); + bool isSorted() const; + +#ifdef DEBUG + public: + bool contains(TenuredChunk* chunk) const; + bool verify() const; + void verifyChunks() const; +#endif + + public: + // Pool mutation does not invalidate an Iter unless the mutation + // is of the TenuredChunk currently being visited by the Iter. + class Iter { + public: + explicit Iter(ChunkPool& pool) : current_(pool.head_) {} + bool done() const { return !current_; } + void next(); + TenuredChunk* get() const { return current_; } + operator TenuredChunk*() const { return get(); } + TenuredChunk* operator->() const { return get(); } + + private: + TenuredChunk* current_; + }; +}; + +class BackgroundMarkTask : public GCParallelTask { + public: + explicit BackgroundMarkTask(GCRuntime* gc); + void setBudget(const SliceBudget& budget) { this->budget = budget; } + void run(AutoLockHelperThreadState& lock) override; + + private: + SliceBudget budget; +}; + +class BackgroundUnmarkTask : public GCParallelTask { + public: + explicit BackgroundUnmarkTask(GCRuntime* gc); + void initZones(); + void run(AutoLockHelperThreadState& lock) override; + + ZoneVector zones; +}; + +class BackgroundSweepTask : public GCParallelTask { + public: + explicit BackgroundSweepTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +class BackgroundFreeTask : public GCParallelTask { + public: + explicit BackgroundFreeTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +// Performs extra allocation off thread so that when memory is required on the +// main thread it will already be available and waiting. +class BackgroundAllocTask : public GCParallelTask { + // Guarded by the GC lock. + GCLockData chunkPool_; + + const bool enabled_; + + public: + BackgroundAllocTask(GCRuntime* gc, ChunkPool& pool); + bool enabled() const { return enabled_; } + + void run(AutoLockHelperThreadState& lock) override; +}; + +// Search the provided chunks for free arenas and decommit them. +class BackgroundDecommitTask : public GCParallelTask { + public: + explicit BackgroundDecommitTask(GCRuntime* gc); + void run(AutoLockHelperThreadState& lock) override; +}; + +template +struct Callback { + F op; + void* data; + + Callback() : op(nullptr), data(nullptr) {} + Callback(F op, void* data) : op(op), data(data) {} +}; + +template +using CallbackVector = Vector, 4, SystemAllocPolicy>; + +typedef HashMap, SystemAllocPolicy> + RootedValueMap; + +using AllocKinds = mozilla::EnumSet; + +// A singly linked list of zones. +class ZoneList { + static Zone* const End; + + Zone* head; + Zone* tail; + + public: + ZoneList(); + ~ZoneList(); + + bool isEmpty() const; + Zone* front() const; + + void prepend(Zone* zone); + void append(Zone* zone); + void prependList(ZoneList&& other); + void appendList(ZoneList&& other); + Zone* removeFront(); + void clear(); + + private: + explicit ZoneList(Zone* singleZone); + void check() const; + + ZoneList(const ZoneList& other) = delete; + ZoneList& operator=(const ZoneList& other) = delete; +}; + +struct WeakCacheToSweep { + JS::detail::WeakCacheBase* cache; + JS::Zone* zone; +}; + +class WeakCacheSweepIterator { + using WeakCacheBase = JS::detail::WeakCacheBase; + + JS::Zone* sweepZone; + WeakCacheBase* sweepCache; + + public: + explicit WeakCacheSweepIterator(JS::Zone* sweepGroup); + + bool done() const; + WeakCacheToSweep get() const; + void next(); + + private: + void settle(); +}; + +struct SweepingTracer final : public GenericTracerImpl { + explicit SweepingTracer(JSRuntime* rt); + + private: + template + void onEdge(T** thingp, const char* name); + friend class GenericTracerImpl; +}; + +class GCRuntime { + public: + explicit GCRuntime(JSRuntime* rt); + [[nodiscard]] bool init(uint32_t maxbytes); + bool wasInitialized() const { return initialized; } + void finishRoots(); + void finish(); + + Zone* atomsZone() { + Zone* zone = zones()[0]; + MOZ_ASSERT(JS::shadow::Zone::from(zone)->isAtomsZone()); + return zone; + } + Zone* maybeSharedAtomsZone() { return sharedAtomsZone_; } + + [[nodiscard]] bool freezeSharedAtomsZone(); + void restoreSharedAtomsZone(); + + JS::HeapState heapState() const { return heapState_; } + + inline bool hasZealMode(ZealMode mode); + inline void clearZealMode(ZealMode mode); + inline bool needZealousGC(); + inline bool hasIncrementalTwoSliceZealMode(); + + [[nodiscard]] bool addRoot(Value* vp, const char* name); + void removeRoot(Value* vp); + + [[nodiscard]] bool setParameter(JSContext* cx, JSGCParamKey key, + uint32_t value); + void resetParameter(JSContext* cx, JSGCParamKey key); + uint32_t getParameter(JSGCParamKey key); + + void setPerformanceHint(PerformanceHint hint); + bool isInPageLoad() const { return inPageLoadCount != 0; } + + [[nodiscard]] bool triggerGC(JS::GCReason reason); + // Check whether to trigger a zone GC after allocating GC cells. + void maybeTriggerGCAfterAlloc(Zone* zone); + // Check whether to trigger a zone GC after malloc memory. + void maybeTriggerGCAfterMalloc(Zone* zone); + bool maybeTriggerGCAfterMalloc(Zone* zone, const HeapSize& heap, + const HeapThreshold& threshold, + JS::GCReason reason); + // The return value indicates if we were able to do the GC. + bool triggerZoneGC(Zone* zone, JS::GCReason reason, size_t usedBytes, + size_t thresholdBytes); + + void maybeGC(); + + // Return whether we want to run a major GC. If eagerOk is true, include eager + // triggers (eg EAGER_ALLOC_TRIGGER) in this determination, and schedule all + // zones that exceed the eager thresholds. + JS::GCReason wantMajorGC(bool eagerOk); + bool checkEagerAllocTrigger(const HeapSize& size, + const HeapThreshold& threshold); + + // Do a minor GC if requested, followed by a major GC if requested. The return + // value indicates whether a major GC was performed. + bool gcIfRequested() { return gcIfRequestedImpl(false); } + + // Internal function to do a GC if previously requested. But if not and + // eagerOk, do an eager GC for all Zones that have exceeded the eager + // thresholds. + // + // Return whether a major GC was performed or started. + bool gcIfRequestedImpl(bool eagerOk); + + void gc(JS::GCOptions options, JS::GCReason reason); + void startGC(JS::GCOptions options, JS::GCReason reason, + const SliceBudget& budget); + void gcSlice(JS::GCReason reason, const SliceBudget& budget); + void finishGC(JS::GCReason reason); + void abortGC(); + void startDebugGC(JS::GCOptions options, const SliceBudget& budget); + void debugGCSlice(const SliceBudget& budget); + + void runDebugGC(); + void notifyRootsRemoved(); + + enum TraceOrMarkRuntime { TraceRuntime, MarkRuntime }; + void traceRuntime(JSTracer* trc, AutoTraceSession& session); + void traceRuntimeForMinorGC(JSTracer* trc, AutoGCSession& session); + + void purgeRuntimeForMinorGC(); + + void shrinkBuffers(); + void onOutOfMallocMemory(); + void onOutOfMallocMemory(const AutoLockGC& lock); + + Nursery& nursery() { return nursery_.ref(); } + gc::StoreBuffer& storeBuffer() { return storeBuffer_.ref(); } + + void minorGC(JS::GCReason reason, + gcstats::PhaseKind phase = gcstats::PhaseKind::MINOR_GC) + JS_HAZ_GC_CALL; + void evictNursery(JS::GCReason reason = JS::GCReason::EVICT_NURSERY) { + minorGC(reason, gcstats::PhaseKind::EVICT_NURSERY); + } + + void* addressOfNurseryPosition() { + return nursery_.refNoCheck().addressOfPosition(); + } + + const void* addressOfLastBufferedWholeCell() { + return storeBuffer_.refNoCheck().addressOfLastBufferedWholeCell(); + } + +#ifdef JS_GC_ZEAL + const uint32_t* addressOfZealModeBits() { return &zealModeBits.refNoCheck(); } + void getZealBits(uint32_t* zealBits, uint32_t* frequency, + uint32_t* nextScheduled); + void setZeal(uint8_t zeal, uint32_t frequency); + void unsetZeal(uint8_t zeal); + bool parseAndSetZeal(const char* str); + void setNextScheduled(uint32_t count); + void verifyPreBarriers(); + void maybeVerifyPreBarriers(bool always); + bool selectForMarking(JSObject* object); + void clearSelectedForMarking(); + void setDeterministic(bool enable); + void setMarkStackLimit(size_t limit, AutoLockGC& lock); +#endif + + uint64_t nextCellUniqueId() { + MOZ_ASSERT(nextCellUniqueId_ > 0); + uint64_t uid = ++nextCellUniqueId_; + return uid; + } + + void setLowMemoryState(bool newState) { lowMemoryState = newState; } + bool systemHasLowMemory() const { return lowMemoryState; } + + public: + // Internal public interface + ZoneVector& zones() { return zones_.ref(); } + gcstats::Statistics& stats() { return stats_.ref(); } + const gcstats::Statistics& stats() const { return stats_.ref(); } + State state() const { return incrementalState; } + bool isHeapCompacting() const { return state() == State::Compact; } + bool isForegroundSweeping() const { return state() == State::Sweep; } + bool isBackgroundSweeping() const { return sweepTask.wasStarted(); } + bool isBackgroundMarking() const { return markTask.wasStarted(); } + void waitBackgroundSweepEnd(); + void waitBackgroundAllocEnd() { allocTask.cancelAndWait(); } + void waitBackgroundFreeEnd(); + void waitForBackgroundTasks(); + bool isWaitingOnBackgroundTask() const; + + void lockGC() { lock.lock(); } + bool tryLockGC() { return lock.tryLock(); } + void unlockGC() { lock.unlock(); } + +#ifdef DEBUG + void assertCurrentThreadHasLockedGC() const { + lock.assertOwnedByCurrentThread(); + } +#endif // DEBUG + + void setAlwaysPreserveCode() { alwaysPreserveCode = true; } + + bool isIncrementalGCAllowed() const { return incrementalAllowed; } + void disallowIncrementalGC() { incrementalAllowed = false; } + + void setIncrementalGCEnabled(bool enabled); + + bool isIncrementalGCEnabled() const { return incrementalGCEnabled; } + bool isPerZoneGCEnabled() const { return perZoneGCEnabled; } + bool isCompactingGCEnabled() const; + bool isParallelMarkingEnabled() const { return parallelMarkingEnabled; } + + bool isIncrementalGCInProgress() const { + return state() != State::NotActive && !isVerifyPreBarriersEnabled(); + } + + bool hasForegroundWork() const; + + bool isShrinkingGC() const { return gcOptions() == JS::GCOptions::Shrink; } + + bool isShutdownGC() const { return gcOptions() == JS::GCOptions::Shutdown; } + +#ifdef DEBUG + bool isShuttingDown() const { return hadShutdownGC; } +#endif + + bool initSweepActions(); + + void setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data); + [[nodiscard]] bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); + void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); + void clearBlackAndGrayRootTracers(); + + void setGCCallback(JSGCCallback callback, void* data); + void callGCCallback(JSGCStatus status, JS::GCReason reason) const; + void setObjectsTenuredCallback(JSObjectsTenuredCallback callback, void* data); + void callObjectsTenuredCallback(); + [[nodiscard]] bool addFinalizeCallback(JSFinalizeCallback callback, + void* data); + void removeFinalizeCallback(JSFinalizeCallback func); + void setHostCleanupFinalizationRegistryCallback( + JSHostCleanupFinalizationRegistryCallback callback, void* data); + void callHostCleanupFinalizationRegistryCallback( + JSFunction* doCleanup, GlobalObject* incumbentGlobal); + [[nodiscard]] bool addWeakPointerZonesCallback( + JSWeakPointerZonesCallback callback, void* data); + void removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback); + [[nodiscard]] bool addWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback, void* data); + void removeWeakPointerCompartmentCallback( + JSWeakPointerCompartmentCallback callback); + JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); + JS::GCNurseryCollectionCallback setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback callback); + JS::DoCycleCollectionCallback setDoCycleCollectionCallback( + JS::DoCycleCollectionCallback callback); + + bool addFinalizationRegistry(JSContext* cx, + Handle registry); + bool registerWithFinalizationRegistry(JSContext* cx, HandleObject target, + HandleObject record); + void queueFinalizationRegistryForCleanup(FinalizationQueueObject* queue); + + void nukeFinalizationRecordWrapper(JSObject* wrapper, + FinalizationRecordObject* record); + void nukeWeakRefWrapper(JSObject* wrapper, WeakRefObject* record); + + void setFullCompartmentChecks(bool enable); + + // Get the main marking tracer. + GCMarker& marker() { return *markers[0]; } + + JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; } + unsigned getCurrentSweepGroupIndex() { + MOZ_ASSERT_IF(unsigned(state()) < unsigned(State::Sweep), + sweepGroupIndex == 0); + return sweepGroupIndex; + } + + uint64_t gcNumber() const { return number; } + void incGcNumber() { ++number; } + + uint64_t minorGCCount() const { return minorGCNumber; } + void incMinorGcNumber() { ++minorGCNumber; } + + uint64_t majorGCCount() const { return majorGCNumber; } + void incMajorGcNumber() { ++majorGCNumber; } + + uint64_t gcSliceCount() const { return sliceNumber; } + void incGcSliceNumber() { ++sliceNumber; } + + int64_t defaultSliceBudgetMS() const { return defaultTimeBudgetMS_; } + + bool isIncrementalGc() const { return isIncremental; } + bool isFullGc() const { return isFull; } + bool isCompactingGc() const { return isCompacting; } + bool didCompactZones() const { return isCompacting && zonesCompacted; } + + bool areGrayBitsValid() const { return grayBitsValid; } + void setGrayBitsInvalid() { grayBitsValid = false; } + + mozilla::TimeStamp lastGCStartTime() const { return lastGCStartTime_; } + mozilla::TimeStamp lastGCEndTime() const { return lastGCEndTime_; } + + bool majorGCRequested() const { + return majorGCTriggerReason != JS::GCReason::NO_REASON; + } + + double computeHeapGrowthFactor(size_t lastBytes); + size_t computeTriggerBytes(double growthFactor, size_t lastBytes); + + inline void updateOnFreeArenaAlloc(const TenuredChunkInfo& info); + void updateOnArenaFree() { ++numArenasFreeCommitted; } + + ChunkPool& fullChunks(const AutoLockGC& lock) { return fullChunks_.ref(); } + ChunkPool& availableChunks(const AutoLockGC& lock) { + return availableChunks_.ref(); + } + ChunkPool& emptyChunks(const AutoLockGC& lock) { return emptyChunks_.ref(); } + const ChunkPool& fullChunks(const AutoLockGC& lock) const { + return fullChunks_.ref(); + } + const ChunkPool& availableChunks(const AutoLockGC& lock) const { + return availableChunks_.ref(); + } + const ChunkPool& emptyChunks(const AutoLockGC& lock) const { + return emptyChunks_.ref(); + } + using NonEmptyChunksIter = ChainedIterator; + NonEmptyChunksIter allNonEmptyChunks(const AutoLockGC& lock) { + return NonEmptyChunksIter(availableChunks(lock), fullChunks(lock)); + } + uint32_t minEmptyChunkCount(const AutoLockGC& lock) const { + return minEmptyChunkCount_; + } + uint32_t maxEmptyChunkCount(const AutoLockGC& lock) const { + return maxEmptyChunkCount_; + } +#ifdef DEBUG + void verifyAllChunks(); +#endif + + TenuredChunk* getOrAllocChunk(AutoLockGCBgAlloc& lock); + void recycleChunk(TenuredChunk* chunk, const AutoLockGC& lock); + +#ifdef JS_GC_ZEAL + void startVerifyPreBarriers(); + void endVerifyPreBarriers(); + void finishVerifier(); + bool isVerifyPreBarriersEnabled() const { return verifyPreData.refNoCheck(); } + bool shouldYieldForZeal(ZealMode mode); +#else + bool isVerifyPreBarriersEnabled() const { return false; } +#endif + +#ifdef JSGC_HASH_TABLE_CHECKS + void checkHashTablesAfterMovingGC(); +#endif + +#ifdef DEBUG + // Crawl the heap to check whether an arbitary pointer is within a cell of + // the given kind. + bool isPointerWithinTenuredCell(void* ptr, JS::TraceKind traceKind); + + bool hasZone(Zone* target); +#endif + + // Queue memory memory to be freed on a background thread if possible. + void queueUnusedLifoBlocksForFree(LifoAlloc* lifo); + void queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo); + void queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers); + + // Public here for ReleaseArenaLists and FinalizeTypedArenas. + void releaseArena(Arena* arena, const AutoLockGC& lock); + + // Allocator + template + [[nodiscard]] bool checkAllocatorState(JSContext* cx, AllocKind kind); + template + void* tryNewNurseryCell(JSContext* cx, size_t thingSize, AllocSite* site); + template + static void* tryNewTenuredThing(JSContext* cx, AllocKind kind, + size_t thingSize); + static void* refillFreeListInGC(Zone* zone, AllocKind thingKind); + + // Delayed marking. + void delayMarkingChildren(gc::Cell* cell, MarkColor color); + bool hasDelayedMarking() const; + void markAllDelayedChildren(ShouldReportMarkTime reportTime); + + /* + * Concurrent sweep infrastructure. + */ + void startTask(GCParallelTask& task, AutoLockHelperThreadState& lock); + void joinTask(GCParallelTask& task, AutoLockHelperThreadState& lock); + void updateHelperThreadCount(); + bool updateMarkersVector(); + size_t parallelWorkerCount() const; + size_t markingWorkerCount() const; + + // WeakRefs + bool registerWeakRef(HandleObject target, HandleObject weakRef); + void traceKeptObjects(JSTracer* trc); + + JS::GCReason lastStartReason() const { return initialReason; } + + void updateAllocationRates(); + +#ifdef DEBUG + const GCVector, 0, SystemAllocPolicy>& getTestMarkQueue() + const; + [[nodiscard]] bool appendTestMarkQueue(const JS::Value& value); + void clearTestMarkQueue(); + size_t testMarkQueuePos() const; +#endif + + private: + enum IncrementalResult { ResetIncremental = 0, Ok }; + + [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, + AutoLockGC& lock); + void resetParameter(JSGCParamKey key, AutoLockGC& lock); + uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); + bool setThreadParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); + void resetThreadParameter(JSGCParamKey key, AutoLockGC& lock); + void updateThreadDataStructures(AutoLockGC& lock); + + JS::GCOptions gcOptions() const { return maybeGcOptions.ref().ref(); } + + TriggerResult checkHeapThreshold(Zone* zone, const HeapSize& heapSize, + const HeapThreshold& heapThreshold); + + void updateSchedulingStateOnGCStart(); + void updateSchedulingStateAfterCollection(mozilla::TimeStamp currentTime); + void updateAllGCStartThresholds(); + + // For ArenaLists::allocateFromArena() + friend class ArenaLists; + TenuredChunk* pickChunk(AutoLockGCBgAlloc& lock); + Arena* allocateArena(TenuredChunk* chunk, Zone* zone, AllocKind kind, + ShouldCheckThresholds checkThresholds, + const AutoLockGC& lock); + + // Allocator internals + void gcIfNeededAtAllocation(JSContext* cx); + static void* refillFreeList(JSContext* cx, AllocKind thingKind); + void attemptLastDitchGC(JSContext* cx); +#ifdef DEBUG + static void checkIncrementalZoneState(JSContext* cx, void* ptr); +#endif + + /* + * Return the list of chunks that can be released outside the GC lock. + * Must be called either during the GC or with the GC lock taken. + */ + friend class BackgroundDecommitTask; + bool tooManyEmptyChunks(const AutoLockGC& lock); + ChunkPool expireEmptyChunkPool(const AutoLockGC& lock); + void freeEmptyChunks(const AutoLockGC& lock); + void prepareToFreeChunk(TenuredChunkInfo& info); + void setMinEmptyChunkCount(uint32_t value, const AutoLockGC& lock); + void setMaxEmptyChunkCount(uint32_t value, const AutoLockGC& lock); + + friend class BackgroundAllocTask; + bool wantBackgroundAllocation(const AutoLockGC& lock) const; + void startBackgroundAllocTaskIfIdle(); + + void requestMajorGC(JS::GCReason reason); + SliceBudget defaultBudget(JS::GCReason reason, int64_t millis); + bool maybeIncreaseSliceBudget(SliceBudget& budget); + bool maybeIncreaseSliceBudgetForLongCollections(SliceBudget& budget); + bool maybeIncreaseSliceBudgetForUrgentCollections(SliceBudget& budget); + IncrementalResult budgetIncrementalGC(bool nonincrementalByAPI, + JS::GCReason reason, + SliceBudget& budget); + void checkZoneIsScheduled(Zone* zone, JS::GCReason reason, + const char* trigger); + IncrementalResult resetIncrementalGC(GCAbortReason reason); + + // Assert if the system state is such that we should never + // receive a request to do GC work. + void checkCanCallAPI(); + + // Check if the system state is such that GC has been supressed + // or otherwise delayed. + [[nodiscard]] bool checkIfGCAllowedInCurrentState(JS::GCReason reason); + + gcstats::ZoneGCStats scanZonesBeforeGC(); + + void setGCOptions(JS::GCOptions options); + + void collect(bool nonincrementalByAPI, const SliceBudget& budget, + JS::GCReason reason) JS_HAZ_GC_CALL; + + /* + * Run one GC "cycle" (either a slice of incremental GC or an entire + * non-incremental GC). + * + * Returns: + * * ResetIncremental if we "reset" an existing incremental GC, which would + * force us to run another cycle or + * * Ok otherwise. + */ + [[nodiscard]] IncrementalResult gcCycle(bool nonincrementalByAPI, + const SliceBudget& budgetArg, + JS::GCReason reason); + bool shouldRepeatForDeadZone(JS::GCReason reason); + + void incrementalSlice(SliceBudget& budget, JS::GCReason reason, + bool budgetWasIncreased); + + bool mightSweepInThisSlice(bool nonIncremental); + void collectNurseryFromMajorGC(JS::GCReason reason); + void collectNursery(JS::GCOptions options, JS::GCReason reason, + gcstats::PhaseKind phase); + + friend class AutoCallGCCallbacks; + void maybeCallGCCallback(JSGCStatus status, JS::GCReason reason); + + void startCollection(JS::GCReason reason); + + void purgeRuntime(); + [[nodiscard]] bool beginPreparePhase(JS::GCReason reason, + AutoGCSession& session); + bool prepareZonesForCollection(JS::GCReason reason, bool* isFullOut); + void unmarkWeakMaps(); + void endPreparePhase(JS::GCReason reason); + void beginMarkPhase(AutoGCSession& session); + bool shouldPreserveJITCode(JS::Realm* realm, + const mozilla::TimeStamp& currentTime, + JS::GCReason reason, bool canAllocateMoreCode, + bool isActiveCompartment); + void discardJITCodeForGC(); + void startBackgroundFreeAfterMinorGC(); + void relazifyFunctionsForShrinkingGC(); + void purgePropMapTablesForShrinkingGC(); + void purgeSourceURLsForShrinkingGC(); + void traceRuntimeForMajorGC(JSTracer* trc, AutoGCSession& session); + void traceRuntimeAtoms(JSTracer* trc); + void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark); + void traceEmbeddingBlackRoots(JSTracer* trc); + void traceEmbeddingGrayRoots(JSTracer* trc); + IncrementalProgress traceEmbeddingGrayRoots(JSTracer* trc, + SliceBudget& budget); + void checkNoRuntimeRoots(AutoGCSession& session); + void maybeDoCycleCollection(); + void findDeadCompartments(); + + friend class BackgroundMarkTask; + enum ParallelMarking : bool { + SingleThreadedMarking = false, + AllowParallelMarking = true + }; + IncrementalProgress markUntilBudgetExhausted( + SliceBudget& sliceBudget, + ParallelMarking allowParallelMarking = SingleThreadedMarking, + ShouldReportMarkTime reportTime = ReportMarkTime); + bool canMarkInParallel() const; + + bool hasMarkingWork(MarkColor color) const; + + void drainMarkStack(); + +#ifdef DEBUG + void assertNoMarkingWork() const; +#else + void assertNoMarkingWork() const {} +#endif + + void markDelayedChildren(gc::Arena* arena, MarkColor color); + void processDelayedMarkingList(gc::MarkColor color); + void rebuildDelayedMarkingList(); + void appendToDelayedMarkingList(gc::Arena** listTail, gc::Arena* arena); + void resetDelayedMarking(); + template + void forEachDelayedMarkingArena(F&& f); + + template + IncrementalProgress markWeakReferences(SliceBudget& budget); + IncrementalProgress markWeakReferencesInCurrentGroup(SliceBudget& budget); + template + IncrementalProgress markGrayRoots(SliceBudget& budget, + gcstats::PhaseKind phase); + void markBufferedGrayRoots(JS::Zone* zone); + IncrementalProgress markAllWeakReferences(); + void markAllGrayReferences(gcstats::PhaseKind phase); + + // The mark queue is a testing-only feature for controlling mark ordering and + // yield timing. + enum MarkQueueProgress { + QueueYielded, // End this incremental GC slice, if possible + QueueComplete, // Done with the queue + QueueSuspended // Continue the GC without ending the slice + }; + MarkQueueProgress processTestMarkQueue(); + + // GC Sweeping. Implemented in Sweeping.cpp. + void beginSweepPhase(JS::GCReason reason, AutoGCSession& session); + void dropStringWrappers(); + void groupZonesForSweeping(JS::GCReason reason); + [[nodiscard]] bool findSweepGroupEdges(); + [[nodiscard]] bool addEdgesForMarkQueue(); + void getNextSweepGroup(); + void resetGrayList(Compartment* comp); + IncrementalProgress beginMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress markGrayRootsInCurrentGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress markGray(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress endMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + void markIncomingGrayCrossCompartmentPointers(); + IncrementalProgress beginSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + void initBackgroundSweep(Zone* zone, JS::GCContext* gcx, + const FinalizePhase& phase); + IncrementalProgress markDuringSweeping(JS::GCContext* gcx, + SliceBudget& budget); + void updateAtomsBitmap(); + void sweepCCWrappers(); + void sweepRealmGlobals(); + void sweepEmbeddingWeakPointers(JS::GCContext* gcx); + void sweepMisc(); + void sweepCompressionTasks(); + void sweepWeakMaps(); + void sweepUniqueIds(); + void sweepDebuggerOnMainThread(JS::GCContext* gcx); + void sweepJitDataOnMainThread(JS::GCContext* gcx); + void sweepFinalizationObserversOnMainThread(); + void traceWeakFinalizationObserverEdges(JSTracer* trc, Zone* zone); + void sweepWeakRefs(); + IncrementalProgress endSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget); + IncrementalProgress performSweepActions(SliceBudget& sliceBudget); + void startSweepingAtomsTable(); + IncrementalProgress sweepAtomsTable(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress sweepWeakCaches(JS::GCContext* gcx, SliceBudget& budget); + IncrementalProgress finalizeAllocKind(JS::GCContext* gcx, + SliceBudget& budget); + bool foregroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind thingKind, + js::SliceBudget& sliceBudget, + SortedArenaList& sweepList); + IncrementalProgress sweepPropMapTree(JS::GCContext* gcx, SliceBudget& budget); + void endSweepPhase(bool lastGC); + void queueZonesAndStartBackgroundSweep(ZoneList&& zones); + void sweepFromBackgroundThread(AutoLockHelperThreadState& lock); + void startBackgroundFree(); + void freeFromBackgroundThread(AutoLockHelperThreadState& lock); + void sweepBackgroundThings(ZoneList& zones); + void backgroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind kind, + Arena** empty); + void assertBackgroundSweepingFinished(); + + bool allCCVisibleZonesWereCollected(); + void sweepZones(JS::GCContext* gcx, bool destroyingRuntime); + bool shouldDecommit() const; + void startDecommit(); + void decommitEmptyChunks(const bool& cancel, AutoLockGC& lock); + void decommitFreeArenas(const bool& cancel, AutoLockGC& lock); + void decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock); + + // Compacting GC. Implemented in Compacting.cpp. + bool shouldCompact(); + void beginCompactPhase(); + IncrementalProgress compactPhase(JS::GCReason reason, + SliceBudget& sliceBudget, + AutoGCSession& session); + void endCompactPhase(); + void sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone); + bool canRelocateZone(Zone* zone) const; + [[nodiscard]] bool relocateArenas(Zone* zone, JS::GCReason reason, + Arena*& relocatedListOut, + SliceBudget& sliceBudget); + void updateCellPointers(Zone* zone, AllocKinds kinds); + void updateAllCellPointers(MovingTracer* trc, Zone* zone); + void updateZonePointersToRelocatedCells(Zone* zone); + void updateRuntimePointersToRelocatedCells(AutoGCSession& session); + void clearRelocatedArenas(Arena* arenaList, JS::GCReason reason); + void clearRelocatedArenasWithoutUnlocking(Arena* arenaList, + JS::GCReason reason, + const AutoLockGC& lock); + void releaseRelocatedArenas(Arena* arenaList); + void releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, + const AutoLockGC& lock); +#ifdef DEBUG + void protectOrReleaseRelocatedArenas(Arena* arenaList, JS::GCReason reason); + void protectAndHoldArenas(Arena* arenaList); + void unprotectHeldRelocatedArenas(const AutoLockGC& lock); + void releaseHeldRelocatedArenas(); + void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock); +#endif + + /* + * Whether to immediately trigger a slice after a background task + * finishes. This may not happen at a convenient time, so the consideration is + * whether the slice will run quickly or may take a long time. + */ + enum ShouldTriggerSliceWhenFinished : bool { + DontTriggerSliceWhenFinished = false, + TriggerSliceWhenFinished = true + }; + + IncrementalProgress waitForBackgroundTask( + GCParallelTask& task, const SliceBudget& budget, bool shouldPauseMutator, + ShouldTriggerSliceWhenFinished triggerSlice); + + void maybeRequestGCAfterBackgroundTask(const AutoLockHelperThreadState& lock); + void cancelRequestedGCAfterBackgroundTask(); + void finishCollection(JS::GCReason reason); + void maybeStopPretenuring(); + void checkGCStateNotInUse(); + IncrementalProgress joinBackgroundMarkTask(); + +#ifdef JS_GC_ZEAL + void computeNonIncrementalMarkingForValidation(AutoGCSession& session); + void validateIncrementalMarking(); + void finishMarkingValidation(); +#endif + +#ifdef DEBUG + void checkForCompartmentMismatches(); +#endif + + void callFinalizeCallbacks(JS::GCContext* gcx, JSFinalizeStatus status) const; + void callWeakPointerZonesCallbacks(JSTracer* trc) const; + void callWeakPointerCompartmentCallbacks(JSTracer* trc, + JS::Compartment* comp) const; + void callDoCycleCollectionCallback(JSContext* cx); + + public: + JSRuntime* const rt; + + // Embedders can use this zone however they wish. + MainThreadData systemZone; + + MainThreadData mainThreadContext; + + private: + // For parent runtimes, a zone containing atoms that is shared by child + // runtimes. + MainThreadData sharedAtomsZone_; + + // All zones in the runtime. The first element is always the atoms zone. + MainThreadOrGCTaskData zones_; + + // Any activity affecting the heap. + MainThreadOrGCTaskData heapState_; + friend class AutoHeapSession; + friend class JS::AutoEnterCycleCollection; + + UnprotectedData stats_; + + public: + js::StringStats stringStats; + + Vector, 1, SystemAllocPolicy> markers; + + // Delayed marking support in case we OOM pushing work onto the mark stack. + MainThreadOrGCTaskData delayedMarkingList; + MainThreadOrGCTaskData delayedMarkingWorkAdded; +#ifdef DEBUG + /* Count of arenas that are currently in the stack. */ + MainThreadOrGCTaskData markLaterArenas; +#endif + + SweepingTracer sweepingTracer; + + /* Track total GC heap size for this runtime. */ + HeapSize heapSize; + + /* GC scheduling state and parameters. */ + GCSchedulingTunables tunables; + GCSchedulingState schedulingState; + MainThreadData fullGCRequested; + + // Helper thread configuration. + MainThreadData helperThreadRatio; + MainThreadData maxHelperThreads; + MainThreadOrGCTaskData helperThreadCount; + MainThreadData markingThreadCount; + + // State used for managing atom mark bitmaps in each zone. + AtomMarkingRuntime atomMarking; + + /* + * Pointer to a callback that, if set, will be used to create a + * budget for internally-triggered GCs. + */ + MainThreadData createBudgetCallback; + + private: + // Arenas used for permanent things created at startup and shared by child + // runtimes. + MainThreadData permanentAtoms; + MainThreadData permanentFatInlineAtoms; + MainThreadData permanentWellKnownSymbols; + + // When chunks are empty, they reside in the emptyChunks pool and are + // re-used as needed or eventually expired if not re-used. The emptyChunks + // pool gets refilled from the background allocation task heuristically so + // that empty chunks should always be available for immediate allocation + // without syscalls. + GCLockData emptyChunks_; + + // Chunks which have had some, but not all, of their arenas allocated live + // in the available chunk lists. When all available arenas in a chunk have + // been allocated, the chunk is removed from the available list and moved + // to the fullChunks pool. During a GC, if all arenas are free, the chunk + // is moved back to the emptyChunks pool and scheduled for eventual + // release. + GCLockData availableChunks_; + + // When all arenas in a chunk are used, it is moved to the fullChunks pool + // so as to reduce the cost of operations on the available lists. + GCLockData fullChunks_; + + /* + * JSGC_MIN_EMPTY_CHUNK_COUNT + * JSGC_MAX_EMPTY_CHUNK_COUNT + * + * Controls the number of empty chunks reserved for future allocation. + * + * They can be read off main thread by the background allocation task and the + * background decommit task. + */ + GCLockData minEmptyChunkCount_; + GCLockData maxEmptyChunkCount_; + + MainThreadData rootsHash; + + // An incrementing id used to assign unique ids to cells that require one. + MainThreadData nextCellUniqueId_; + + /* + * Number of the committed arenas in all GC chunks including empty chunks. + */ + mozilla::Atomic numArenasFreeCommitted; + MainThreadData verifyPreData; + + MainThreadData lastGCStartTime_; + MainThreadData lastGCEndTime_; + + WriteOnceData initialized; + MainThreadData incrementalGCEnabled; + MainThreadData perZoneGCEnabled; + + mozilla::Atomic numActiveZoneIters; + + /* During shutdown, the GC needs to clean up every possible object. */ + MainThreadData cleanUpEverything; + + /* + * The gray bits can become invalid if UnmarkGray overflows the stack. A + * full GC will reset this bit, since it fills in all the gray bits. + */ + UnprotectedData grayBitsValid; + + mozilla::Atomic majorGCTriggerReason; + + /* Incremented at the start of every minor GC. */ + MainThreadData minorGCNumber; + + /* Incremented at the start of every major GC. */ + MainThreadData majorGCNumber; + + /* Incremented on every GC slice or minor collection. */ + MainThreadData number; + + /* Incremented on every GC slice. */ + MainThreadData sliceNumber; + + /* Whether the currently running GC can finish in multiple slices. */ + MainThreadOrGCTaskData isIncremental; + + /* Whether all zones are being collected in first GC slice. */ + MainThreadData isFull; + + /* Whether the heap will be compacted at the end of GC. */ + MainThreadData isCompacting; + + /* The invocation kind of the current GC, set at the start of collection. */ + MainThreadOrGCTaskData> maybeGcOptions; + + /* The initial GC reason, taken from the first slice. */ + MainThreadData initialReason; + + /* + * The current incremental GC phase. This is also used internally in + * non-incremental GC. + */ + MainThreadOrGCTaskData incrementalState; + + /* The incremental state at the start of this slice. */ + MainThreadOrGCTaskData initialState; + + /* Whether to pay attention the zeal settings in this incremental slice. */ +#ifdef JS_GC_ZEAL + MainThreadData useZeal; +#else + const bool useZeal; +#endif + + /* Indicates that the last incremental slice exhausted the mark stack. */ + MainThreadData lastMarkSlice; + + // Whether it's currently safe to yield to the mutator in an incremental GC. + MainThreadData safeToYield; + + // Whether to do any marking caused by barriers on a background thread during + // incremental sweeping, while also sweeping zones which have finished + // marking. + MainThreadData markOnBackgroundThreadDuringSweeping; + + // Whether any sweeping and decommitting will run on a separate GC helper + // thread. + MainThreadData useBackgroundThreads; + +#ifdef DEBUG + /* Shutdown has started. Further collections must be shutdown collections. */ + MainThreadData hadShutdownGC; +#endif + + /* Singly linked list of zones to be swept in the background. */ + HelperThreadLockData backgroundSweepZones; + + /* + * Whether to trigger a GC slice after a background task is complete, so that + * the collector can continue or finsish collecting. This is only used for the + * tasks that run concurrently with the mutator, which are background + * finalization and background decommit. + */ + HelperThreadLockData requestSliceAfterBackgroundTask; + + /* + * Free LIFO blocks are transferred to these allocators before being freed on + * a background thread. + */ + HelperThreadLockData lifoBlocksToFree; + MainThreadData lifoBlocksToFreeAfterMinorGC; + HelperThreadLockData buffersToFreeAfterMinorGC; + + /* Index of current sweep group (for stats). */ + MainThreadData sweepGroupIndex; + + /* + * Incremental sweep state. + */ + MainThreadData sweepGroups; + MainThreadOrGCTaskData currentSweepGroup; + MainThreadData> sweepActions; + MainThreadOrGCTaskData sweepZone; + MainThreadOrGCTaskData sweepAllocKind; + MainThreadData> maybeAtomsToSweep; + MainThreadOrGCTaskData> + weakCachesToSweep; + MainThreadData abortSweepAfterCurrentGroup; + MainThreadOrGCTaskData sweepMarkResult; + +#ifdef DEBUG + /* + * List of objects to mark at the beginning of a GC for testing purposes. May + * also contain string directives to change mark color or wait until different + * phases of the GC. + * + * This is a WeakCache because not everything in this list is guaranteed to + * end up marked (eg if you insert an object from an already-processed sweep + * group in the middle of an incremental GC). Also, the mark queue is not + * used during shutdown GCs. In either case, unmarked objects may need to be + * discarded. + */ + JS::WeakCache, 0, SystemAllocPolicy>> + testMarkQueue; + + /* Position within the test mark queue. */ + size_t queuePos; + + /* The test marking queue might want to be marking a particular color. */ + mozilla::Maybe queueMarkColor; + + // During gray marking, delay AssertCellIsNotGray checks by + // recording the cell pointers here and checking after marking has + // finished. + MainThreadData> + cellsToAssertNotGray; + friend void js::gc::detail::AssertCellIsNotGray(const Cell*); +#endif + + friend class SweepGroupsIter; + + /* + * Incremental compacting state. + */ + MainThreadData startedCompacting; + MainThreadData zonesToMaybeCompact; + MainThreadData zonesCompacted; +#ifdef DEBUG + GCLockData relocatedArenasToRelease; +#endif + +#ifdef JS_GC_ZEAL + MainThreadData markingValidator; +#endif + + /* + * Default budget for incremental GC slice. See js/SliceBudget.h. + * + * JSGC_SLICE_TIME_BUDGET_MS + * pref: javascript.options.mem.gc_incremental_slice_ms, + */ + MainThreadData defaultTimeBudgetMS_; + + /* + * We disable incremental GC if we encounter a Class with a trace hook + * that does not implement write barriers. + */ + MainThreadData incrementalAllowed; + + /* + * Whether compacting GC can is enabled globally. + * + * JSGC_COMPACTING_ENABLED + * pref: javascript.options.mem.gc_compacting + */ + MainThreadData compactingEnabled; + + /* + * Whether parallel marking is enabled globally. + * + * JSGC_PARALLEL_MARKING_ENABLED + * pref: javascript.options.mem.gc_parallel_marking + */ + MainThreadData parallelMarkingEnabled; + + MainThreadData rootsRemoved; + + /* + * These options control the zealousness of the GC. At every allocation, + * nextScheduled is decremented. When it reaches zero we do a full GC. + * + * At this point, if zeal_ is one of the types that trigger periodic + * collection, then nextScheduled is reset to the value of zealFrequency. + * Otherwise, no additional GCs take place. + * + * You can control these values in several ways: + * - Set the JS_GC_ZEAL environment variable + * - Call gczeal() or schedulegc() from inside shell-executed JS code + * (see the help for details) + * + * If gcZeal_ == 1 then we perform GCs in select places (during MaybeGC and + * whenever we are notified that GC roots have been removed). This option is + * mainly useful to embedders. + * + * We use zeal_ == 4 to enable write barrier verification. See the comment + * in gc/Verifier.cpp for more information about this. + * + * zeal_ values from 8 to 10 periodically run different types of + * incremental GC. + * + * zeal_ value 14 performs periodic shrinking collections. + */ +#ifdef JS_GC_ZEAL + static_assert(size_t(ZealMode::Count) <= 32, + "Too many zeal modes to store in a uint32_t"); + MainThreadData zealModeBits; + MainThreadData zealFrequency; + MainThreadData nextScheduled; + MainThreadData deterministicOnly; + MainThreadData zealSliceBudget; + MainThreadData maybeMarkStackLimit; + + MainThreadData>> + selectedForMarking; +#endif + + MainThreadData fullCompartmentChecks; + + MainThreadData gcCallbackDepth; + + MainThreadData> gcCallback; + MainThreadData> + gcDoCycleCollectionCallback; + MainThreadData> tenuredCallback; + MainThreadData> finalizeCallbacks; + MainThreadOrGCTaskData> + hostCleanupFinalizationRegistryCallback; + MainThreadData> + updateWeakPointerZonesCallbacks; + MainThreadData> + updateWeakPointerCompartmentCallbacks; + + /* + * The trace operations to trace embedding-specific GC roots. One is for + * tracing through black roots and the other is for tracing through gray + * roots. The black/gray distinction is only relevant to the cycle + * collector. + */ + MainThreadData> blackRootTracers; + MainThreadOrGCTaskData> grayRootTracer; + + /* Always preserve JIT code during GCs, for testing. */ + MainThreadData alwaysPreserveCode; + + /* Count of the number of zones that are currently in page load. */ + MainThreadData inPageLoadCount; + + MainThreadData lowMemoryState; + + /* + * General purpose GC lock, used for synchronising operations on + * arenas and during parallel marking. + */ + friend class js::AutoLockGC; + friend class js::AutoLockGCBgAlloc; + js::Mutex lock MOZ_UNANNOTATED; + + /* Lock used to synchronise access to delayed marking state. */ + js::Mutex delayedMarkingLock MOZ_UNANNOTATED; + + friend class BackgroundSweepTask; + friend class BackgroundFreeTask; + + BackgroundAllocTask allocTask; + BackgroundUnmarkTask unmarkTask; + BackgroundMarkTask markTask; + BackgroundSweepTask sweepTask; + BackgroundFreeTask freeTask; + BackgroundDecommitTask decommitTask; + + /* + * During incremental sweeping, this field temporarily holds the arenas of + * the current AllocKind being swept in order of increasing free space. + */ + MainThreadData incrementalSweepList; + + MainThreadData nursery_; + + // The store buffer used to track tenured to nursery edges for generational + // GC. This is accessed off main thread when sweeping WeakCaches. + MainThreadOrGCTaskData storeBuffer_; + + mozilla::TimeStamp lastLastDitchTime; + + // The last time per-zone allocation rates were updated. + MainThreadData lastAllocRateUpdateTime; + + // Total collector time since per-zone allocation rates were last updated. + MainThreadData collectorTimeSinceAllocRateUpdate; + + friend class MarkingValidator; + friend class AutoEnterIteration; +}; + +/* Prevent compartments and zones from being collected during iteration. */ +class MOZ_RAII AutoEnterIteration { + GCRuntime* gc; + + public: + explicit AutoEnterIteration(GCRuntime* gc_) : gc(gc_) { + ++gc->numActiveZoneIters; + } + + ~AutoEnterIteration() { + MOZ_ASSERT(gc->numActiveZoneIters); + --gc->numActiveZoneIters; + } +}; + +#ifdef JS_GC_ZEAL + +inline bool GCRuntime::hasZealMode(ZealMode mode) { + static_assert(size_t(ZealMode::Limit) < sizeof(zealModeBits) * 8, + "Zeal modes must fit in zealModeBits"); + return zealModeBits & (1 << uint32_t(mode)); +} + +inline void GCRuntime::clearZealMode(ZealMode mode) { + zealModeBits &= ~(1 << uint32_t(mode)); + MOZ_ASSERT(!hasZealMode(mode)); +} + +inline bool GCRuntime::needZealousGC() { + if (nextScheduled > 0 && --nextScheduled == 0) { + if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::GenerationalGC) || + hasZealMode(ZealMode::IncrementalMultipleSlices) || + hasZealMode(ZealMode::Compact) || hasIncrementalTwoSliceZealMode()) { + nextScheduled = zealFrequency; + } + return true; + } + return false; +} + +inline bool GCRuntime::hasIncrementalTwoSliceZealMode() { + return hasZealMode(ZealMode::YieldBeforeRootMarking) || + hasZealMode(ZealMode::YieldBeforeMarking) || + hasZealMode(ZealMode::YieldBeforeSweeping) || + hasZealMode(ZealMode::YieldBeforeSweepingAtoms) || + hasZealMode(ZealMode::YieldBeforeSweepingCaches) || + hasZealMode(ZealMode::YieldBeforeSweepingObjects) || + hasZealMode(ZealMode::YieldBeforeSweepingNonObjects) || + hasZealMode(ZealMode::YieldBeforeSweepingPropMapTrees) || + hasZealMode(ZealMode::YieldWhileGrayMarking); +} + +#else +inline bool GCRuntime::hasZealMode(ZealMode mode) { return false; } +inline void GCRuntime::clearZealMode(ZealMode mode) {} +inline bool GCRuntime::needZealousGC() { return false; } +inline bool GCRuntime::hasIncrementalTwoSliceZealMode() { return false; } +#endif + +bool IsCurrentlyAnimating(const mozilla::TimeStamp& lastAnimationTime, + const mozilla::TimeStamp& currentTime); + +} /* namespace gc */ +} /* namespace js */ + +#endif diff --git a/js/src/gc/GenerateStatsPhases.py b/js/src/gc/GenerateStatsPhases.py new file mode 100644 index 0000000000..355abaf09d --- /dev/null +++ b/js/src/gc/GenerateStatsPhases.py @@ -0,0 +1,404 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# flake8: noqa: F821 + +# Generate graph structures for GC statistics recording. +# +# Stats phases are nested and form a directed acyclic graph starting +# from a set of root phases. Importantly, a phase may appear under more +# than one parent phase. +# +# For example, the following arrangement is possible: +# +# +---+ +# | A | +# +---+ +# | +# +-------+-------+ +# | | | +# v v v +# +---+ +---+ +---+ +# | B | | C | | D | +# +---+ +---+ +---+ +# | | +# +---+---+ +# | +# v +# +---+ +# | E | +# +---+ +# +# This graph is expanded into a tree (or really a forest) and phases +# with multiple parents are duplicated. +# +# For example, the input example above would be expanded to: +# +# +---+ +# | A | +# +---+ +# | +# +-------+-------+ +# | | | +# v v v +# +---+ +---+ +---+ +# | B | | C | | D | +# +---+ +---+ +---+ +# | | +# v v +# +---+ +---+ +# | E | | E'| +# +---+ +---+ + +# NOTE: If you add new phases here the current next phase kind number can be +# found at the end of js/src/gc/StatsPhasesGenerated.inc + +import collections +import re + + +class PhaseKind: + def __init__(self, name, descr, bucket, children=[]): + self.name = name + self.descr = descr + # For telemetry + self.bucket = bucket + self.children = children + + +AllPhaseKinds = [] +PhaseKindsByName = dict() + + +def addPhaseKind(name, descr, bucket, children=[]): + assert name not in PhaseKindsByName + phaseKind = PhaseKind(name, descr, bucket, children) + AllPhaseKinds.append(phaseKind) + PhaseKindsByName[name] = phaseKind + return phaseKind + + +def getPhaseKind(name): + return PhaseKindsByName[name] + + +PhaseKindGraphRoots = [ + addPhaseKind("MUTATOR", "Mutator Running", 0), + addPhaseKind("GC_BEGIN", "Begin Callback", 1), + addPhaseKind( + "EVICT_NURSERY_FOR_MAJOR_GC", + "Evict Nursery For Major GC", + 70, + [ + addPhaseKind( + "MARK_ROOTS", + "Mark Roots", + 48, + [ + addPhaseKind("MARK_CCWS", "Mark Cross Compartment Wrappers", 50), + addPhaseKind("MARK_STACK", "Mark C and JS stacks", 51), + addPhaseKind("MARK_RUNTIME_DATA", "Mark Runtime-wide Data", 52), + addPhaseKind("MARK_EMBEDDING", "Mark Embedding", 53), + ], + ) + ], + ), + addPhaseKind("WAIT_BACKGROUND_THREAD", "Wait Background Thread", 2), + addPhaseKind( + "PREPARE", + "Prepare For Collection", + 69, + [ + addPhaseKind("UNMARK", "Unmark", 7), + addPhaseKind("UNMARK_WEAKMAPS", "Unmark WeakMaps", 76), + addPhaseKind("MARK_DISCARD_CODE", "Mark Discard Code", 3), + addPhaseKind("RELAZIFY_FUNCTIONS", "Relazify Functions", 4), + addPhaseKind("PURGE", "Purge", 5), + addPhaseKind("PURGE_PROP_MAP_TABLES", "Purge PropMapTables", 60), + addPhaseKind("PURGE_SOURCE_URLS", "Purge Source URLs", 73), + addPhaseKind("JOIN_PARALLEL_TASKS", "Join Parallel Tasks", 67), + ], + ), + addPhaseKind( + "MARK", + "Mark", + 6, + [ + getPhaseKind("MARK_ROOTS"), + addPhaseKind("MARK_DELAYED", "Mark Delayed", 8), + addPhaseKind( + "MARK_WEAK", + "Mark Weak", + 13, + [ + getPhaseKind("MARK_DELAYED"), + addPhaseKind("MARK_GRAY_WEAK", "Mark Gray and Weak", 16), + ], + ), + addPhaseKind("MARK_INCOMING_GRAY", "Mark Incoming Gray Pointers", 14), + addPhaseKind("MARK_GRAY", "Mark Gray", 15), + addPhaseKind( + "PARALLEL_MARK", + "Parallel marking", + 78, + [ + getPhaseKind("JOIN_PARALLEL_TASKS"), + # The following are only used for parallel phase times: + addPhaseKind("PARALLEL_MARK_MARK", "Parallel marking work", 79), + addPhaseKind("PARALLEL_MARK_WAIT", "Waiting for work", 80), + ], + ), + ], + ), + addPhaseKind( + "SWEEP", + "Sweep", + 9, + [ + getPhaseKind("MARK"), + addPhaseKind( + "FINALIZE_START", + "Finalize Start Callbacks", + 17, + [ + addPhaseKind("WEAK_ZONES_CALLBACK", "Per-Slice Weak Callback", 57), + addPhaseKind( + "WEAK_COMPARTMENT_CALLBACK", "Per-Compartment Weak Callback", 58 + ), + ], + ), + addPhaseKind("UPDATE_ATOMS_BITMAP", "Sweep Atoms Bitmap", 68), + addPhaseKind("SWEEP_ATOMS_TABLE", "Sweep Atoms Table", 18), + addPhaseKind( + "SWEEP_COMPARTMENTS", + "Sweep Compartments", + 20, + [ + addPhaseKind("SWEEP_DISCARD_CODE", "Sweep Discard Code", 21), + addPhaseKind("SWEEP_INNER_VIEWS", "Sweep Inner Views", 22), + addPhaseKind( + "SWEEP_CC_WRAPPER", "Sweep Cross Compartment Wrappers", 23 + ), + addPhaseKind("SWEEP_BASE_SHAPE", "Sweep Base Shapes", 24), + addPhaseKind("SWEEP_INITIAL_SHAPE", "Sweep Initial Shapes", 25), + addPhaseKind("SWEEP_REGEXP", "Sweep Regexps", 28), + addPhaseKind("SWEEP_COMPRESSION", "Sweep Compression Tasks", 62), + addPhaseKind("SWEEP_WEAKMAPS", "Sweep WeakMaps", 63), + addPhaseKind("SWEEP_UNIQUEIDS", "Sweep Unique IDs", 64), + addPhaseKind( + "SWEEP_FINALIZATION_OBSERVERS", + "Sweep FinalizationRegistries and WeakRefs", + 74, + ), + addPhaseKind("SWEEP_JIT_DATA", "Sweep JIT Data", 65), + addPhaseKind("SWEEP_WEAK_CACHES", "Sweep Weak Caches", 66), + addPhaseKind("SWEEP_MISC", "Sweep Miscellaneous", 29), + getPhaseKind("JOIN_PARALLEL_TASKS"), + ], + ), + addPhaseKind("FINALIZE_OBJECT", "Finalize Objects", 33), + addPhaseKind("FINALIZE_NON_OBJECT", "Finalize Non-objects", 34), + addPhaseKind("SWEEP_PROP_MAP", "Sweep PropMap Tree", 77), + addPhaseKind("FINALIZE_END", "Finalize End Callback", 38), + addPhaseKind("DESTROY", "Deallocate", 39), + getPhaseKind("JOIN_PARALLEL_TASKS"), + addPhaseKind("FIND_DEAD_COMPARTMENTS", "Find Dead Compartments", 54), + ], + ), + addPhaseKind( + "COMPACT", + "Compact", + 40, + [ + addPhaseKind("COMPACT_MOVE", "Compact Move", 41), + addPhaseKind( + "COMPACT_UPDATE", + "Compact Update", + 42, + [ + getPhaseKind("MARK_ROOTS"), + addPhaseKind("COMPACT_UPDATE_CELLS", "Compact Update Cells", 43), + getPhaseKind("JOIN_PARALLEL_TASKS"), + ], + ), + ], + ), + addPhaseKind("DECOMMIT", "Decommit", 72), + addPhaseKind("GC_END", "End Callback", 44), + addPhaseKind( + "MINOR_GC", + "All Minor GCs", + 45, + [ + getPhaseKind("MARK_ROOTS"), + ], + ), + addPhaseKind( + "EVICT_NURSERY", + "Minor GCs to Evict Nursery", + 46, + [ + getPhaseKind("MARK_ROOTS"), + ], + ), + addPhaseKind( + "TRACE_HEAP", + "Trace Heap", + 47, + [ + getPhaseKind("MARK_ROOTS"), + ], + ), +] + + +class Phase: + # Expand the DAG into a tree, duplicating phases which have more than + # one parent. + def __init__(self, phaseKind, parent): + self.phaseKind = phaseKind + self.parent = parent + self.depth = parent.depth + 1 if parent else 0 + self.children = [] + self.nextSibling = None + self.nextInPhaseKind = None + + self.path = re.sub(r"\W+", "_", phaseKind.name.lower()) + if parent is not None: + self.path = parent.path + "." + self.path + + +def expandPhases(): + phases = [] + phasesForKind = collections.defaultdict(list) + + def traverse(phaseKind, parent): + ep = Phase(phaseKind, parent) + phases.append(ep) + + # Update list of expanded phases for this phase kind. + if phasesForKind[phaseKind]: + phasesForKind[phaseKind][-1].nextInPhaseKind = ep + phasesForKind[phaseKind].append(ep) + + # Recurse over children. + for child in phaseKind.children: + child_ep = traverse(child, ep) + if ep.children: + ep.children[-1].nextSibling = child_ep + ep.children.append(child_ep) + return ep + + for phaseKind in PhaseKindGraphRoots: + traverse(phaseKind, None) + + return phases, phasesForKind + + +AllPhases, PhasesForPhaseKind = expandPhases() + +# Name phases based on phase kind name and index if there are multiple phases +# corresponding to a single phase kind. + +for phaseKind in AllPhaseKinds: + phases = PhasesForPhaseKind[phaseKind] + if len(phases) == 1: + phases[0].name = "%s" % phaseKind.name + else: + for index, phase in enumerate(phases): + phase.name = "%s_%d" % (phaseKind.name, index + 1) + +# Find the maximum phase nesting. +MaxPhaseNesting = max(phase.depth for phase in AllPhases) + 1 + +# And the maximum bucket number. +MaxBucket = max(kind.bucket for kind in AllPhaseKinds) + +# Generate code. + + +def writeList(out, items): + if items: + out.write(",\n".join(" " + item for item in items) + "\n") + + +def writeEnumClass(out, name, type, items, extraItems): + items = ["FIRST"] + list(items) + ["LIMIT"] + list(extraItems) + items[1] += " = " + items[0] + out.write("enum class %s : %s {\n" % (name, type)) + writeList(out, items) + out.write("};\n") + + +def generateHeader(out): + # + # Generate PhaseKind enum. + # + phaseKindNames = map(lambda phaseKind: phaseKind.name, AllPhaseKinds) + extraPhaseKinds = [ + "NONE = LIMIT", + "EXPLICIT_SUSPENSION = LIMIT", + "IMPLICIT_SUSPENSION", + ] + writeEnumClass(out, "PhaseKind", "uint8_t", phaseKindNames, extraPhaseKinds) + out.write("\n") + + # + # Generate Phase enum. + # + phaseNames = map(lambda phase: phase.name, AllPhases) + extraPhases = ["NONE = LIMIT", "EXPLICIT_SUSPENSION = LIMIT", "IMPLICIT_SUSPENSION"] + writeEnumClass(out, "Phase", "uint8_t", phaseNames, extraPhases) + out.write("\n") + + # + # Generate MAX_PHASE_NESTING constant. + # + out.write("static const size_t MAX_PHASE_NESTING = %d;\n" % MaxPhaseNesting) + + +def generateCpp(out): + # + # Generate the PhaseKindInfo table. + # + out.write("static constexpr PhaseKindTable phaseKinds = {\n") + for phaseKind in AllPhaseKinds: + phase = PhasesForPhaseKind[phaseKind][0] + out.write( + ' /* PhaseKind::%s */ PhaseKindInfo { Phase::%s, %d, "%s" },\n' + % (phaseKind.name, phase.name, phaseKind.bucket, phaseKind.name) + ) + out.write("};\n") + out.write("\n") + + # + # Generate the PhaseInfo tree. + # + def name(phase): + return "Phase::" + phase.name if phase else "Phase::NONE" + + out.write("static constexpr PhaseTable phases = {\n") + for phase in AllPhases: + firstChild = phase.children[0] if phase.children else None + phaseKind = phase.phaseKind + out.write( + ' /* %s */ PhaseInfo { %s, %s, %s, %s, PhaseKind::%s, %d, "%s", "%s" },\n' + % ( # NOQA: E501 + name(phase), + name(phase.parent), + name(firstChild), + name(phase.nextSibling), + name(phase.nextInPhaseKind), + phaseKind.name, + phase.depth, + phaseKind.descr, + phase.path, + ) + ) + out.write("};\n") + + # + # Print in a comment the next available phase kind number. + # + out.write("// The next available phase kind number is: %d\n" % (MaxBucket + 1)) diff --git a/js/src/gc/HashUtil.h b/js/src/gc/HashUtil.h new file mode 100644 index 0000000000..52a3058f8a --- /dev/null +++ b/js/src/gc/HashUtil.h @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_HashUtil_h +#define gc_HashUtil_h + +#include + +#include "vm/JSContext.h" + +namespace js { + +/* + * Used to add entries to a js::HashMap or HashSet where the key depends on a GC + * thing that may be moved by generational or compacting GC between the call to + * lookupForAdd() and relookupOrAdd(). + */ +template +struct DependentAddPtr { + typedef typename T::AddPtr AddPtr; + typedef typename T::Entry Entry; + + template + DependentAddPtr(const JSContext* cx, T& table, const Lookup& lookup) + : addPtr(table.lookupForAdd(lookup)), + originalGcNumber(cx->runtime()->gc.gcNumber()) {} + + DependentAddPtr(DependentAddPtr&& other) + : addPtr(other.addPtr), originalGcNumber(other.originalGcNumber) {} + + template + bool add(JSContext* cx, T& table, const KeyInput& key, + const ValueInput& value) { + refreshAddPtr(cx, table, key); + if (!table.relookupOrAdd(addPtr, key, value)) { + ReportOutOfMemory(cx); + return false; + } + return true; + } + + template + void remove(JSContext* cx, T& table, const KeyInput& key) { + refreshAddPtr(cx, table, key); + if (addPtr) { + table.remove(addPtr); + } + } + + bool found() const { return addPtr.found(); } + explicit operator bool() const { return found(); } + const Entry& operator*() const { return *addPtr; } + const Entry* operator->() const { return &*addPtr; } + + private: + AddPtr addPtr; + const uint64_t originalGcNumber; + + template + void refreshAddPtr(JSContext* cx, T& table, const KeyInput& key) { + bool gcHappened = originalGcNumber != cx->runtime()->gc.gcNumber(); + if (gcHappened) { + addPtr = table.lookupForAdd(key); + } + } + + DependentAddPtr() = delete; + DependentAddPtr(const DependentAddPtr&) = delete; + DependentAddPtr& operator=(const DependentAddPtr&) = delete; +}; + +template +inline auto MakeDependentAddPtr(const JSContext* cx, T& table, + const Lookup& lookup) { + using Ptr = DependentAddPtr>; + return Ptr(cx, table, lookup); +} + +} // namespace js + +#endif diff --git a/js/src/gc/Heap-inl.h b/js/src/gc/Heap-inl.h new file mode 100644 index 0000000000..87dcdc900c --- /dev/null +++ b/js/src/gc/Heap-inl.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Heap_inl_h +#define gc_Heap_inl_h + +#include "gc/Heap.h" + +#include "gc/StoreBuffer.h" +#include "gc/Zone.h" +#include "util/Poison.h" +#include "vm/Runtime.h" + +inline void js::gc::Arena::init(JS::Zone* zoneArg, AllocKind kind, + const AutoLockGC& lock) { +#ifdef DEBUG + MOZ_MAKE_MEM_DEFINED(&zone, sizeof(zone)); + MOZ_ASSERT((uintptr_t(zone) & 0xff) == JS_FREED_ARENA_PATTERN); +#endif + + MOZ_ASSERT(firstFreeSpan.isEmpty()); + MOZ_ASSERT(!allocated()); + MOZ_ASSERT(!onDelayedMarkingList_); + MOZ_ASSERT(!hasDelayedBlackMarking_); + MOZ_ASSERT(!hasDelayedGrayMarking_); + MOZ_ASSERT(!nextDelayedMarkingArena_); + + MOZ_MAKE_MEM_UNDEFINED(this, ArenaSize); + + zone = zoneArg; + allocKind = kind; + isNewlyCreated_ = 1; + onDelayedMarkingList_ = 0; + hasDelayedBlackMarking_ = 0; + hasDelayedGrayMarking_ = 0; + nextDelayedMarkingArena_ = 0; + if (zone->isAtomsZone()) { + zone->runtimeFromAnyThread()->gc.atomMarking.registerArena(this, lock); + } else { + bufferedCells() = &ArenaCellSet::Empty; + } + + setAsFullyUnused(); +} + +inline void js::gc::Arena::release(const AutoLockGC& lock) { + if (zone->isAtomsZone()) { + zone->runtimeFromAnyThread()->gc.atomMarking.unregisterArena(this, lock); + } + setAsNotAllocated(); +} + +inline js::gc::ArenaCellSet*& js::gc::Arena::bufferedCells() { + MOZ_ASSERT(zone && !zone->isAtomsZone()); + return bufferedCells_; +} + +inline size_t& js::gc::Arena::atomBitmapStart() { + MOZ_ASSERT(zone && zone->isAtomsZone()); + return atomBitmapStart_; +} + +inline js::gc::NurseryCellHeader::NurseryCellHeader(AllocSite* site, + JS::TraceKind kind) + : allocSiteAndTraceKind(MakeValue(site, kind)) {} + +#endif diff --git a/js/src/gc/Heap.cpp b/js/src/gc/Heap.cpp new file mode 100644 index 0000000000..fdf24fbb7c --- /dev/null +++ b/js/src/gc/Heap.cpp @@ -0,0 +1,635 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Tenured heap management. + * + * This file contains method definitions for the following classes for code that + * is not specific to a particular phase of GC: + * + * - Arena + * - ArenaList + * - FreeLists + * - ArenaLists + * - TenuredChunk + * - ChunkPool + */ + +#include "gc/Heap-inl.h" + +#include "gc/GCLock.h" +#include "gc/Memory.h" +#include "jit/Assembler.h" +#include "vm/BigIntType.h" +#include "vm/RegExpShared.h" +#include "vm/Scope.h" + +#include "gc/ArenaList-inl.h" +#include "gc/PrivateIterators-inl.h" + +using namespace js; +using namespace js::gc; + +// Check that reserved bits of a Cell are compatible with our typical allocators +// since most derived classes will store a pointer in the first word. +static const size_t MinFirstWordAlignment = 1u << CellFlagBitsReservedForGC; +static_assert(js::detail::LIFO_ALLOC_ALIGN >= MinFirstWordAlignment, + "CellFlagBitsReservedForGC should support LifoAlloc"); +static_assert(CellAlignBytes >= MinFirstWordAlignment, + "CellFlagBitsReservedForGC should support gc::Cell"); +static_assert(js::jit::CodeAlignment >= MinFirstWordAlignment, + "CellFlagBitsReservedForGC should support JIT code"); +static_assert(js::gc::JSClassAlignBytes >= MinFirstWordAlignment, + "CellFlagBitsReservedForGC should support JSClass pointers"); +static_assert(js::ScopeDataAlignBytes >= MinFirstWordAlignment, + "CellFlagBitsReservedForGC should support scope data pointers"); + +#define CHECK_THING_SIZE(allocKind, traceKind, type, sizedType, bgFinal, \ + nursery, compact) \ + static_assert(sizeof(sizedType) >= SortedArenaList::MinThingSize, \ + #sizedType " is smaller than SortedArenaList::MinThingSize!"); \ + static_assert(sizeof(sizedType) >= sizeof(FreeSpan), \ + #sizedType " is smaller than FreeSpan"); \ + static_assert(sizeof(sizedType) % CellAlignBytes == 0, \ + "Size of " #sizedType " is not a multiple of CellAlignBytes"); \ + static_assert(sizeof(sizedType) >= MinCellSize, \ + "Size of " #sizedType " is smaller than the minimum size"); +FOR_EACH_ALLOCKIND(CHECK_THING_SIZE); +#undef CHECK_THING_SIZE + +FreeSpan FreeLists::emptySentinel; + +template +struct ArenaLayout { + static constexpr size_t thingSize() { return sizeof(T); } + static constexpr size_t thingsPerArena() { + return (ArenaSize - ArenaHeaderSize) / thingSize(); + } + static constexpr size_t firstThingOffset() { + return ArenaSize - thingSize() * thingsPerArena(); + } +}; + +const uint8_t Arena::ThingSizes[] = { +#define EXPAND_THING_SIZE(_1, _2, _3, sizedType, _4, _5, _6) \ + ArenaLayout::thingSize(), + FOR_EACH_ALLOCKIND(EXPAND_THING_SIZE) +#undef EXPAND_THING_SIZE +}; + +const uint8_t Arena::FirstThingOffsets[] = { +#define EXPAND_FIRST_THING_OFFSET(_1, _2, _3, sizedType, _4, _5, _6) \ + ArenaLayout::firstThingOffset(), + FOR_EACH_ALLOCKIND(EXPAND_FIRST_THING_OFFSET) +#undef EXPAND_FIRST_THING_OFFSET +}; + +const uint8_t Arena::ThingsPerArena[] = { +#define EXPAND_THINGS_PER_ARENA(_1, _2, _3, sizedType, _4, _5, _6) \ + ArenaLayout::thingsPerArena(), + FOR_EACH_ALLOCKIND(EXPAND_THINGS_PER_ARENA) +#undef EXPAND_THINGS_PER_ARENA +}; + +void Arena::unmarkAll() { + MarkBitmapWord* arenaBits = chunk()->markBits.arenaBits(this); + for (size_t i = 0; i < ArenaBitmapWords; i++) { + arenaBits[i] = 0; + } +} + +void Arena::unmarkPreMarkedFreeCells() { + for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) { + MOZ_ASSERT(cell->isMarkedBlack()); + cell->unmark(); + } +} + +#ifdef DEBUG + +void Arena::checkNoMarkedFreeCells() { + for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) { + MOZ_ASSERT(!cell->isMarkedAny()); + } +} + +void Arena::checkAllCellsMarkedBlack() { + for (ArenaCellIter cell(this); !cell.done(); cell.next()) { + MOZ_ASSERT(cell->isMarkedBlack()); + } +} + +#endif + +#if defined(DEBUG) || defined(JS_GC_ZEAL) +void Arena::checkNoMarkedCells() { + for (ArenaCellIter cell(this); !cell.done(); cell.next()) { + MOZ_ASSERT(!cell->isMarkedAny()); + } +} +#endif + +/* static */ +void Arena::staticAsserts() { + static_assert(size_t(AllocKind::LIMIT) <= 255, + "All AllocKinds and AllocKind::LIMIT must fit in a uint8_t."); + static_assert(std::size(ThingSizes) == AllocKindCount, + "We haven't defined all thing sizes."); + static_assert(std::size(FirstThingOffsets) == AllocKindCount, + "We haven't defined all offsets."); + static_assert(std::size(ThingsPerArena) == AllocKindCount, + "We haven't defined all counts."); +} + +/* static */ +void Arena::checkLookupTables() { +#ifdef DEBUG + for (size_t i = 0; i < AllocKindCount; i++) { + MOZ_ASSERT( + FirstThingOffsets[i] + ThingsPerArena[i] * ThingSizes[i] == ArenaSize, + "Inconsistent arena lookup table data"); + } +#endif +} + +#ifdef DEBUG +void js::gc::ArenaList::dump() { + fprintf(stderr, "ArenaList %p:", this); + if (cursorp_ == &head_) { + fprintf(stderr, " *"); + } + for (Arena* arena = head(); arena; arena = arena->next) { + fprintf(stderr, " %p", arena); + if (cursorp_ == &arena->next) { + fprintf(stderr, " *"); + } + } + fprintf(stderr, "\n"); +} +#endif + +Arena* ArenaList::removeRemainingArenas(Arena** arenap) { + // This is only ever called to remove arenas that are after the cursor, so + // we don't need to update it. +#ifdef DEBUG + for (Arena* arena = *arenap; arena; arena = arena->next) { + MOZ_ASSERT(cursorp_ != &arena->next); + } +#endif + Arena* remainingArenas = *arenap; + *arenap = nullptr; + check(); + return remainingArenas; +} + +FreeLists::FreeLists() { + for (auto i : AllAllocKinds()) { + freeLists_[i] = &emptySentinel; + } +} + +ArenaLists::ArenaLists(Zone* zone) + : zone_(zone), + incrementalSweptArenaKind(AllocKind::LIMIT), + gcCompactPropMapArenasToUpdate(nullptr), + gcNormalPropMapArenasToUpdate(nullptr), + savedEmptyArenas(nullptr) { + for (auto i : AllAllocKinds()) { + concurrentUse(i) = ConcurrentUse::None; + } +} + +void ReleaseArenas(JSRuntime* rt, Arena* arena, const AutoLockGC& lock) { + Arena* next; + for (; arena; arena = next) { + next = arena->next; + rt->gc.releaseArena(arena, lock); + } +} + +void ReleaseArenaList(JSRuntime* rt, ArenaList& arenaList, + const AutoLockGC& lock) { + ReleaseArenas(rt, arenaList.head(), lock); + arenaList.clear(); +} + +ArenaLists::~ArenaLists() { + AutoLockGC lock(runtime()); + + for (auto i : AllAllocKinds()) { + /* + * We can only call this during the shutdown after the last GC when + * the background finalization is disabled. + */ + MOZ_ASSERT(concurrentUse(i) == ConcurrentUse::None); + ReleaseArenaList(runtime(), arenaList(i), lock); + } + ReleaseArenaList(runtime(), incrementalSweptArenas.ref(), lock); + + ReleaseArenas(runtime(), savedEmptyArenas, lock); +} + +void ArenaLists::moveArenasToCollectingLists() { + checkEmptyFreeLists(); + for (AllocKind kind : AllAllocKinds()) { + MOZ_ASSERT(collectingArenaList(kind).isEmpty()); + collectingArenaList(kind) = std::move(arenaList(kind)); + MOZ_ASSERT(arenaList(kind).isEmpty()); + } +} + +void ArenaLists::mergeArenasFromCollectingLists() { + for (AllocKind kind : AllAllocKinds()) { + collectingArenaList(kind).insertListWithCursorAtEnd(arenaList(kind)); + arenaList(kind) = std::move(collectingArenaList(kind)); + MOZ_ASSERT(collectingArenaList(kind).isEmpty()); + } +} + +Arena* ArenaLists::takeSweptEmptyArenas() { + Arena* arenas = savedEmptyArenas; + savedEmptyArenas = nullptr; + return arenas; +} + +void ArenaLists::setIncrementalSweptArenas(AllocKind kind, + SortedArenaList& arenas) { + incrementalSweptArenaKind = kind; + incrementalSweptArenas.ref().clear(); + incrementalSweptArenas = arenas.toArenaList(); +} + +void ArenaLists::clearIncrementalSweptArenas() { + incrementalSweptArenaKind = AllocKind::LIMIT; + incrementalSweptArenas.ref().clear(); +} + +void ArenaLists::checkGCStateNotInUse() { + // Called before and after collection to check the state is as expected. +#ifdef DEBUG + checkSweepStateNotInUse(); + for (auto i : AllAllocKinds()) { + MOZ_ASSERT(collectingArenaList(i).isEmpty()); + } +#endif +} + +void ArenaLists::checkSweepStateNotInUse() { +#ifdef DEBUG + checkNoArenasToUpdate(); + MOZ_ASSERT(incrementalSweptArenaKind == AllocKind::LIMIT); + MOZ_ASSERT(incrementalSweptArenas.ref().isEmpty()); + MOZ_ASSERT(!savedEmptyArenas); + for (auto i : AllAllocKinds()) { + MOZ_ASSERT(concurrentUse(i) == ConcurrentUse::None); + } +#endif +} + +void ArenaLists::checkNoArenasToUpdate() { + MOZ_ASSERT(!gcCompactPropMapArenasToUpdate); + MOZ_ASSERT(!gcNormalPropMapArenasToUpdate); +} + +void ArenaLists::checkNoArenasToUpdateForKind(AllocKind kind) { +#ifdef DEBUG + switch (kind) { + case AllocKind::COMPACT_PROP_MAP: + MOZ_ASSERT(!gcCompactPropMapArenasToUpdate); + break; + case AllocKind::NORMAL_PROP_MAP: + MOZ_ASSERT(!gcNormalPropMapArenasToUpdate); + break; + default: + break; + } +#endif +} + +inline bool TenuredChunk::canDecommitPage(size_t pageIndex) const { + if (decommittedPages[pageIndex]) { + return false; + } + + size_t arenaIndex = pageIndex * ArenasPerPage; + for (size_t i = 0; i < ArenasPerPage; i++) { + if (!freeCommittedArenas[arenaIndex + i]) { + return false; + } + } + + return true; +} + +void TenuredChunk::decommitFreeArenas(GCRuntime* gc, const bool& cancel, + AutoLockGC& lock) { + MOZ_ASSERT(DecommitEnabled()); + + for (size_t i = 0; i < PagesPerChunk; i++) { + if (cancel) { + break; + } + + if (canDecommitPage(i) && !decommitOneFreePage(gc, i, lock)) { + break; + } + } +} + +void TenuredChunk::recycleArena(Arena* arena, SortedArenaList& dest, + size_t thingsPerArena) { + arena->setAsFullyUnused(); + dest.insertAt(arena, thingsPerArena); +} + +void TenuredChunk::releaseArena(GCRuntime* gc, Arena* arena, + const AutoLockGC& lock) { + MOZ_ASSERT(!arena->allocated()); + MOZ_ASSERT(!freeCommittedArenas[arenaIndex(arena)]); + + freeCommittedArenas[arenaIndex(arena)] = true; + ++info.numArenasFreeCommitted; + ++info.numArenasFree; + gc->updateOnArenaFree(); + + verify(); + + updateChunkListAfterFree(gc, 1, lock); +} + +bool TenuredChunk::decommitOneFreePage(GCRuntime* gc, size_t pageIndex, + AutoLockGC& lock) { + MOZ_ASSERT(DecommitEnabled()); + MOZ_ASSERT(canDecommitPage(pageIndex)); + MOZ_ASSERT(info.numArenasFreeCommitted >= ArenasPerPage); + + // Temporarily mark the page as allocated while we decommit. + for (size_t i = 0; i < ArenasPerPage; i++) { + size_t arenaIndex = pageIndex * ArenasPerPage + i; + MOZ_ASSERT(freeCommittedArenas[arenaIndex]); + freeCommittedArenas[arenaIndex] = false; + } + info.numArenasFreeCommitted -= ArenasPerPage; + info.numArenasFree -= ArenasPerPage; + updateChunkListAfterAlloc(gc, lock); + + verify(); + + bool ok; + { + AutoUnlockGC unlock(lock); + ok = !oom::ShouldFailWithOOM() && + MarkPagesUnusedSoft(pageAddress(pageIndex), PageSize); + } + + // Mark the page as decommited if successful or restore the original free + // state. + if (ok) { + decommittedPages[pageIndex] = true; + } else { + for (size_t i = 0; i < ArenasPerPage; i++) { + size_t arenaIndex = pageIndex * ArenasPerPage + i; + MOZ_ASSERT(!freeCommittedArenas[arenaIndex]); + freeCommittedArenas[arenaIndex] = true; + } + info.numArenasFreeCommitted += ArenasPerPage; + } + + info.numArenasFree += ArenasPerPage; + updateChunkListAfterFree(gc, ArenasPerPage, lock); + + verify(); + + return ok; +} + +void TenuredChunk::decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock) { + MOZ_ASSERT(DecommitEnabled()); + + for (size_t i = 0; i < PagesPerChunk; i++) { + if (!canDecommitPage(i)) { + continue; + } + + MOZ_ASSERT(!decommittedPages[i]); + MOZ_ASSERT(info.numArenasFreeCommitted >= ArenasPerPage); + + if (js::oom::ShouldFailWithOOM() || + !MarkPagesUnusedSoft(pageAddress(i), SystemPageSize())) { + break; + } + + decommittedPages[i] = true; + for (size_t j = 0; j < ArenasPerPage; ++j) { + size_t arenaIndex = i * ArenasPerPage + j; + MOZ_ASSERT(freeCommittedArenas[arenaIndex]); + freeCommittedArenas[arenaIndex] = false; + } + info.numArenasFreeCommitted -= ArenasPerPage; + } + + verify(); +} + +void TenuredChunk::updateChunkListAfterAlloc(GCRuntime* gc, + const AutoLockGC& lock) { + if (MOZ_UNLIKELY(!hasAvailableArenas())) { + gc->availableChunks(lock).remove(this); + gc->fullChunks(lock).push(this); + } +} + +void TenuredChunk::updateChunkListAfterFree(GCRuntime* gc, size_t numArenasFree, + const AutoLockGC& lock) { + if (info.numArenasFree == numArenasFree) { + gc->fullChunks(lock).remove(this); + gc->availableChunks(lock).push(this); + } else if (!unused()) { + MOZ_ASSERT(gc->availableChunks(lock).contains(this)); + } else { + MOZ_ASSERT(unused()); + gc->availableChunks(lock).remove(this); + gc->recycleChunk(this, lock); + } +} + +TenuredChunk* ChunkPool::pop() { + MOZ_ASSERT(bool(head_) == bool(count_)); + if (!count_) { + return nullptr; + } + return remove(head_); +} + +void ChunkPool::push(TenuredChunk* chunk) { + MOZ_ASSERT(!chunk->info.next); + MOZ_ASSERT(!chunk->info.prev); + + chunk->info.next = head_; + if (head_) { + head_->info.prev = chunk; + } + head_ = chunk; + ++count_; +} + +TenuredChunk* ChunkPool::remove(TenuredChunk* chunk) { + MOZ_ASSERT(count_ > 0); + MOZ_ASSERT(contains(chunk)); + + if (head_ == chunk) { + head_ = chunk->info.next; + } + if (chunk->info.prev) { + chunk->info.prev->info.next = chunk->info.next; + } + if (chunk->info.next) { + chunk->info.next->info.prev = chunk->info.prev; + } + chunk->info.next = chunk->info.prev = nullptr; + --count_; + + return chunk; +} + +// We could keep the chunk pool sorted, but that's likely to be more expensive. +// This sort is nlogn, but keeping it sorted is likely to be m*n, with m being +// the number of operations (likely higher than n). +void ChunkPool::sort() { + // Only sort if the list isn't already sorted. + if (!isSorted()) { + head_ = mergeSort(head(), count()); + + // Fixup prev pointers. + TenuredChunk* prev = nullptr; + for (TenuredChunk* cur = head_; cur; cur = cur->info.next) { + cur->info.prev = prev; + prev = cur; + } + } + + MOZ_ASSERT(verify()); + MOZ_ASSERT(isSorted()); +} + +TenuredChunk* ChunkPool::mergeSort(TenuredChunk* list, size_t count) { + MOZ_ASSERT(bool(list) == bool(count)); + + if (count < 2) { + return list; + } + + size_t half = count / 2; + + // Split; + TenuredChunk* front = list; + TenuredChunk* back; + { + TenuredChunk* cur = list; + for (size_t i = 0; i < half - 1; i++) { + MOZ_ASSERT(cur); + cur = cur->info.next; + } + back = cur->info.next; + cur->info.next = nullptr; + } + + front = mergeSort(front, half); + back = mergeSort(back, count - half); + + // Merge + list = nullptr; + TenuredChunk** cur = &list; + while (front || back) { + if (!front) { + *cur = back; + break; + } + if (!back) { + *cur = front; + break; + } + + // Note that the sort is stable due to the <= here. Nothing depends on + // this but it could. + if (front->info.numArenasFree <= back->info.numArenasFree) { + *cur = front; + front = front->info.next; + cur = &(*cur)->info.next; + } else { + *cur = back; + back = back->info.next; + cur = &(*cur)->info.next; + } + } + + return list; +} + +bool ChunkPool::isSorted() const { + uint32_t last = 1; + for (TenuredChunk* cursor = head_; cursor; cursor = cursor->info.next) { + if (cursor->info.numArenasFree < last) { + return false; + } + last = cursor->info.numArenasFree; + } + return true; +} + +#ifdef DEBUG + +bool ChunkPool::contains(TenuredChunk* chunk) const { + verify(); + for (TenuredChunk* cursor = head_; cursor; cursor = cursor->info.next) { + if (cursor == chunk) { + return true; + } + } + return false; +} + +bool ChunkPool::verify() const { + MOZ_ASSERT(bool(head_) == bool(count_)); + uint32_t count = 0; + for (TenuredChunk* cursor = head_; cursor; + cursor = cursor->info.next, ++count) { + MOZ_ASSERT_IF(cursor->info.prev, cursor->info.prev->info.next == cursor); + MOZ_ASSERT_IF(cursor->info.next, cursor->info.next->info.prev == cursor); + } + MOZ_ASSERT(count_ == count); + return true; +} + +void ChunkPool::verifyChunks() const { + for (TenuredChunk* chunk = head_; chunk; chunk = chunk->info.next) { + chunk->verify(); + } +} + +void TenuredChunk::verify() const { + MOZ_ASSERT(info.numArenasFree <= ArenasPerChunk); + MOZ_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree); + + size_t decommittedCount = decommittedPages.Count() * ArenasPerPage; + size_t freeCommittedCount = freeCommittedArenas.Count(); + size_t freeCount = freeCommittedCount + decommittedCount; + + MOZ_ASSERT(freeCount == info.numArenasFree); + MOZ_ASSERT(freeCommittedCount == info.numArenasFreeCommitted); + + for (size_t i = 0; i < ArenasPerChunk; ++i) { + MOZ_ASSERT(!(decommittedPages[pageIndex(i)] && freeCommittedArenas[i])); + MOZ_ASSERT_IF(freeCommittedArenas[i], !arenas[i].allocated()); + } +} + +#endif + +void ChunkPool::Iter::next() { + MOZ_ASSERT(!done()); + current_ = current_->info.next; +} diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h new file mode 100644 index 0000000000..9a1b25d508 --- /dev/null +++ b/js/src/gc/Heap.h @@ -0,0 +1,846 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Heap_h +#define gc_Heap_h + +#include "mozilla/DebugOnly.h" + +#include "gc/AllocKind.h" +#include "gc/Pretenuring.h" +#include "js/HeapAPI.h" +#include "js/TypeDecls.h" +#include "util/Poison.h" + +namespace js { + +class AutoLockGC; +class AutoLockGCBgAlloc; +class Nursery; + +// To prevent false sharing, some data structures are aligned to a typical cache +// line size. +static constexpr size_t TypicalCacheLineSize = 64; + +namespace gc { + +class Arena; +class ArenaCellSet; +class ArenaList; +class GCRuntime; +class MarkingValidator; +class SortedArenaList; +class TenuredCell; + +// Cells are aligned to CellAlignShift, so the largest tagged null pointer is: +const uintptr_t LargestTaggedNullCellPointer = (1 << CellAlignShift) - 1; + +/* + * The minimum cell size ends up as twice the cell alignment because the mark + * bitmap contains one bit per CellBytesPerMarkBit bytes (which is equal to + * CellAlignBytes) and we need two mark bits per cell. + */ +const size_t MinCellSize = CellBytesPerMarkBit * MarkBitsPerCell; + +static_assert(ArenaSize % CellAlignBytes == 0, + "Arena size must be a multiple of cell alignment"); + +/* + * A FreeSpan represents a contiguous sequence of free cells in an Arena. It + * can take two forms. + * + * - In an empty span, |first| and |last| are both zero. + * + * - In a non-empty span, |first| is the address of the first free thing in the + * span, and |last| is the address of the last free thing in the span. + * Furthermore, the memory pointed to by |last| holds a FreeSpan structure + * that points to the next span (which may be empty); this works because + * sizeof(FreeSpan) is less than the smallest thingSize. + */ +class FreeSpan { + friend class Arena; + friend class ArenaCellIter; + friend class ArenaFreeCellIter; + + uint16_t first; + uint16_t last; + + public: + // This inits just |first| and |last|; if the span is non-empty it doesn't + // do anything with the next span stored at |last|. + void initBounds(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + checkRange(firstArg, lastArg, arena); + first = firstArg; + last = lastArg; + } + + void initAsEmpty() { + first = 0; + last = 0; + } + + // This sets |first| and |last|, and also sets the next span stored at + // |last| as empty. (As a result, |firstArg| and |lastArg| cannot represent + // an empty span.) + void initFinal(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + initBounds(firstArg, lastArg, arena); + FreeSpan* last = nextSpanUnchecked(arena); + last->initAsEmpty(); + checkSpan(arena); + } + + bool isEmpty() const { return !first; } + + Arena* getArenaUnchecked() { return reinterpret_cast(this); } + inline Arena* getArena(); + + static size_t offsetOfFirst() { return offsetof(FreeSpan, first); } + + static size_t offsetOfLast() { return offsetof(FreeSpan, last); } + + // Like nextSpan(), but no checking of the following span is done. + FreeSpan* nextSpanUnchecked(const Arena* arena) const { + MOZ_ASSERT(arena && !isEmpty()); + return reinterpret_cast(uintptr_t(arena) + last); + } + + const FreeSpan* nextSpan(const Arena* arena) const { + checkSpan(arena); + return nextSpanUnchecked(arena); + } + + MOZ_ALWAYS_INLINE TenuredCell* allocate(size_t thingSize) { + // Eschew the usual checks, because this might be the placeholder span. + // If this is somehow an invalid, non-empty span, checkSpan() will catch it. + Arena* arena = getArenaUnchecked(); + checkSpan(arena); + uintptr_t thing = uintptr_t(arena) + first; + if (first < last) { + // We have space for at least two more things, so do a simple + // bump-allocate. + first += thingSize; + } else if (MOZ_LIKELY(first)) { + // The last space points to the next free span (which may be empty). + const FreeSpan* next = nextSpan(arena); + first = next->first; + last = next->last; + } else { + return nullptr; // The span is empty. + } + checkSpan(arena); + DebugOnlyPoison(reinterpret_cast(thing), + JS_ALLOCATED_TENURED_PATTERN, thingSize, + MemCheckKind::MakeUndefined); + return reinterpret_cast(thing); + } + + inline void checkSpan(const Arena* arena) const; + inline void checkRange(uintptr_t first, uintptr_t last, + const Arena* arena) const; +}; + +/* + * Arenas are the allocation units of the tenured heap in the GC. An arena + * is 4kiB in size and 4kiB-aligned. It starts with several header fields + * followed by some bytes of padding. The remainder of the arena is filled + * with GC things of a particular AllocKind. The padding ensures that the + * GC thing array ends exactly at the end of the arena: + * + * <----------------------------------------------> = ArenaSize bytes + * +---------------+---------+----+----+-----+----+ + * | header fields | padding | T0 | T1 | ... | Tn | + * +---------------+---------+----+----+-----+----+ + * <-------------------------> = first thing offset + */ +class alignas(ArenaSize) Arena { + static JS_PUBLIC_DATA const uint8_t ThingSizes[]; + static JS_PUBLIC_DATA const uint8_t FirstThingOffsets[]; + static JS_PUBLIC_DATA const uint8_t ThingsPerArena[]; + /* + * The first span of free things in the arena. Most of these spans are + * stored as offsets in free regions of the data array, and most operations + * on FreeSpans take an Arena pointer for safety. However, the FreeSpans + * used for allocation are stored here, at the start of an Arena, and use + * their own address to grab the next span within the same Arena. + */ + FreeSpan firstFreeSpan; + + public: + /* + * One of the AllocKind constants or AllocKind::LIMIT when the arena does + * not contain any GC things and is on the list of empty arenas in the GC + * chunk. + */ + AllocKind allocKind; + + /* + * The zone that this Arena is contained within, when allocated. The offset + * of this field must match the ArenaZoneOffset stored in js/HeapAPI.h, + * as is statically asserted below. + */ + JS::Zone* zone; + + /* + * Arena::next has two purposes: when unallocated, it points to the next + * available Arena. When allocated, it points to the next Arena in the same + * zone and with the same alloc kind. + */ + Arena* next; + + private: + static const size_t ARENA_FLAG_BITS = 4; + static const size_t DELAYED_MARKING_ARENA_BITS = + JS_BITS_PER_WORD - ArenaShift; + static_assert( + ARENA_FLAG_BITS + DELAYED_MARKING_ARENA_BITS <= JS_BITS_PER_WORD, + "Not enough space to pack flags and nextDelayedMarkingArena_ pointer " + "into a single word."); + + /* + * True until the arena is swept for the first time. + */ + size_t isNewlyCreated_ : 1; + + /* + * When recursive marking uses too much stack we delay marking of arenas and + * link them into a list for later processing. This uses the following fields. + */ + size_t onDelayedMarkingList_ : 1; + size_t hasDelayedBlackMarking_ : 1; + size_t hasDelayedGrayMarking_ : 1; + size_t nextDelayedMarkingArena_ : DELAYED_MARKING_ARENA_BITS; + + union { + /* + * For arenas in zones other than the atoms zone, if non-null, points + * to an ArenaCellSet that represents the set of cells in this arena + * that are in the nursery's store buffer. + */ + ArenaCellSet* bufferedCells_; + + /* + * For arenas in the atoms zone, the starting index into zone atom + * marking bitmaps (see AtomMarking.h) of the things in this zone. + * Atoms never refer to nursery things, so no store buffer index is + * needed. + */ + size_t atomBitmapStart_; + }; + + public: + /* + * The size of data should be |ArenaSize - offsetof(data)|, but the offset + * is not yet known to the compiler, so we do it by hand. |firstFreeSpan| + * takes up 8 bytes on 64-bit due to alignment requirements; the rest are + * obvious. This constant is stored in js/HeapAPI.h. + */ + uint8_t data[ArenaSize - ArenaHeaderSize]; + + void init(JS::Zone* zoneArg, AllocKind kind, const AutoLockGC& lock); + + // Sets |firstFreeSpan| to the Arena's entire valid range, and + // also sets the next span stored at |firstFreeSpan.last| as empty. + void setAsFullyUnused() { + AllocKind kind = getAllocKind(); + firstFreeSpan.first = firstThingOffset(kind); + firstFreeSpan.last = lastThingOffset(kind); + FreeSpan* last = firstFreeSpan.nextSpanUnchecked(this); + last->initAsEmpty(); + } + + // Initialize an arena to its unallocated state. For arenas that were + // previously allocated for some zone, use release() instead. + void setAsNotAllocated() { + firstFreeSpan.initAsEmpty(); + + // Poison zone pointer to highlight UAF on released arenas in crash data. + AlwaysPoison(&zone, JS_FREED_ARENA_PATTERN, sizeof(zone), + MemCheckKind::MakeNoAccess); + + allocKind = AllocKind::LIMIT; + onDelayedMarkingList_ = 0; + hasDelayedBlackMarking_ = 0; + hasDelayedGrayMarking_ = 0; + nextDelayedMarkingArena_ = 0; + bufferedCells_ = nullptr; + + MOZ_ASSERT(!allocated()); + } + + // Return an allocated arena to its unallocated state. + inline void release(const AutoLockGC& lock); + + uintptr_t address() const { + checkAddress(); + return uintptr_t(this); + } + + inline void checkAddress() const; + + inline TenuredChunk* chunk() const; + + bool allocated() const { + MOZ_ASSERT(IsAllocKind(AllocKind(allocKind))); + return IsValidAllocKind(AllocKind(allocKind)); + } + + AllocKind getAllocKind() const { + MOZ_ASSERT(allocated()); + return allocKind; + } + + FreeSpan* getFirstFreeSpan() { return &firstFreeSpan; } + + static size_t thingSize(AllocKind kind) { return ThingSizes[size_t(kind)]; } + static size_t thingsPerArena(AllocKind kind) { + return ThingsPerArena[size_t(kind)]; + } + static size_t thingsSpan(AllocKind kind) { + return thingsPerArena(kind) * thingSize(kind); + } + + static size_t firstThingOffset(AllocKind kind) { + return FirstThingOffsets[size_t(kind)]; + } + static size_t lastThingOffset(AllocKind kind) { + return ArenaSize - thingSize(kind); + } + + size_t getThingSize() const { return thingSize(getAllocKind()); } + size_t getThingsPerArena() const { return thingsPerArena(getAllocKind()); } + size_t getThingsSpan() const { return getThingsPerArena() * getThingSize(); } + size_t getFirstThingOffset() const { + return firstThingOffset(getAllocKind()); + } + + uintptr_t thingsStart() const { return address() + getFirstThingOffset(); } + uintptr_t thingsEnd() const { return address() + ArenaSize; } + + bool isEmpty() const { + // Arena is empty if its first span covers the whole arena. + firstFreeSpan.checkSpan(this); + AllocKind kind = getAllocKind(); + return firstFreeSpan.first == firstThingOffset(kind) && + firstFreeSpan.last == lastThingOffset(kind); + } + + bool hasFreeThings() const { return !firstFreeSpan.isEmpty(); } + + size_t numFreeThings(size_t thingSize) const { + firstFreeSpan.checkSpan(this); + size_t numFree = 0; + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) { + numFree += (span->last - span->first) / thingSize + 1; + } + return numFree; + } + + size_t countFreeCells() { return numFreeThings(getThingSize()); } + size_t countUsedCells() { return getThingsPerArena() - countFreeCells(); } + +#ifdef DEBUG + bool inFreeList(uintptr_t thing) { + uintptr_t base = address(); + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) { + // If the thing comes before the current span, it's not free. + if (thing < base + span->first) { + return false; + } + + // If we find it before the end of the span, it's free. + if (thing <= base + span->last) { + return true; + } + } + return false; + } +#endif + + static bool isAligned(uintptr_t thing, size_t thingSize) { + /* Things ends at the arena end. */ + uintptr_t tailOffset = ArenaSize - (thing & ArenaMask); + return tailOffset % thingSize == 0; + } + + bool isNewlyCreated() const { return isNewlyCreated_; } + + bool onDelayedMarkingList() const { return onDelayedMarkingList_; } + + Arena* getNextDelayedMarking() const { + MOZ_ASSERT(onDelayedMarkingList_); + return reinterpret_cast(nextDelayedMarkingArena_ << ArenaShift); + } + + void setNextDelayedMarkingArena(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(!onDelayedMarkingList_); + MOZ_ASSERT(!hasDelayedBlackMarking_); + MOZ_ASSERT(!hasDelayedGrayMarking_); + MOZ_ASSERT(!nextDelayedMarkingArena_); + onDelayedMarkingList_ = 1; + if (arena) { + nextDelayedMarkingArena_ = arena->address() >> ArenaShift; + } + } + + void updateNextDelayedMarkingArena(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(onDelayedMarkingList_); + nextDelayedMarkingArena_ = arena ? arena->address() >> ArenaShift : 0; + } + + bool hasDelayedMarking(MarkColor color) const { + MOZ_ASSERT(onDelayedMarkingList_); + return color == MarkColor::Black ? hasDelayedBlackMarking_ + : hasDelayedGrayMarking_; + } + + bool hasAnyDelayedMarking() const { + MOZ_ASSERT(onDelayedMarkingList_); + return hasDelayedBlackMarking_ || hasDelayedGrayMarking_; + } + + void setHasDelayedMarking(MarkColor color, bool value) { + MOZ_ASSERT(onDelayedMarkingList_); + if (color == MarkColor::Black) { + hasDelayedBlackMarking_ = value; + } else { + hasDelayedGrayMarking_ = value; + } + } + + void clearDelayedMarkingState() { + MOZ_ASSERT(onDelayedMarkingList_); + onDelayedMarkingList_ = 0; + hasDelayedBlackMarking_ = 0; + hasDelayedGrayMarking_ = 0; + nextDelayedMarkingArena_ = 0; + } + + inline ArenaCellSet*& bufferedCells(); + inline size_t& atomBitmapStart(); + + template + size_t finalize(JS::GCContext* gcx, AllocKind thingKind, size_t thingSize); + + static void staticAsserts(); + static void checkLookupTables(); + + void unmarkAll(); + void unmarkPreMarkedFreeCells(); + + void arenaAllocatedDuringGC(); + +#ifdef DEBUG + void checkNoMarkedFreeCells(); + void checkAllCellsMarkedBlack(); +#endif + +#if defined(DEBUG) || defined(JS_GC_ZEAL) + void checkNoMarkedCells(); +#endif +}; + +static_assert(ArenaZoneOffset == offsetof(Arena, zone), + "The hardcoded API zone offset must match the actual offset."); + +static_assert(sizeof(Arena) == ArenaSize, + "ArenaSize must match the actual size of the Arena structure."); + +static_assert( + offsetof(Arena, data) == ArenaHeaderSize, + "ArenaHeaderSize must match the actual size of the header fields."); + +inline Arena* FreeSpan::getArena() { + Arena* arena = getArenaUnchecked(); + arena->checkAddress(); + return arena; +} + +inline void FreeSpan::checkSpan(const Arena* arena) const { +#ifdef DEBUG + if (!first) { + MOZ_ASSERT(!first && !last); + return; + } + + arena->checkAddress(); + checkRange(first, last, arena); + + // If there's a following span, it must have a higher address, + // and the gap must be at least 2 * thingSize. + const FreeSpan* next = nextSpanUnchecked(arena); + if (next->first) { + checkRange(next->first, next->last, arena); + size_t thingSize = arena->getThingSize(); + MOZ_ASSERT(last + 2 * thingSize <= next->first); + } +#endif +} + +inline void FreeSpan::checkRange(uintptr_t first, uintptr_t last, + const Arena* arena) const { +#ifdef DEBUG + MOZ_ASSERT(arena); + MOZ_ASSERT(first <= last); + AllocKind thingKind = arena->getAllocKind(); + MOZ_ASSERT(first >= Arena::firstThingOffset(thingKind)); + MOZ_ASSERT(last <= Arena::lastThingOffset(thingKind)); + MOZ_ASSERT((last - first) % Arena::thingSize(thingKind) == 0); +#endif +} + +// Mark bitmap API: + +MOZ_ALWAYS_INLINE bool MarkBitmap::markBit(const TenuredCell* cell, + ColorBit colorBit) { + MarkBitmapWord* word; + uintptr_t mask; + getMarkWordAndMask(cell, colorBit, &word, &mask); + return *word & mask; +} + +MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedAny(const TenuredCell* cell) { + return markBit(cell, ColorBit::BlackBit) || + markBit(cell, ColorBit::GrayOrBlackBit); +} + +MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedBlack(const TenuredCell* cell) { + return markBit(cell, ColorBit::BlackBit); +} + +MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedGray(const TenuredCell* cell) { + return !markBit(cell, ColorBit::BlackBit) && + markBit(cell, ColorBit::GrayOrBlackBit); +} + +// The following methods that update the mark bits are not thread safe and must +// not be called in parallel with each other. +// +// They use separate read and write operations to avoid an unnecessarily strict +// atomic update on the marking bitmap. +// +// They may be called in parallel with read operations on the mark bitmap where +// there is no required ordering between the operations. This happens when gray +// unmarking occurs in parallel with background sweeping. + +// The return value indicates if the cell went from unmarked to marked. +MOZ_ALWAYS_INLINE bool MarkBitmap::markIfUnmarked(const TenuredCell* cell, + MarkColor color) { + MarkBitmapWord* word; + uintptr_t mask; + getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask); + if (*word & mask) { + return false; + } + if (color == MarkColor::Black) { + uintptr_t bits = *word; + *word = bits | mask; + } else { + // We use getMarkWordAndMask to recalculate both mask and word as doing just + // mask << color may overflow the mask. + getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask); + if (*word & mask) { + return false; + } + uintptr_t bits = *word; + *word = bits | mask; + } + return true; +} + +MOZ_ALWAYS_INLINE bool MarkBitmap::markIfUnmarkedAtomic(const TenuredCell* cell, + MarkColor color) { + // This version of the method is safe in the face of concurrent writes to the + // mark bitmap but may return false positives. The extra synchronisation + // necessary to avoid this resulted in worse performance overall. + + MarkBitmapWord* word; + uintptr_t mask; + getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask); + if (*word & mask) { + return false; + } + if (color == MarkColor::Black) { + *word |= mask; + } else { + // We use getMarkWordAndMask to recalculate both mask and word as doing just + // mask << color may overflow the mask. + getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask); + if (*word & mask) { + return false; + } + *word |= mask; + } + return true; +} + +MOZ_ALWAYS_INLINE void MarkBitmap::markBlack(const TenuredCell* cell) { + MarkBitmapWord* word; + uintptr_t mask; + getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask); + uintptr_t bits = *word; + *word = bits | mask; +} + +MOZ_ALWAYS_INLINE void MarkBitmap::markBlackAtomic(const TenuredCell* cell) { + MarkBitmapWord* word; + uintptr_t mask; + getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask); + *word |= mask; +} + +MOZ_ALWAYS_INLINE void MarkBitmap::copyMarkBit(TenuredCell* dst, + const TenuredCell* src, + ColorBit colorBit) { + TenuredChunkBase* srcChunk = detail::GetCellChunkBase(src); + MarkBitmapWord* srcWord; + uintptr_t srcMask; + srcChunk->markBits.getMarkWordAndMask(src, colorBit, &srcWord, &srcMask); + + MarkBitmapWord* dstWord; + uintptr_t dstMask; + getMarkWordAndMask(dst, colorBit, &dstWord, &dstMask); + + uintptr_t bits = *dstWord; + bits &= ~dstMask; + if (*srcWord & srcMask) { + bits |= dstMask; + } + *dstWord = bits; +} + +MOZ_ALWAYS_INLINE void MarkBitmap::unmark(const TenuredCell* cell) { + MarkBitmapWord* word; + uintptr_t mask; + uintptr_t bits; + getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask); + bits = *word; + *word = bits & ~mask; + getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask); + bits = *word; + *word = bits & ~mask; +} + +inline MarkBitmapWord* MarkBitmap::arenaBits(Arena* arena) { + static_assert( + ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD, + "We assume that the part of the bitmap corresponding to the arena " + "has the exact number of words so we do not need to deal with a word " + "that covers bits from two arenas."); + + MarkBitmapWord* word; + uintptr_t unused; + getMarkWordAndMask(reinterpret_cast(arena->address()), + ColorBit::BlackBit, &word, &unused); + return word; +} + +/* + * A chunk in the tenured heap. TenuredChunks contain arenas and associated data + * structures (mark bitmap, delayed marking state). + */ +class TenuredChunk : public TenuredChunkBase { + Arena arenas[ArenasPerChunk]; + + friend class GCRuntime; + friend class MarkingValidator; + + public: + static TenuredChunk* fromAddress(uintptr_t addr) { + addr &= ~ChunkMask; + return reinterpret_cast(addr); + } + + static bool withinValidRange(uintptr_t addr) { + uintptr_t offset = addr & ChunkMask; + if (TenuredChunk::fromAddress(addr)->isNurseryChunk()) { + return offset >= sizeof(ChunkBase) && offset < ChunkSize; + } + return offset >= offsetof(TenuredChunk, arenas) && offset < ChunkSize; + } + + static size_t arenaIndex(const Arena* arena) { + uintptr_t addr = arena->address(); + MOZ_ASSERT(!TenuredChunk::fromAddress(addr)->isNurseryChunk()); + MOZ_ASSERT(withinValidRange(addr)); + uintptr_t offset = addr & ChunkMask; + return (offset - offsetof(TenuredChunk, arenas)) >> ArenaShift; + } + + explicit TenuredChunk(JSRuntime* runtime) : TenuredChunkBase(runtime) {} + + uintptr_t address() const { + uintptr_t addr = reinterpret_cast(this); + MOZ_ASSERT(!(addr & ChunkMask)); + return addr; + } + + bool unused() const { return info.numArenasFree == ArenasPerChunk; } + + bool hasAvailableArenas() const { return info.numArenasFree != 0; } + + bool isNurseryChunk() const { return storeBuffer; } + + Arena* allocateArena(GCRuntime* gc, JS::Zone* zone, AllocKind kind, + const AutoLockGC& lock); + + void releaseArena(GCRuntime* gc, Arena* arena, const AutoLockGC& lock); + void recycleArena(Arena* arena, SortedArenaList& dest, size_t thingsPerArena); + + void decommitFreeArenas(GCRuntime* gc, const bool& cancel, AutoLockGC& lock); + [[nodiscard]] bool decommitOneFreePage(GCRuntime* gc, size_t pageIndex, + AutoLockGC& lock); + void decommitAllArenas(); + + // This will decommit each unused not-already decommitted arena. It performs a + // system call for each arena but is only used during OOM. + void decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock); + + static void* allocate(GCRuntime* gc); + static TenuredChunk* emplace(void* ptr, GCRuntime* gc, + bool allMemoryCommitted); + + /* Unlink and return the freeArenasHead. */ + Arena* fetchNextFreeArena(GCRuntime* gc); + +#ifdef DEBUG + void verify() const; +#else + void verify() const {} +#endif + + private: + void commitOnePage(GCRuntime* gc); + + void updateChunkListAfterAlloc(GCRuntime* gc, const AutoLockGC& lock); + void updateChunkListAfterFree(GCRuntime* gc, size_t numArenasFree, + const AutoLockGC& lock); + + // Check if all arenas in a page are free. + bool canDecommitPage(size_t pageIndex) const; + + // Check the arena from freeArenasList is located in a free page. + // Unlike the isPageFree(size_t) version, this isPageFree(Arena*) will see the + // following arenas from the freeArenasHead are also located in the same page, + // to prevent not to access the arenas mprotect'ed during compaction in debug + // build. + bool isPageFree(const Arena* arena) const; + + // Get the page index of the arena. + size_t pageIndex(const Arena* arena) const { + return pageIndex(arenaIndex(arena)); + } + size_t pageIndex(size_t arenaIndex) const { + return arenaIndex / ArenasPerPage; + } + + Arena* pageAddress(size_t pageIndex) { + return &arenas[pageIndex * ArenasPerPage]; + } +}; + +inline void Arena::checkAddress() const { + mozilla::DebugOnly addr = uintptr_t(this); + MOZ_ASSERT(addr); + MOZ_ASSERT(!(addr & ArenaMask)); + MOZ_ASSERT(TenuredChunk::withinValidRange(addr)); +} + +inline TenuredChunk* Arena::chunk() const { + return TenuredChunk::fromAddress(address()); +} + +// Cell header stored before all nursery cells. +struct alignas(gc::CellAlignBytes) NurseryCellHeader { + // Store zone pointer with the trace kind in the lowest three bits. + const uintptr_t allocSiteAndTraceKind; + + // We only need to store a subset of trace kinds so this doesn't cover the + // full range. + static const uintptr_t TraceKindMask = 3; + + static uintptr_t MakeValue(AllocSite* const site, JS::TraceKind kind) { + MOZ_ASSERT(uintptr_t(kind) < TraceKindMask); + MOZ_ASSERT((uintptr_t(site) & TraceKindMask) == 0); + return uintptr_t(site) | uintptr_t(kind); + } + + inline NurseryCellHeader(AllocSite* site, JS::TraceKind kind); + + AllocSite* allocSite() const { + return reinterpret_cast(allocSiteAndTraceKind & ~TraceKindMask); + } + + JS::Zone* zone() const { return allocSite()->zone(); } + + JS::TraceKind traceKind() const { + return JS::TraceKind(allocSiteAndTraceKind & TraceKindMask); + } + + static const NurseryCellHeader* from(const Cell* cell) { + MOZ_ASSERT(IsInsideNursery(cell)); + return reinterpret_cast( + uintptr_t(cell) - sizeof(NurseryCellHeader)); + } +}; + +static_assert(uintptr_t(JS::TraceKind::Object) <= + NurseryCellHeader::TraceKindMask); +static_assert(uintptr_t(JS::TraceKind::String) <= + NurseryCellHeader::TraceKindMask); +static_assert(uintptr_t(JS::TraceKind::BigInt) <= + NurseryCellHeader::TraceKindMask); + +} /* namespace gc */ + +namespace debug { + +// Utility functions meant to be called from an interactive debugger. +enum class MarkInfo : int { + BLACK = 0, + GRAY = 1, + UNMARKED = -1, + NURSERY = -2, +}; + +// Get the mark color for a cell, in a way easily usable from a debugger. +MOZ_NEVER_INLINE MarkInfo GetMarkInfo(js::gc::Cell* cell); + +// Sample usage from gdb: +// +// (gdb) p $word = js::debug::GetMarkWordAddress(obj) +// $1 = (uintptr_t *) 0x7fa56d5fe360 +// (gdb) p/x $mask = js::debug::GetMarkMask(obj, js::gc::GRAY) +// $2 = 0x200000000 +// (gdb) watch *$word +// Hardware watchpoint 7: *$word +// (gdb) cond 7 *$word & $mask +// (gdb) cont +// +// Note that this is *not* a watchpoint on a single bit. It is a watchpoint on +// the whole word, which will trigger whenever the word changes and the +// selected bit is set after the change. +// +// So if the bit changing is the desired one, this is exactly what you want. +// But if a different bit changes (either set or cleared), you may still stop +// execution if the $mask bit happened to already be set. gdb does not expose +// enough information to restrict the watchpoint to just a single bit. + +// Return the address of the word containing the mark bits for the given cell, +// or nullptr if the cell is in the nursery. +MOZ_NEVER_INLINE uintptr_t* GetMarkWordAddress(js::gc::Cell* cell); + +// Return the mask for the given cell and color bit, or 0 if the cell is in the +// nursery. +MOZ_NEVER_INLINE uintptr_t GetMarkMask(js::gc::Cell* cell, uint32_t colorBit); + +} /* namespace debug */ +} /* namespace js */ + +#endif /* gc_Heap_h */ diff --git a/js/src/gc/IteratorUtils.h b/js/src/gc/IteratorUtils.h new file mode 100644 index 0000000000..614fd12100 --- /dev/null +++ b/js/src/gc/IteratorUtils.h @@ -0,0 +1,121 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_IteratorUtils_h +#define gc_IteratorUtils_h + +#include "mozilla/Array.h" +#include "mozilla/Maybe.h" + +#include + +namespace js { + +/* + * Create an iterator that yields the values from IteratorB(a) for all a in + * IteratorA(). Equivalent to nested for loops over IteratorA and IteratorB + * where IteratorB is constructed with a value from IteratorA. + */ +template +class NestedIterator { + using T = decltype(std::declval().get()); + + IteratorA a; + mozilla::Maybe b; + + public: + template + explicit NestedIterator(Args&&... args) : a(std::forward(args)...) { + settle(); + } + + bool done() const { return b.isNothing(); } + + T get() const { + MOZ_ASSERT(!done()); + return b.ref().get(); + } + + void next() { + MOZ_ASSERT(!done()); + b->next(); + if (b->done()) { + b.reset(); + a.next(); + settle(); + } + } + + const IteratorB& ref() const { return *b; } + + operator T() const { return get(); } + + T operator->() const { return get(); } + + private: + void settle() { + MOZ_ASSERT(b.isNothing()); + while (!a.done()) { + b.emplace(a.get()); + if (!b->done()) { + break; + } + b.reset(); + a.next(); + } + } +}; + +/* + * An iterator the yields values from each of N of instances of Iterator in + * sequence. + */ +template +class ChainedIterator { + using T = decltype(std::declval().get()); + + mozilla::Array iterators; + size_t index = 0; + + public: + template + MOZ_IMPLICIT ChainedIterator(Args&&... args) + : iterators(Iterator(std::forward(args))...) { + static_assert(N > 1); + settle(); + } + + bool done() const { return index == N; } + + void next() { + MOZ_ASSERT(!done()); + iterators[index].next(); + settle(); + } + + T get() const { + MOZ_ASSERT(!done()); + return iterators[index].get(); + } + + operator T() const { return get(); } + T operator->() const { return get(); } + + private: + void settle() { + MOZ_ASSERT(!done()); + while (iterators[index].done()) { + index++; + if (done()) { + break; + } + } + } +}; + +} /* namespace js */ + +#endif // gc_IteratorUtils_h diff --git a/js/src/gc/MallocedBlockCache.cpp b/js/src/gc/MallocedBlockCache.cpp new file mode 100644 index 0000000000..602d1a80b0 --- /dev/null +++ b/js/src/gc/MallocedBlockCache.cpp @@ -0,0 +1,144 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/MallocedBlockCache.h" +#include "mozilla/MemoryChecking.h" + +using js::PointerAndUint7; +using js::gc::MallocedBlockCache; + +MallocedBlockCache::~MallocedBlockCache() { clear(); } + +PointerAndUint7 MallocedBlockCache::alloc(size_t size) { + // Figure out which free list can give us a block of size `size`, after it + // has been rounded up to a multiple of `step`. + // + // Example mapping for STEP = 16 and NUM_LISTS = 8, after rounding up: + // 0 never holds any blocks (denotes "too large") + // 1 holds blocks of size 16 + // 2 holds blocks of size 32 + // 3 holds blocks of size 48 + // 4 holds blocks of size 64 + // 5 holds blocks of size 80 + // 6 holds blocks of size 96 + // 7 holds blocks of size 112 + // + // For a request of size n: + // * if n == 0, fail + // * else + // round n up to a multiple of STEP + // let i = n / STEP + // if i >= NUM_LISTS + // alloc direct from js_malloc, and return listID = 0 + // if lists[i] is nonempty, use lists[i] and return listID = i. + // else + // let p = js_malloc(n) + // return p and listID = i. + + // We're never expected to handle zero-sized blocks. + MOZ_ASSERT(size > 0); + + size = js::RoundUp(size, STEP); + size_t i = size / STEP; + + // Too large to cache; go straight to js_malloc. + if (MOZ_UNLIKELY(i >= NUM_LISTS)) { + void* p = js_malloc(size); + // If p is nullptr, that fact is carried into the PointerAndUint7, and the + // caller is expected to check that. + return PointerAndUint7(p, OVERSIZE_BLOCK_LIST_ID); + } + + // The case we hope is common. First, see if we can pull a block from the + // relevant list. + MOZ_ASSERT(i >= 1 && i < NUM_LISTS); + // Check that i is the right list + MOZ_ASSERT(i * STEP == size); + if (MOZ_LIKELY(!lists[i].empty())) { + void* block = lists[i].popCopy(); + return PointerAndUint7(block, i); + } + + // No luck. + void* p = js_malloc(size); + if (MOZ_UNLIKELY(!p)) { + return PointerAndUint7(nullptr, 0); // OOM + } + return PointerAndUint7(p, i); +} + +void MallocedBlockCache::free(PointerAndUint7 blockAndListID) { + // This is a whole lot simpler than the ::alloc case, since we are given the + // listId and don't have to compute it (not that we have any way to). + void* block = blockAndListID.pointer(); + uint32_t listID = blockAndListID.uint7(); + MOZ_ASSERT(block); + MOZ_ASSERT(listID < NUM_LISTS); + if (MOZ_UNLIKELY(listID == OVERSIZE_BLOCK_LIST_ID)) { + // It was too large for recycling; go straight to js_free. + js_free(block); + return; + } + + // Put it back on list `listId`, first poisoning it for safety. + memset(block, JS_NOTINUSE_TRAILER_PATTERN, listID * STEP); + MOZ_MAKE_MEM_UNDEFINED(block, listID * STEP); + if (MOZ_UNLIKELY(!lists[listID].append(block))) { + // OOM'd while doing admin. Hand it off to js_free and forget about the + // OOM. + js_free(block); + } +} + +void MallocedBlockCache::preen(float percentOfBlocksToDiscard) { + MOZ_ASSERT(percentOfBlocksToDiscard >= 0.0 && + percentOfBlocksToDiscard <= 100.0); + MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty()); + for (size_t listID = 1; listID < NUM_LISTS; listID++) { + MallocedBlockVector& list = lists[listID]; + size_t numToFree = + size_t(float(list.length()) * (percentOfBlocksToDiscard / 100.0)); + MOZ_RELEASE_ASSERT(numToFree <= list.length()); + while (numToFree > 0) { + void* block = list.popCopy(); + MOZ_ASSERT(block); + js_free(block); + numToFree--; + } + } +} + +void MallocedBlockCache::clear() { + MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty()); + for (size_t i = 1; i < NUM_LISTS; i++) { + MallocedBlockVector& list = lists[i]; + for (size_t j = 0; j < list.length(); j++) { + MOZ_ASSERT(list[j]); + js_free(list[j]); + list[j] = nullptr; // for safety + } + list.clear(); + } +} + +size_t MallocedBlockCache::sizeOfExcludingThis( + mozilla::MallocSizeOf mallocSizeOf) const { + MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty()); + size_t nBytes = 0; + for (size_t listID = 0; listID < NUM_LISTS; listID++) { + const MallocedBlockVector& list = lists[listID]; + nBytes += list.sizeOfExcludingThis(mallocSizeOf); + // The payload size of each block in `list` is the same. Hence, we could + // possibly do better here (measure once and multiply by the length) if we + // believe that the metadata size for each block is also the same. + for (size_t i = 0; i < list.length(); i++) { + MOZ_ASSERT(list[i]); + nBytes += mallocSizeOf(list[i]); + } + } + return nBytes; +} diff --git a/js/src/gc/MallocedBlockCache.h b/js/src/gc/MallocedBlockCache.h new file mode 100644 index 0000000000..546d7e50af --- /dev/null +++ b/js/src/gc/MallocedBlockCache.h @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_MallocedBlockCache_h +#define gc_MallocedBlockCache_h + +#include "ds/PointerAndUint7.h" +#include "js/AllocPolicy.h" +#include "js/Vector.h" +#include "util/Poison.h" + +namespace js { +namespace gc { + +// MallocedBlockCache implements a lightweight wrapper around js_malloc/js_free. +// +// Blocks are requested by ::alloc and must be returned with ::free, at which +// point the cache may decide to hold on to the block rather than hand it back +// to js_free. Subsequent ::alloc calls may be satisfied from the cached +// blocks rather than calling js_malloc. The mechanism is designed to be much +// cheaper than calling js_malloc/js_free directly. One consequence is that +// there is no locking; it is essential therefore to use each cache only from +// a single thread. +// +// The intended use is for lightweight management of OOL (malloc'd) storage +// associated with WasmStructObject and WasmArrayObject. The mechanism is +// general and potentially has other uses. Blocks of size STEP * NUM_LISTS +// and larger are never cached, though. +// +// Request sizes are rounded up to a multiple of STEP. There are NUM_LISTS-1 +// free lists, with a "List ID" indicating the multiple of STEP stored on the +// list. So for example, blocks of size 3 * STEP (after rounding up) are +// stored on the list with ID 3. List ID 0 indicates blocks which are too +// large to live on any freelist. With the default settings, this gives +// separate freelists for blocks of size 16, 32, 48, .. 496. Blocks of size +// zero are not supported, and `lists[0]` will always be empty. +// +// Allocation of a block produces not only the block's address but also its +// List ID. When freeing, both values must be presented, because there is +// otherwise no way for ::free to know the size of the original allocation, +// and hence which freelist it should go on. For this reason, the ::alloc and +// ::free methods produce and take a `PointerAndUint7`, not a `void*`. +// +// Resizing of blocks is not supported. + +class MallocedBlockCache { + public: + static const size_t STEP = 16; + + static const size_t NUM_LISTS = 32; + // This limitation exists because allocation returns a PointerAndUint7, and + // a List-ID value (viz, 0 .. NUM_LISTS-1) is stored in the Uint7 part. + static_assert(NUM_LISTS <= (1 << 7)); + + // list[0] must always remain empty. List ID 0 indicates a block which + // cannot participate in the freelist machinery because it is too large. + // + // list[i], for 1 <= i < NUM_LISTS, holds blocks of size i * STEP only. + // All requests are rounded up to multiple of SIZE. + // + // We do not expect to be required to issue or accept blocks of size zero. + static const size_t OVERSIZE_BLOCK_LIST_ID = 0; + using MallocedBlockVector = Vector; + + MallocedBlockVector lists[NUM_LISTS]; + + ~MallocedBlockCache(); + + // Allocation and freeing. + [[nodiscard]] PointerAndUint7 alloc(size_t size); + void free(PointerAndUint7 blockAndListID); + + // Allows users to gradually hand blocks back to js_free, so as to avoid + // space leaks in long-running scenarios. The specified percentage of + // blocks in each list is discarded. + void preen(float percentOfBlocksToDiscard); + + // Return all blocks in the cache to js_free. + void clear(); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +} // namespace gc +} // namespace js + +#endif // gc_MallocedBlockCache_h diff --git a/js/src/gc/Marking-inl.h b/js/src/gc/Marking-inl.h new file mode 100644 index 0000000000..07cfffb8da --- /dev/null +++ b/js/src/gc/Marking-inl.h @@ -0,0 +1,196 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Marking_inl_h +#define gc_Marking_inl_h + +#include "gc/Marking.h" + +#include + +#include "gc/RelocationOverlay.h" +#include "js/Id.h" +#include "js/Value.h" +#include "vm/StringType.h" +#include "vm/TaggedProto.h" + +#include "gc/Nursery-inl.h" + +namespace js { +namespace gc { + +// An abstraction to re-wrap any kind of typed pointer back to the tagged +// pointer it came from with |TaggedPtr::wrap(sourcePtr)|. +template +struct TaggedPtr {}; + +template <> +struct TaggedPtr { + static JS::Value wrap(JSObject* obj) { + if (!obj) { + return JS::NullValue(); + } +#ifdef ENABLE_RECORD_TUPLE + if (MaybeForwardedIsExtendedPrimitive(*obj)) { + return JS::ExtendedPrimitiveValue(*obj); + } +#endif + return JS::ObjectValue(*obj); + } + static JS::Value wrap(JSString* str) { return JS::StringValue(str); } + static JS::Value wrap(JS::Symbol* sym) { return JS::SymbolValue(sym); } + static JS::Value wrap(JS::BigInt* bi) { return JS::BigIntValue(bi); } + template + static JS::Value wrap(T* priv) { + static_assert(std::is_base_of_v, + "Type must be a GC thing derived from js::gc::Cell"); + return JS::PrivateGCThingValue(priv); + } + static JS::Value empty() { return JS::UndefinedValue(); } +}; + +template <> +struct TaggedPtr { + static jsid wrap(JSString* str) { return JS::PropertyKey::NonIntAtom(str); } + static jsid wrap(JS::Symbol* sym) { return PropertyKey::Symbol(sym); } + static jsid empty() { return JS::PropertyKey::Void(); } +}; + +template <> +struct TaggedPtr { + static TaggedProto wrap(JSObject* obj) { return TaggedProto(obj); } + static TaggedProto empty() { return TaggedProto(); } +}; + +template +struct MightBeForwarded { + static_assert(std::is_base_of_v); + static_assert(!std::is_same_v && !std::is_same_v); + +#define CAN_FORWARD_KIND_OR(_1, _2, Type, _3, _4, _5, canCompact) \ + std::is_base_of_v ? canCompact: + + // FOR_EACH_ALLOCKIND doesn't cover every possible type: make sure + // to default to `true` for unknown types. + static constexpr bool value = FOR_EACH_ALLOCKIND(CAN_FORWARD_KIND_OR) true; +#undef CAN_FORWARD_KIND_OR +}; + +template +inline bool IsForwarded(const T* t) { + if (!MightBeForwarded::value) { + MOZ_ASSERT(!t->isForwarded()); + return false; + } + + return t->isForwarded(); +} + +template +inline T* Forwarded(const T* t) { + const RelocationOverlay* overlay = RelocationOverlay::fromCell(t); + MOZ_ASSERT(overlay->isForwarded()); + return reinterpret_cast(overlay->forwardingAddress()); +} + +template +inline T MaybeForwarded(T t) { + if (IsForwarded(t)) { + t = Forwarded(t); + } + MOZ_ASSERT(!IsForwarded(t)); + return t; +} + +inline const JSClass* MaybeForwardedObjectClass(const JSObject* obj) { + Shape* shape = MaybeForwarded(obj->shapeMaybeForwarded()); + BaseShape* baseShape = MaybeForwarded(shape->base()); + return baseShape->clasp(); +} + +template +inline bool MaybeForwardedObjectIs(const JSObject* obj) { + MOZ_ASSERT(!obj->isForwarded()); + return MaybeForwardedObjectClass(obj) == &T::class_; +} + +template +inline T& MaybeForwardedObjectAs(JSObject* obj) { + MOZ_ASSERT(MaybeForwardedObjectIs(obj)); + return *static_cast(obj); +} + +inline RelocationOverlay::RelocationOverlay(Cell* dst) { + MOZ_ASSERT(dst->flags() == 0); + uintptr_t ptr = uintptr_t(dst); + header_.setForwardingAddress(ptr); +} + +/* static */ +inline RelocationOverlay* RelocationOverlay::forwardCell(Cell* src, Cell* dst) { + MOZ_ASSERT(!src->isForwarded()); + MOZ_ASSERT(!dst->isForwarded()); + return new (src) RelocationOverlay(dst); +} + +inline bool IsAboutToBeFinalizedDuringMinorSweep(Cell** cellp) { + MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting()); + + if ((*cellp)->isTenured()) { + return false; + } + + return !Nursery::getForwardedPointer(cellp); +} + +// Special case pre-write barrier for strings used during rope flattening. This +// avoids eager marking of ropes which does not immediately mark the cells if we +// hit OOM. This does not traverse ropes and is instead called on every node in +// a rope during flattening. +inline void PreWriteBarrierDuringFlattening(JSString* str) { + MOZ_ASSERT(str); + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + + if (IsInsideNursery(str)) { + return; + } + + auto* cell = reinterpret_cast(str); + JS::shadow::Zone* zone = cell->shadowZoneFromAnyThread(); + if (!zone->needsIncrementalBarrier()) { + return; + } + + MOZ_ASSERT(!str->isPermanentAndMayBeShared()); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(zone->runtimeFromAnyThread())); + PerformIncrementalBarrierDuringFlattening(str); +} + +#ifdef JSGC_HASH_TABLE_CHECKS + +template +inline bool IsGCThingValidAfterMovingGC(T* t) { + return !IsInsideNursery(t) && !t->isForwarded(); +} + +template +inline void CheckGCThingAfterMovingGC(T* t) { + if (t) { + MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t)); + } +} + +template +inline void CheckGCThingAfterMovingGC(const WeakHeapPtr& t) { + CheckGCThingAfterMovingGC(t.unbarrieredGet()); +} + +#endif // JSGC_HASH_TABLE_CHECKS + +} /* namespace gc */ +} /* namespace js */ + +#endif // gc_Marking_inl_h diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp new file mode 100644 index 0000000000..b92cd5f3ac --- /dev/null +++ b/js/src/gc/Marking.cpp @@ -0,0 +1,2774 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Marking-inl.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerRange.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/Maybe.h" +#include "mozilla/PodOperations.h" +#include "mozilla/ScopeExit.h" + +#include +#include + +#include "gc/GCInternals.h" +#include "gc/ParallelMarking.h" +#include "gc/TraceKind.h" +#include "jit/JitCode.h" +#include "js/GCTypeMacros.h" // JS_FOR_EACH_PUBLIC_{,TAGGED_}GC_POINTER_TYPE +#include "js/SliceBudget.h" +#include "util/Poison.h" +#include "vm/GeneratorObject.h" + +#include "gc/GC-inl.h" +#include "gc/PrivateIterators-inl.h" +#include "gc/TraceMethods-inl.h" +#include "gc/WeakMap-inl.h" +#include "vm/GeckoProfiler-inl.h" + +using namespace js; +using namespace js::gc; + +using JS::MapTypeToTraceKind; + +using mozilla::DebugOnly; +using mozilla::IntegerRange; +using mozilla::PodCopy; + +// [SMDOC] GC Tracing +// +// Tracing Overview +// ================ +// +// Tracing, in this context, refers to an abstract visitation of some or all of +// the GC-controlled heap. The effect of tracing an edge of the graph depends +// on the subclass of the JSTracer on whose behalf we are tracing. +// +// Marking +// ------- +// +// The primary JSTracer is the GCMarker. The marking tracer causes the target +// of each traversed edge to be marked black and the target edge's children to +// be marked either gray (in the gc algorithm sense) or immediately black. +// +// Callback +// -------- +// +// The secondary JSTracer is the CallbackTracer. This simply invokes a callback +// on each edge in a child. +// +// The following is a rough outline of the general struture of the tracing +// internals. +// +/* clang-format off */ +// +// +-------------------+ ...................... +// | | : : +// | v v +---+---+ +// | TraceRoot TraceEdge TraceRange GCMarker:: | | +// | | | | processMarkStackTop | Mark | +// | +-----------------------+ | | Stack | +// | | | | | +// | v | +---+---+ +// | TraceEdgeInternal | ^ +// | | +<-------------+ : +// | | | | : +// | v v | : +// | CallbackTracer:: markAndTraverseEdge | : +// | onSomeEdge | | : +// | | | | : +// | | | | : +// | +-------------+---------------+ | : +// | | | : +// | v | : +// | markAndTraverse | : +// | | | : +// | | | : +// | traverse | : +// | | | : +// | +--------------------------------------+ | : +// | | | | | : +// | v v v | : +// | markAndTraceChildren markAndPush eagerlyMarkChildren | : +// | | : | | : +// | v : +-----------+ : +// | T::traceChildren : : +// | | : : +// +-------------+ ...................................... +// +// Legend: +// ------- Direct calls +// ....... Data flow +// +/* clang-format on */ + +/*** Tracing Invariants *****************************************************/ + +template +static inline bool IsOwnedByOtherRuntime(JSRuntime* rt, T thing) { + bool other = thing->runtimeFromAnyThread() != rt; + MOZ_ASSERT_IF(other, thing->isPermanentAndMayBeShared()); + return other; +} + +#ifdef DEBUG + +static inline bool IsInFreeList(TenuredCell* cell) { + Arena* arena = cell->arena(); + uintptr_t addr = reinterpret_cast(cell); + MOZ_ASSERT(Arena::isAligned(addr, arena->getThingSize())); + return arena->inFreeList(addr); +} + +template +void js::CheckTracedThing(JSTracer* trc, T* thing) { + MOZ_ASSERT(trc); + MOZ_ASSERT(thing); + + if (IsForwarded(thing)) { + JS::TracerKind kind = trc->kind(); + MOZ_ASSERT(kind == JS::TracerKind::Tenuring || + kind == JS::TracerKind::MinorSweeping || + kind == JS::TracerKind::Moving); + thing = Forwarded(thing); + } + + /* This function uses data that's not available in the nursery. */ + if (IsInsideNursery(thing)) { + return; + } + + /* + * Permanent shared things that are not associated with this runtime will be + * ignored during marking. + */ + Zone* zone = thing->zoneFromAnyThread(); + if (IsOwnedByOtherRuntime(trc->runtime(), thing)) { + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(thing->isMarkedBlack()); + return; + } + + JSRuntime* rt = trc->runtime(); + MOZ_ASSERT(zone->runtimeFromAnyThread() == rt); + + bool isGcMarkingTracer = trc->isMarkingTracer(); + bool isUnmarkGrayTracer = IsTracerKind(trc, JS::TracerKind::UnmarkGray); + bool isClearEdgesTracer = IsTracerKind(trc, JS::TracerKind::ClearEdges); + + if (TlsContext.get()) { + // If we're on the main thread we must have access to the runtime and zone. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + } else { + MOZ_ASSERT(isGcMarkingTracer || isUnmarkGrayTracer || isClearEdgesTracer || + IsTracerKind(trc, JS::TracerKind::Moving) || + IsTracerKind(trc, JS::TracerKind::Sweeping)); + MOZ_ASSERT_IF(!isClearEdgesTracer, CurrentThreadIsPerformingGC()); + } + + MOZ_ASSERT(thing->isAligned()); + MOZ_ASSERT(MapTypeToTraceKind>::kind == + thing->getTraceKind()); + + /* + * Check that we only mark allocated cells. + * + * This check is restricted to marking for two reasons: Firstly, if background + * sweeping is running and concurrently modifying the free list then it is not + * safe. Secondly, it was thought to be slow so this is a compromise so as to + * not affect test times too much. + */ + MOZ_ASSERT_IF(zone->isGCMarking(), !IsInFreeList(&thing->asTenured())); +} + +template +void js::CheckTracedThing(JSTracer* trc, const T& thing) { + ApplyGCThingTyped(thing, [trc](auto t) { CheckTracedThing(trc, t); }); +} + +template +static void CheckMarkedThing(GCMarker* gcMarker, T* thing) { + Zone* zone = thing->zoneFromAnyThread(); + + MOZ_ASSERT(zone->shouldMarkInZone(gcMarker->markColor())); + + MOZ_ASSERT_IF(gcMarker->shouldCheckCompartments(), + zone->isCollectingFromAnyThread() || zone->isAtomsZone()); + + MOZ_ASSERT_IF(gcMarker->markColor() == MarkColor::Gray, + !zone->isGCMarkingBlackOnly() || zone->isAtomsZone()); + + MOZ_ASSERT(!(zone->isGCSweeping() || zone->isGCFinished() || + zone->isGCCompacting())); + + // Check that we don't stray from the current compartment and zone without + // using TraceCrossCompartmentEdge. + Compartment* comp = thing->maybeCompartment(); + MOZ_ASSERT_IF(gcMarker->tracingCompartment && comp, + gcMarker->tracingCompartment == comp); + MOZ_ASSERT_IF(gcMarker->tracingZone, + gcMarker->tracingZone == zone || zone->isAtomsZone()); +} + +namespace js { + +# define IMPL_CHECK_TRACED_THING(_, type, _1, _2) \ + template void CheckTracedThing(JSTracer*, type*); +JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING); +# undef IMPL_CHECK_TRACED_THING + +template void CheckTracedThing(JSTracer*, const Value&); + +} // namespace js + +#endif + +static inline bool ShouldMarkCrossCompartment(GCMarker* marker, JSObject* src, + Cell* dstCell) { + MarkColor color = marker->markColor(); + + if (!dstCell->isTenured()) { +#ifdef DEBUG + // Bug 1743098: This shouldn't be possible but it does seem to happen. Log + // some useful information in debug builds. + if (color != MarkColor::Black) { + fprintf(stderr, + "ShouldMarkCrossCompartment: cross compartment edge from gray " + "object to nursery thing\n"); + fprintf(stderr, "src: "); + src->dump(); + fprintf(stderr, "dst: "); + dstCell->dump(); + } +#endif + MOZ_ASSERT(color == MarkColor::Black); + return false; + } + TenuredCell& dst = dstCell->asTenured(); + + JS::Zone* dstZone = dst.zone(); + if (!src->zone()->isGCMarking() && !dstZone->isGCMarking()) { + return false; + } + + if (color == MarkColor::Black) { + // Check our sweep groups are correct: we should never have to + // mark something in a zone that we have started sweeping. + MOZ_ASSERT_IF(!dst.isMarkedBlack(), !dstZone->isGCSweeping()); + + /* + * Having black->gray edges violates our promise to the cycle collector so + * we ensure that gray things we encounter when marking black end up getting + * marked black. + * + * This can happen for two reasons: + * + * 1) If we're collecting a compartment and it has an edge to an uncollected + * compartment it's possible that the source and destination of the + * cross-compartment edge should be gray, but the source was marked black by + * the write barrier. + * + * 2) If we yield during gray marking and the write barrier marks a gray + * thing black. + * + * We handle the first case before returning whereas the second case happens + * as part of normal marking. + */ + if (dst.isMarkedGray() && !dstZone->isGCMarking()) { + UnmarkGrayGCThingUnchecked(marker, + JS::GCCellPtr(&dst, dst.getTraceKind())); + return false; + } + + return dstZone->isGCMarking(); + } else { + // Check our sweep groups are correct as above. + MOZ_ASSERT_IF(!dst.isMarkedAny(), !dstZone->isGCSweeping()); + + if (dstZone->isGCMarkingBlackOnly()) { + /* + * The destination compartment is being not being marked gray now, + * but it will be later, so record the cell so it can be marked gray + * at the appropriate time. + */ + if (!dst.isMarkedAny()) { + DelayCrossCompartmentGrayMarking(marker, src); + } + return false; + } + + return dstZone->isGCMarkingBlackAndGray(); + } +} + +static bool ShouldTraceCrossCompartment(JSTracer* trc, JSObject* src, + Cell* dstCell) { + if (!trc->isMarkingTracer()) { + return true; + } + + return ShouldMarkCrossCompartment(GCMarker::fromTracer(trc), src, dstCell); +} + +static bool ShouldTraceCrossCompartment(JSTracer* trc, JSObject* src, + const Value& val) { + return val.isGCThing() && + ShouldTraceCrossCompartment(trc, src, val.toGCThing()); +} + +#ifdef DEBUG + +inline void js::gc::AssertShouldMarkInZone(GCMarker* marker, Cell* thing) { + if (!thing->isMarkedBlack()) { + Zone* zone = thing->zone(); + MOZ_ASSERT(zone->isAtomsZone() || + zone->shouldMarkInZone(marker->markColor())); + } +} + +void js::gc::AssertRootMarkingPhase(JSTracer* trc) { + MOZ_ASSERT_IF(trc->isMarkingTracer(), + trc->runtime()->gc.state() == State::NotActive || + trc->runtime()->gc.state() == State::MarkRoots); +} + +#endif // DEBUG + +/*** Tracing Interface ******************************************************/ + +template +static void TraceExternalEdgeHelper(JSTracer* trc, T* thingp, + const char* name) { + MOZ_ASSERT(InternalBarrierMethods::isMarkable(*thingp)); + TraceEdgeInternal(trc, ConvertToBase(thingp), name); +} + +JS_PUBLIC_API void js::UnsafeTraceManuallyBarrieredEdge(JSTracer* trc, + JSObject** thingp, + const char* name) { + TraceEdgeInternal(trc, ConvertToBase(thingp), name); +} + +template +static void TraceRootHelper(JSTracer* trc, T* thingp, const char* name) { + MOZ_ASSERT(thingp); + js::TraceNullableRoot(trc, thingp, name); +} + +namespace js { +class AbstractGeneratorObject; +class SavedFrame; +} // namespace js + +#define DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION(type) \ + JS_PUBLIC_API void js::gc::TraceExternalEdge(JSTracer* trc, type* thingp, \ + const char* name) { \ + TraceExternalEdgeHelper(trc, thingp, name); \ + } + +// Define TraceExternalEdge for each public GC pointer type. +JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION) +JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION) + +#undef DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION + +#define DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(type) \ + JS_PUBLIC_API void JS::TraceRoot(JSTracer* trc, type* thingp, \ + const char* name) { \ + TraceRootHelper(trc, thingp, name); \ + } + +// Define TraceRoot for each public GC pointer type. +JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(DEFINE_UNSAFE_TRACE_ROOT_FUNCTION) +JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(DEFINE_UNSAFE_TRACE_ROOT_FUNCTION) + +// Also, for the moment, define TraceRoot for internal GC pointer types. +DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(AbstractGeneratorObject*) +DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(SavedFrame*) + +#undef DEFINE_UNSAFE_TRACE_ROOT_FUNCTION + +namespace js { +namespace gc { + +#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type) \ + template void TraceRangeInternal(JSTracer*, size_t len, type*, \ + const char*); + +#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND(_1, type, _2, _3) \ + INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type*) + +JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND) +JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS) +INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(TaggedProto) + +#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND +#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS + +} // namespace gc +} // namespace js + +// In debug builds, makes a note of the current compartment before calling a +// trace hook or traceChildren() method on a GC thing. +class MOZ_RAII AutoSetTracingSource { +#ifndef DEBUG + public: + template + AutoSetTracingSource(JSTracer* trc, T* thing) {} + ~AutoSetTracingSource() {} +#else + GCMarker* marker = nullptr; + + public: + template + AutoSetTracingSource(JSTracer* trc, T* thing) { + if (trc->isMarkingTracer() && thing) { + marker = GCMarker::fromTracer(trc); + MOZ_ASSERT(!marker->tracingZone); + marker->tracingZone = thing->asTenured().zone(); + MOZ_ASSERT(!marker->tracingCompartment); + marker->tracingCompartment = thing->maybeCompartment(); + } + } + + ~AutoSetTracingSource() { + if (marker) { + marker->tracingZone = nullptr; + marker->tracingCompartment = nullptr; + } + } +#endif +}; + +// In debug builds, clear the trace hook compartment. This happens after the +// trace hook has called back into one of our trace APIs and we've checked the +// traced thing. +class MOZ_RAII AutoClearTracingSource { +#ifndef DEBUG + public: + explicit AutoClearTracingSource(GCMarker* marker) {} + explicit AutoClearTracingSource(JSTracer* trc) {} + ~AutoClearTracingSource() {} +#else + GCMarker* marker = nullptr; + JS::Zone* prevZone = nullptr; + Compartment* prevCompartment = nullptr; + + public: + explicit AutoClearTracingSource(JSTracer* trc) { + if (trc->isMarkingTracer()) { + marker = GCMarker::fromTracer(trc); + prevZone = marker->tracingZone; + marker->tracingZone = nullptr; + prevCompartment = marker->tracingCompartment; + marker->tracingCompartment = nullptr; + } + } + ~AutoClearTracingSource() { + if (marker) { + marker->tracingZone = prevZone; + marker->tracingCompartment = prevCompartment; + } + } +#endif +}; + +template +void js::TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, + JSObject* src, T* dst, + const char* name) { + // Clear expected compartment for cross-compartment edge. + AutoClearTracingSource acts(trc); + + if (ShouldTraceCrossCompartment(trc, src, *dst)) { + TraceEdgeInternal(trc, dst, name); + } +} +template void js::TraceManuallyBarrieredCrossCompartmentEdge( + JSTracer*, JSObject*, Value*, const char*); +template void js::TraceManuallyBarrieredCrossCompartmentEdge( + JSTracer*, JSObject*, JSObject**, const char*); +template void js::TraceManuallyBarrieredCrossCompartmentEdge( + JSTracer*, JSObject*, BaseScript**, const char*); + +template +void js::TraceSameZoneCrossCompartmentEdge(JSTracer* trc, + const WriteBarriered* dst, + const char* name) { +#ifdef DEBUG + if (trc->isMarkingTracer()) { + MOZ_ASSERT((*dst)->maybeCompartment(), + "Use TraceEdge for GC things without a compartment"); + + GCMarker* gcMarker = GCMarker::fromTracer(trc); + MOZ_ASSERT_IF(gcMarker->tracingZone, + (*dst)->zone() == gcMarker->tracingZone); + } + + // Skip compartment checks for this edge. + if (trc->kind() == JS::TracerKind::CompartmentCheck) { + return; + } +#endif + + // Clear expected compartment for cross-compartment edge. + AutoClearTracingSource acts(trc); + TraceEdgeInternal(trc, ConvertToBase(dst->unbarrieredAddress()), name); +} +template void js::TraceSameZoneCrossCompartmentEdge( + JSTracer*, const WriteBarriered*, const char*); + +template +void js::TraceWeakMapKeyEdgeInternal(JSTracer* trc, Zone* weakMapZone, + T** thingp, const char* name) { + // We can't use ShouldTraceCrossCompartment here because that assumes the + // source of the edge is a CCW object which could be used to delay gray + // marking. Instead, assert that the weak map zone is in the same marking + // state as the target thing's zone and therefore we can go ahead and mark it. +#ifdef DEBUG + auto thing = *thingp; + if (trc->isMarkingTracer()) { + MOZ_ASSERT(weakMapZone->isGCMarking()); + MOZ_ASSERT(weakMapZone->gcState() == thing->zone()->gcState()); + } +#endif + + // Clear expected compartment for cross-compartment edge. + AutoClearTracingSource acts(trc); + + TraceEdgeInternal(trc, thingp, name); +} + +template void js::TraceWeakMapKeyEdgeInternal(JSTracer*, Zone*, + JSObject**, + const char*); +template void js::TraceWeakMapKeyEdgeInternal(JSTracer*, Zone*, + BaseScript**, + const char*); + +static Cell* TraceGenericPointerRootAndType(JSTracer* trc, Cell* thing, + JS::TraceKind kind, + const char* name) { + return MapGCThingTyped(thing, kind, [trc, name](auto t) -> Cell* { + TraceRoot(trc, &t, name); + return t; + }); +} + +void js::TraceGenericPointerRoot(JSTracer* trc, Cell** thingp, + const char* name) { + MOZ_ASSERT(thingp); + Cell* thing = *thingp; + if (!thing) { + return; + } + + Cell* traced = + TraceGenericPointerRootAndType(trc, thing, thing->getTraceKind(), name); + if (traced != thing) { + *thingp = traced; + } +} + +void js::TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, Cell** thingp, + const char* name) { + MOZ_ASSERT(thingp); + Cell* thing = *thingp; + if (!*thingp) { + return; + } + + auto traced = MapGCThingTyped(thing, thing->getTraceKind(), + [trc, name](auto t) -> Cell* { + TraceManuallyBarrieredEdge(trc, &t, name); + return t; + }); + if (traced != thing) { + *thingp = traced; + } +} + +void js::TraceGCCellPtrRoot(JSTracer* trc, JS::GCCellPtr* thingp, + const char* name) { + Cell* thing = thingp->asCell(); + if (!thing) { + return; + } + + Cell* traced = + TraceGenericPointerRootAndType(trc, thing, thingp->kind(), name); + + if (!traced) { + *thingp = JS::GCCellPtr(); + } else if (traced != thingp->asCell()) { + *thingp = JS::GCCellPtr(traced, thingp->kind()); + } +} + +void js::TraceManuallyBarrieredGCCellPtr(JSTracer* trc, JS::GCCellPtr* thingp, + const char* name) { + Cell* thing = thingp->asCell(); + if (!thing) { + return; + } + + Cell* traced = MapGCThingTyped(thing, thing->getTraceKind(), + [trc, name](auto t) -> Cell* { + TraceManuallyBarrieredEdge(trc, &t, name); + return t; + }); + + if (!traced) { + // If we are clearing edges, also erase the type. This happens when using + // ClearEdgesTracer. + *thingp = JS::GCCellPtr(); + } else if (traced != thingp->asCell()) { + *thingp = JS::GCCellPtr(traced, thingp->kind()); + } +} + +template +inline bool TraceTaggedPtrEdge(JSTracer* trc, T* thingp, const char* name) { + // Return true by default. For some types the lambda below won't be called. + bool ret = true; + auto thing = MapGCThingTyped(*thingp, [&](auto thing) { + if (!TraceEdgeInternal(trc, &thing, name)) { + ret = false; + return TaggedPtr::empty(); + } + + return TaggedPtr::wrap(thing); + }); + + // Only update *thingp if the value changed, to avoid TSan false positives for + // template objects when using DumpHeapTracer or UbiNode tracers while Ion + // compiling off-thread. + if (thing.isSome() && thing.value() != *thingp) { + *thingp = thing.value(); + } + + return ret; +} + +bool js::gc::TraceEdgeInternal(JSTracer* trc, Value* thingp, const char* name) { + return TraceTaggedPtrEdge(trc, thingp, name); +} +bool js::gc::TraceEdgeInternal(JSTracer* trc, jsid* thingp, const char* name) { + return TraceTaggedPtrEdge(trc, thingp, name); +} +bool js::gc::TraceEdgeInternal(JSTracer* trc, TaggedProto* thingp, + const char* name) { + return TraceTaggedPtrEdge(trc, thingp, name); +} + +template +void js::gc::TraceRangeInternal(JSTracer* trc, size_t len, T* vec, + const char* name) { + JS::AutoTracingIndex index(trc); + for (auto i : IntegerRange(len)) { + if (InternalBarrierMethods::isMarkable(vec[i])) { + TraceEdgeInternal(trc, &vec[i], name); + } + ++index; + } +} + +/*** GC Marking Interface ***************************************************/ + +namespace js { + +using HasNoImplicitEdgesType = bool; + +template +struct ImplicitEdgeHolderType { + using Type = HasNoImplicitEdgesType; +}; + +// For now, we only handle JSObject* and BaseScript* keys, but the linear time +// algorithm can be easily extended by adding in more types here, then making +// GCMarker::traverse call markImplicitEdges. +template <> +struct ImplicitEdgeHolderType { + using Type = JSObject*; +}; + +template <> +struct ImplicitEdgeHolderType { + using Type = BaseScript*; +}; + +void GCMarker::markEphemeronEdges(EphemeronEdgeVector& edges, + gc::CellColor srcColor) { + // This is called as part of GC weak marking or by barriers outside of GC. + MOZ_ASSERT_IF(CurrentThreadIsPerformingGC(), + state == MarkingState::WeakMarking); + + DebugOnly initialLength = edges.length(); + + for (auto& edge : edges) { + CellColor targetColor = std::min(srcColor, edge.color); + MOZ_ASSERT(CellColor(markColor()) >= targetColor); + if (targetColor == markColor()) { + ApplyGCThingTyped( + edge.target, edge.target->getTraceKind(), + [this](auto t) { markAndTraverse(t); }); + } + } + + // The above marking always goes through markAndPush, which will not cause + // 'edges' to be appended to while iterating. + MOZ_ASSERT(edges.length() == initialLength); + + // This is not just an optimization. When nuking a CCW, we conservatively + // mark through the related edges and then lose the CCW->target connection + // that induces a sweep group edge. As a result, it is possible for the + // delegate zone to get marked later, look up an edge in this table, and + // then try to mark something in a Zone that is no longer marking. + if (srcColor == CellColor::Black && markColor() == MarkColor::Black) { + edges.eraseIf([](auto& edge) { return edge.color == MarkColor::Black; }); + } +} + +// 'delegate' is no longer the delegate of 'key'. +void GCMarker::severWeakDelegate(JSObject* key, JSObject* delegate) { + MOZ_ASSERT(CurrentThreadIsMainThread()); + + JS::Zone* zone = delegate->zone(); + if (!zone->needsIncrementalBarrier()) { + MOZ_ASSERT( + !zone->gcEphemeronEdges(delegate).get(delegate), + "non-collecting zone should not have populated gcEphemeronEdges"); + return; + } + auto* p = zone->gcEphemeronEdges(delegate).get(delegate); + if (!p) { + return; + } + + // We are losing 3 edges here: key -> delegate, delegate -> key, and + // -> value. Maintain snapshot-at-beginning (hereafter, + // S-A-B) by conservatively assuming the delegate will end up black and + // marking through the latter 2 edges. + // + // Note that this does not fully give S-A-B: + // + // 1. If the map is gray, then the value will only be marked gray here even + // though the map could later be discovered to be black. + // + // 2. If the map has not yet been marked, we won't have any entries to mark + // here in the first place. + // + // 3. We're not marking the delegate, since that would cause eg nukeAllCCWs + // to keep everything alive for another collection. + // + // We can't even assume that the delegate passed in here is live, because we + // could have gotten here from nukeAllCCWs, which iterates over all CCWs + // including dead ones. + // + // This is ok because S-A-B is only needed to prevent the case where an + // unmarked object is removed from the graph and then re-inserted where it is + // reachable only by things that have already been marked. None of the 3 + // target objects will be re-inserted anywhere as a result of this action. + + EphemeronEdgeVector& edges = p->value; + MOZ_ASSERT(markColor() == MarkColor::Black); + markEphemeronEdges(edges, MarkColor::Black); +} + +// 'delegate' is now the delegate of 'key'. Update weakmap marking state. +void GCMarker::restoreWeakDelegate(JSObject* key, JSObject* delegate) { + MOZ_ASSERT(CurrentThreadIsMainThread()); + + if (!key->zone()->needsIncrementalBarrier()) { + // Temporary diagnostic printouts for when this would have asserted. + if (key->zone()->gcEphemeronEdges(key).has(key)) { + fprintf(stderr, "key zone: %d\n", int(key->zone()->gcState())); +#ifdef DEBUG + key->dump(); +#endif + fprintf(stderr, "delegate zone: %d\n", int(delegate->zone()->gcState())); +#ifdef DEBUG + delegate->dump(); +#endif + } + MOZ_ASSERT( + !key->zone()->gcEphemeronEdges(key).has(key), + "non-collecting zone should not have populated gcEphemeronEdges"); + return; + } + if (!delegate->zone()->needsIncrementalBarrier()) { + // Normally we should not have added the key -> value edge if the delegate + // zone is not marking (because the delegate would have been seen as black, + // so we would mark the key immediately instead). But if there wasn't a + // delegate (the key was nuked), then we won't have consulted it. So we + // can't do the same assertion as above. + // + // Specifically, the sequence would be: + // 1. Nuke the key. + // 2. Start the incremental GC. + // 3. Mark the WeakMap. Insert a key->value edge with a DeadObjectProxy key. + // 4. Un-nuke the key with a delegate in a nonmarking Zone. + // + // The result is an ephemeron edge (from to value, but stored + // as key to value) involving a key with a delegate in a nonmarking Zone, + // something that ordinarily would not happen. + return; + } + auto* p = key->zone()->gcEphemeronEdges(key).get(key); + if (!p) { + return; + } + + // Similar to severWeakDelegate above, mark through the key -> value edge. + EphemeronEdgeVector& edges = p->value; + MOZ_ASSERT(markColor() == MarkColor::Black); + markEphemeronEdges(edges, MarkColor::Black); +} + +template +void GCMarker::markImplicitEdgesHelper(T markedThing) { + if (!isWeakMarking()) { + return; + } + + Zone* zone = markedThing->asTenured().zone(); + MOZ_ASSERT(zone->isGCMarking()); + MOZ_ASSERT(!zone->isGCSweeping()); + + auto p = zone->gcEphemeronEdges().get(markedThing); + if (!p) { + return; + } + EphemeronEdgeVector& edges = p->value; + + // markedThing might be a key in a debugger weakmap, which can end up marking + // values that are in a different compartment. + AutoClearTracingSource acts(tracer()); + + CellColor thingColor = gc::detail::GetEffectiveColor(this, markedThing); + markEphemeronEdges(edges, thingColor); +} + +template <> +void GCMarker::markImplicitEdgesHelper(HasNoImplicitEdgesType) {} + +template +void GCMarker::markImplicitEdges(T* thing) { + markImplicitEdgesHelper::Type>(thing); +} + +template void GCMarker::markImplicitEdges(JSObject*); +template void GCMarker::markImplicitEdges(BaseScript*); + +} // namespace js + +template +static inline bool ShouldMark(GCMarker* gcmarker, T* thing) { + // We may encounter nursery things during normal marking since we don't + // collect the nursery at the start of every GC slice. + if (!thing->isTenured()) { + return false; + } + + // Don't mark things outside a zone if we are in a per-zone GC. Don't mark + // permanent shared things owned by other runtimes (we will never observe + // their zone being collected). + Zone* zone = thing->asTenured().zoneFromAnyThread(); + return zone->shouldMarkInZone(gcmarker->markColor()); +} + +template +MarkingTracerT::MarkingTracerT(JSRuntime* runtime, GCMarker* marker) + : GenericTracerImpl>( + runtime, JS::TracerKind::Marking, + JS::TraceOptions(JS::WeakMapTraceAction::Expand, + JS::WeakEdgeTraceAction::Skip)) { + // Marking tracers are owned by (and part of) a GCMarker. + MOZ_ASSERT(this == marker->tracer()); + MOZ_ASSERT(getMarker() == marker); +} + +template +MOZ_ALWAYS_INLINE GCMarker* MarkingTracerT::getMarker() { + return GCMarker::fromTracer(this); +} + +template +template +void MarkingTracerT::onEdge(T** thingp, const char* name) { + T* thing = *thingp; + + // Do per-type marking precondition checks. + GCMarker* marker = getMarker(); + if (!ShouldMark(marker, thing)) { + MOZ_ASSERT(gc::detail::GetEffectiveColor(marker, thing) == + js::gc::CellColor::Black); + return; + } + + MOZ_ASSERT(!IsOwnedByOtherRuntime(this->runtime(), thing)); + +#ifdef DEBUG + CheckMarkedThing(marker, thing); +#endif + + AutoClearTracingSource acts(this); + marker->markAndTraverse(thing); +} + +#define INSTANTIATE_ONEDGE_METHOD(name, type, _1, _2) \ + template void MarkingTracerT::onEdge( \ + type * *thingp, const char* name); \ + template void \ + MarkingTracerT::onEdge( \ + type * *thingp, const char* name); +JS_FOR_EACH_TRACEKIND(INSTANTIATE_ONEDGE_METHOD) +#undef INSTANTIATE_ONEDGE_METHOD + +static void TraceEdgeForBarrier(GCMarker* gcmarker, TenuredCell* thing, + JS::TraceKind kind) { + // Dispatch to markAndTraverse without checking ShouldMark. + ApplyGCThingTyped(thing, kind, [gcmarker](auto thing) { + MOZ_ASSERT(ShouldMark(gcmarker, thing)); + CheckTracedThing(gcmarker->tracer(), thing); + AutoClearTracingSource acts(gcmarker->tracer()); + gcmarker->markAndTraverse(thing); + }); +} + +JS_PUBLIC_API void js::gc::PerformIncrementalReadBarrier(JS::GCCellPtr thing) { + // Optimized marking for read barriers. This is called from + // ExposeGCThingToActiveJS which has already checked the prerequisites for + // performing a read barrier. This means we can skip a bunch of checks and + // call into the tracer directly. + + MOZ_ASSERT(thing); + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + + TenuredCell* cell = &thing.asCell()->asTenured(); + MOZ_ASSERT(!cell->isMarkedBlack()); + + Zone* zone = cell->zone(); + MOZ_ASSERT(zone->needsIncrementalBarrier()); + + // Skip dispatching on known tracer type. + GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer()); + TraceEdgeForBarrier(gcmarker, cell, thing.kind()); +} + +void js::gc::PerformIncrementalReadBarrier(TenuredCell* cell) { + // Internal version of previous function. + + MOZ_ASSERT(cell); + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + + if (cell->isMarkedBlack()) { + return; + } + + Zone* zone = cell->zone(); + MOZ_ASSERT(zone->needsIncrementalBarrier()); + + // Skip dispatching on known tracer type. + GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer()); + TraceEdgeForBarrier(gcmarker, cell, cell->getTraceKind()); +} + +void js::gc::PerformIncrementalPreWriteBarrier(TenuredCell* cell) { + // The same as PerformIncrementalReadBarrier except for an extra check on the + // runtime for cells in atoms zone. + + Zone* zone = cell->zoneFromAnyThread(); + MOZ_ASSERT(zone->needsIncrementalBarrier()); + + MOZ_ASSERT(cell); + if (cell->isMarkedBlack()) { + return; + } + + // Barriers can be triggered off the main thread by background finalization of + // HeapPtrs to the atoms zone. We don't want to trigger the barrier in this + // case. + bool checkThread = zone->isAtomsZone(); + JSRuntime* runtime = cell->runtimeFromAnyThread(); + if (checkThread && !CurrentThreadCanAccessRuntime(runtime)) { + MOZ_ASSERT(CurrentThreadIsGCFinalizing()); + return; + } + + MOZ_ASSERT(CurrentThreadIsMainThread()); + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + + // Skip dispatching on known tracer type. + GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer()); + TraceEdgeForBarrier(gcmarker, cell, cell->getTraceKind()); +} + +void js::gc::PerformIncrementalBarrierDuringFlattening(JSString* str) { + TenuredCell* cell = &str->asTenured(); + + // Skip eager marking of ropes during flattening. Their children will also be + // barriered by flattening process so we don't need to traverse them. + if (str->isRope()) { + cell->markBlack(); + return; + } + + PerformIncrementalPreWriteBarrier(cell); +} + +template +void js::GCMarker::markAndTraverse(T* thing) { + if (mark(thing)) { + // We only mark permanent things during initialization. + MOZ_ASSERT_IF(thing->isPermanentAndMayBeShared(), + !runtime()->permanentAtomsPopulated()); + + // We don't need to pass MarkRootCompartments options on to children. + constexpr uint32_t traverseOpts = + opts & ~MarkingOptions::MarkRootCompartments; + + traverse(thing); + + if constexpr (bool(opts & MarkingOptions::MarkRootCompartments)) { + // Mark the compartment as live. + SetCompartmentHasMarkedCells(thing); + } + } +} + +// The |traverse| method overloads select the traversal strategy for each kind. +// +// There are three possible strategies: +// +// 1. traceChildren +// +// The simplest traversal calls out to the fully generic traceChildren +// function to visit the child edges. In the absence of other traversal +// mechanisms, this function will rapidly grow the stack past its bounds and +// crash the process. Thus, this generic tracing should only be used in cases +// where subsequent tracing will not recurse. +// +// 2. scanChildren +// +// Strings, Shapes, and Scopes are extremely common, but have simple patterns +// of recursion. We traverse trees of these edges immediately, with +// aggressive, manual inlining, implemented by eagerlyTraceChildren. +// +// 3. pushThing +// +// Objects are extremely common and can contain arbitrarily nested graphs, so +// are not trivially inlined. In this case we use the mark stack to control +// recursion. JitCode shares none of these properties, but is included for +// historical reasons. JSScript normally cannot recurse, but may be used as a +// weakmap key and thereby recurse into weakmapped values. + +template +void GCMarker::traverse(BaseShape* thing) { + traceChildren(thing); +} +template +void GCMarker::traverse(GetterSetter* thing) { + traceChildren(thing); +} +template +void GCMarker::traverse(JS::Symbol* thing) { + traceChildren(thing); +} +template +void GCMarker::traverse(JS::BigInt* thing) { + traceChildren(thing); +} +template +void GCMarker::traverse(RegExpShared* thing) { + traceChildren(thing); +} +template +void GCMarker::traverse(JSString* thing) { + scanChildren(thing); +} +template +void GCMarker::traverse(Shape* thing) { + scanChildren(thing); +} +template +void GCMarker::traverse(PropMap* thing) { + scanChildren(thing); +} +template +void GCMarker::traverse(js::Scope* thing) { + scanChildren(thing); +} +template +void GCMarker::traverse(JSObject* thing) { + pushThing(thing); +} +template +void GCMarker::traverse(jit::JitCode* thing) { + pushThing(thing); +} +template +void GCMarker::traverse(BaseScript* thing) { + pushThing(thing); +} + +template +void js::GCMarker::traceChildren(T* thing) { + MOZ_ASSERT(!thing->isPermanentAndMayBeShared()); + MOZ_ASSERT(thing->isMarkedAny()); + AutoSetTracingSource asts(tracer(), thing); + thing->traceChildren(tracer()); +} + +template +void js::GCMarker::scanChildren(T* thing) { + MOZ_ASSERT(!thing->isPermanentAndMayBeShared()); + MOZ_ASSERT(thing->isMarkedAny()); + eagerlyMarkChildren(thing); +} + +template +void js::GCMarker::pushThing(T* thing) { + MOZ_ASSERT(!thing->isPermanentAndMayBeShared()); + MOZ_ASSERT(thing->isMarkedAny()); + pushTaggedPtr(thing); +} + +template void js::GCMarker::markAndTraverse( + JSObject* thing); +template void js::GCMarker::markAndTraverse< + MarkingOptions::MarkRootCompartments, JSObject>(JSObject* thing); + +#ifdef DEBUG +void GCMarker::setCheckAtomMarking(bool check) { + MOZ_ASSERT(check != checkAtomMarking); + checkAtomMarking = check; +} +#endif + +template +inline void GCMarker::checkTraversedEdge(S source, T* target) { +#ifdef DEBUG + // Atoms and Symbols do not have or mark their internal pointers, + // respectively. + MOZ_ASSERT(!source->isPermanentAndMayBeShared()); + + // Shared things are already black so we will not mark them. + if (target->isPermanentAndMayBeShared()) { + Zone* zone = target->zoneFromAnyThread(); + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + MOZ_ASSERT(target->isMarkedBlack()); + MOZ_ASSERT(!target->maybeCompartment()); + return; + } + + Zone* sourceZone = source->zone(); + Zone* targetZone = target->zone(); + + // Atoms and Symbols do not have access to a compartment pointer, or we'd need + // to adjust the subsequent check to catch that case. + MOZ_ASSERT_IF(targetZone->isAtomsZone(), !target->maybeCompartment()); + + // The Zones must match, unless the target is an atom. + MOZ_ASSERT(targetZone == sourceZone || targetZone->isAtomsZone()); + + // If we are marking an atom, that atom must be marked in the source zone's + // atom bitmap. + if (checkAtomMarking && !sourceZone->isAtomsZone() && + targetZone->isAtomsZone()) { + MOZ_ASSERT(target->runtimeFromAnyThread()->gc.atomMarking.atomIsMarked( + sourceZone, reinterpret_cast(target))); + } + + // If we have access to a compartment pointer for both things, they must + // match. + MOZ_ASSERT_IF(source->maybeCompartment() && target->maybeCompartment(), + source->maybeCompartment() == target->maybeCompartment()); +#endif +} + +template +void js::GCMarker::markAndTraverseEdge(S source, T* target) { + checkTraversedEdge(source, target); + markAndTraverse(target); +} + +template +void js::GCMarker::markAndTraverseEdge(S source, const T& thing) { + ApplyGCThingTyped(thing, [this, source](auto t) { + this->markAndTraverseEdge(source, t); + }); +} + +template +bool js::GCMarker::mark(T* thing) { + if (!thing->isTenured()) { + return false; + } + + AssertShouldMarkInZone(this, thing); + + MarkColor color = + TraceKindCanBeGray::value ? markColor() : MarkColor::Black; + + if constexpr (bool(opts & MarkingOptions::ParallelMarking)) { + return thing->asTenured().markIfUnmarkedAtomic(color); + } + + return thing->asTenured().markIfUnmarked(color); +} + +/*** Mark-stack Marking *****************************************************/ + +// Call the trace hook set on the object, if present. +static inline void CallTraceHook(JSTracer* trc, JSObject* obj) { + const JSClass* clasp = obj->getClass(); + MOZ_ASSERT(clasp); + + if (clasp->hasTrace()) { + AutoSetTracingSource asts(trc, obj); + clasp->doTrace(trc, obj); + } +} + +static gcstats::PhaseKind GrayMarkingPhaseForCurrentPhase( + const gcstats::Statistics& stats) { + using namespace gcstats; + switch (stats.currentPhaseKind()) { + case PhaseKind::MARK: + return PhaseKind::MARK_GRAY; + case PhaseKind::MARK_WEAK: + return PhaseKind::MARK_GRAY_WEAK; + default: + MOZ_CRASH("Unexpected current phase"); + } +} + +void GCMarker::moveWork(GCMarker* dst, GCMarker* src) { + MOZ_ASSERT(dst->stack.isEmpty()); + MOZ_ASSERT(src->canDonateWork()); + + MarkStack::moveWork(dst->stack, src->stack); +} + +bool GCMarker::markUntilBudgetExhausted(SliceBudget& budget, + ShouldReportMarkTime reportTime) { +#ifdef DEBUG + MOZ_ASSERT(!strictCompartmentChecking); + strictCompartmentChecking = true; + auto acc = mozilla::MakeScopeExit([&] { strictCompartmentChecking = false; }); +#endif + + if (budget.isOverBudget()) { + return false; + } + + return doMarking(budget, reportTime); +} + +template +bool GCMarker::doMarking(SliceBudget& budget, ShouldReportMarkTime reportTime) { + GCRuntime& gc = runtime()->gc; + + // This method leaves the mark color as it found it. + + if (hasBlackEntries() && !markOneColor(budget)) { + return false; + } + + if (hasGrayEntries()) { + mozilla::Maybe ap; + if (reportTime) { + auto& stats = runtime()->gc.stats(); + ap.emplace(stats, GrayMarkingPhaseForCurrentPhase(stats)); + } + + if (!markOneColor(budget)) { + return false; + } + } + + // Mark children of things that caused too deep recursion during the above + // tracing. All normal marking happens before any delayed marking. + if (gc.hasDelayedMarking()) { + gc.markAllDelayedChildren(reportTime); + } + + MOZ_ASSERT(!gc.hasDelayedMarking()); + MOZ_ASSERT(isDrained()); + + return true; +} + +template +bool GCMarker::markOneColor(SliceBudget& budget) { + AutoSetMarkColor setColor(*this, color); + + while (processMarkStackTop(budget)) { + if (stack.isEmpty()) { + return true; + } + } + + return false; +} + +bool GCMarker::markCurrentColorInParallel(SliceBudget& budget) { + ParallelMarker::AtomicCount& waitingTaskCount = + parallelMarker_->waitingTaskCountRef(); + + while (processMarkStackTop(budget)) { + if (stack.isEmpty()) { + return true; + } + + // TODO: It might be better to only check this occasionally, possibly + // combined with the slice budget check. Experiments with giving this its + // own counter resulted in worse performance. + if (waitingTaskCount && canDonateWork()) { + parallelMarker_->donateWorkFrom(this); + } + } + + return false; +} + +static inline void CheckForCompartmentMismatch(JSObject* obj, JSObject* obj2) { +#ifdef DEBUG + if (MOZ_UNLIKELY(obj->compartment() != obj2->compartment())) { + fprintf( + stderr, + "Compartment mismatch in pointer from %s object slot to %s object\n", + obj->getClass()->name, obj2->getClass()->name); + MOZ_CRASH("Compartment mismatch"); + } +#endif +} + +static inline size_t NumUsedFixedSlots(NativeObject* obj) { + return std::min(obj->numFixedSlots(), obj->slotSpan()); +} + +static inline size_t NumUsedDynamicSlots(NativeObject* obj) { + size_t nfixed = obj->numFixedSlots(); + size_t nslots = obj->slotSpan(); + if (nslots < nfixed) { + return 0; + } + + return nslots - nfixed; +} + +template +inline bool GCMarker::processMarkStackTop(SliceBudget& budget) { + /* + * This function uses explicit goto and scans objects directly. This allows us + * to eliminate tail recursion and significantly improve the marking + * performance, see bug 641025. + * + * Note that the mutator can change the size and layout of objects between + * marking slices, so we must check slots and element ranges read from the + * stack. + */ + + MOZ_ASSERT(!stack.isEmpty()); + MOZ_ASSERT_IF(markColor() == MarkColor::Gray, !hasBlackEntries()); + + JSObject* obj; // The object being scanned. + SlotsOrElementsKind kind; // The kind of slot range being scanned, if any. + HeapSlot* base; // Slot range base pointer. + size_t index; // Index of the next slot to mark. + size_t end; // End of slot range to mark. + + if (stack.peekTag() == MarkStack::SlotsOrElementsRangeTag) { + auto range = stack.popSlotsOrElementsRange(); + obj = range.ptr().asRangeObject(); + NativeObject* nobj = &obj->as(); + kind = range.kind(); + index = range.start(); + + switch (kind) { + case SlotsOrElementsKind::FixedSlots: { + base = nobj->fixedSlots(); + end = NumUsedFixedSlots(nobj); + break; + } + + case SlotsOrElementsKind::DynamicSlots: { + base = nobj->slots_; + end = NumUsedDynamicSlots(nobj); + break; + } + + case SlotsOrElementsKind::Elements: { + base = nobj->getDenseElements(); + + // Account for shifted elements. + size_t numShifted = nobj->getElementsHeader()->numShiftedElements(); + size_t initlen = nobj->getDenseInitializedLength(); + index = std::max(index, numShifted) - numShifted; + end = initlen; + break; + } + + case SlotsOrElementsKind::Unused: { + MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("Unused SlotsOrElementsKind"); + } + } + + goto scan_value_range; + } + + budget.step(); + if (budget.isOverBudget()) { + return false; + } + + { + MarkStack::TaggedPtr ptr = stack.popPtr(); + switch (ptr.tag()) { + case MarkStack::ObjectTag: { + obj = ptr.as(); + AssertShouldMarkInZone(this, obj); + goto scan_obj; + } + + case MarkStack::JitCodeTag: { + auto* code = ptr.as(); + AutoSetTracingSource asts(tracer(), code); + code->traceChildren(tracer()); + return true; + } + + case MarkStack::ScriptTag: { + auto* script = ptr.as(); + if constexpr (bool(opts & MarkingOptions::MarkImplicitEdges)) { + markImplicitEdges(script); + } + AutoSetTracingSource asts(tracer(), script); + script->traceChildren(tracer()); + return true; + } + + default: + MOZ_CRASH("Invalid tag in mark stack"); + } + } + + return true; + +scan_value_range: + while (index < end) { + budget.step(); + if (budget.isOverBudget()) { + pushValueRange(obj, kind, index, end); + return false; + } + + const Value& v = base[index]; + index++; + + if (v.isString()) { + markAndTraverseEdge(obj, v.toString()); + } else if (v.hasObjectPayload()) { + JSObject* obj2 = &v.getObjectPayload(); +#ifdef DEBUG + if (!obj2) { + fprintf(stderr, + "processMarkStackTop found ObjectValue(nullptr) " + "at %zu Values from end of range in object:\n", + size_t(end - (index - 1))); + obj->dump(); + } +#endif + CheckForCompartmentMismatch(obj, obj2); + if (mark(obj2)) { + // Save the rest of this value range for later and start scanning obj2's + // children. + pushValueRange(obj, kind, index, end); + obj = obj2; + goto scan_obj; + } + } else if (v.isSymbol()) { + markAndTraverseEdge(obj, v.toSymbol()); + } else if (v.isBigInt()) { + markAndTraverseEdge(obj, v.toBigInt()); + } else if (v.isPrivateGCThing()) { + // v.toGCCellPtr cannot be inlined, so construct one manually. + Cell* cell = v.toGCThing(); + markAndTraverseEdge(obj, JS::GCCellPtr(cell, cell->getTraceKind())); + } + } + + return true; + +scan_obj : { + AssertShouldMarkInZone(this, obj); + + if constexpr (bool(opts & MarkingOptions::MarkImplicitEdges)) { + markImplicitEdges(obj); + } + markAndTraverseEdge(obj, obj->shape()); + + CallTraceHook(tracer(), obj); + + if (!obj->is()) { + return true; + } + + NativeObject* nobj = &obj->as(); + + unsigned nslots = nobj->slotSpan(); + + do { + if (nobj->hasEmptyElements()) { + break; + } + + base = nobj->getDenseElements(); + kind = SlotsOrElementsKind::Elements; + index = 0; + end = nobj->getDenseInitializedLength(); + + if (!nslots) { + goto scan_value_range; + } + pushValueRange(nobj, kind, index, end); + } while (false); + + unsigned nfixed = nobj->numFixedSlots(); + + base = nobj->fixedSlots(); + kind = SlotsOrElementsKind::FixedSlots; + index = 0; + + if (nslots > nfixed) { + pushValueRange(nobj, kind, index, nfixed); + kind = SlotsOrElementsKind::DynamicSlots; + base = nobj->slots_; + end = nslots - nfixed; + goto scan_value_range; + } + + MOZ_ASSERT(nslots <= nobj->numFixedSlots()); + end = nslots; + goto scan_value_range; +} +} + +/*** Mark Stack *************************************************************/ + +static_assert(sizeof(MarkStack::TaggedPtr) == sizeof(uintptr_t), + "A TaggedPtr should be the same size as a pointer"); +static_assert((sizeof(MarkStack::SlotsOrElementsRange) % sizeof(uintptr_t)) == + 0, + "SlotsOrElementsRange size should be a multiple of " + "the pointer size"); + +static const size_t ValueRangeWords = + sizeof(MarkStack::SlotsOrElementsRange) / sizeof(uintptr_t); + +template +struct MapTypeToMarkStackTag {}; +template <> +struct MapTypeToMarkStackTag { + static const auto value = MarkStack::ObjectTag; +}; +template <> +struct MapTypeToMarkStackTag { + static const auto value = MarkStack::JitCodeTag; +}; +template <> +struct MapTypeToMarkStackTag { + static const auto value = MarkStack::ScriptTag; +}; + +#ifdef DEBUG +static inline bool TagIsRangeTag(MarkStack::Tag tag) { + return tag == MarkStack::SlotsOrElementsRangeTag; +} +#endif + +inline MarkStack::TaggedPtr::TaggedPtr(Tag tag, Cell* ptr) + : bits(tag | uintptr_t(ptr)) { + assertValid(); +} + +inline uintptr_t MarkStack::TaggedPtr::tagUnchecked() const { + return bits & TagMask; +} + +inline MarkStack::Tag MarkStack::TaggedPtr::tag() const { + auto tag = Tag(bits & TagMask); + MOZ_ASSERT(tag <= LastTag); + return tag; +} + +inline Cell* MarkStack::TaggedPtr::ptr() const { + return reinterpret_cast(bits & ~TagMask); +} + +inline void MarkStack::TaggedPtr::assertValid() const { + (void)tag(); + MOZ_ASSERT(IsCellPointerValid(ptr())); +} + +template +inline T* MarkStack::TaggedPtr::as() const { + MOZ_ASSERT(tag() == MapTypeToMarkStackTag::value); + MOZ_ASSERT(ptr()->isTenured()); + MOZ_ASSERT(ptr()->is()); + return static_cast(ptr()); +} + +inline JSObject* MarkStack::TaggedPtr::asRangeObject() const { + MOZ_ASSERT(TagIsRangeTag(tag())); + MOZ_ASSERT(ptr()->isTenured()); + return ptr()->as(); +} + +inline JSRope* MarkStack::TaggedPtr::asTempRope() const { + MOZ_ASSERT(tag() == TempRopeTag); + return &ptr()->as()->asRope(); +} + +inline MarkStack::SlotsOrElementsRange::SlotsOrElementsRange( + SlotsOrElementsKind kindArg, JSObject* obj, size_t startArg) + : startAndKind_((startArg << StartShift) | size_t(kindArg)), + ptr_(SlotsOrElementsRangeTag, obj) { + assertValid(); + MOZ_ASSERT(kind() == kindArg); + MOZ_ASSERT(start() == startArg); +} + +inline void MarkStack::SlotsOrElementsRange::assertValid() const { + ptr_.assertValid(); + MOZ_ASSERT(TagIsRangeTag(ptr_.tag())); +} + +inline SlotsOrElementsKind MarkStack::SlotsOrElementsRange::kind() const { + return SlotsOrElementsKind(startAndKind_ & KindMask); +} + +inline size_t MarkStack::SlotsOrElementsRange::start() const { + return startAndKind_ >> StartShift; +} + +inline MarkStack::TaggedPtr MarkStack::SlotsOrElementsRange::ptr() const { + return ptr_; +} + +MarkStack::MarkStack() { MOZ_ASSERT(isEmpty()); } + +MarkStack::~MarkStack() { MOZ_ASSERT(isEmpty()); } + +MarkStack::MarkStack(const MarkStack& other) { + MOZ_CRASH("Compiler requires this but doesn't call it"); +} + +MarkStack& MarkStack::operator=(const MarkStack& other) { + new (this) MarkStack(other); + return *this; +} + +MarkStack::MarkStack(MarkStack&& other) + : stack_(std::move(other.stack_.ref())), + topIndex_(other.topIndex_.ref()) +#ifdef JS_GC_ZEAL + , + maxCapacity_(other.maxCapacity_) +#endif +{ + other.topIndex_ = 0; +} + +MarkStack& MarkStack::operator=(MarkStack&& other) { + new (this) MarkStack(std::move(other)); + return *this; +} + +bool MarkStack::init() { return resetStackCapacity(); } + +bool MarkStack::resetStackCapacity() { + MOZ_ASSERT(isEmpty()); + + size_t capacity = MARK_STACK_BASE_CAPACITY; + +#ifdef JS_GC_ZEAL + capacity = std::min(capacity, maxCapacity_.ref()); +#endif + + return resize(capacity); +} + +#ifdef JS_GC_ZEAL +void MarkStack::setMaxCapacity(size_t maxCapacity) { + MOZ_ASSERT(maxCapacity != 0); + MOZ_ASSERT(isEmpty()); + + maxCapacity_ = maxCapacity; + if (capacity() > maxCapacity_) { + // If the realloc fails, just keep using the existing stack; it's + // not ideal but better than failing. + (void)resize(maxCapacity_); + } +} +#endif + +MOZ_ALWAYS_INLINE bool MarkStack::indexIsEntryBase(size_t index) const { + // The mark stack holds both TaggedPtr and SlotsOrElementsRange entries, which + // are one or two words long respectively. Determine whether |index| points to + // the base of an entry (i.e. the lowest word in memory). + // + // The possible cases are that |index| points to: + // 1. a single word TaggedPtr entry => true + // 2. the startAndKind_ word of SlotsOrElementsRange => true + // (startAndKind_ is a uintptr_t tagged with SlotsOrElementsKind) + // 3. the ptr_ word of SlotsOrElementsRange (itself a TaggedPtr) => false + // + // To check for case 3, interpret the word as a TaggedPtr: if it is tagged as + // a SlotsOrElementsRange tagged pointer then we are inside such a range and + // |index| does not point to the base of an entry. This requires that no + // startAndKind_ word can be interpreted as such, which is arranged by making + // SlotsOrElementsRangeTag zero and all SlotsOrElementsKind tags non-zero. + + MOZ_ASSERT(index < position()); + return stack()[index].tagUnchecked() != SlotsOrElementsRangeTag; +} + +/* static */ +void MarkStack::moveWork(MarkStack& dst, MarkStack& src) { + // Move some work from |src| to |dst|. Assumes |dst| is empty. + // + // When this method runs during parallel marking, we are on the thread that + // owns |src|, and the thread that owns |dst| is blocked waiting on the + // ParallelMarkTask::resumed condition variable. + + // Limit the size of moves to stop threads with work spending too much time + // donating. + static const size_t MaxWordsToMove = 4096; + + size_t totalWords = src.position(); + size_t wordsToMove = std::min(totalWords / 2, MaxWordsToMove); + + size_t targetPos = src.position() - wordsToMove; + + // Adjust the target position in case it points to the middle of a two word + // entry. + if (!src.indexIsEntryBase(targetPos)) { + targetPos--; + wordsToMove++; + } + MOZ_ASSERT(src.indexIsEntryBase(targetPos)); + MOZ_ASSERT(targetPos < src.position()); + MOZ_ASSERT(targetPos > 0); + MOZ_ASSERT(wordsToMove == src.position() - targetPos); + + if (!dst.ensureSpace(wordsToMove)) { + return; + } + + // TODO: This doesn't have good cache behaviour when moving work between + // threads. It might be better if the original thread ended up with the top + // part of the stack, in src words if this method stole from the bottom of + // the stack rather than the top. + + mozilla::PodCopy(dst.topPtr(), src.stack().begin() + targetPos, wordsToMove); + dst.topIndex_ += wordsToMove; + dst.peekPtr().assertValid(); + + src.topIndex_ = targetPos; +#ifdef DEBUG + src.poisonUnused(); +#endif + src.peekPtr().assertValid(); +} + +void MarkStack::clearAndResetCapacity() { + // Fall back to the smaller initial capacity so we don't hold on to excess + // memory between GCs. + stack().clear(); + topIndex_ = 0; + (void)resetStackCapacity(); +} + +void MarkStack::clearAndFreeStack() { + // Free all stack memory so we don't hold on to excess memory between GCs. + stack().clearAndFree(); + topIndex_ = 0; +} + +inline MarkStack::TaggedPtr* MarkStack::topPtr() { return &stack()[topIndex_]; } + +template +inline bool MarkStack::push(T* ptr) { + return push(TaggedPtr(MapTypeToMarkStackTag::value, ptr)); +} + +inline bool MarkStack::pushTempRope(JSRope* rope) { + return push(TaggedPtr(TempRopeTag, rope)); +} + +inline bool MarkStack::push(const TaggedPtr& ptr) { + if (!ensureSpace(1)) { + return false; + } + + infalliblePush(ptr); + return true; +} + +inline void MarkStack::infalliblePush(const TaggedPtr& ptr) { + *topPtr() = ptr; + topIndex_++; + MOZ_ASSERT(position() <= capacity()); +} + +inline bool MarkStack::push(JSObject* obj, SlotsOrElementsKind kind, + size_t start) { + return push(SlotsOrElementsRange(kind, obj, start)); +} + +inline bool MarkStack::push(const SlotsOrElementsRange& array) { + array.assertValid(); + + if (!ensureSpace(ValueRangeWords)) { + return false; + } + + infalliblePush(array); + return true; +} + +inline void MarkStack::infalliblePush(const SlotsOrElementsRange& array) { + *reinterpret_cast(topPtr()) = array; + topIndex_ += ValueRangeWords; + MOZ_ASSERT(position() <= capacity()); + MOZ_ASSERT(TagIsRangeTag(peekTag())); +} + +inline const MarkStack::TaggedPtr& MarkStack::peekPtr() const { + MOZ_ASSERT(!isEmpty()); + return stack()[topIndex_ - 1]; +} + +inline MarkStack::Tag MarkStack::peekTag() const { + MOZ_ASSERT(!isEmpty()); + return peekPtr().tag(); +} + +inline MarkStack::TaggedPtr MarkStack::popPtr() { + MOZ_ASSERT(!isEmpty()); + MOZ_ASSERT(!TagIsRangeTag(peekTag())); + peekPtr().assertValid(); + topIndex_--; + return *topPtr(); +} + +inline MarkStack::SlotsOrElementsRange MarkStack::popSlotsOrElementsRange() { + MOZ_ASSERT(!isEmpty()); + MOZ_ASSERT(TagIsRangeTag(peekTag())); + MOZ_ASSERT(position() >= ValueRangeWords); + + topIndex_ -= ValueRangeWords; + const auto& array = *reinterpret_cast(topPtr()); + array.assertValid(); + return array; +} + +inline bool MarkStack::ensureSpace(size_t count) { + if (MOZ_LIKELY((topIndex_ + count) <= capacity())) { + return !js::oom::ShouldFailWithOOM(); + } + + return enlarge(count); +} + +MOZ_NEVER_INLINE bool MarkStack::enlarge(size_t count) { + size_t required = capacity() + count; + size_t newCapacity = mozilla::RoundUpPow2(required); + +#ifdef JS_GC_ZEAL + newCapacity = std::min(newCapacity, maxCapacity_.ref()); + if (newCapacity < required) { + return false; + } +#endif + + return resize(newCapacity); +} + +bool MarkStack::resize(size_t newCapacity) { + MOZ_ASSERT(newCapacity != 0); + MOZ_ASSERT(newCapacity >= position()); + + if (!stack().resize(newCapacity)) { + return false; + } + + poisonUnused(); + return true; +} + +inline void MarkStack::poisonUnused() { + static_assert((JS_FRESH_MARK_STACK_PATTERN & TagMask) > LastTag, + "The mark stack poison pattern must not look like a valid " + "tagged pointer"); + + AlwaysPoison(stack().begin() + topIndex_, JS_FRESH_MARK_STACK_PATTERN, + stack().capacity() - topIndex_, MemCheckKind::MakeUndefined); +} + +size_t MarkStack::sizeOfExcludingThis( + mozilla::MallocSizeOf mallocSizeOf) const { + return stack().sizeOfExcludingThis(mallocSizeOf); +} + +/*** GCMarker ***************************************************************/ + +/* + * WeakMapTraceAction::Expand: the GC is recomputing the liveness of WeakMap + * entries by expanding each live WeakMap into its constituent key->value edges, + * a table of which will be consulted in a later phase whenever marking a + * potential key. + */ +GCMarker::GCMarker(JSRuntime* rt) + : tracer_(mozilla::VariantType(), rt, this), + runtime_(rt), + haveSwappedStacks(false), + markColor_(MarkColor::Black), + state(NotActive), + incrementalWeakMapMarkingEnabled( + TuningDefaults::IncrementalWeakMapMarkingEnabled) +#ifdef DEBUG + , + checkAtomMarking(true), + strictCompartmentChecking(false) +#endif +{ +} + +bool GCMarker::init() { return stack.init(); } + +void GCMarker::start() { + MOZ_ASSERT(state == NotActive); + MOZ_ASSERT(stack.isEmpty()); + state = RegularMarking; + haveAllImplicitEdges = true; + setMarkColor(MarkColor::Black); +} + +static void ClearEphemeronEdges(JSRuntime* rt) { + AutoEnterOOMUnsafeRegion oomUnsafe; + for (GCZonesIter zone(rt); !zone.done(); zone.next()) { + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing weak keys in GCMarker::stop()"); + } + if (!zone->gcNurseryEphemeronEdges().clear()) { + oomUnsafe.crash("clearing (nursery) weak keys in GCMarker::stop()"); + } + } +} + +void GCMarker::stop() { + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(markColor() == MarkColor::Black); + MOZ_ASSERT(!haveSwappedStacks); + + if (state == NotActive) { + return; + } + state = NotActive; + + (void)stack.resetStackCapacity(); + otherStack.clearAndFreeStack(); + ClearEphemeronEdges(runtime()); + unmarkGrayStack.clearAndFree(); +} + +void GCMarker::reset() { + stack.clearAndResetCapacity(); + otherStack.clearAndFreeStack(); + ClearEphemeronEdges(runtime()); + MOZ_ASSERT(isDrained()); + + setMarkColor(MarkColor::Black); + MOZ_ASSERT(!haveSwappedStacks); + + unmarkGrayStack.clearAndFree(); +} + +void GCMarker::setMarkColor(gc::MarkColor newColor) { + if (markColor_ == newColor) { + return; + } + + // We don't support gray marking while there is black marking work to do. + MOZ_ASSERT(!hasBlackEntries()); + + markColor_ = newColor; + + // Switch stacks. We only need to do this if there are any stack entries (as + // empty stacks are interchangeable) or to swtich back to the original stack. + if (!isDrained() || haveSwappedStacks) { + std::swap(stack, otherStack); + haveSwappedStacks = !haveSwappedStacks; + } +} + +bool GCMarker::hasEntries(MarkColor color) const { + const MarkStack& stackForColor = color == markColor() ? stack : otherStack; + return stackForColor.hasEntries(); +} + +template +inline void GCMarker::pushTaggedPtr(T* ptr) { + checkZone(ptr); + if (!stack.push(ptr)) { + delayMarkingChildrenOnOOM(ptr); + } +} + +inline void GCMarker::pushValueRange(JSObject* obj, SlotsOrElementsKind kind, + size_t start, size_t end) { + checkZone(obj); + MOZ_ASSERT(obj->is()); + MOZ_ASSERT(start <= end); + + if (start == end) { + return; + } + + if (MOZ_UNLIKELY(!stack.push(obj, kind, start))) { + delayMarkingChildrenOnOOM(obj); + } +} + +void GCMarker::repush(JSObject* obj) { + MOZ_ASSERT(obj->asTenured().isMarkedAtLeast(markColor())); + pushTaggedPtr(obj); +} + +void GCMarker::setRootMarkingMode(bool newState) { + if (newState) { + setMarkingStateAndTracer(RegularMarking, RootMarking); + } else { + setMarkingStateAndTracer(RootMarking, RegularMarking); + } +} + +void GCMarker::enterParallelMarkingMode(ParallelMarker* pm) { + MOZ_ASSERT(pm); + MOZ_ASSERT(!parallelMarker_); + setMarkingStateAndTracer(RegularMarking, + ParallelMarking); + parallelMarker_ = pm; +} + +void GCMarker::leaveParallelMarkingMode() { + MOZ_ASSERT(parallelMarker_); + setMarkingStateAndTracer(ParallelMarking, RegularMarking); + parallelMarker_ = nullptr; +} + +bool GCMarker::canDonateWork() const { + // It's not worth the overhead of donating very few entries. For some + // (non-parallelizable) workloads this can lead to constantly interrupting + // marking work and makes parallel marking slower than single threaded. + constexpr size_t MinWordCount = 12; + + static_assert(MinWordCount >= ValueRangeWords, + "We must always leave at least one stack entry."); + + return stack.position() > MinWordCount; +} + +template +void GCMarker::setMarkingStateAndTracer(MarkingState prev, MarkingState next) { + MOZ_ASSERT(state == prev); + state = next; + tracer_.emplace(runtime(), this); +} + +bool GCMarker::enterWeakMarkingMode() { + MOZ_ASSERT(tracer()->weakMapAction() == JS::WeakMapTraceAction::Expand); + MOZ_ASSERT(state == RegularMarking); + if (!haveAllImplicitEdges) { + return false; + } + + // During weak marking mode, we maintain a table mapping weak keys to + // entries in known-live weakmaps. Initialize it with the keys of marked + // weakmaps -- or more precisely, the keys of marked weakmaps that are + // mapped to not yet live values. (Once bug 1167452 implements incremental + // weakmap marking, this initialization step will become unnecessary, as + // the table will already hold all such keys.) + + // Set state before doing anything else, so any new key that is marked + // during the following gcEphemeronEdges scan will itself be looked up in + // gcEphemeronEdges and marked according to ephemeron rules. + state = WeakMarking; + + return true; +} + +IncrementalProgress JS::Zone::enterWeakMarkingMode(GCMarker* marker, + SliceBudget& budget) { + MOZ_ASSERT(marker->isWeakMarking()); + + if (!marker->incrementalWeakMapMarkingEnabled) { + for (WeakMapBase* m : gcWeakMapList()) { + if (m->mapColor) { + (void)m->markEntries(marker); + } + } + return IncrementalProgress::Finished; + } + + // gcEphemeronEdges contains the keys from all weakmaps marked so far, or at + // least the keys that might still need to be marked through. Scan through + // gcEphemeronEdges and mark all values whose keys are marked. This marking + // may recursively mark through other weakmap entries (immediately since we + // are now in WeakMarking mode). The end result is a consistent state where + // all values are marked if both their map and key are marked -- though note + // that we may later leave weak marking mode, do some more marking, and then + // enter back in. + if (!isGCMarking()) { + return IncrementalProgress::Finished; + } + + MOZ_ASSERT(gcNurseryEphemeronEdges().count() == 0); + + // An OrderedHashMap::MutableRange stays valid even when the underlying table + // (zone->gcEphemeronEdges) is mutated, which is useful here since we may add + // additional entries while iterating over the Range. + EphemeronEdgeTable::MutableRange r = gcEphemeronEdges().mutableAll(); + while (!r.empty()) { + Cell* src = r.front().key; + CellColor srcColor = gc::detail::GetEffectiveColor(marker, src); + auto& edges = r.front().value; + r.popFront(); // Pop before any mutations happen. + + if (edges.length() > 0) { + uint32_t steps = edges.length(); + marker->markEphemeronEdges(edges, srcColor); + budget.step(steps); + if (budget.isOverBudget()) { + return NotFinished; + } + } + } + + return IncrementalProgress::Finished; +} + +void GCMarker::leaveWeakMarkingMode() { + MOZ_ASSERT(state == WeakMarking || state == RegularMarking); + state = RegularMarking; + + // The gcEphemeronEdges table is still populated and may be used during a + // future weak marking mode within this GC. +} + +void GCMarker::abortLinearWeakMarking() { + haveAllImplicitEdges = false; + if (state == WeakMarking) { + leaveWeakMarkingMode(); + } +} + +MOZ_NEVER_INLINE void GCMarker::delayMarkingChildrenOnOOM(Cell* cell) { + runtime()->gc.delayMarkingChildren(cell, markColor()); +} + +bool GCRuntime::hasDelayedMarking() const { + bool result = delayedMarkingList; + MOZ_ASSERT(result == (markLaterArenas != 0)); + return result; +} + +void GCRuntime::delayMarkingChildren(Cell* cell, MarkColor color) { + // Synchronize access to delayed marking state during parallel marking. + LockGuard lock(delayedMarkingLock); + + Arena* arena = cell->asTenured().arena(); + if (!arena->onDelayedMarkingList()) { + arena->setNextDelayedMarkingArena(delayedMarkingList); + delayedMarkingList = arena; +#ifdef DEBUG + markLaterArenas++; +#endif + } + + if (!arena->hasDelayedMarking(color)) { + arena->setHasDelayedMarking(color, true); + delayedMarkingWorkAdded = true; + } +} + +void GCRuntime::markDelayedChildren(Arena* arena, MarkColor color) { + JSTracer* trc = marker().tracer(); + JS::TraceKind kind = MapAllocToTraceKind(arena->getAllocKind()); + MarkColor colorToCheck = + TraceKindCanBeMarkedGray(kind) ? color : MarkColor::Black; + + for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) { + if (cell->isMarked(colorToCheck)) { + ApplyGCThingTyped(cell, kind, [trc, this](auto t) { + t->traceChildren(trc); + marker().markImplicitEdges(t); + }); + } + } +} + +/* + * Process arenas from |delayedMarkingList| by marking the unmarked children of + * marked cells of color |color|. + * + * This is called twice, first to mark gray children and then to mark black + * children. + */ +void GCRuntime::processDelayedMarkingList(MarkColor color) { + // Marking delayed children may add more arenas to the list, including arenas + // we are currently processing or have previously processed. Handle this by + // clearing a flag on each arena before marking its children. This flag will + // be set again if the arena is re-added. Iterate the list until no new arenas + // were added. + + AutoSetMarkColor setColor(marker(), color); + + do { + delayedMarkingWorkAdded = false; + for (Arena* arena = delayedMarkingList; arena; + arena = arena->getNextDelayedMarking()) { + if (arena->hasDelayedMarking(color)) { + arena->setHasDelayedMarking(color, false); + markDelayedChildren(arena, color); + } + } + while (marker().hasEntriesForCurrentColor()) { + SliceBudget budget = SliceBudget::unlimited(); + MOZ_ALWAYS_TRUE( + marker().processMarkStackTop(budget)); + } + } while (delayedMarkingWorkAdded); + + MOZ_ASSERT(marker().isDrained()); +} + +void GCRuntime::markAllDelayedChildren(ShouldReportMarkTime reportTime) { + MOZ_ASSERT(CurrentThreadIsMainThread() || CurrentThreadIsPerformingGC()); + MOZ_ASSERT(marker().isDrained()); + MOZ_ASSERT(hasDelayedMarking()); + + mozilla::Maybe ap; + if (reportTime) { + ap.emplace(stats(), gcstats::PhaseKind::MARK_DELAYED); + } + + // We have a list of arenas containing marked cells with unmarked children + // where we ran out of stack space during marking. Both black and gray cells + // in these arenas may have unmarked children. Mark black children first. + + const MarkColor colors[] = {MarkColor::Black, MarkColor::Gray}; + for (MarkColor color : colors) { + processDelayedMarkingList(color); + rebuildDelayedMarkingList(); + } + + MOZ_ASSERT(!hasDelayedMarking()); +} + +void GCRuntime::rebuildDelayedMarkingList() { + // Rebuild the delayed marking list, removing arenas which do not need further + // marking. + + Arena* listTail = nullptr; + forEachDelayedMarkingArena([&](Arena* arena) { + if (!arena->hasAnyDelayedMarking()) { + arena->clearDelayedMarkingState(); +#ifdef DEBUG + MOZ_ASSERT(markLaterArenas); + markLaterArenas--; +#endif + return; + } + + appendToDelayedMarkingList(&listTail, arena); + }); + appendToDelayedMarkingList(&listTail, nullptr); +} + +void GCRuntime::resetDelayedMarking() { + MOZ_ASSERT(CurrentThreadIsMainThread()); + + forEachDelayedMarkingArena([&](Arena* arena) { + MOZ_ASSERT(arena->onDelayedMarkingList()); + arena->clearDelayedMarkingState(); +#ifdef DEBUG + MOZ_ASSERT(markLaterArenas); + markLaterArenas--; +#endif + }); + delayedMarkingList = nullptr; + MOZ_ASSERT(!markLaterArenas); +} + +inline void GCRuntime::appendToDelayedMarkingList(Arena** listTail, + Arena* arena) { + if (*listTail) { + (*listTail)->updateNextDelayedMarkingArena(arena); + } else { + delayedMarkingList = arena; + } + *listTail = arena; +} + +template +inline void GCRuntime::forEachDelayedMarkingArena(F&& f) { + Arena* arena = delayedMarkingList; + Arena* next; + while (arena) { + next = arena->getNextDelayedMarking(); + f(arena); + arena = next; + } +} + +#ifdef DEBUG +void GCMarker::checkZone(void* p) { + MOZ_ASSERT(state != NotActive); + DebugOnly cell = static_cast(p); + MOZ_ASSERT_IF(cell->isTenured(), + cell->asTenured().zone()->isCollectingFromAnyThread()); +} +#endif + +size_t GCMarker::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return mallocSizeOf(this) + stack.sizeOfExcludingThis(mallocSizeOf) + + otherStack.sizeOfExcludingThis(mallocSizeOf); +} + +/*** IsMarked / IsAboutToBeFinalized ****************************************/ + +template +static inline void CheckIsMarkedThing(T* thing) { +#define IS_SAME_TYPE_OR(name, type, _, _1) std::is_same_v || + static_assert(JS_FOR_EACH_TRACEKIND(IS_SAME_TYPE_OR) false, + "Only the base cell layout types are allowed into " + "marking/tracing internals"); +#undef IS_SAME_TYPE_OR + +#ifdef DEBUG + MOZ_ASSERT(thing); + + // Allow any thread access to uncollected things. + Zone* zone = thing->zoneFromAnyThread(); + if (thing->isPermanentAndMayBeShared()) { + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + MOZ_ASSERT(thing->isMarkedBlack()); + return; + } + + // Allow the current thread access if it is sweeping or in sweep-marking, but + // try to check the zone. Some threads have access to all zones when sweeping. + JS::GCContext* gcx = TlsGCContext.get(); + MOZ_ASSERT(gcx->gcUse() != GCUse::Finalizing); + if (gcx->gcUse() == GCUse::Sweeping || gcx->gcUse() == GCUse::Marking) { + MOZ_ASSERT_IF(gcx->gcSweepZone(), + gcx->gcSweepZone() == zone || zone->isAtomsZone()); + return; + } + + // Otherwise only allow access from the main thread or this zone's associated + // thread. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread()) || + CurrentThreadCanAccessZone(thing->zoneFromAnyThread())); +#endif +} + +template +bool js::gc::IsMarkedInternal(JSRuntime* rt, T* thing) { + // Don't depend on the mark state of other cells during finalization. + MOZ_ASSERT(!CurrentThreadIsGCFinalizing()); + MOZ_ASSERT(rt->heapState() != JS::HeapState::MinorCollecting); + MOZ_ASSERT(thing); + CheckIsMarkedThing(thing); + + // This is not used during minor sweeping nor used to update moved GC things. + MOZ_ASSERT(!IsForwarded(thing)); + + // Permanent things are never marked by non-owning runtimes. + TenuredCell* cell = &thing->asTenured(); + Zone* zone = cell->zoneFromAnyThread(); +#ifdef DEBUG + if (IsOwnedByOtherRuntime(rt, thing)) { + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(thing->isMarkedBlack()); + } +#endif + + return !zone->isGCMarking() || TenuredThingIsMarkedAny(thing); +} + +template +bool js::gc::IsAboutToBeFinalizedInternal(T* thing) { + // Don't depend on the mark state of other cells during finalization. + MOZ_ASSERT(!CurrentThreadIsGCFinalizing()); + MOZ_ASSERT(thing); + CheckIsMarkedThing(thing); + + // This is not used during minor sweeping nor used to update moved GC things. + MOZ_ASSERT(!IsForwarded(thing)); + + if (!thing->isTenured()) { + return false; + } + + // Permanent things are never finalized by non-owning runtimes. + TenuredCell* cell = &thing->asTenured(); + Zone* zone = cell->zoneFromAnyThread(); +#ifdef DEBUG + JSRuntime* rt = TlsGCContext.get()->runtimeFromAnyThread(); + if (IsOwnedByOtherRuntime(rt, thing)) { + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(thing->isMarkedBlack()); + } +#endif + + return zone->isGCSweeping() && !TenuredThingIsMarkedAny(thing); +} + +template +bool js::gc::IsAboutToBeFinalizedInternal(const T& thing) { + bool dying = false; + ApplyGCThingTyped( + thing, [&dying](auto t) { dying = IsAboutToBeFinalizedInternal(t); }); + return dying; +} + +SweepingTracer::SweepingTracer(JSRuntime* rt) + : GenericTracerImpl(rt, JS::TracerKind::Sweeping, + JS::WeakMapTraceAction::TraceKeysAndValues) {} + +template +inline void SweepingTracer::onEdge(T** thingp, const char* name) { + T* thing = *thingp; + CheckIsMarkedThing(thing); + + if (!thing->isTenured()) { + return; + } + + // Permanent things are never finalized by non-owning runtimes. + TenuredCell* cell = &thing->asTenured(); + Zone* zone = cell->zoneFromAnyThread(); +#ifdef DEBUG + if (IsOwnedByOtherRuntime(runtime(), thing)) { + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(thing->isMarkedBlack()); + } +#endif + + // It would be nice if we could assert that the zone of the tenured cell is in + // the Sweeping state, but that isn't always true for: + // - atoms + // - the jitcode map + // - the mark queue + if (zone->isGCSweeping() && !cell->isMarkedAny()) { + *thingp = nullptr; + } +} + +namespace js { +namespace gc { + +template +JS_PUBLIC_API bool TraceWeakEdge(JSTracer* trc, JS::Heap* thingp) { + return TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unsafeGet()), + "JS::Heap edge"); +} + +template +JS_PUBLIC_API bool EdgeNeedsSweepUnbarrieredSlow(T* thingp) { + return IsAboutToBeFinalizedInternal(*ConvertToBase(thingp)); +} + +// Instantiate a copy of the Tracing templates for each public GC type. +#define INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS(type) \ + template JS_PUBLIC_API bool TraceWeakEdge(JSTracer * trc, \ + JS::Heap*); \ + template JS_PUBLIC_API bool EdgeNeedsSweepUnbarrieredSlow(type*); +JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS) +JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE( + INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS) + +#define INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION(type) \ + template bool IsMarkedInternal(JSRuntime* rt, type thing); + +#define INSTANTIATE_INTERNAL_IATBF_FUNCTION(type) \ + template bool IsAboutToBeFinalizedInternal(type thingp); + +#define INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND(_1, type, _2, \ + _3) \ + INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION(type*) \ + INSTANTIATE_INTERNAL_IATBF_FUNCTION(type*) + +JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND) + +#define INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER(type) \ + INSTANTIATE_INTERNAL_IATBF_FUNCTION(const type&) + +JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE( + INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER) + +#undef INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION +#undef INSTANTIATE_INTERNAL_IATBF_FUNCTION +#undef INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND +#undef INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER + +} /* namespace gc */ +} /* namespace js */ + +/*** Cycle Collector Barrier Implementation *********************************/ + +/* + * The GC and CC are run independently. Consequently, the following sequence of + * events can occur: + * 1. GC runs and marks an object gray. + * 2. The mutator runs (specifically, some C++ code with access to gray + * objects) and creates a pointer from a JS root or other black object to + * the gray object. If we re-ran a GC at this point, the object would now be + * black. + * 3. Now we run the CC. It may think it can collect the gray object, even + * though it's reachable from the JS heap. + * + * To prevent this badness, we unmark the gray bit of an object when it is + * accessed by callers outside XPConnect. This would cause the object to go + * black in step 2 above. This must be done on everything reachable from the + * object being returned. The following code takes care of the recursive + * re-coloring. + * + * There is an additional complication for certain kinds of edges that are not + * contained explicitly in the source object itself, such as from a weakmap key + * to its value. These "implicit edges" are represented in some other + * container object, such as the weakmap itself. In these + * cases, calling unmark gray on an object won't find all of its children. + * + * Handling these implicit edges has two parts: + * - A special pass enumerating all of the containers that know about the + * implicit edges to fix any black-gray edges that have been created. This + * is implemented in nsXPConnect::FixWeakMappingGrayBits. + * - To prevent any incorrectly gray objects from escaping to live JS outside + * of the containers, we must add unmark-graying read barriers to these + * containers. + */ + +#ifdef DEBUG +struct AssertNonGrayTracer final : public JS::CallbackTracer { + // This is used by the UnmarkGray tracer only, and needs to report itself as + // the non-gray tracer to not trigger assertions. Do not use it in another + // context without making this more generic. + explicit AssertNonGrayTracer(JSRuntime* rt) + : JS::CallbackTracer(rt, JS::TracerKind::UnmarkGray) {} + void onChild(JS::GCCellPtr thing, const char* name) override { + MOZ_ASSERT(!thing.asCell()->isMarkedGray()); + } +}; +#endif + +class js::gc::UnmarkGrayTracer final : public JS::CallbackTracer { + public: + // We set weakMapAction to WeakMapTraceAction::Skip because the cycle + // collector will fix up any color mismatches involving weakmaps when it runs. + explicit UnmarkGrayTracer(GCMarker* marker) + : JS::CallbackTracer(marker->runtime(), JS::TracerKind::UnmarkGray, + JS::WeakMapTraceAction::Skip), + unmarkedAny(false), + oom(false), + marker(marker), + stack(marker->unmarkGrayStack) {} + + void unmark(JS::GCCellPtr cell); + + // Whether we unmarked anything. + bool unmarkedAny; + + // Whether we ran out of memory. + bool oom; + + private: + // Marker to use if we need to unmark in zones that are currently being + // marked. + GCMarker* marker; + + // Stack of cells to traverse. + Vector& stack; + + void onChild(JS::GCCellPtr thing, const char* name) override; +}; + +void UnmarkGrayTracer::onChild(JS::GCCellPtr thing, const char* name) { + Cell* cell = thing.asCell(); + + // Cells in the nursery cannot be gray, and nor can certain kinds of tenured + // cells. These must necessarily point only to black edges. + if (!cell->isTenured() || !TraceKindCanBeMarkedGray(thing.kind())) { +#ifdef DEBUG + MOZ_ASSERT(!cell->isMarkedGray()); + AssertNonGrayTracer nongray(runtime()); + JS::TraceChildren(&nongray, thing); +#endif + return; + } + + TenuredCell& tenured = cell->asTenured(); + Zone* zone = tenured.zone(); + + // If the cell is in a zone whose mark bits are being cleared, then it will + // end up white. + if (zone->isGCPreparing()) { + return; + } + + // If the cell is in a zone that we're currently marking, then it's possible + // that it is currently white but will end up gray. To handle this case, + // trigger the barrier for any cells in zones that are currently being + // marked. This will ensure they will eventually get marked black. + if (zone->isGCMarking()) { + if (!cell->isMarkedBlack()) { + TraceEdgeForBarrier(marker, &tenured, thing.kind()); + unmarkedAny = true; + } + return; + } + + if (!tenured.isMarkedGray()) { + return; + } + + // TODO: It may be a small improvement to only use the atomic version during + // parallel marking. + tenured.markBlackAtomic(); + unmarkedAny = true; + + if (!stack.append(thing)) { + oom = true; + } +} + +void UnmarkGrayTracer::unmark(JS::GCCellPtr cell) { + MOZ_ASSERT(stack.empty()); + + onChild(cell, "unmarking root"); + + while (!stack.empty() && !oom) { + TraceChildren(this, stack.popCopy()); + } + + if (oom) { + // If we run out of memory, we take a drastic measure: require that we + // GC again before the next CC. + stack.clear(); + runtime()->gc.setGrayBitsInvalid(); + return; + } +} + +bool js::gc::UnmarkGrayGCThingUnchecked(GCMarker* marker, JS::GCCellPtr thing) { + MOZ_ASSERT(thing); + MOZ_ASSERT(thing.asCell()->isMarkedGray()); + + mozilla::Maybe profilingStackFrame; + if (JSContext* cx = TlsContext.get()) { + profilingStackFrame.emplace(cx, "UnmarkGrayGCThing", + JS::ProfilingCategoryPair::GCCC_UnmarkGray); + } + + UnmarkGrayTracer unmarker(marker); + unmarker.unmark(thing); + return unmarker.unmarkedAny; +} + +JS_PUBLIC_API bool JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) { + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + MOZ_ASSERT(!JS::RuntimeHeapIsCycleCollecting()); + + JSRuntime* rt = thing.asCell()->runtimeFromMainThread(); + if (thing.asCell()->zone()->isGCPreparing()) { + // Mark bits are being cleared in preparation for GC. + return false; + } + + return UnmarkGrayGCThingUnchecked(&rt->gc.marker(), thing); +} + +void js::gc::UnmarkGrayGCThingRecursively(TenuredCell* cell) { + JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(cell, cell->getTraceKind())); +} + +bool js::UnmarkGrayShapeRecursively(Shape* shape) { + return JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(shape)); +} + +#ifdef DEBUG +Cell* js::gc::UninlinedForwarded(const Cell* cell) { return Forwarded(cell); } +#endif + +namespace js { +namespace debug { + +MarkInfo GetMarkInfo(Cell* rawCell) { + if (!rawCell->isTenured()) { + return MarkInfo::NURSERY; + } + + TenuredCell* cell = &rawCell->asTenured(); + if (cell->isMarkedGray()) { + return MarkInfo::GRAY; + } + if (cell->isMarkedBlack()) { + return MarkInfo::BLACK; + } + return MarkInfo::UNMARKED; +} + +uintptr_t* GetMarkWordAddress(Cell* cell) { + if (!cell->isTenured()) { + return nullptr; + } + + MarkBitmapWord* wordp; + uintptr_t mask; + TenuredChunkBase* chunk = gc::detail::GetCellChunkBase(&cell->asTenured()); + chunk->markBits.getMarkWordAndMask(&cell->asTenured(), ColorBit::BlackBit, + &wordp, &mask); + return reinterpret_cast(wordp); +} + +uintptr_t GetMarkMask(Cell* cell, uint32_t colorBit) { + MOZ_ASSERT(colorBit == 0 || colorBit == 1); + + if (!cell->isTenured()) { + return 0; + } + + ColorBit bit = colorBit == 0 ? ColorBit::BlackBit : ColorBit::GrayOrBlackBit; + MarkBitmapWord* wordp; + uintptr_t mask; + TenuredChunkBase* chunk = gc::detail::GetCellChunkBase(&cell->asTenured()); + chunk->markBits.getMarkWordAndMask(&cell->asTenured(), bit, &wordp, &mask); + return mask; +} + +} // namespace debug +} // namespace js diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h new file mode 100644 index 0000000000..7043eccc53 --- /dev/null +++ b/js/src/gc/Marking.h @@ -0,0 +1,150 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Marking and sweeping APIs for use by implementations of different GC cell + * kinds. + */ + +#ifndef gc_Marking_h +#define gc_Marking_h + +#include "gc/Barrier.h" +#include "js/TypeDecls.h" + +class JSTracer; +struct JSClass; + +namespace js { +class GCMarker; +class Shape; +class WeakMapBase; + +namespace gc { + +struct Cell; + +/*** Liveness ***/ + +// The IsMarkedInternal and IsAboutToBeFinalizedInternal function templates are +// used to implement the IsMarked and IsAboutToBeFinalized set of functions. +// These internal functions are instantiated for the base GC types and should +// not be called directly. +// +// Note that there are two function templates declared for each, not one +// template and a specialization. This is necessary so that pointer arguments +// (e.g. JSObject**) and tagged value arguments (e.g. JS::Value*) are routed to +// separate implementations. + +template +bool IsMarkedInternal(JSRuntime* rt, T* thing); + +template +bool IsAboutToBeFinalizedInternal(T* thing); +template +bool IsAboutToBeFinalizedInternal(const T& thing); + +// Report whether a GC thing has been marked with any color. Things which are in +// zones that are not currently being collected or are owned by another runtime +// are always reported as being marked. +template +inline bool IsMarked(JSRuntime* rt, const BarrieredBase& thing) { + return IsMarkedInternal(rt, *ConvertToBase(thing.unbarrieredAddress())); +} +template +inline bool IsMarkedUnbarriered(JSRuntime* rt, T thing) { + return IsMarkedInternal(rt, *ConvertToBase(&thing)); +} + +// Report whether a GC thing is dead and will be finalized in the current sweep +// group. This is mainly used in read barriers for incremental sweeping. +// +// This no longer updates pointers moved by the GC (tracing should be used for +// this instead). +template +inline bool IsAboutToBeFinalized(const BarrieredBase& thing) { + return IsAboutToBeFinalizedInternal( + *ConvertToBase(thing.unbarrieredAddress())); +} +template +inline bool IsAboutToBeFinalizedUnbarriered(T thing) { + return IsAboutToBeFinalizedInternal(*ConvertToBase(&thing)); +} + +inline bool IsAboutToBeFinalizedDuringMinorSweep(Cell* cell); + +inline Cell* ToMarkable(const Value& v) { + if (v.isGCThing()) { + return (Cell*)v.toGCThing(); + } + return nullptr; +} + +inline Cell* ToMarkable(Cell* cell) { return cell; } + +bool UnmarkGrayGCThingUnchecked(GCMarker* marker, JS::GCCellPtr thing); + +} /* namespace gc */ + +// The return value indicates if anything was unmarked. +bool UnmarkGrayShapeRecursively(Shape* shape); + +namespace gc { + +// Functions for checking and updating GC thing pointers that might have been +// moved by compacting GC. Overloads are also provided that work with Values. +// +// IsForwarded - check whether a pointer refers to an GC thing that has been +// moved. +// +// Forwarded - return a pointer to the new location of a GC thing given a +// pointer to old location. +// +// MaybeForwarded - used before dereferencing a pointer that may refer to a +// moved GC thing without updating it. For JSObjects this will +// also update the object's shape pointer if it has been moved +// to allow slots to be accessed. + +template +inline bool IsForwarded(const T* t); + +template +inline T* Forwarded(const T* t); + +inline Value Forwarded(const JS::Value& value); + +template +inline T MaybeForwarded(T t); + +// Helper functions for use in situations where the object's group might be +// forwarded, for example while marking. + +inline const JSClass* MaybeForwardedObjectClass(const JSObject* obj); + +template +inline bool MaybeForwardedObjectIs(const JSObject* obj); + +template +inline T& MaybeForwardedObjectAs(JSObject* obj); + +#ifdef JSGC_HASH_TABLE_CHECKS + +template +inline bool IsGCThingValidAfterMovingGC(T* t); + +template +inline void CheckGCThingAfterMovingGC(T* t); + +template +inline void CheckGCThingAfterMovingGC(const WeakHeapPtr& t); + +#endif // JSGC_HASH_TABLE_CHECKS + +} /* namespace gc */ + +} /* namespace js */ + +#endif /* gc_Marking_h */ diff --git a/js/src/gc/MaybeRooted.h b/js/src/gc/MaybeRooted.h new file mode 100644 index 0000000000..fd615a1efb --- /dev/null +++ b/js/src/gc/MaybeRooted.h @@ -0,0 +1,141 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Template types for use in generic code: to use Rooted/Handle/MutableHandle in + * cases where GC may occur, or to use mock versions of those types that perform + * no rooting or root list manipulation when GC cannot occur. + */ + +#ifndef gc_MaybeRooted_h +#define gc_MaybeRooted_h + +#include "mozilla/Attributes.h" // MOZ_IMPLICIT, MOZ_RAII + +#include // std::true_type + +#include "gc/Allocator.h" // js::AllowGC, js::CanGC, js::NoGC +#include "js/ComparisonOperators.h" // JS::detail::DefineComparisonOps +#include "js/RootingAPI.h" // js::{Rooted,MutableHandle}Base, JS::SafelyInitialized, DECLARE_POINTER_{CONSTREF,ASSIGN}_OPS, DECLARE_NONPOINTER_{,MUTABLE_}ACCESSOR_METHODS, JS::Rooted, JS::{,Mutable}Handle + +namespace js { + +/** + * Interface substitute for Rooted which does not root the variable's + * memory. + */ +template +class MOZ_RAII FakeRooted : public RootedOperations> { + public: + using ElementType = T; + + explicit FakeRooted(JSContext* cx) + : ptr(JS::SafelyInitialized::create()) {} + + FakeRooted(JSContext* cx, T initial) : ptr(initial) {} + + DECLARE_POINTER_CONSTREF_OPS(T); + DECLARE_POINTER_ASSIGN_OPS(FakeRooted, T); + DECLARE_NONPOINTER_ACCESSOR_METHODS(ptr); + DECLARE_NONPOINTER_MUTABLE_ACCESSOR_METHODS(ptr); + + private: + T ptr; + + void set(const T& value) { ptr = value; } + + FakeRooted(const FakeRooted&) = delete; +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::FakeRooted& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +/** + * Interface substitute for MutableHandle which is not required to point to + * rooted memory. + */ +template +class FakeMutableHandle + : public js::MutableHandleOperations> { + public: + using ElementType = T; + + MOZ_IMPLICIT FakeMutableHandle(T* t) : ptr(t) {} + + MOZ_IMPLICIT FakeMutableHandle(FakeRooted* root) : ptr(root->address()) {} + + void set(const T& v) { *ptr = v; } + + DECLARE_POINTER_CONSTREF_OPS(T); + DECLARE_NONPOINTER_ACCESSOR_METHODS(*ptr); + DECLARE_NONPOINTER_MUTABLE_ACCESSOR_METHODS(*ptr); + + private: + FakeMutableHandle() : ptr(nullptr) {} + DELETE_ASSIGNMENT_OPS(FakeMutableHandle, T); + + T* ptr; +}; + +} // namespace js + +namespace JS { + +namespace detail { + +template +struct DefineComparisonOps> : std::true_type { + static const T& get(const js::FakeMutableHandle& v) { return v.get(); } +}; + +} // namespace detail + +} // namespace JS + +namespace js { + +/** + * Types for a variable that either should or shouldn't be rooted, depending on + * the template parameter allowGC. Used for implementing functions that can + * operate on either rooted or unrooted data. + */ + +template +class MaybeRooted; + +template +class MaybeRooted { + public: + using HandleType = JS::Handle; + using RootType = JS::Rooted; + using MutableHandleType = JS::MutableHandle; +}; + +template +class MaybeRooted { + public: + using HandleType = const T&; + using RootType = FakeRooted; + using MutableHandleType = FakeMutableHandle; +}; + +} // namespace js + +#endif // gc_MaybeRooted_h diff --git a/js/src/gc/Memory.cpp b/js/src/gc/Memory.cpp new file mode 100644 index 0000000000..c5bdaa14bf --- /dev/null +++ b/js/src/gc/Memory.cpp @@ -0,0 +1,1050 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Memory.h" + +#include "mozilla/Atomics.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/RandomNum.h" +#include "mozilla/TaggedAnonymousMemory.h" + +#include "jit/JitOptions.h" +#include "js/HeapAPI.h" +#include "js/Utility.h" +#include "util/Memory.h" + +#ifdef XP_WIN + +# include "util/WindowsWrapper.h" +# include + +#else + +# include +# include +# include + +# if !defined(__wasi__) +# include +# include +# include +# include +# endif // !defined(__wasi__) + +#endif // !XP_WIN + +namespace js { +namespace gc { + +/* + * System allocation functions generally require the allocation size + * to be an integer multiple of the page size of the running process. + */ +static size_t pageSize = 0; + +/* The OS allocation granularity may not match the page size. */ +static size_t allocGranularity = 0; + +/* The number of bits used by addresses on this platform. */ +static size_t numAddressBits = 0; + +/* An estimate of the number of bytes available for virtual memory. */ +static size_t virtualMemoryLimit = size_t(-1); + +/* + * System allocation functions may hand out regions of memory in increasing or + * decreasing order. This ordering is used as a hint during chunk alignment to + * reduce the number of system calls. On systems with 48-bit addresses, our + * workarounds to obtain 47-bit pointers cause addresses to be handed out in + * increasing order. + * + * We do not use the growth direction on Windows, as constraints on VirtualAlloc + * would make its application failure prone and complex. Tests indicate that + * VirtualAlloc always hands out regions of memory in increasing order. + */ +#if defined(XP_DARWIN) +static mozilla::Atomic growthDirection(1); +#elif defined(XP_UNIX) +static mozilla::Atomic growthDirection(0); +#endif + +/* + * Data from OOM crashes shows there may be up to 24 chunk-sized but unusable + * chunks available in low memory situations. These chunks may all need to be + * used up before we gain access to remaining *alignable* chunk-sized regions, + * so we use a generous limit of 32 unusable chunks to ensure we reach them. + */ +static const int MaxLastDitchAttempts = 32; + +#ifdef JS_64BIT +/* + * On some 64-bit platforms we can use a random, scattershot allocator that + * tries addresses from the available range at random. If the address range + * is large enough this will have a high chance of success and additionally + * makes the memory layout of our process less predictable. + * + * However, not all 64-bit platforms have a very large address range. For + * example, AArch64 on Linux defaults to using 39-bit addresses to limit the + * number of translation tables used. On such configurations the scattershot + * approach to allocation creates a conflict with our desire to reserve large + * regions of memory for applications like WebAssembly: Small allocations may + * inadvertently block off all available 4-6GiB regions, and conversely + * reserving such regions may lower the success rate for smaller allocations to + * unacceptable levels. + * + * So we make a compromise: Instead of using the scattershot on all 64-bit + * platforms, we only use it on platforms that meet a minimum requirement for + * the available address range. In addition we split the address range, + * reserving the upper half for huge allocations and the lower half for smaller + * allocations. We use a limit of 43 bits so that at least 42 bits are available + * for huge allocations - this matches the 8TiB per process address space limit + * that we're already subject to on Windows. + */ +static const size_t MinAddressBitsForRandomAlloc = 43; + +/* The lower limit for huge allocations. This is fairly arbitrary. */ +static const size_t HugeAllocationSize = 1024 * 1024 * 1024; + +/* The minimum and maximum valid addresses that can be allocated into. */ +static size_t minValidAddress = 0; +static size_t maxValidAddress = 0; + +/* The upper limit for smaller allocations and the lower limit for huge ones. */ +static size_t hugeSplit = 0; +#endif + +size_t SystemPageSize() { return pageSize; } + +size_t SystemAddressBits() { return numAddressBits; } + +size_t VirtualMemoryLimit() { return virtualMemoryLimit; } + +bool UsingScattershotAllocator() { +#ifdef JS_64BIT + return numAddressBits >= MinAddressBitsForRandomAlloc; +#else + return false; +#endif +} + +enum class Commit : bool { + No = false, + Yes = true, +}; + +#ifdef XP_WIN +enum class PageAccess : DWORD { + None = PAGE_NOACCESS, + Read = PAGE_READONLY, + ReadWrite = PAGE_READWRITE, + Execute = PAGE_EXECUTE, + ReadExecute = PAGE_EXECUTE_READ, + ReadWriteExecute = PAGE_EXECUTE_READWRITE, +}; +#elif defined(__wasi__) +enum class PageAccess : int { + None = 0, + Read = 0, + ReadWrite = 0, + Execute = 0, + ReadExecute = 0, + ReadWriteExecute = 0, +}; +#else +enum class PageAccess : int { + None = PROT_NONE, + Read = PROT_READ, + ReadWrite = PROT_READ | PROT_WRITE, + Execute = PROT_EXEC, + ReadExecute = PROT_READ | PROT_EXEC, + ReadWriteExecute = PROT_READ | PROT_WRITE | PROT_EXEC, +}; +#endif + +template +static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion, + size_t length, size_t alignment); + +#ifndef __wasi__ +static void* MapAlignedPagesSlow(size_t length, size_t alignment); +#endif // wasi +static void* MapAlignedPagesLastDitch(size_t length, size_t alignment); + +#ifdef JS_64BIT +static void* MapAlignedPagesRandom(size_t length, size_t alignment); +#endif + +void* TestMapAlignedPagesLastDitch(size_t length, size_t alignment) { + return MapAlignedPagesLastDitch(length, alignment); +} + +bool DecommitEnabled() { return SystemPageSize() == PageSize; } + +/* Returns the offset from the nearest aligned address at or below |region|. */ +static inline size_t OffsetFromAligned(void* region, size_t alignment) { + return uintptr_t(region) % alignment; +} + +template +static inline void* MapInternal(void* desired, size_t length) { + void* region = nullptr; +#ifdef XP_WIN + DWORD flags = + (commit == Commit::Yes ? MEM_RESERVE | MEM_COMMIT : MEM_RESERVE); + region = VirtualAlloc(desired, length, flags, DWORD(prot)); +#elif defined(__wasi__) + if (int err = posix_memalign(®ion, gc::SystemPageSize(), length)) { + MOZ_RELEASE_ASSERT(err == ENOMEM); + return nullptr; + } + if (region) { + memset(region, 0, length); + } +#else + int flags = MAP_PRIVATE | MAP_ANON; + region = MozTaggedAnonymousMmap(desired, length, int(prot), flags, -1, 0, + "js-gc-heap"); + if (region == MAP_FAILED) { + return nullptr; + } +#endif + return region; +} + +static inline void UnmapInternal(void* region, size_t length) { + MOZ_ASSERT(region && OffsetFromAligned(region, allocGranularity) == 0); + MOZ_ASSERT(length > 0 && length % pageSize == 0); + +#ifdef XP_WIN + MOZ_RELEASE_ASSERT(VirtualFree(region, 0, MEM_RELEASE) != 0); +#elif defined(__wasi__) + free(region); +#else + if (munmap(region, length)) { + MOZ_RELEASE_ASSERT(errno == ENOMEM); + } +#endif +} + +template +static inline void* MapMemory(size_t length) { + MOZ_ASSERT(length > 0); + + return MapInternal(nullptr, length); +} + +/* + * Attempts to map memory at the given address, but allows the system + * to return a different address that may still be suitable. + */ +template +static inline void* MapMemoryAtFuzzy(void* desired, size_t length) { + MOZ_ASSERT(desired && OffsetFromAligned(desired, allocGranularity) == 0); + MOZ_ASSERT(length > 0); + + // Note that some platforms treat the requested address as a hint, so the + // returned address might not match the requested address. + return MapInternal(desired, length); +} + +/* + * Attempts to map memory at the given address, returning nullptr if + * the system returns any address other than the requested one. + */ +template +static inline void* MapMemoryAt(void* desired, size_t length) { + MOZ_ASSERT(desired && OffsetFromAligned(desired, allocGranularity) == 0); + MOZ_ASSERT(length > 0); + + void* region = MapInternal(desired, length); + if (!region) { + return nullptr; + } + + // On some platforms mmap treats the desired address as a hint, so + // check that the address we got is the address we requested. + if (region != desired) { + UnmapInternal(region, length); + return nullptr; + } + return region; +} + +#ifdef JS_64BIT + +/* Returns a random number in the given range. */ +static inline uint64_t GetNumberInRange(uint64_t minNum, uint64_t maxNum) { + const uint64_t MaxRand = UINT64_C(0xffffffffffffffff); + maxNum -= minNum; + uint64_t binSize = 1 + (MaxRand - maxNum) / (maxNum + 1); + + uint64_t rndNum; + do { + mozilla::Maybe result; + do { + result = mozilla::RandomUint64(); + } while (!result); + rndNum = result.value() / binSize; + } while (rndNum > maxNum); + + return minNum + rndNum; +} + +# ifndef XP_WIN +static inline uint64_t FindAddressLimitInner(size_t highBit, size_t tries); + +/* + * The address range available to applications depends on both hardware and + * kernel configuration. For example, AArch64 on Linux uses addresses with + * 39 significant bits by default, but can be configured to use addresses with + * 48 significant bits by enabling a 4th translation table. Unfortunately, + * there appears to be no standard way to query the limit at runtime + * (Windows exposes this via GetSystemInfo()). + * + * This function tries to find the address limit by performing a binary search + * on the index of the most significant set bit in the addresses it attempts to + * allocate. As the requested address is often treated as a hint by the + * operating system, we use the actual returned addresses to narrow the range. + * We return the number of bits of an address that may be set. + */ +static size_t FindAddressLimit() { + // Use 32 bits as a lower bound in case we keep getting nullptr. + uint64_t low = 31; + uint64_t highestSeen = (UINT64_C(1) << 32) - allocGranularity - 1; + + // Exclude 48-bit and 47-bit addresses first. + uint64_t high = 47; + for (; high >= std::max(low, UINT64_C(46)); --high) { + highestSeen = std::max(FindAddressLimitInner(high, 4), highestSeen); + low = mozilla::FloorLog2(highestSeen); + } + // If those didn't work, perform a modified binary search. + while (high - 1 > low) { + uint64_t middle = low + (high - low) / 2; + highestSeen = std::max(FindAddressLimitInner(middle, 4), highestSeen); + low = mozilla::FloorLog2(highestSeen); + if (highestSeen < (UINT64_C(1) << middle)) { + high = middle; + } + } + // We can be sure of the lower bound, but check the upper bound again. + do { + high = low + 1; + highestSeen = std::max(FindAddressLimitInner(high, 8), highestSeen); + low = mozilla::FloorLog2(highestSeen); + } while (low >= high); + + // `low` is the highest set bit, so `low + 1` is the number of bits. + return low + 1; +} + +static inline uint64_t FindAddressLimitInner(size_t highBit, size_t tries) { + const size_t length = allocGranularity; // Used as both length and alignment. + + uint64_t highestSeen = 0; + uint64_t startRaw = UINT64_C(1) << highBit; + uint64_t endRaw = 2 * startRaw - length - 1; + uint64_t start = (startRaw + length - 1) / length; + uint64_t end = (endRaw - (length - 1)) / length; + for (size_t i = 0; i < tries; ++i) { + uint64_t desired = length * GetNumberInRange(start, end); + void* address = MapMemoryAtFuzzy(reinterpret_cast(desired), length); + uint64_t actual = uint64_t(address); + if (address) { + UnmapInternal(address, length); + } + if (actual > highestSeen) { + highestSeen = actual; + if (actual >= startRaw) { + break; + } + } + } + return highestSeen; +} +# endif // !defined(XP_WIN) + +#endif // defined(JS_64BIT) + +void InitMemorySubsystem() { + if (pageSize == 0) { +#ifdef XP_WIN + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + pageSize = sysinfo.dwPageSize; + allocGranularity = sysinfo.dwAllocationGranularity; +#else + pageSize = size_t(sysconf(_SC_PAGESIZE)); + allocGranularity = pageSize; +#endif +#ifdef JS_64BIT +# ifdef XP_WIN + minValidAddress = size_t(sysinfo.lpMinimumApplicationAddress); + maxValidAddress = size_t(sysinfo.lpMaximumApplicationAddress); + numAddressBits = mozilla::FloorLog2(maxValidAddress) + 1; +# else + // No standard way to determine these, so fall back to FindAddressLimit(). + numAddressBits = FindAddressLimit(); + minValidAddress = allocGranularity; + maxValidAddress = (UINT64_C(1) << numAddressBits) - 1 - allocGranularity; +# endif + // Sanity check the address to ensure we don't use more than 47 bits. + uint64_t maxJSAddress = UINT64_C(0x00007fffffffffff) - allocGranularity; + if (maxValidAddress > maxJSAddress) { + maxValidAddress = maxJSAddress; + hugeSplit = UINT64_C(0x00003fffffffffff) - allocGranularity; + } else { + hugeSplit = (UINT64_C(1) << (numAddressBits - 1)) - 1 - allocGranularity; + } +#else // !defined(JS_64BIT) + numAddressBits = 32; +#endif +#ifdef RLIMIT_AS + if (jit::HasJitBackend()) { + rlimit as_limit; + if (getrlimit(RLIMIT_AS, &as_limit) == 0 && + as_limit.rlim_max != RLIM_INFINITY) { + virtualMemoryLimit = as_limit.rlim_max; + } + } +#endif + } +} + +#ifdef JS_64BIT +/* The JS engine uses 47-bit pointers; all higher bits must be clear. */ +static inline bool IsInvalidRegion(void* region, size_t length) { + const uint64_t invalidPointerMask = UINT64_C(0xffff800000000000); + return (uintptr_t(region) + length - 1) & invalidPointerMask; +} +#endif + +void* MapAlignedPages(size_t length, size_t alignment) { + MOZ_RELEASE_ASSERT(length > 0 && alignment > 0); + MOZ_RELEASE_ASSERT(length % pageSize == 0); + MOZ_RELEASE_ASSERT(std::max(alignment, allocGranularity) % + std::min(alignment, allocGranularity) == + 0); + + // Smaller alignments aren't supported by the allocation functions. + if (alignment < allocGranularity) { + alignment = allocGranularity; + } + +#ifdef __wasi__ + void* region = nullptr; + if (int err = posix_memalign(®ion, alignment, length)) { + MOZ_ASSERT(err == ENOMEM); + return nullptr; + } + MOZ_ASSERT(region != nullptr); + memset(region, 0, length); + return region; +#else + +# ifdef JS_64BIT + // Use the scattershot allocator if the address range is large enough. + if (UsingScattershotAllocator()) { + void* region = MapAlignedPagesRandom(length, alignment); + + MOZ_RELEASE_ASSERT(!IsInvalidRegion(region, length)); + MOZ_ASSERT(OffsetFromAligned(region, alignment) == 0); + + return region; + } +# endif + + // Try to allocate the region. If the returned address is aligned, + // either we OOMed (region is nullptr) or we're done. + void* region = MapMemory(length); + if (OffsetFromAligned(region, alignment) == 0) { + return region; + } + + // Try to align the region. On success, TryToAlignChunk() returns + // true and we can return the aligned region immediately. + void* retainedRegion; + if (TryToAlignChunk(®ion, &retainedRegion, length, alignment)) { + MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0); + MOZ_ASSERT(!retainedRegion); + return region; + } + + // On failure, the unaligned region is retained unless we OOMed. We don't + // use the retained region on this path (see the last ditch allocator). + if (retainedRegion) { + UnmapInternal(retainedRegion, length); + } + + // If it fails to align the given region, TryToAlignChunk() returns the + // next valid region that we might be able to align (unless we OOMed). + if (region) { + MOZ_ASSERT(OffsetFromAligned(region, alignment) != 0); + UnmapInternal(region, length); + } + + // Since we couldn't align the first region, fall back to allocating a + // region large enough that we can definitely align it. + region = MapAlignedPagesSlow(length, alignment); + if (!region) { + // If there wasn't enough contiguous address space left for that, + // try to find an alignable region using the last ditch allocator. + region = MapAlignedPagesLastDitch(length, alignment); + } + + // At this point we should either have an aligned region or nullptr. + MOZ_ASSERT(OffsetFromAligned(region, alignment) == 0); + return region; +#endif // !__wasi__ +} + +#ifdef JS_64BIT + +/* + * This allocator takes advantage of the large address range on some 64-bit + * platforms to allocate in a scattershot manner, choosing addresses at random + * from the range. By controlling the range we can avoid returning addresses + * that have more than 47 significant bits (as required by SpiderMonkey). + * This approach also has some other advantages over the methods employed by + * the other allocation functions in this file: + * 1) Allocations are extremely likely to succeed on the first try. + * 2) The randomness makes our memory layout becomes harder to predict. + * 3) The low probability of reusing regions guards against use-after-free. + * + * The main downside is that detecting physical OOM situations becomes more + * difficult; to guard against this, we occasionally try a regular allocation. + * In addition, sprinkling small allocations throughout the full address range + * might get in the way of large address space reservations such as those + * employed by WebAssembly. To avoid this (or the opposite problem of such + * reservations reducing the chance of success for smaller allocations) we + * split the address range in half, with one half reserved for huge allocations + * and the other for regular (usually chunk sized) allocations. + */ +static void* MapAlignedPagesRandom(size_t length, size_t alignment) { + uint64_t minNum, maxNum; + if (length < HugeAllocationSize) { + // Use the lower half of the range. + minNum = (minValidAddress + alignment - 1) / alignment; + maxNum = (hugeSplit - (length - 1)) / alignment; + } else { + // Use the upper half of the range. + minNum = (hugeSplit + 1 + alignment - 1) / alignment; + maxNum = (maxValidAddress - (length - 1)) / alignment; + } + + // Try to allocate in random aligned locations. + void* region = nullptr; + for (size_t i = 1; i <= 1024; ++i) { + if (i & 0xf) { + uint64_t desired = alignment * GetNumberInRange(minNum, maxNum); + region = MapMemoryAtFuzzy(reinterpret_cast(desired), length); + if (!region) { + continue; + } + } else { + // Check for OOM. + region = MapMemory(length); + if (!region) { + return nullptr; + } + } + if (IsInvalidRegion(region, length)) { + UnmapInternal(region, length); + continue; + } + if (OffsetFromAligned(region, alignment) == 0) { + return region; + } + void* retainedRegion = nullptr; + if (TryToAlignChunk(®ion, &retainedRegion, length, alignment)) { + MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0); + MOZ_ASSERT(!retainedRegion); + return region; + } + MOZ_ASSERT(region && !retainedRegion); + UnmapInternal(region, length); + } + + if (numAddressBits < 48) { + // Try the reliable fallback of overallocating. + // Note: This will not respect the address space split. + region = MapAlignedPagesSlow(length, alignment); + if (region) { + return region; + } + } + if (length < HugeAllocationSize) { + MOZ_CRASH("Couldn't allocate even after 1000 tries!"); + } + + return nullptr; +} + +#endif // defined(JS_64BIT) + +#ifndef __wasi__ +static void* MapAlignedPagesSlow(size_t length, size_t alignment) { + void* alignedRegion = nullptr; + do { + size_t reserveLength = length + alignment - pageSize; +# ifdef XP_WIN + // Don't commit the requested pages as we won't use the region directly. + void* region = MapMemory(reserveLength); +# else + void* region = MapMemory(reserveLength); +# endif + if (!region) { + return nullptr; + } + alignedRegion = + reinterpret_cast(AlignBytes(uintptr_t(region), alignment)); +# ifdef XP_WIN + // Windows requires that map and unmap calls be matched, so deallocate + // and immediately reallocate at the desired (aligned) address. + UnmapInternal(region, reserveLength); + alignedRegion = MapMemoryAt(alignedRegion, length); +# else + // munmap allows us to simply unmap the pages that don't interest us. + if (alignedRegion != region) { + UnmapInternal(region, uintptr_t(alignedRegion) - uintptr_t(region)); + } + void* regionEnd = + reinterpret_cast(uintptr_t(region) + reserveLength); + void* alignedEnd = + reinterpret_cast(uintptr_t(alignedRegion) + length); + if (alignedEnd != regionEnd) { + UnmapInternal(alignedEnd, uintptr_t(regionEnd) - uintptr_t(alignedEnd)); + } +# endif + // On Windows we may have raced with another thread; if so, try again. + } while (!alignedRegion); + + return alignedRegion; +} +#endif // wasi + +/* + * In a low memory or high fragmentation situation, alignable chunks of the + * desired length may still be available, even if there are no more contiguous + * free chunks that meet the |length + alignment - pageSize| requirement of + * MapAlignedPagesSlow. In this case, try harder to find an alignable chunk + * by temporarily holding onto the unaligned parts of each chunk until the + * allocator gives us a chunk that either is, or can be aligned. + */ +static void* MapAlignedPagesLastDitch(size_t length, size_t alignment) { + void* tempMaps[MaxLastDitchAttempts]; + int attempt = 0; + void* region = MapMemory(length); + if (OffsetFromAligned(region, alignment) == 0) { + return region; + } + for (; attempt < MaxLastDitchAttempts; ++attempt) { + if (TryToAlignChunk(®ion, tempMaps + attempt, length, alignment)) { + MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0); + MOZ_ASSERT(!tempMaps[attempt]); + break; // Success! + } + if (!region || !tempMaps[attempt]) { + break; // We ran out of memory, so give up. + } + } + if (OffsetFromAligned(region, alignment)) { + UnmapInternal(region, length); + region = nullptr; + } + while (--attempt >= 0) { + UnmapInternal(tempMaps[attempt], length); + } + return region; +} + +#ifdef XP_WIN + +/* + * On Windows, map and unmap calls must be matched, so we deallocate the + * unaligned chunk, then reallocate the unaligned part to block off the + * old address and force the allocator to give us a new one. + */ +template +static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion, + size_t length, size_t alignment) { + void* region = *aRegion; + MOZ_ASSERT(region && OffsetFromAligned(region, alignment) != 0); + + size_t retainedLength = 0; + void* retainedRegion = nullptr; + do { + size_t offset = OffsetFromAligned(region, alignment); + if (offset == 0) { + // If the address is aligned, either we hit OOM or we're done. + break; + } + UnmapInternal(region, length); + retainedLength = alignment - offset; + retainedRegion = MapMemoryAt(region, retainedLength); + region = MapMemory(length); + + // If retainedRegion is null here, we raced with another thread. + } while (!retainedRegion); + + bool result = OffsetFromAligned(region, alignment) == 0; + if (result && retainedRegion) { + UnmapInternal(retainedRegion, retainedLength); + retainedRegion = nullptr; + } + + *aRegion = region; + *aRetainedRegion = retainedRegion; + return region && result; +} + +#else // !defined(XP_WIN) + +/* + * mmap calls don't have to be matched with calls to munmap, so we can unmap + * just the pages we don't need. However, as we don't know a priori if addresses + * are handed out in increasing or decreasing order, we have to try both + * directions (depending on the environment, one will always fail). + */ +template +static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion, + size_t length, size_t alignment) { + void* regionStart = *aRegion; + MOZ_ASSERT(regionStart && OffsetFromAligned(regionStart, alignment) != 0); + + bool addressesGrowUpward = growthDirection > 0; + bool directionUncertain = -8 < growthDirection && growthDirection <= 8; + size_t offsetLower = OffsetFromAligned(regionStart, alignment); + size_t offsetUpper = alignment - offsetLower; + for (size_t i = 0; i < 2; ++i) { + if (addressesGrowUpward) { + void* upperStart = + reinterpret_cast(uintptr_t(regionStart) + offsetUpper); + void* regionEnd = + reinterpret_cast(uintptr_t(regionStart) + length); + if (MapMemoryAt(regionEnd, offsetUpper)) { + UnmapInternal(regionStart, offsetUpper); + if (directionUncertain) { + ++growthDirection; + } + regionStart = upperStart; + break; + } + } else { + auto* lowerStart = + reinterpret_cast(uintptr_t(regionStart) - offsetLower); + auto* lowerEnd = reinterpret_cast(uintptr_t(lowerStart) + length); + if (MapMemoryAt(lowerStart, offsetLower)) { + UnmapInternal(lowerEnd, offsetLower); + if (directionUncertain) { + --growthDirection; + } + regionStart = lowerStart; + break; + } + } + // If we're confident in the growth direction, don't try the other. + if (!directionUncertain) { + break; + } + addressesGrowUpward = !addressesGrowUpward; + } + + void* retainedRegion = nullptr; + bool result = OffsetFromAligned(regionStart, alignment) == 0; + if (AlwaysGetNew && !result) { + // If our current chunk cannot be aligned, just get a new one. + retainedRegion = regionStart; + regionStart = MapMemory(length); + // Our new region might happen to already be aligned. + result = OffsetFromAligned(regionStart, alignment) == 0; + if (result) { + UnmapInternal(retainedRegion, length); + retainedRegion = nullptr; + } + } + + *aRegion = regionStart; + *aRetainedRegion = retainedRegion; + return regionStart && result; +} + +#endif + +void UnmapPages(void* region, size_t length) { + MOZ_RELEASE_ASSERT(region && + OffsetFromAligned(region, allocGranularity) == 0); + MOZ_RELEASE_ASSERT(length > 0 && length % pageSize == 0); + + // ASan does not automatically unpoison memory, so we have to do this here. + MOZ_MAKE_MEM_UNDEFINED(region, length); + + UnmapInternal(region, length); +} + +static void CheckDecommit(void* region, size_t length) { + MOZ_RELEASE_ASSERT(region); + MOZ_RELEASE_ASSERT(length > 0); + + // pageSize == ArenaSize doesn't necessarily hold, but this function is + // used by the GC to decommit unused Arenas, so we don't want to assert + // if pageSize > ArenaSize. + MOZ_ASSERT(OffsetFromAligned(region, ArenaSize) == 0); + MOZ_ASSERT(length % ArenaSize == 0); + + MOZ_RELEASE_ASSERT(OffsetFromAligned(region, pageSize) == 0); + MOZ_RELEASE_ASSERT(length % pageSize == 0); +} + +bool MarkPagesUnusedSoft(void* region, size_t length) { + MOZ_ASSERT(DecommitEnabled()); + CheckDecommit(region, length); + + MOZ_MAKE_MEM_NOACCESS(region, length); + +#if defined(XP_WIN) + return VirtualAlloc(region, length, MEM_RESET, + DWORD(PageAccess::ReadWrite)) == region; +#elif defined(__wasi__) + return 0; +#else + int status; + do { +# if defined(XP_DARWIN) + status = madvise(region, length, MADV_FREE_REUSABLE); +# elif defined(XP_SOLARIS) + status = posix_madvise(region, length, POSIX_MADV_DONTNEED); +# else + status = madvise(region, length, MADV_DONTNEED); +# endif + } while (status == -1 && errno == EAGAIN); + return status == 0; +#endif +} + +bool MarkPagesUnusedHard(void* region, size_t length) { + CheckDecommit(region, length); + + MOZ_MAKE_MEM_NOACCESS(region, length); + + if (!DecommitEnabled()) { + return true; + } + +#if defined(XP_WIN) + return VirtualFree(region, length, MEM_DECOMMIT); +#else + return MarkPagesUnusedSoft(region, length); +#endif +} + +void MarkPagesInUseSoft(void* region, size_t length) { + MOZ_ASSERT(DecommitEnabled()); + CheckDecommit(region, length); + +#if defined(XP_DARWIN) + while (madvise(region, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { + } +#endif + + MOZ_MAKE_MEM_UNDEFINED(region, length); +} + +bool MarkPagesInUseHard(void* region, size_t length) { + if (js::oom::ShouldFailWithOOM()) { + return false; + } + + CheckDecommit(region, length); + + MOZ_MAKE_MEM_UNDEFINED(region, length); + + if (!DecommitEnabled()) { + return true; + } + +#if defined(XP_WIN) + return VirtualAlloc(region, length, MEM_COMMIT, + DWORD(PageAccess::ReadWrite)) == region; +#else + return true; +#endif +} + +size_t GetPageFaultCount() { +#ifdef XP_WIN + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) == 0) { + return 0; + } + return pmc.PageFaultCount; +#elif defined(__wasi__) + return 0; +#else + struct rusage usage; + int err = getrusage(RUSAGE_SELF, &usage); + if (err) { + return 0; + } + return usage.ru_majflt; +#endif +} + +void* AllocateMappedContent(int fd, size_t offset, size_t length, + size_t alignment) { +#ifdef __wasi__ + MOZ_CRASH("Not yet supported for WASI"); +#else + if (length == 0 || alignment == 0 || offset % alignment != 0 || + std::max(alignment, allocGranularity) % + std::min(alignment, allocGranularity) != + 0) { + return nullptr; + } + + size_t alignedOffset = offset - (offset % allocGranularity); + size_t alignedLength = length + (offset % allocGranularity); + + // We preallocate the mapping using MapAlignedPages, which expects + // the length parameter to be an integer multiple of the page size. + size_t mappedLength = alignedLength; + if (alignedLength % pageSize != 0) { + mappedLength += pageSize - alignedLength % pageSize; + } + +# ifdef XP_WIN + HANDLE hFile = reinterpret_cast(intptr_t(fd)); + + // This call will fail if the file does not exist. + HANDLE hMap = + CreateFileMappingW(hFile, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (!hMap) { + return nullptr; + } + + DWORD offsetH = uint32_t(uint64_t(alignedOffset) >> 32); + DWORD offsetL = uint32_t(alignedOffset); + + uint8_t* map = nullptr; + for (;;) { + // The value of a pointer is technically only defined while the region + // it points to is allocated, so explicitly treat this one as a number. + uintptr_t region = uintptr_t(MapAlignedPages(mappedLength, alignment)); + if (region == 0) { + break; + } + UnmapInternal(reinterpret_cast(region), mappedLength); + // If the offset or length are out of bounds, this call will fail. + map = static_cast( + MapViewOfFileEx(hMap, FILE_MAP_COPY, offsetH, offsetL, alignedLength, + reinterpret_cast(region))); + + // Retry if another thread mapped the address we were trying to use. + if (map || GetLastError() != ERROR_INVALID_ADDRESS) { + break; + } + } + + // This just decreases the file mapping object's internal reference count; + // it won't actually be destroyed until we unmap the associated view. + CloseHandle(hMap); + + if (!map) { + return nullptr; + } +# else // !defined(XP_WIN) + // Sanity check the offset and length, as mmap does not do this for us. + struct stat st; + if (fstat(fd, &st) || offset >= uint64_t(st.st_size) || + length > uint64_t(st.st_size) - offset) { + return nullptr; + } + + void* region = MapAlignedPages(mappedLength, alignment); + if (!region) { + return nullptr; + } + + // Calling mmap with MAP_FIXED will replace the previous mapping, allowing + // us to reuse the region we obtained without racing with other threads. + uint8_t* map = + static_cast(mmap(region, alignedLength, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED, fd, alignedOffset)); + if (map == MAP_FAILED) { + UnmapInternal(region, mappedLength); + return nullptr; + } +# endif + +# ifdef DEBUG + // Zero out data before and after the desired mapping to catch errors early. + if (offset != alignedOffset) { + memset(map, 0, offset - alignedOffset); + } + if (alignedLength % pageSize) { + memset(map + alignedLength, 0, pageSize - (alignedLength % pageSize)); + } +# endif + + return map + (offset - alignedOffset); +#endif // __wasi__ +} + +void DeallocateMappedContent(void* region, size_t length) { +#ifdef __wasi__ + MOZ_CRASH("Not yet supported for WASI"); +#else + if (!region) { + return; + } + + // Due to bug 1502562, the following assertion does not currently hold. + // MOZ_RELEASE_ASSERT(length > 0); + + // Calculate the address originally returned by the system call. + // This is needed because AllocateMappedContent returns a pointer + // that might be offset from the mapping, as the beginning of a + // mapping must be aligned with the allocation granularity. + uintptr_t map = uintptr_t(region) - (uintptr_t(region) % allocGranularity); +# ifdef XP_WIN + MOZ_RELEASE_ASSERT(UnmapViewOfFile(reinterpret_cast(map)) != 0); +# else + size_t alignedLength = length + (uintptr_t(region) % allocGranularity); + if (munmap(reinterpret_cast(map), alignedLength)) { + MOZ_RELEASE_ASSERT(errno == ENOMEM); + } +# endif +#endif // __wasi__ +} + +static inline void ProtectMemory(void* region, size_t length, PageAccess prot) { + MOZ_RELEASE_ASSERT(region && OffsetFromAligned(region, pageSize) == 0); + MOZ_RELEASE_ASSERT(length > 0 && length % pageSize == 0); +#ifdef XP_WIN + DWORD oldProtect; + MOZ_RELEASE_ASSERT(VirtualProtect(region, length, DWORD(prot), &oldProtect) != + 0); +#elif defined(__wasi__) + /* nothing */ +#else + MOZ_RELEASE_ASSERT(mprotect(region, length, int(prot)) == 0); +#endif +} + +void ProtectPages(void* region, size_t length) { + ProtectMemory(region, length, PageAccess::None); +} + +void MakePagesReadOnly(void* region, size_t length) { + ProtectMemory(region, length, PageAccess::Read); +} + +void UnprotectPages(void* region, size_t length) { + ProtectMemory(region, length, PageAccess::ReadWrite); +} + +} // namespace gc +} // namespace js diff --git a/js/src/gc/Memory.h b/js/src/gc/Memory.h new file mode 100644 index 0000000000..e49a10f04b --- /dev/null +++ b/js/src/gc/Memory.h @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Memory_h +#define gc_Memory_h + +#include + +namespace js { +namespace gc { + +// Sanity check that our compiled configuration matches the currently +// running instance and initialize any runtime data needed for allocation. +void InitMemorySubsystem(); + +// The page size as reported by the operating system. +size_t SystemPageSize(); + +// The number of bits that may be set in a valid address, as +// reported by the operating system or measured at startup. +size_t SystemAddressBits(); + +// The number of bytes of virtual memory that may be allocated or mapped, as +// reported by the operating system on certain platforms. If no limit was able +// to be determined, then it will be size_t(-1). +size_t VirtualMemoryLimit(); + +// The scattershot allocator is used on platforms that have a large address +// range. On these platforms we allocate at random addresses. +bool UsingScattershotAllocator(); + +// Allocate or deallocate pages from the system with the given alignment. +// Pages will be read/write-able. +void* MapAlignedPages(size_t length, size_t alignment); +void UnmapPages(void* region, size_t length); + +// We can only decommit unused pages if the page size is less than or equal to +// the hardcoded Arena size for the running process. +bool DecommitEnabled(); + +// Tell the OS that the given pages are not in use, so they should not be +// written to a paging file. This may be a no-op on some platforms. +bool MarkPagesUnusedSoft(void* region, size_t length); + +// Tell the OS that the given pages are not in use and it can decommit them +// immediately. This may defer to MarkPagesUnusedSoft and must be paired with +// MarkPagesInUse to use the pages again. +bool MarkPagesUnusedHard(void* region, size_t length); + +// Undo |MarkPagesUnusedSoft|: tell the OS that the given pages are of interest +// and should be paged in and out normally. This may be a no-op on some +// platforms. May make pages read/write-able. +void MarkPagesInUseSoft(void* region, size_t length); + +// Undo |MarkPagesUnusedHard|: tell the OS that the given pages are of interest +// and should be paged in and out normally. This may be a no-op on some +// platforms. Callers must check the result, false could mean that the pages +// are not available. May make pages read/write. +[[nodiscard]] bool MarkPagesInUseHard(void* region, size_t length); + +// Returns #(hard faults) + #(soft faults) +size_t GetPageFaultCount(); + +// Allocate memory mapped content. +// The offset must be aligned according to alignment requirement. +void* AllocateMappedContent(int fd, size_t offset, size_t length, + size_t alignment); + +// Deallocate memory mapped content. +void DeallocateMappedContent(void* region, size_t length); + +void* TestMapAlignedPagesLastDitch(size_t size, size_t alignment); + +void ProtectPages(void* region, size_t length); +void MakePagesReadOnly(void* region, size_t length); +void UnprotectPages(void* region, size_t length); + +} // namespace gc +} // namespace js + +#endif /* gc_Memory_h */ diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h new file mode 100644 index 0000000000..48f25a536e --- /dev/null +++ b/js/src/gc/Nursery-inl.h @@ -0,0 +1,172 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=4 sw=2 et tw=80 ft=cpp: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Nursery_inl_h +#define gc_Nursery_inl_h + +#include "gc/Nursery.h" + +#include "gc/RelocationOverlay.h" +#include "js/TracingAPI.h" +#include "vm/JSContext.h" +#include "vm/Runtime.h" + +#include "vm/JSContext-inl.h" + +inline JSRuntime* js::Nursery::runtime() const { return gc->rt; } + +template +bool js::Nursery::isInside(const SharedMem& p) const { + return isInside(p.unwrap(/*safe - used for value in comparison above*/)); +} + +MOZ_ALWAYS_INLINE /* static */ bool js::Nursery::getForwardedPointer( + js::gc::Cell** ref) { + js::gc::Cell* cell = (*ref); + MOZ_ASSERT(IsInsideNursery(cell)); + if (!cell->isForwarded()) { + return false; + } + const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(cell); + *ref = overlay->forwardingAddress(); + return true; +} + +inline void js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData, + void* newData, bool direct) { + if (trc->isTenuringTracer()) { + setForwardingPointerWhileTenuring(oldData, newData, direct); + } +} + +inline void js::Nursery::setForwardingPointerWhileTenuring(void* oldData, + void* newData, + bool direct) { + if (isInside(oldData)) { + setForwardingPointer(oldData, newData, direct); + } +} + +inline void js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, + HeapSlot* newSlots, + uint32_t nslots) { + // Slot arrays always have enough space for a forwarding pointer, since the + // number of slots is never zero. + MOZ_ASSERT(nslots > 0); + setDirectForwardingPointer(oldSlots, newSlots); +} + +inline void js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, + ObjectElements* newHeader, + uint32_t capacity) { + // Only use a direct forwarding pointer if there is enough space for one. + setForwardingPointer(oldHeader->elements(), newHeader->elements(), + capacity > 0); +} + +inline void js::Nursery::setForwardingPointer(void* oldData, void* newData, + bool direct) { + if (direct) { + setDirectForwardingPointer(oldData, newData); + return; + } + + setIndirectForwardingPointer(oldData, newData); +} + +inline void js::Nursery::setDirectForwardingPointer(void* oldData, + void* newData) { + MOZ_ASSERT(isInside(oldData)); + MOZ_ASSERT(!isInside(newData)); + + new (oldData) BufferRelocationOverlay{newData}; +} + +namespace js { + +// The allocation methods below will not run the garbage collector. If the +// nursery cannot accomodate the allocation, the malloc heap will be used +// instead. + +template +static inline T* AllocateObjectBuffer(JSContext* cx, uint32_t count) { + size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value)); + auto* buffer = + static_cast(cx->nursery().allocateBuffer(cx->zone(), nbytes)); + if (!buffer) { + ReportOutOfMemory(cx); + } + return buffer; +} + +template +static inline T* AllocateObjectBuffer(JSContext* cx, JSObject* obj, + uint32_t count) { + MOZ_ASSERT(cx->isMainThreadContext()); + + size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value)); + auto* buffer = + static_cast(cx->nursery().allocateBuffer(cx->zone(), obj, nbytes)); + if (!buffer) { + ReportOutOfMemory(cx); + } + return buffer; +} + +// If this returns null then the old buffer will be left alone. +template +static inline T* ReallocateObjectBuffer(JSContext* cx, JSObject* obj, + T* oldBuffer, uint32_t oldCount, + uint32_t newCount) { + MOZ_ASSERT(cx->isMainThreadContext()); + + T* buffer = static_cast(cx->nursery().reallocateBuffer( + obj->zone(), obj, oldBuffer, oldCount * sizeof(T), newCount * sizeof(T))); + if (!buffer) { + ReportOutOfMemory(cx); + } + + return buffer; +} + +static inline JS::BigInt::Digit* AllocateBigIntDigits(JSContext* cx, + JS::BigInt* bi, + uint32_t length) { + MOZ_ASSERT(cx->isMainThreadContext()); + + size_t nbytes = RoundUp(length * sizeof(JS::BigInt::Digit), sizeof(Value)); + auto* digits = + static_cast(cx->nursery().allocateBuffer(bi, nbytes)); + if (!digits) { + ReportOutOfMemory(cx); + } + + return digits; +} + +static inline JS::BigInt::Digit* ReallocateBigIntDigits( + JSContext* cx, JS::BigInt* bi, JS::BigInt::Digit* oldDigits, + uint32_t oldLength, uint32_t newLength) { + MOZ_ASSERT(cx->isMainThreadContext()); + + size_t oldBytes = + RoundUp(oldLength * sizeof(JS::BigInt::Digit), sizeof(Value)); + size_t newBytes = + RoundUp(newLength * sizeof(JS::BigInt::Digit), sizeof(Value)); + + auto* digits = static_cast(cx->nursery().reallocateBuffer( + bi->zone(), bi, oldDigits, oldBytes, newBytes)); + if (!digits) { + ReportOutOfMemory(cx); + } + + return digits; +} + +} // namespace js + +#endif /* gc_Nursery_inl_h */ diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp new file mode 100644 index 0000000000..17831b7f97 --- /dev/null +++ b/js/src/gc/Nursery.cpp @@ -0,0 +1,2083 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Nursery-inl.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Sprintf.h" +#include "mozilla/TimeStamp.h" + +#include +#include +#include + +#include "builtin/MapObject.h" +#include "debugger/DebugAPI.h" +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/GCParallelTask.h" +#include "gc/GCProbes.h" +#include "gc/Memory.h" +#include "gc/PublicIterators.h" +#include "gc/Tenuring.h" +#include "jit/JitFrames.h" +#include "jit/JitRealm.h" +#include "js/Printer.h" +#include "util/DifferentialTesting.h" +#include "util/GetPidProvider.h" // getpid() +#include "util/Poison.h" +#include "vm/JSONPrinter.h" +#include "vm/Realm.h" +#include "vm/Time.h" + +#include "gc/Heap-inl.h" +#include "gc/Marking-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "vm/GeckoProfiler-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::DebugOnly; +using mozilla::PodCopy; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +namespace js { + +struct NurseryChunk : public ChunkBase { + char data[Nursery::NurseryChunkUsableSize]; + + static NurseryChunk* fromChunk(gc::TenuredChunk* chunk); + + explicit NurseryChunk(JSRuntime* runtime) + : ChunkBase(runtime, &runtime->gc.storeBuffer()) {} + + void poisonAndInit(JSRuntime* rt, size_t size = ChunkSize); + void poisonRange(size_t from, size_t size, uint8_t value, + MemCheckKind checkKind); + void poisonAfterEvict(size_t extent = ChunkSize); + + // Mark pages from startOffset to the end of the chunk as unused. The start + // offset must be after the first page, which contains the chunk header and is + // not marked as unused. + void markPagesUnusedHard(size_t startOffset); + + // Mark pages from the second page of the chunk to endOffset as in use, + // following a call to markPagesUnusedHard. + [[nodiscard]] bool markPagesInUseHard(size_t endOffset); + + uintptr_t start() const { return uintptr_t(&data); } + uintptr_t end() const { return uintptr_t(this) + ChunkSize; } +}; +static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize, + "Nursery chunk size must match gc::Chunk size."); + +class NurseryDecommitTask : public GCParallelTask { + public: + explicit NurseryDecommitTask(gc::GCRuntime* gc); + bool reserveSpaceForBytes(size_t nbytes); + + bool isEmpty(const AutoLockHelperThreadState& lock) const; + + void queueChunk(NurseryChunk* chunk, const AutoLockHelperThreadState& lock); + void queueRange(size_t newCapacity, NurseryChunk& chunk, + const AutoLockHelperThreadState& lock); + + private: + using NurseryChunkVector = Vector; + + void run(AutoLockHelperThreadState& lock) override; + + NurseryChunkVector& chunksToDecommit() { return chunksToDecommit_.ref(); } + const NurseryChunkVector& chunksToDecommit() const { + return chunksToDecommit_.ref(); + } + + MainThreadOrGCTaskData chunksToDecommit_; + + MainThreadOrGCTaskData partialChunk; + MainThreadOrGCTaskData partialCapacity; +}; + +} // namespace js + +inline void js::NurseryChunk::poisonAndInit(JSRuntime* rt, size_t size) { + MOZ_ASSERT(size >= sizeof(ChunkBase)); + MOZ_ASSERT(size <= ChunkSize); + poisonRange(0, size, JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined); + new (this) NurseryChunk(rt); +} + +inline void js::NurseryChunk::poisonRange(size_t from, size_t size, + uint8_t value, + MemCheckKind checkKind) { + MOZ_ASSERT(from + size <= ChunkSize); + + auto* start = reinterpret_cast(this) + from; + + // We can poison the same chunk more than once, so first make sure memory + // sanitizers will let us poison it. + MOZ_MAKE_MEM_UNDEFINED(start, size); + Poison(start, value, size, checkKind); +} + +inline void js::NurseryChunk::poisonAfterEvict(size_t extent) { + MOZ_ASSERT(extent <= ChunkSize); + poisonRange(sizeof(ChunkBase), extent - sizeof(ChunkBase), + JS_SWEPT_NURSERY_PATTERN, MemCheckKind::MakeNoAccess); +} + +inline void js::NurseryChunk::markPagesUnusedHard(size_t startOffset) { + MOZ_ASSERT(startOffset >= sizeof(ChunkBase)); // Don't touch the header. + MOZ_ASSERT(startOffset >= SystemPageSize()); + MOZ_ASSERT(startOffset <= ChunkSize); + uintptr_t start = uintptr_t(this) + startOffset; + size_t length = ChunkSize - startOffset; + MarkPagesUnusedHard(reinterpret_cast(start), length); +} + +inline bool js::NurseryChunk::markPagesInUseHard(size_t endOffset) { + MOZ_ASSERT(endOffset >= sizeof(ChunkBase)); + MOZ_ASSERT(endOffset >= SystemPageSize()); + MOZ_ASSERT(endOffset <= ChunkSize); + uintptr_t start = uintptr_t(this) + SystemPageSize(); + size_t length = endOffset - SystemPageSize(); + return MarkPagesInUseHard(reinterpret_cast(start), length); +} + +// static +inline js::NurseryChunk* js::NurseryChunk::fromChunk(TenuredChunk* chunk) { + return reinterpret_cast(chunk); +} + +js::NurseryDecommitTask::NurseryDecommitTask(gc::GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::NONE) { + // This can occur outside GCs so doesn't have a stats phase. +} + +bool js::NurseryDecommitTask::isEmpty( + const AutoLockHelperThreadState& lock) const { + return chunksToDecommit().empty() && !partialChunk; +} + +bool js::NurseryDecommitTask::reserveSpaceForBytes(size_t nbytes) { + MOZ_ASSERT(isIdle()); + size_t nchunks = HowMany(nbytes, ChunkSize); + return chunksToDecommit().reserve(nchunks); +} + +void js::NurseryDecommitTask::queueChunk( + NurseryChunk* chunk, const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isIdle(lock)); + MOZ_ALWAYS_TRUE(chunksToDecommit().append(chunk)); +} + +void js::NurseryDecommitTask::queueRange( + size_t newCapacity, NurseryChunk& newChunk, + const AutoLockHelperThreadState& lock) { + MOZ_ASSERT(isIdle(lock)); + MOZ_ASSERT(!partialChunk); + MOZ_ASSERT(newCapacity < ChunkSize); + MOZ_ASSERT(newCapacity % SystemPageSize() == 0); + + partialChunk = &newChunk; + partialCapacity = newCapacity; +} + +void js::NurseryDecommitTask::run(AutoLockHelperThreadState& lock) { + while (!chunksToDecommit().empty()) { + NurseryChunk* nurseryChunk = chunksToDecommit().popCopy(); + AutoUnlockHelperThreadState unlock(lock); + nurseryChunk->~NurseryChunk(); + TenuredChunk* tenuredChunk = TenuredChunk::emplace( + nurseryChunk, gc, /* allMemoryCommitted = */ false); + AutoLockGC lock(gc); + gc->recycleChunk(tenuredChunk, lock); + } + + if (partialChunk) { + { + AutoUnlockHelperThreadState unlock(lock); + partialChunk->markPagesUnusedHard(partialCapacity); + } + partialChunk = nullptr; + partialCapacity = 0; + } +} + +js::Nursery::Nursery(GCRuntime* gc) + : position_(0), + currentEnd_(0), + gc(gc), + currentChunk_(0), + currentStartChunk_(0), + currentStartPosition_(0), + capacity_(0), + timeInChunkAlloc_(0), + enableProfiling_(false), + profileThreshold_(0), + canAllocateStrings_(true), + canAllocateBigInts_(true), + reportDeduplications_(false), + reportPretenuring_(false), + reportPretenuringThreshold_(0), + minorGCTriggerReason_(JS::GCReason::NO_REASON), + hasRecentGrowthData(false), + smoothedTargetSize(0.0) { + const char* env = getenv("MOZ_NURSERY_STRINGS"); + if (env && *env) { + canAllocateStrings_ = (*env == '1'); + } + env = getenv("MOZ_NURSERY_BIGINTS"); + if (env && *env) { + canAllocateBigInts_ = (*env == '1'); + } +} + +static void PrintAndExit(const char* message) { + fprintf(stderr, "%s", message); + exit(0); +} + +static const char* GetEnvVar(const char* name, const char* helpMessage) { + const char* value = getenv(name); + if (!value) { + return nullptr; + } + + if (strcmp(value, "help") == 0) { + PrintAndExit(helpMessage); + } + + return value; +} + +static bool GetBoolEnvVar(const char* name, const char* helpMessage) { + const char* env = GetEnvVar(name, helpMessage); + return env && bool(atoi(env)); +} + +static void ReadReportPretenureEnv(const char* name, const char* helpMessage, + bool* enabled, size_t* threshold) { + *enabled = false; + *threshold = 0; + + const char* env = GetEnvVar(name, helpMessage); + if (!env) { + return; + } + + char* end; + *threshold = strtol(env, &end, 10); + if (end == env || *end) { + PrintAndExit(helpMessage); + } + + *enabled = true; +} + +bool js::Nursery::init(AutoLockGCBgAlloc& lock) { + ReadProfileEnv("JS_GC_PROFILE_NURSERY", + "Report minor GCs taking at least N microseconds.\n", + &enableProfiling_, &profileWorkers_, &profileThreshold_); + + reportDeduplications_ = GetBoolEnvVar( + "JS_GC_REPORT_STATS", + "JS_GC_REPORT_STATS=1\n" + "\tAfter a minor GC, report how many strings were deduplicated.\n"); + + ReadReportPretenureEnv( + "JS_GC_REPORT_PRETENURE", + "JS_GC_REPORT_PRETENURE=N\n" + "\tAfter a minor GC, report information about pretenuring, including\n" + "\tallocation sites with at least N allocations.\n", + &reportPretenuring_, &reportPretenuringThreshold_); + + decommitTask = MakeUnique(gc); + if (!decommitTask) { + return false; + } + + if (!gc->storeBuffer().enable()) { + return false; + } + + return initFirstChunk(lock); +} + +js::Nursery::~Nursery() { disable(); } + +void js::Nursery::enable() { + MOZ_ASSERT(isEmpty()); + MOZ_ASSERT(!gc->isVerifyPreBarriersEnabled()); + if (isEnabled()) { + return; + } + + { + AutoLockGCBgAlloc lock(gc); + if (!initFirstChunk(lock)) { + // If we fail to allocate memory, the nursery will not be enabled. + return; + } + } + +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + enterZealMode(); + } +#endif + + updateAllZoneAllocFlags(); + + // This should always succeed after the first time it's called. + MOZ_ALWAYS_TRUE(gc->storeBuffer().enable()); +} + +bool js::Nursery::initFirstChunk(AutoLockGCBgAlloc& lock) { + MOZ_ASSERT(!isEnabled()); + + capacity_ = tunables().gcMinNurseryBytes(); + + if (!decommitTask->reserveSpaceForBytes(capacity_) || + !allocateNextChunk(0, lock)) { + capacity_ = 0; + return false; + } + + setCurrentChunk(0); + setStartPosition(); + poisonAndInitCurrentChunk(); + + // Clear any information about previous collections. + clearRecentGrowthData(); + + return true; +} + +void js::Nursery::disable() { + MOZ_ASSERT(isEmpty()); + if (!isEnabled()) { + return; + } + + // Free all chunks. + decommitTask->join(); + freeChunksFrom(0); + decommitTask->runFromMainThread(); + + capacity_ = 0; + + // We must reset currentEnd_ so that there is no space for anything in the + // nursery. JIT'd code uses this even if the nursery is disabled. + currentEnd_ = 0; + position_ = 0; + gc->storeBuffer().disable(); + + if (gc->wasInitialized()) { + // This assumes there is an atoms zone. + updateAllZoneAllocFlags(); + } +} + +void js::Nursery::enableStrings() { + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = true; + updateAllZoneAllocFlags(); +} + +void js::Nursery::disableStrings() { + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = false; + updateAllZoneAllocFlags(); +} + +void js::Nursery::enableBigInts() { + MOZ_ASSERT(isEmpty()); + canAllocateBigInts_ = true; + updateAllZoneAllocFlags(); +} + +void js::Nursery::disableBigInts() { + MOZ_ASSERT(isEmpty()); + canAllocateBigInts_ = false; + updateAllZoneAllocFlags(); +} + +void js::Nursery::updateAllZoneAllocFlags() { + // The alloc flags are not relevant for the atoms zone, and flushing + // jit-related information can be problematic for the atoms zone. + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + updateAllocFlagsForZone(zone); + } +} + +void js::Nursery::getAllocFlagsForZone(JS::Zone* zone, bool* allocObjectsOut, + bool* allocStringsOut, + bool* allocBigIntsOut) { + *allocObjectsOut = isEnabled(); + *allocStringsOut = + isEnabled() && canAllocateStrings() && !zone->nurseryStringsDisabled; + *allocBigIntsOut = + isEnabled() && canAllocateBigInts() && !zone->nurseryBigIntsDisabled; +} + +void js::Nursery::setAllocFlagsForZone(JS::Zone* zone) { + bool allocObjects; + bool allocStrings; + bool allocBigInts; + + getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts); + zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts); +} + +void js::Nursery::updateAllocFlagsForZone(JS::Zone* zone) { + bool allocObjects; + bool allocStrings; + bool allocBigInts; + + getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts); + + if (allocObjects != zone->allocNurseryObjects() || + allocStrings != zone->allocNurseryStrings() || + allocBigInts != zone->allocNurseryBigInts()) { + CancelOffThreadIonCompile(zone); + zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts); + discardCodeAndSetJitFlagsForZone(zone); + } +} + +void js::Nursery::discardCodeAndSetJitFlagsForZone(JS::Zone* zone) { + zone->forceDiscardJitCode(runtime()->gcContext()); + + for (RealmsInZoneIter r(zone); !r.done(); r.next()) { + if (jit::JitRealm* jitRealm = r->jitRealm()) { + jitRealm->discardStubs(); + jitRealm->setStringsCanBeInNursery(zone->allocNurseryStrings()); + } + } +} + +bool js::Nursery::isEmpty() const { + if (!isEnabled()) { + return true; + } + + if (!gc->hasZealMode(ZealMode::GenerationalGC)) { + MOZ_ASSERT(currentStartChunk_ == 0); + MOZ_ASSERT(currentStartPosition_ == chunk(0).start()); + } + return position() == currentStartPosition_; +} + +#ifdef JS_GC_ZEAL +void js::Nursery::enterZealMode() { + if (!isEnabled()) { + return; + } + + MOZ_ASSERT(isEmpty()); + + decommitTask->join(); + + AutoEnterOOMUnsafeRegion oomUnsafe; + + if (isSubChunkMode()) { + { + if (!chunk(0).markPagesInUseHard(ChunkSize)) { + oomUnsafe.crash("Out of memory trying to extend chunk for zeal mode"); + } + } + + // It'd be simpler to poison the whole chunk, but we can't do that + // because the nursery might be partially used. + chunk(0).poisonRange(capacity_, ChunkSize - capacity_, + JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined); + } + + capacity_ = RoundUp(tunables().gcMaxNurseryBytes(), ChunkSize); + + if (!decommitTask->reserveSpaceForBytes(capacity_)) { + oomUnsafe.crash("Nursery::enterZealMode"); + } + + setCurrentEnd(); +} + +void js::Nursery::leaveZealMode() { + if (!isEnabled()) { + return; + } + + MOZ_ASSERT(isEmpty()); + + setCurrentChunk(0); + setStartPosition(); + poisonAndInitCurrentChunk(); +} +#endif // JS_GC_ZEAL + +void* js::Nursery::allocateCell(gc::AllocSite* site, size_t size, + JS::TraceKind kind) { + // Ensure there's enough space to replace the contents with a + // RelocationOverlay. + MOZ_ASSERT(size >= sizeof(RelocationOverlay)); + MOZ_ASSERT(size % CellAlignBytes == 0); + MOZ_ASSERT(size_t(kind) < NurseryTraceKinds); + MOZ_ASSERT_IF(kind == JS::TraceKind::String, canAllocateStrings()); + MOZ_ASSERT_IF(kind == JS::TraceKind::BigInt, canAllocateBigInts()); + + void* ptr = allocate(sizeof(NurseryCellHeader) + size); + if (!ptr) { + return nullptr; + } + + new (ptr) NurseryCellHeader(site, kind); + + void* cell = + reinterpret_cast(uintptr_t(ptr) + sizeof(NurseryCellHeader)); + + // Update the allocation site. This code is also inlined in + // MacroAssembler::updateAllocSite. + uint32_t allocCount = site->incAllocCount(); + if (allocCount == 1) { + pretenuringNursery.insertIntoAllocatedList(site); + } else { + MOZ_ASSERT_IF(site->isNormal(), site->isInAllocatedList()); + } + + gcprobes::NurseryAlloc(cell, kind); + return cell; +} + +inline void* js::Nursery::allocate(size_t size) { + MOZ_ASSERT(isEnabled()); + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_, + position() >= currentStartPosition_); + MOZ_ASSERT(position() % CellAlignBytes == 0); + MOZ_ASSERT(size % CellAlignBytes == 0); + + if (MOZ_UNLIKELY(currentEnd() < position() + size)) { + return moveToNextChunkAndAllocate(size); + } + + void* thing = (void*)position(); + position_ = position() + size; + + DebugOnlyPoison(thing, JS_ALLOCATED_NURSERY_PATTERN, size, + MemCheckKind::MakeUndefined); + + return thing; +} + +void* Nursery::moveToNextChunkAndAllocate(size_t size) { + MOZ_ASSERT(currentEnd() < position() + size); + + unsigned chunkno = currentChunk_ + 1; + MOZ_ASSERT(chunkno <= maxChunkCount()); + MOZ_ASSERT(chunkno <= allocatedChunkCount()); + if (chunkno == maxChunkCount()) { + return nullptr; + } + if (chunkno == allocatedChunkCount()) { + TimeStamp start = TimeStamp::Now(); + { + AutoLockGCBgAlloc lock(gc); + if (!allocateNextChunk(chunkno, lock)) { + return nullptr; + } + } + timeInChunkAlloc_ += TimeStamp::Now() - start; + MOZ_ASSERT(chunkno < allocatedChunkCount()); + } + setCurrentChunk(chunkno); + poisonAndInitCurrentChunk(); + + // We know there's enough space to allocate now so we can call allocate() + // recursively. + MOZ_ASSERT(currentEnd() >= position() + size); + return allocate(size); +} +void* js::Nursery::allocateBuffer(Zone* zone, size_t nbytes) { + MOZ_ASSERT(nbytes > 0); + + if (nbytes <= MaxNurseryBufferSize) { + void* buffer = allocate(nbytes); + if (buffer) { + return buffer; + } + } + + void* buffer = zone->pod_malloc(nbytes); + if (buffer && !registerMallocedBuffer(buffer, nbytes)) { + js_free(buffer); + return nullptr; + } + return buffer; +} + +void* js::Nursery::allocateBuffer(Zone* zone, JSObject* obj, size_t nbytes) { + MOZ_ASSERT(obj); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(obj)) { + return zone->pod_malloc(nbytes); + } + + return allocateBuffer(zone, nbytes); +} + +void* js::Nursery::allocateBufferSameLocation(JSObject* obj, size_t nbytes) { + MOZ_ASSERT(obj); + MOZ_ASSERT(nbytes > 0); + MOZ_ASSERT(nbytes <= MaxNurseryBufferSize); + + if (!IsInsideNursery(obj)) { + return obj->zone()->pod_malloc(nbytes); + } + + return allocate(nbytes); +} + +void* js::Nursery::allocateZeroedBuffer( + Zone* zone, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) { + MOZ_ASSERT(nbytes > 0); + + if (nbytes <= MaxNurseryBufferSize) { + void* buffer = allocate(nbytes); + if (buffer) { + memset(buffer, 0, nbytes); + return buffer; + } + } + + void* buffer = zone->pod_arena_calloc(arena, nbytes); + if (buffer && !registerMallocedBuffer(buffer, nbytes)) { + js_free(buffer); + return nullptr; + } + return buffer; +} + +void* js::Nursery::allocateZeroedBuffer( + JSObject* obj, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) { + MOZ_ASSERT(obj); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(obj)) { + return obj->zone()->pod_arena_calloc(arena, nbytes); + } + return allocateZeroedBuffer(obj->zone(), nbytes, arena); +} + +void* js::Nursery::reallocateBuffer(Zone* zone, Cell* cell, void* oldBuffer, + size_t oldBytes, size_t newBytes) { + if (!IsInsideNursery(cell)) { + MOZ_ASSERT(!isInside(oldBuffer)); + return zone->pod_realloc((uint8_t*)oldBuffer, oldBytes, newBytes); + } + + if (!isInside(oldBuffer)) { + MOZ_ASSERT(mallocedBufferBytes >= oldBytes); + void* newBuffer = + zone->pod_realloc((uint8_t*)oldBuffer, oldBytes, newBytes); + if (newBuffer) { + if (oldBuffer != newBuffer) { + MOZ_ALWAYS_TRUE( + mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer)); + } + mallocedBufferBytes -= oldBytes; + mallocedBufferBytes += newBytes; + } + return newBuffer; + } + + // The nursery cannot make use of the returned slots data. + if (newBytes < oldBytes) { + return oldBuffer; + } + + void* newBuffer = allocateBuffer(zone, newBytes); + if (newBuffer) { + PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes); + } + return newBuffer; +} + +void* js::Nursery::allocateBuffer(JS::BigInt* bi, size_t nbytes) { + MOZ_ASSERT(bi); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(bi)) { + return bi->zone()->pod_malloc(nbytes); + } + return allocateBuffer(bi->zone(), nbytes); +} + +void js::Nursery::freeBuffer(void* buffer, size_t nbytes) { + if (!isInside(buffer)) { + removeMallocedBuffer(buffer, nbytes); + js_free(buffer); + } +} + +#ifdef DEBUG +/* static */ +inline bool Nursery::checkForwardingPointerLocation(void* ptr, + bool expectedInside) { + if (isInside(ptr) == expectedInside) { + return true; + } + + // If a zero-capacity elements header lands right at the end of a chunk then + // elements data will appear to be in the next chunk. If we have a pointer to + // the very start of a chunk, check the previous chunk. + if ((uintptr_t(ptr) & ChunkMask) == 0 && + isInside(reinterpret_cast(ptr) - 1) == expectedInside) { + return true; + } + + return false; +} +#endif + +void Nursery::setIndirectForwardingPointer(void* oldData, void* newData) { + MOZ_ASSERT(checkForwardingPointerLocation(oldData, true)); + MOZ_ASSERT(checkForwardingPointerLocation(newData, false)); + + AutoEnterOOMUnsafeRegion oomUnsafe; +#ifdef DEBUG + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData)) { + MOZ_ASSERT(p->value() == newData); + } +#endif + if (!forwardedBuffers.put(oldData, newData)) { + oomUnsafe.crash("Nursery::setForwardingPointer"); + } +} + +#ifdef DEBUG +static bool IsWriteableAddress(void* ptr) { + auto* vPtr = reinterpret_cast(ptr); + *vPtr = *vPtr; + return true; +} +#endif + +void js::Nursery::forwardBufferPointer(uintptr_t* pSlotsElems) { + // Read the current pointer value which may be one of: + // - Non-nursery pointer + // - Nursery-allocated buffer + // - A BufferRelocationOverlay inside the nursery + // + // Note: The buffer has already be relocated. We are just patching stale + // pointers now. + auto* buffer = reinterpret_cast(*pSlotsElems); + + if (!isInside(buffer)) { + return; + } + + // The new location for this buffer is either stored inline with it or in + // the forwardedBuffers table. + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(buffer)) { + buffer = p->value(); + // It's not valid to assert IsWriteableAddress for indirect forwarding + // pointers because the size of the allocation could be less than a word. + } else { + BufferRelocationOverlay* reloc = + static_cast(buffer); + buffer = *reloc; + MOZ_ASSERT(IsWriteableAddress(buffer)); + } + + MOZ_ASSERT(!isInside(buffer)); + *pSlotsElems = reinterpret_cast(buffer); +} + +inline double js::Nursery::calcPromotionRate(bool* validForTenuring) const { + MOZ_ASSERT(validForTenuring); + + if (previousGC.nurseryUsedBytes == 0) { + *validForTenuring = false; + return 0.0; + } + + double used = double(previousGC.nurseryUsedBytes); + double capacity = double(previousGC.nurseryCapacity); + double tenured = double(previousGC.tenuredBytes); + + // We should only use the promotion rate to make tenuring decisions if it's + // likely to be valid. The criterion we use is that the nursery was at least + // 90% full. + *validForTenuring = used > capacity * 0.9; + + return tenured / used; +} + +void js::Nursery::renderProfileJSON(JSONPrinter& json) const { + if (!isEnabled()) { + json.beginObject(); + json.property("status", "nursery disabled"); + json.endObject(); + return; + } + + if (previousGC.reason == JS::GCReason::NO_REASON) { + // If the nursery was empty when the last minorGC was requested, then + // no nursery collection will have been performed but JSON may still be + // requested. (And as a public API, this function should not crash in + // such a case.) + json.beginObject(); + json.property("status", "nursery empty"); + json.endObject(); + return; + } + + json.beginObject(); + + json.property("status", "complete"); + + json.property("reason", JS::ExplainGCReason(previousGC.reason)); + json.property("bytes_tenured", previousGC.tenuredBytes); + json.property("cells_tenured", previousGC.tenuredCells); + json.property("strings_tenured", + stats().getStat(gcstats::STAT_STRINGS_TENURED)); + json.property("strings_deduplicated", + stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED)); + json.property("bigints_tenured", + stats().getStat(gcstats::STAT_BIGINTS_TENURED)); + json.property("bytes_used", previousGC.nurseryUsedBytes); + json.property("cur_capacity", previousGC.nurseryCapacity); + const size_t newCapacity = capacity(); + if (newCapacity != previousGC.nurseryCapacity) { + json.property("new_capacity", newCapacity); + } + if (previousGC.nurseryCommitted != previousGC.nurseryCapacity) { + json.property("lazy_capacity", previousGC.nurseryCommitted); + } + if (!timeInChunkAlloc_.IsZero()) { + json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS); + } + + // These counters only contain consistent data if the profiler is enabled, + // and then there's no guarentee. + if (runtime()->geckoProfiler().enabled()) { + json.property("cells_allocated_nursery", + pretenuringNursery.totalAllocCount()); + json.property("cells_allocated_tenured", + stats().allocsSinceMinorGCTenured()); + } + + json.beginObjectProperty("phase_times"); + +#define EXTRACT_NAME(name, text) #name, + static const char* const names[] = { + FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME) +#undef EXTRACT_NAME + ""}; + + size_t i = 0; + for (auto time : profileDurations_) { + json.property(names[i++], time, json.MICROSECONDS); + } + + json.endObject(); // timings value + + json.endObject(); +} + +// The following macros define nursery GC profile metadata fields that are +// printed before the timing information defined by +// FOR_EACH_NURSERY_PROFILE_TIME. + +#define FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \ + _("PID", 7, "%7zu", pid) \ + _("Runtime", 14, "0x%12p", runtime) + +#define FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_) \ + _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \ + _("Reason", 20, "%-20.20s", reasonStr) \ + _("PRate", 6, "%5.1f%%", promotionRatePercent) \ + _("OldKB", 6, "%6zu", oldSizeKB) \ + _("NewKB", 6, "%6zu", newSizeKB) \ + _("Dedup", 6, "%6zu", dedupCount) + +#define FOR_EACH_NURSERY_PROFILE_METADATA(_) \ + FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \ + FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_) + +void js::Nursery::printCollectionProfile(JS::GCReason reason, + double promotionRate) { + stats().maybePrintProfileHeaders(); + + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) { + return; + } + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + TimeDuration timestamp = collectionStartTime() - stats().creationTime(); + const char* reasonStr = ExplainGCReason(reason); + double promotionRatePercent = promotionRate * 100; + size_t oldSizeKB = previousGC.nurseryCapacity / 1024; + size_t newSizeKB = capacity() / 1024; + size_t dedupCount = stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED); + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + if (!sprinter.jsprintf(" " format, value)) { \ + return; \ + } + FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + printProfileDurations(profileDurations_, sprinter); + + fputs(sprinter.string(), stats().profileFile()); +} + +void js::Nursery::printProfileHeader() { + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) { + return; + } + +#define PRINT_FIELD_NAME(name, width, _1, _2) \ + if (!sprinter.jsprintf(" %-*s", width, name)) { \ + return; \ + } + FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_NAME) +#undef PRINT_FIELD_NAME + +#define PRINT_PROFILE_NAME(_1, text) \ + if (!sprinter.jsprintf(" %-6.6s", text)) { \ + return; \ + } + FOR_EACH_NURSERY_PROFILE_TIME(PRINT_PROFILE_NAME) +#undef PRINT_PROFILE_NAME + + if (!sprinter.put("\n")) { + return; + } + + fputs(sprinter.string(), stats().profileFile()); +} + +// static +bool js::Nursery::printProfileDurations(const ProfileDurations& times, + Sprinter& sprinter) { + for (auto time : times) { + int64_t micros = int64_t(time.ToMicroseconds()); + if (!sprinter.jsprintf(" %6" PRIi64, micros)) { + return false; + } + } + + return sprinter.put("\n"); +} + +static constexpr size_t NurserySliceMetadataFormatWidth() { + size_t fieldCount = 0; + size_t totalWidth = 0; + +#define UPDATE_COUNT_AND_WIDTH(_1, width, _2, _3) \ + fieldCount++; \ + totalWidth += width; + FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(UPDATE_COUNT_AND_WIDTH) +#undef UPDATE_COUNT_AND_WIDTH + + // Add padding between fields. + totalWidth += fieldCount - 1; + + return totalWidth; +} + +void js::Nursery::printTotalProfileTimes() { + if (!enableProfiling_) { + return; + } + + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) { + return; + } + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + + char collections[32]; + DebugOnly r = SprintfLiteral( + collections, "TOTALS: %7" PRIu64 " collections:", gc->minorGCCount()); + MOZ_ASSERT(r > 0 && r < int(sizeof(collections))); + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + if (!sprinter.jsprintf(" " format, value)) { \ + return; \ + } + FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + // Use whole width of per-slice metadata to print total slices so the profile + // totals that follow line up. + size_t width = NurserySliceMetadataFormatWidth(); + if (!sprinter.jsprintf(" %-*s", int(width), collections)) { + return; + } + + if (!printProfileDurations(totalDurations_, sprinter)) { + return; + } + + fputs(sprinter.string(), stats().profileFile()); +} + +void js::Nursery::maybeClearProfileDurations() { + for (auto& duration : profileDurations_) { + duration = mozilla::TimeDuration(); + } +} + +inline void js::Nursery::startProfile(ProfileKey key) { + startTimes_[key] = TimeStamp::Now(); +} + +inline void js::Nursery::endProfile(ProfileKey key) { + profileDurations_[key] = TimeStamp::Now() - startTimes_[key]; + totalDurations_[key] += profileDurations_[key]; +} + +inline TimeStamp js::Nursery::collectionStartTime() const { + return startTimes_[ProfileKey::Total]; +} + +inline TimeStamp js::Nursery::lastCollectionEndTime() const { + return previousGC.endTime; +} + +bool js::Nursery::shouldCollect() const { + if (!isEnabled()) { + return false; + } + + if (isEmpty() && capacity() == tunables().gcMinNurseryBytes()) { + return false; + } + + if (minorGCRequested()) { + return true; + } + + // Eagerly collect the nursery in idle time if it's nearly full. + if (isNearlyFull()) { + return true; + } + + // If the nursery is not being collected often then it may be taking up more + // space than necessary. + return isUnderused(); +} + +inline bool js::Nursery::isNearlyFull() const { + bool belowBytesThreshold = + freeSpace() < tunables().nurseryFreeThresholdForIdleCollection(); + bool belowFractionThreshold = + double(freeSpace()) / double(capacity()) < + tunables().nurseryFreeThresholdForIdleCollectionFraction(); + + // We want to use belowBytesThreshold when the nursery is sufficiently large, + // and belowFractionThreshold when it's small. + // + // When the nursery is small then belowBytesThreshold is a lower threshold + // (triggered earlier) than belowFractionThreshold. So if the fraction + // threshold is true, the bytes one will be true also. The opposite is true + // when the nursery is large. + // + // Therefore, by the time we cross the threshold we care about, we've already + // crossed the other one, and we can boolean AND to use either condition + // without encoding any "is the nursery big/small" test/threshold. The point + // at which they cross is when the nursery is: BytesThreshold / + // FractionThreshold large. + // + // With defaults that's: + // + // 1MB = 256KB / 0.25 + // + return belowBytesThreshold && belowFractionThreshold; +} + +inline bool js::Nursery::isUnderused() const { + if (js::SupportDifferentialTesting() || !previousGC.endTime) { + return false; + } + + if (capacity() == tunables().gcMinNurseryBytes()) { + return false; + } + + // If the nursery is above its minimum size, collect it every so often if we + // have idle time. This allows the nursery to shrink when it's not being + // used. There are other heuristics we could use for this, but this is the + // simplest. + TimeDuration timeSinceLastCollection = TimeStamp::Now() - previousGC.endTime; + return timeSinceLastCollection > tunables().nurseryTimeoutForIdleCollection(); +} + +// typeReason is the gcReason for specified type, for example, +// FULL_CELL_PTR_OBJ_BUFFER is the gcReason for JSObject. +static inline bool IsFullStoreBufferReason(JS::GCReason reason, + JS::GCReason typeReason) { + return reason == typeReason || + reason == JS::GCReason::FULL_WHOLE_CELL_BUFFER || + reason == JS::GCReason::FULL_GENERIC_BUFFER || + reason == JS::GCReason::FULL_VALUE_BUFFER || + reason == JS::GCReason::FULL_SLOT_BUFFER || + reason == JS::GCReason::FULL_SHAPE_BUFFER; +} + +void js::Nursery::collect(JS::GCOptions options, JS::GCReason reason) { + JSRuntime* rt = runtime(); + MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC); + + if (!isEnabled() || isEmpty()) { + // Our barriers are not always exact, and there may be entries in the + // storebuffer even when the nursery is disabled or empty. It's not safe + // to keep these entries as they may refer to tenured cells which may be + // freed after this point. + gc->storeBuffer().clear(); + + MOZ_ASSERT(!pretenuringNursery.hasAllocatedSites()); + } + + if (!isEnabled()) { + return; + } + + AutoGCSession session(gc, JS::HeapState::MinorCollecting); + + stats().beginNurseryCollection(reason); + gcprobes::MinorGCStart(); + + maybeClearProfileDurations(); + startProfile(ProfileKey::Total); + + previousGC.reason = JS::GCReason::NO_REASON; + previousGC.nurseryUsedBytes = usedSpace(); + previousGC.nurseryCapacity = capacity(); + previousGC.nurseryCommitted = committed(); + previousGC.nurseryUsedChunkCount = currentChunk_ + 1; + previousGC.tenuredBytes = 0; + previousGC.tenuredCells = 0; + + // If it isn't empty, it will call doCollection, and possibly after that + // isEmpty() will become true, so use another variable to keep track of the + // old empty state. + bool wasEmpty = isEmpty(); + if (!wasEmpty) { + CollectionResult result = doCollection(session, options, reason); + // Don't include chunk headers when calculating nursery space, since this + // space does not represent data that can be tenured + MOZ_ASSERT(result.tenuredBytes <= + (previousGC.nurseryUsedBytes - + (sizeof(ChunkBase) * previousGC.nurseryUsedChunkCount))); + + previousGC.reason = reason; + previousGC.tenuredBytes = result.tenuredBytes; + previousGC.tenuredCells = result.tenuredCells; + previousGC.nurseryUsedChunkCount = currentChunk_ + 1; + } + + // Resize the nursery. + maybeResizeNursery(options, reason); + + // Poison/initialise the first chunk. + if (previousGC.nurseryUsedBytes) { + // In most cases Nursery::clear() has not poisoned this chunk or marked it + // as NoAccess; so we only need to poison the region used during the last + // cycle. Also, if the heap was recently expanded we don't want to + // re-poison the new memory. In both cases we only need to poison until + // previousGC.nurseryUsedBytes. + // + // In cases where this is not true, like generational zeal mode or subchunk + // mode, poisonAndInitCurrentChunk() will ignore its parameter. It will + // also clamp the parameter. + poisonAndInitCurrentChunk(previousGC.nurseryUsedBytes); + } + + bool validPromotionRate; + const double promotionRate = calcPromotionRate(&validPromotionRate); + + startProfile(ProfileKey::Pretenure); + size_t sitesPretenured = 0; + if (!wasEmpty) { + sitesPretenured = + doPretenuring(rt, reason, validPromotionRate, promotionRate); + } + endProfile(ProfileKey::Pretenure); + + // We ignore gcMaxBytes when allocating for minor collection. However, if we + // overflowed, we disable the nursery. The next time we allocate, we'll fail + // because bytes >= gcMaxBytes. + if (gc->heapSize.bytes() >= tunables().gcMaxBytes()) { + disable(); + } + + previousGC.endTime = + TimeStamp::Now(); // Must happen after maybeResizeNursery. + endProfile(ProfileKey::Total); + gc->incMinorGcNumber(); + + TimeDuration totalTime = profileDurations_[ProfileKey::Total]; + sendTelemetry(reason, totalTime, wasEmpty, promotionRate, sitesPretenured); + + stats().endNurseryCollection(reason); // Calls GCNurseryCollectionCallback. + gcprobes::MinorGCEnd(); + + timeInChunkAlloc_ = mozilla::TimeDuration(); + + js::StringStats prevStats = gc->stringStats; + js::StringStats& currStats = gc->stringStats; + currStats = js::StringStats(); + for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) { + currStats += zone->stringStats; + zone->previousGCStringStats = zone->stringStats; + } + stats().setStat( + gcstats::STAT_STRINGS_DEDUPLICATED, + currStats.deduplicatedStrings - prevStats.deduplicatedStrings); + if (ShouldPrintProfile(runtime(), enableProfiling_, profileWorkers_, + profileThreshold_, totalTime)) { + printCollectionProfile(reason, promotionRate); + } + + if (reportDeduplications_) { + printDeduplicationData(prevStats, currStats); + } +} + +void js::Nursery::sendTelemetry(JS::GCReason reason, TimeDuration totalTime, + bool wasEmpty, double promotionRate, + size_t sitesPretenured) { + JSRuntime* rt = runtime(); + rt->metrics().GC_MINOR_REASON(uint32_t(reason)); + + // Long minor GCs are those that take more than 1ms. + bool wasLongMinorGC = totalTime.ToMilliseconds() > 1.0; + if (wasLongMinorGC) { + rt->metrics().GC_MINOR_REASON_LONG(uint32_t(reason)); + } + rt->metrics().GC_MINOR_US(totalTime); + rt->metrics().GC_NURSERY_BYTES_2(committed()); + + if (!wasEmpty) { + rt->metrics().GC_PRETENURE_COUNT_2(sitesPretenured); + rt->metrics().GC_NURSERY_PROMOTION_RATE(promotionRate * 100); + } +} + +void js::Nursery::printDeduplicationData(js::StringStats& prev, + js::StringStats& curr) { + if (curr.deduplicatedStrings > prev.deduplicatedStrings) { + fprintf(stderr, + "pid %zu: deduplicated %" PRIi64 " strings, %" PRIu64 + " chars, %" PRIu64 " malloc bytes\n", + size_t(getpid()), + curr.deduplicatedStrings - prev.deduplicatedStrings, + curr.deduplicatedChars - prev.deduplicatedChars, + curr.deduplicatedBytes - prev.deduplicatedBytes); + } +} + +void js::Nursery::freeTrailerBlocks(void) { + // This routine frees those blocks denoted by the set + // + // trailersAdded_ (all of it) + // - trailersRemoved_ (entries with index below trailersRemovedUsed_) + // + // For each block, places it back on the nursery's small-malloced-block pool + // by calling mallocedBlockCache_.free. + + MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length()); + MOZ_ASSERT(trailersRemovedUsed_ <= trailersRemoved_.length()); + + // Sort the removed entries. + std::sort(trailersRemoved_.begin(), + trailersRemoved_.begin() + trailersRemovedUsed_, + [](const void* block1, const void* block2) { + return uintptr_t(block1) < uintptr_t(block2); + }); + + // Use one of two schemes to enumerate the set subtraction. + if (trailersRemovedUsed_ < 1000) { + // If the number of removed items is relatively small, it isn't worth the + // cost of sorting `trailersAdded_`. Instead, walk through the vector in + // whatever order it is and use binary search to establish whether each + // item is present in trailersRemoved_[0 .. trailersRemovedUsed_ - 1]. + const size_t nAdded = trailersAdded_.length(); + for (size_t i = 0; i < nAdded; i++) { + const PointerAndUint7 block = trailersAdded_[i]; + const void* blockPointer = block.pointer(); + if (!std::binary_search(trailersRemoved_.begin(), + trailersRemoved_.begin() + trailersRemovedUsed_, + blockPointer)) { + mallocedBlockCache_.free(block); + } + } + } else { + // The general case, which is algorithmically safer for large inputs. + // Sort the added entries, and then walk through both them and the removed + // entries in lockstep. + std::sort(trailersAdded_.begin(), trailersAdded_.end(), + [](const PointerAndUint7& block1, const PointerAndUint7& block2) { + return uintptr_t(block1.pointer()) < + uintptr_t(block2.pointer()); + }); + // Enumerate the set subtraction. This is somewhat simplified by the fact + // that all elements of the removed set must also be present in the added + // set. (the "inclusion property"). + const size_t nAdded = trailersAdded_.length(); + const size_t nRemoved = trailersRemovedUsed_; + size_t iAdded; + size_t iRemoved = 0; + for (iAdded = 0; iAdded < nAdded; iAdded++) { + if (iRemoved == nRemoved) { + // We've run out of items to skip, so move on to the next loop. + break; + } + const PointerAndUint7 blockAdded = trailersAdded_[iAdded]; + const void* blockRemoved = trailersRemoved_[iRemoved]; + if (blockAdded.pointer() < blockRemoved) { + mallocedBlockCache_.free(blockAdded); + continue; + } + // If this doesn't hold + // (that is, if `blockAdded.pointer() > blockRemoved`), + // then the abovementioned inclusion property doesn't hold. + MOZ_RELEASE_ASSERT(blockAdded.pointer() == blockRemoved); + iRemoved++; + } + MOZ_ASSERT(iRemoved == nRemoved); + // We've used up the removed set, so now finish up the remainder of the + // added set. + for (/*keep going*/; iAdded < nAdded; iAdded++) { + const PointerAndUint7 block = trailersAdded_[iAdded]; + mallocedBlockCache_.free(block); + } + } + + // And empty out both sets, but preserve the underlying storage. + trailersAdded_.clear(); + trailersRemoved_.clear(); + trailersRemovedUsed_ = 0; + trailerBytes_ = 0; + + // Discard blocks from the cache at 0.05% per megabyte of nursery capacity, + // that is, 0.8% of blocks for a 16-megabyte nursery. This allows the cache + // to gradually discard unneeded blocks in long running applications. + mallocedBlockCache_.preen(0.05 * float(capacity() / (1024 * 1024))); +} + +size_t Nursery::sizeOfTrailerBlockSets( + mozilla::MallocSizeOf mallocSizeOf) const { + return trailersAdded_.sizeOfExcludingThis(mallocSizeOf) + + trailersRemoved_.sizeOfExcludingThis(mallocSizeOf); +} + +js::Nursery::CollectionResult js::Nursery::doCollection(AutoGCSession& session, + JS::GCOptions options, + JS::GCReason reason) { + JSRuntime* rt = runtime(); + AutoSetThreadIsPerformingGC performingGC(rt->gcContext()); + AutoStopVerifyingBarriers av(rt, false); + AutoDisableProxyCheck disableStrictProxyChecking; + mozilla::DebugOnly oomUnsafeRegion; + + // Move objects pointed to by roots from the nursery to the major heap. + TenuringTracer mover(rt, this); + + // Trace everything considered as a root by a minor GC. + traceRoots(session, mover); + + startProfile(ProfileKey::SweepCaches); + gc->purgeRuntimeForMinorGC(); + endProfile(ProfileKey::SweepCaches); + + // Most of the work is done here. This loop iterates over objects that have + // been moved to the major heap. If these objects have any outgoing pointers + // to the nursery, then those nursery objects get moved as well, until no + // objects are left to move. That is, we iterate to a fixed point. + startProfile(ProfileKey::CollectToObjFP); + mover.collectToObjectFixedPoint(); + endProfile(ProfileKey::CollectToObjFP); + + startProfile(ProfileKey::CollectToStrFP); + mover.collectToStringFixedPoint(); + endProfile(ProfileKey::CollectToStrFP); + + // Sweep to update any pointers to nursery objects that have now been + // tenured. + startProfile(ProfileKey::Sweep); + sweep(); + endProfile(ProfileKey::Sweep); + + // Update any slot or element pointers whose destination has been tenured. + startProfile(ProfileKey::UpdateJitActivations); + js::jit::UpdateJitActivationsForMinorGC(rt); + forwardedBuffers.clearAndCompact(); + endProfile(ProfileKey::UpdateJitActivations); + + startProfile(ProfileKey::ObjectsTenuredCallback); + gc->callObjectsTenuredCallback(); + endProfile(ProfileKey::ObjectsTenuredCallback); + + // Sweep. + startProfile(ProfileKey::FreeMallocedBuffers); + gc->queueBuffersForFreeAfterMinorGC(mallocedBuffers); + mallocedBufferBytes = 0; + endProfile(ProfileKey::FreeMallocedBuffers); + + // Give trailer blocks associated with non-tenured Wasm{Struct,Array}Objects + // back to our `mallocedBlockCache_`. + startProfile(ProfileKey::FreeTrailerBlocks); + freeTrailerBlocks(); + if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason)) { + mallocedBlockCache_.clear(); + } + endProfile(ProfileKey::FreeTrailerBlocks); + + startProfile(ProfileKey::ClearNursery); + clear(); + endProfile(ProfileKey::ClearNursery); + + startProfile(ProfileKey::ClearStoreBuffer); + gc->storeBuffer().clear(); + endProfile(ProfileKey::ClearStoreBuffer); + + // Purge the StringToAtomCache. This has to happen at the end because the + // cache is used when tenuring strings. + startProfile(ProfileKey::PurgeStringToAtomCache); + runtime()->caches().stringToAtomCache.purge(); + endProfile(ProfileKey::PurgeStringToAtomCache); + + // Make sure hashtables have been updated after the collection. + startProfile(ProfileKey::CheckHashTables); +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) { + runtime()->caches().checkEvalCacheAfterMinorGC(); + gc->checkHashTablesAfterMovingGC(); + } +#endif + endProfile(ProfileKey::CheckHashTables); + + return {mover.getTenuredSize(), mover.getTenuredCells()}; +} + +void js::Nursery::traceRoots(AutoGCSession& session, TenuringTracer& mover) { + { + // Suppress the sampling profiler to prevent it observing moved functions. + AutoSuppressProfilerSampling suppressProfiler( + runtime()->mainContextFromOwnThread()); + + // Trace the store buffer. This must happen first. + StoreBuffer& sb = gc->storeBuffer(); + + // Strings in the whole cell buffer must be traced first, in order to mark + // tenured dependent strings' bases as non-deduplicatable. The rest of + // nursery collection (whole non-string cells, edges, etc.) can happen + // later. + startProfile(ProfileKey::TraceWholeCells); + sb.traceWholeCells(mover); + endProfile(ProfileKey::TraceWholeCells); + + startProfile(ProfileKey::TraceValues); + sb.traceValues(mover); + endProfile(ProfileKey::TraceValues); + + startProfile(ProfileKey::TraceCells); + sb.traceCells(mover); + endProfile(ProfileKey::TraceCells); + + startProfile(ProfileKey::TraceSlots); + sb.traceSlots(mover); + endProfile(ProfileKey::TraceSlots); + + startProfile(ProfileKey::TraceGenericEntries); + sb.traceGenericEntries(&mover); + endProfile(ProfileKey::TraceGenericEntries); + + startProfile(ProfileKey::MarkRuntime); + gc->traceRuntimeForMinorGC(&mover, session); + endProfile(ProfileKey::MarkRuntime); + } + + startProfile(ProfileKey::MarkDebugger); + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS); + DebugAPI::traceAllForMovingGC(&mover); + } + endProfile(ProfileKey::MarkDebugger); +} + +size_t js::Nursery::doPretenuring(JSRuntime* rt, JS::GCReason reason, + bool validPromotionRate, + double promotionRate) { + size_t sitesPretenured = pretenuringNursery.doPretenuring( + gc, reason, validPromotionRate, promotionRate, reportPretenuring_, + reportPretenuringThreshold_); + + bool highPromotionRate = + validPromotionRate && promotionRate > tunables().pretenureThreshold(); + + bool pretenureStr = false; + bool pretenureBigInt = false; + if (tunables().attemptPretenuring()) { + // Should we check for pretenuring regardless of GCReason? + // Use 3MB as the threshold so the pretenuring can be applied on Android. + bool pretenureAll = + highPromotionRate && previousGC.nurseryUsedBytes >= 3 * 1024 * 1024; + + pretenureStr = + pretenureAll || + IsFullStoreBufferReason(reason, JS::GCReason::FULL_CELL_PTR_STR_BUFFER); + pretenureBigInt = + pretenureAll || IsFullStoreBufferReason( + reason, JS::GCReason::FULL_CELL_PTR_BIGINT_BUFFER); + } + + uint32_t numStringsTenured = 0; + uint32_t numBigIntsTenured = 0; + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + // For some tests in JetStream2 and Kraken, the tenuredRate is high but the + // number of allocated strings is low. So we calculate the tenuredRate only + // if the number of string allocations is enough. + uint32_t zoneNurseryStrings = + zone->nurseryAllocCount(JS::TraceKind::String); + bool allocThreshold = zoneNurseryStrings > 30000; + uint64_t zoneTenuredStrings = + zone->stringStats.ref().liveNurseryStrings - + zone->previousGCStringStats.ref().liveNurseryStrings; + double tenuredRate = + allocThreshold ? double(zoneTenuredStrings) / double(zoneNurseryStrings) + : 0.0; + bool disableNurseryStrings = + pretenureStr && zone->allocNurseryStrings() && + tenuredRate > tunables().pretenureStringThreshold(); + bool disableNurseryBigInts = pretenureBigInt && + zone->allocNurseryBigInts() && + zone->tenuredBigInts >= 30 * 1000; + if (disableNurseryStrings || disableNurseryBigInts) { + if (disableNurseryStrings) { + zone->nurseryStringsDisabled = true; + } + if (disableNurseryBigInts) { + zone->nurseryBigIntsDisabled = true; + } + updateAllocFlagsForZone(zone); + } + numStringsTenured += zoneTenuredStrings; + numBigIntsTenured += zone->tenuredBigInts; + zone->tenuredBigInts = 0; + } + stats().setStat(gcstats::STAT_STRINGS_TENURED, numStringsTenured); + stats().setStat(gcstats::STAT_BIGINTS_TENURED, numBigIntsTenured); + + return sitesPretenured; +} + +bool js::Nursery::registerMallocedBuffer(void* buffer, size_t nbytes) { + MOZ_ASSERT(buffer); + MOZ_ASSERT(nbytes > 0); + if (!mallocedBuffers.putNew(buffer)) { + return false; + } + + mallocedBufferBytes += nbytes; + if (MOZ_UNLIKELY(mallocedBufferBytes > capacity() * 8)) { + requestMinorGC(JS::GCReason::NURSERY_MALLOC_BUFFERS); + } + + return true; +} + +size_t Nursery::sizeOfMallocedBuffers( + mozilla::MallocSizeOf mallocSizeOf) const { + size_t total = 0; + for (BufferSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront()) { + total += mallocSizeOf(r.front()); + } + total += mallocedBuffers.shallowSizeOfExcludingThis(mallocSizeOf); + return total; +} + +void js::Nursery::sweep() { + // It's important that the context's GCUse is not Finalizing at this point, + // otherwise we will miscount memory attached to nursery objects with + // CellAllocPolicy. + AutoSetThreadIsSweeping setThreadSweeping(runtime()->gcContext()); + + MinorSweepingTracer trc(runtime()); + + // Sweep unique IDs first before we sweep any tables that may be keyed based + // on them. + for (Cell* cell : cellsWithUid_) { + auto* obj = static_cast(cell); + if (!IsForwarded(obj)) { + gc::RemoveUniqueId(obj); + } else { + JSObject* dst = Forwarded(obj); + gc::TransferUniqueId(dst, obj); + } + } + cellsWithUid_.clear(); + + for (ZonesIter zone(runtime(), SkipAtoms); !zone.done(); zone.next()) { + zone->sweepAfterMinorGC(&trc); + } + + sweepMapAndSetObjects(); + + runtime()->caches().sweepAfterMinorGC(&trc); +} + +void js::Nursery::clear() { + // Poison the nursery contents so touching a freed object will crash. + unsigned firstClearChunk; + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + // Poison all the chunks used in this cycle. The new start chunk is + // reposioned in Nursery::collect() but there's no point optimising that in + // this case. + firstClearChunk = currentStartChunk_; + } else { + // In normal mode we start at the second chunk, the first one will be used + // in the next cycle and poisoned in Nusery::collect(); + MOZ_ASSERT(currentStartChunk_ == 0); + firstClearChunk = 1; + } + for (unsigned i = firstClearChunk; i < currentChunk_; ++i) { + chunk(i).poisonAfterEvict(); + } + // Clear only the used part of the chunk because that's the part we touched, + // but only if it's not going to be re-used immediately (>= firstClearChunk). + if (currentChunk_ >= firstClearChunk) { + chunk(currentChunk_) + .poisonAfterEvict(position() - chunk(currentChunk_).start()); + } + + // Reset the start chunk & position if we're not in this zeal mode, or we're + // in it and close to the end of the nursery. + MOZ_ASSERT(maxChunkCount() > 0); + if (!gc->hasZealMode(ZealMode::GenerationalGC) || + (gc->hasZealMode(ZealMode::GenerationalGC) && + currentChunk_ + 1 == maxChunkCount())) { + setCurrentChunk(0); + } + + // Set current start position for isEmpty checks. + setStartPosition(); +} + +size_t js::Nursery::spaceToEnd(unsigned chunkCount) const { + if (chunkCount == 0) { + return 0; + } + + unsigned lastChunk = chunkCount - 1; + + MOZ_ASSERT(lastChunk >= currentStartChunk_); + MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= + NurseryChunkUsableSize); + + size_t bytes; + + if (chunkCount != 1) { + // In the general case we have to add: + // + the bytes used in the first + // chunk which may be less than the total size of a chunk since in some + // zeal modes we start the first chunk at some later position + // (currentStartPosition_). + // + the size of all the other chunks. + bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) + + ((lastChunk - currentStartChunk_) * ChunkSize); + } else { + // In sub-chunk mode, but it also works whenever chunkCount == 1, we need to + // use currentEnd_ since it may not refer to a full chunk. + bytes = currentEnd_ - currentStartPosition_; + } + + MOZ_ASSERT(bytes <= maxChunkCount() * ChunkSize); + + return bytes; +} + +MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) { + MOZ_ASSERT(chunkno < allocatedChunkCount()); + + currentChunk_ = chunkno; + position_ = chunk(chunkno).start(); + setCurrentEnd(); +} + +void js::Nursery::poisonAndInitCurrentChunk(size_t extent) { + if (gc->hasZealMode(ZealMode::GenerationalGC) || !isSubChunkMode()) { + chunk(currentChunk_).poisonAndInit(runtime()); + } else { + extent = std::min(capacity_, extent); + chunk(currentChunk_).poisonAndInit(runtime(), extent); + } +} + +MOZ_ALWAYS_INLINE void js::Nursery::setCurrentEnd() { + MOZ_ASSERT_IF(isSubChunkMode(), + currentChunk_ == 0 && currentEnd_ <= chunk(0).end()); + currentEnd_ = + uintptr_t(&chunk(currentChunk_)) + std::min(capacity_, ChunkSize); +} + +bool js::Nursery::allocateNextChunk(const unsigned chunkno, + AutoLockGCBgAlloc& lock) { + const unsigned priorCount = allocatedChunkCount(); + const unsigned newCount = priorCount + 1; + + MOZ_ASSERT((chunkno == currentChunk_ + 1) || + (chunkno == 0 && allocatedChunkCount() == 0)); + MOZ_ASSERT(chunkno == allocatedChunkCount()); + MOZ_ASSERT(chunkno < HowMany(capacity(), ChunkSize)); + + if (!chunks_.resize(newCount)) { + return false; + } + + TenuredChunk* newChunk; + newChunk = gc->getOrAllocChunk(lock); + if (!newChunk) { + chunks_.shrinkTo(priorCount); + return false; + } + + chunks_[chunkno] = NurseryChunk::fromChunk(newChunk); + return true; +} + +MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() { + currentStartChunk_ = currentChunk_; + currentStartPosition_ = position(); +} + +void js::Nursery::maybeResizeNursery(JS::GCOptions options, + JS::GCReason reason) { +#ifdef JS_GC_ZEAL + // This zeal mode disabled nursery resizing. + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + return; + } +#endif + + decommitTask->join(); + + size_t newCapacity = mozilla::Clamp(targetSize(options, reason), + tunables().gcMinNurseryBytes(), + tunables().gcMaxNurseryBytes()); + + MOZ_ASSERT(roundSize(newCapacity) == newCapacity); + MOZ_ASSERT(newCapacity >= SystemPageSize()); + + if (newCapacity > capacity()) { + growAllocableSpace(newCapacity); + } else if (newCapacity < capacity()) { + shrinkAllocableSpace(newCapacity); + } + + AutoLockHelperThreadState lock; + if (!decommitTask->isEmpty(lock)) { + decommitTask->startOrRunIfIdle(lock); + } +} + +static inline bool ClampDouble(double* value, double min, double max) { + MOZ_ASSERT(!std::isnan(*value) && !std::isnan(min) && !std::isnan(max)); + MOZ_ASSERT(max >= min); + + if (*value <= min) { + *value = min; + return true; + } + + if (*value >= max) { + *value = max; + return true; + } + + return false; +} + +size_t js::Nursery::targetSize(JS::GCOptions options, JS::GCReason reason) { + // Shrink the nursery as much as possible if purging was requested or in low + // memory situations. + if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason) || + gc->systemHasLowMemory()) { + clearRecentGrowthData(); + return 0; + } + + // Don't resize the nursery during shutdown. + if (options == JS::GCOptions::Shutdown) { + clearRecentGrowthData(); + return capacity(); + } + + TimeStamp now = TimeStamp::Now(); + + // If the nursery is completely unused then minimise it. + if (hasRecentGrowthData && previousGC.nurseryUsedBytes == 0 && + now - lastCollectionEndTime() > + tunables().nurseryTimeoutForIdleCollection() && + !js::SupportDifferentialTesting()) { + clearRecentGrowthData(); + return 0; + } + + // Calculate the fraction of the nursery promoted out of its entire + // capacity. This gives better results than using the promotion rate (based on + // the amount of nursery used) in cases where we collect before the nursery is + // full. + double fractionPromoted = + double(previousGC.tenuredBytes) / double(previousGC.nurseryCapacity); + + // Calculate the duty factor, the fraction of time spent collecting the + // nursery. + double dutyFactor = 0.0; + TimeDuration collectorTime = now - collectionStartTime(); + if (hasRecentGrowthData && !js::SupportDifferentialTesting()) { + TimeDuration totalTime = now - lastCollectionEndTime(); + dutyFactor = collectorTime.ToSeconds() / totalTime.ToSeconds(); + } + + // Calculate a growth factor to try to achieve target promotion rate and duty + // factor goals. + static const double PromotionGoal = 0.02; + static const double DutyFactorGoal = 0.01; + double promotionGrowth = fractionPromoted / PromotionGoal; + double dutyGrowth = dutyFactor / DutyFactorGoal; + double growthFactor = std::max(promotionGrowth, dutyGrowth); + + // Decrease the growth factor to try to keep collections shorter than a target + // maximum time. Don't do this during page load. + static const double MaxTimeGoalMs = 4.0; + if (!gc->isInPageLoad() && !js::SupportDifferentialTesting()) { + double timeGrowth = MaxTimeGoalMs / collectorTime.ToMilliseconds(); + growthFactor = std::min(growthFactor, timeGrowth); + } + + // Limit the range of the growth factor to prevent transient high promotion + // rates from affecting the nursery size too far into the future. + static const double GrowthRange = 2.0; + bool wasClamped = ClampDouble(&growthFactor, 1.0 / GrowthRange, GrowthRange); + + // Calculate the target size based on data from this collection. + double target = double(capacity()) * growthFactor; + + // Use exponential smoothing on the target size to take into account data from + // recent previous collections. + if (hasRecentGrowthData && + now - lastCollectionEndTime() < TimeDuration::FromMilliseconds(200) && + !js::SupportDifferentialTesting()) { + // Pay more attention to large changes. + double fraction = wasClamped ? 0.5 : 0.25; + smoothedTargetSize = + (1 - fraction) * smoothedTargetSize + fraction * target; + } else { + smoothedTargetSize = target; + } + hasRecentGrowthData = true; + + // Leave size untouched if we are close to the target. + static const double GoalWidth = 1.5; + growthFactor = smoothedTargetSize / double(capacity()); + if (growthFactor > (1.0 / GoalWidth) && growthFactor < GoalWidth) { + return capacity(); + } + + return roundSize(size_t(smoothedTargetSize)); +} + +void js::Nursery::clearRecentGrowthData() { + if (js::SupportDifferentialTesting()) { + return; + } + + hasRecentGrowthData = false; + smoothedTargetSize = 0.0; +} + +/* static */ +size_t js::Nursery::roundSize(size_t size) { + size_t step = size >= ChunkSize ? ChunkSize : SystemPageSize(); + return Round(size, step); +} + +void js::Nursery::growAllocableSpace(size_t newCapacity) { + MOZ_ASSERT_IF(!isSubChunkMode(), newCapacity > currentChunk_ * ChunkSize); + MOZ_ASSERT(newCapacity <= tunables().gcMaxNurseryBytes()); + MOZ_ASSERT(newCapacity > capacity()); + + if (!decommitTask->reserveSpaceForBytes(newCapacity)) { + return; + } + + if (isSubChunkMode()) { + MOZ_ASSERT(currentChunk_ == 0); + + // The remainder of the chunk may have been decommitted. + if (!chunk(0).markPagesInUseHard(std::min(newCapacity, ChunkSize))) { + // The OS won't give us the memory we need, we can't grow. + return; + } + + // The capacity has changed and since we were in sub-chunk mode we need to + // update the poison values / asan information for the now-valid region of + // this chunk. + size_t size = std::min(newCapacity, ChunkSize) - capacity(); + chunk(0).poisonRange(capacity(), size, JS_FRESH_NURSERY_PATTERN, + MemCheckKind::MakeUndefined); + } + + capacity_ = newCapacity; + + setCurrentEnd(); +} + +void js::Nursery::freeChunksFrom(const unsigned firstFreeChunk) { + MOZ_ASSERT(firstFreeChunk < chunks_.length()); + + // The loop below may need to skip the first chunk, so we may use this so we + // can modify it. + unsigned firstChunkToDecommit = firstFreeChunk; + + if ((firstChunkToDecommit == 0) && isSubChunkMode()) { + // Part of the first chunk may be hard-decommitted, un-decommit it so that + // the GC's normal chunk-handling doesn't segfault. + MOZ_ASSERT(currentChunk_ == 0); + if (!chunk(0).markPagesInUseHard(ChunkSize)) { + // Free the chunk if we can't allocate its pages. + UnmapPages(static_cast(&chunk(0)), ChunkSize); + firstChunkToDecommit = 1; + } + } + + { + AutoLockHelperThreadState lock; + for (size_t i = firstChunkToDecommit; i < chunks_.length(); i++) { + decommitTask->queueChunk(chunks_[i], lock); + } + } + + chunks_.shrinkTo(firstFreeChunk); +} + +void js::Nursery::shrinkAllocableSpace(size_t newCapacity) { +#ifdef JS_GC_ZEAL + if (gc->hasZealMode(ZealMode::GenerationalGC)) { + return; + } +#endif + + // Don't shrink the nursery to zero (use Nursery::disable() instead) + // This can't happen due to the rounding-down performed above because of the + // clamping in maybeResizeNursery(). + MOZ_ASSERT(newCapacity != 0); + // Don't attempt to shrink it to the same size. + if (newCapacity == capacity_) { + return; + } + MOZ_ASSERT(newCapacity < capacity_); + + unsigned newCount = HowMany(newCapacity, ChunkSize); + if (newCount < allocatedChunkCount()) { + freeChunksFrom(newCount); + } + + size_t oldCapacity = capacity_; + capacity_ = newCapacity; + + setCurrentEnd(); + + if (isSubChunkMode()) { + MOZ_ASSERT(currentChunk_ == 0); + size_t size = std::min(oldCapacity, ChunkSize) - newCapacity; + chunk(0).poisonRange(newCapacity, size, JS_SWEPT_NURSERY_PATTERN, + MemCheckKind::MakeNoAccess); + + AutoLockHelperThreadState lock; + decommitTask->queueRange(capacity_, chunk(0), lock); + } +} + +uintptr_t js::Nursery::currentEnd() const { + // These are separate asserts because it can be useful to see which one + // failed. + MOZ_ASSERT_IF(isSubChunkMode(), currentChunk_ == 0); + MOZ_ASSERT_IF(isSubChunkMode(), currentEnd_ <= chunk(currentChunk_).end()); + MOZ_ASSERT_IF(!isSubChunkMode(), currentEnd_ == chunk(currentChunk_).end()); + MOZ_ASSERT(currentEnd_ != chunk(currentChunk_).start()); + return currentEnd_; +} + +gcstats::Statistics& js::Nursery::stats() const { return gc->stats(); } + +MOZ_ALWAYS_INLINE const js::gc::GCSchedulingTunables& js::Nursery::tunables() + const { + return gc->tunables; +} + +bool js::Nursery::isSubChunkMode() const { + return capacity() <= NurseryChunkUsableSize; +} + +void js::Nursery::sweepMapAndSetObjects() { + auto gcx = runtime()->gcContext(); + + for (auto mapobj : mapsWithNurseryMemory_) { + MapObject::sweepAfterMinorGC(gcx, mapobj); + } + mapsWithNurseryMemory_.clearAndFree(); + + for (auto setobj : setsWithNurseryMemory_) { + SetObject::sweepAfterMinorGC(gcx, setobj); + } + setsWithNurseryMemory_.clearAndFree(); +} + +void js::Nursery::joinDecommitTask() { decommitTask->join(); } + +JS_PUBLIC_API void JS::EnableNurseryStrings(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().enableStrings(); +} + +JS_PUBLIC_API void JS::DisableNurseryStrings(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().disableStrings(); +} + +JS_PUBLIC_API void JS::EnableNurseryBigInts(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().enableBigInts(); +} + +JS_PUBLIC_API void JS::DisableNurseryBigInts(JSContext* cx) { + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->gcContext()); + cx->runtime()->gc.nursery().disableBigInts(); +} diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h new file mode 100644 index 0000000000..3dde209d77 --- /dev/null +++ b/js/src/gc/Nursery.h @@ -0,0 +1,645 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Nursery_h +#define gc_Nursery_h + +#include "mozilla/EnumeratedArray.h" +#include "mozilla/TimeStamp.h" + +#include "gc/Heap.h" +#include "gc/MallocedBlockCache.h" +#include "gc/Pretenuring.h" +#include "js/AllocPolicy.h" +#include "js/Class.h" +#include "js/GCAPI.h" +#include "js/TypeDecls.h" +#include "js/UniquePtr.h" +#include "js/Vector.h" + +#define FOR_EACH_NURSERY_PROFILE_TIME(_) \ + /* Key Header text */ \ + _(Total, "total") \ + _(TraceValues, "mkVals") \ + _(TraceCells, "mkClls") \ + _(TraceSlots, "mkSlts") \ + _(TraceWholeCells, "mcWCll") \ + _(TraceGenericEntries, "mkGnrc") \ + _(CheckHashTables, "ckTbls") \ + _(MarkRuntime, "mkRntm") \ + _(MarkDebugger, "mkDbgr") \ + _(SweepCaches, "swpCch") \ + _(CollectToObjFP, "colObj") \ + _(CollectToStrFP, "colStr") \ + _(ObjectsTenuredCallback, "tenCB") \ + _(Sweep, "sweep") \ + _(UpdateJitActivations, "updtIn") \ + _(FreeMallocedBuffers, "frSlts") \ + _(FreeTrailerBlocks, "frTrBs") \ + _(ClearStoreBuffer, "clrSB") \ + _(ClearNursery, "clear") \ + _(PurgeStringToAtomCache, "pStoA") \ + _(Pretenure, "pretnr") + +template +class SharedMem; + +namespace js { + +struct StringStats; +class AutoLockGCBgAlloc; +class ObjectElements; +struct NurseryChunk; +class HeapSlot; +class JSONPrinter; +class MapObject; +class SetObject; +class JS_PUBLIC_API Sprinter; + +namespace gc { +class AutoGCSession; +struct Cell; +class GCSchedulingTunables; +class TenuringTracer; +} // namespace gc + +class Nursery { + public: + explicit Nursery(gc::GCRuntime* gc); + ~Nursery(); + + [[nodiscard]] bool init(AutoLockGCBgAlloc& lock); + + // Number of allocated (ready to use) chunks. + unsigned allocatedChunkCount() const { return chunks_.length(); } + + // Total number of chunks and the capacity of the nursery. Chunks will be + // lazilly allocated and added to the chunks array up to this limit, after + // that the nursery must be collected, this limit may be raised during + // collection. + unsigned maxChunkCount() const { + MOZ_ASSERT(capacity()); + return HowMany(capacity(), gc::ChunkSize); + } + + void enable(); + void disable(); + bool isEnabled() const { return capacity() != 0; } + + void enableStrings(); + void disableStrings(); + bool canAllocateStrings() const { return canAllocateStrings_; } + + void enableBigInts(); + void disableBigInts(); + bool canAllocateBigInts() const { return canAllocateBigInts_; } + + // Return true if no allocations have been made since the last collection. + bool isEmpty() const; + + // Check whether an arbitrary pointer is within the nursery. This is + // slower than IsInsideNursery(Cell*), but works on all types of pointers. + MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete; + MOZ_ALWAYS_INLINE bool isInside(const void* p) const { + for (auto chunk : chunks_) { + if (uintptr_t(p) - uintptr_t(chunk) < gc::ChunkSize) { + return true; + } + } + return false; + } + + template + inline bool isInside(const SharedMem& p) const; + + // Allocate and return a pointer to a new GC thing. Returns nullptr if the + // Nursery is full. + void* allocateCell(gc::AllocSite* site, size_t size, JS::TraceKind kind); + + static size_t nurseryCellHeaderSize() { + return sizeof(gc::NurseryCellHeader); + } + + // Allocate a buffer for a given zone, using the nursery if possible. + void* allocateBuffer(JS::Zone* zone, size_t nbytes); + + // Allocate a buffer for a given object, using the nursery if possible and + // obj is in the nursery. + void* allocateBuffer(JS::Zone* zone, JSObject* obj, size_t nbytes); + + // Allocate a buffer for a given object, always using the nursery if obj is + // in the nursery. The requested size must be less than or equal to + // MaxNurseryBufferSize. + void* allocateBufferSameLocation(JSObject* obj, size_t nbytes); + + // Allocate a zero-initialized buffer for a given zone, using the nursery if + // possible. If the buffer isn't allocated in the nursery, the given arena is + // used. + void* allocateZeroedBuffer(JS::Zone* zone, size_t nbytes, + arena_id_t arena = js::MallocArena); + + // Allocate a zero-initialized buffer for a given object, using the nursery if + // possible and obj is in the nursery. If the buffer isn't allocated in the + // nursery, the given arena is used. + void* allocateZeroedBuffer(JSObject* obj, size_t nbytes, + arena_id_t arena = js::MallocArena); + + // Resize an existing buffer. + void* reallocateBuffer(JS::Zone* zone, gc::Cell* cell, void* oldBuffer, + size_t oldBytes, size_t newBytes); + + // Allocate a digits buffer for a given BigInt, using the nursery if possible + // and |bi| is in the nursery. + void* allocateBuffer(JS::BigInt* bi, size_t nbytes); + + // Free an object buffer. + void freeBuffer(void* buffer, size_t nbytes); + + // The maximum number of bytes allowed to reside in nursery buffers. + static const size_t MaxNurseryBufferSize = 1024; + + // Do a minor collection. + void collect(JS::GCOptions options, JS::GCReason reason); + + // If the thing at |*ref| in the Nursery has been forwarded, set |*ref| to + // the new location and return true. Otherwise return false and leave + // |*ref| unset. + [[nodiscard]] MOZ_ALWAYS_INLINE static bool getForwardedPointer( + js::gc::Cell** ref); + + // Forward a slots/elements pointer stored in an Ion frame. + void forwardBufferPointer(uintptr_t* pSlotsElems); + + inline void maybeSetForwardingPointer(JSTracer* trc, void* oldData, + void* newData, bool direct); + inline void setForwardingPointerWhileTenuring(void* oldData, void* newData, + bool direct); + + // Register a malloced buffer that is held by a nursery object, which + // should be freed at the end of a minor GC. Buffers are unregistered when + // their owning objects are tenured. + [[nodiscard]] bool registerMallocedBuffer(void* buffer, size_t nbytes); + + // Mark a malloced buffer as no longer needing to be freed. + void removeMallocedBuffer(void* buffer, size_t nbytes) { + MOZ_ASSERT(mallocedBuffers.has(buffer)); + MOZ_ASSERT(nbytes > 0); + MOZ_ASSERT(mallocedBufferBytes >= nbytes); + mallocedBuffers.remove(buffer); + mallocedBufferBytes -= nbytes; + } + + // Mark a malloced buffer as no longer needing to be freed during minor + // GC. There's no need to account for the size here since all remaining + // buffers will soon be freed. + void removeMallocedBufferDuringMinorGC(void* buffer) { + MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting()); + MOZ_ASSERT(mallocedBuffers.has(buffer)); + mallocedBuffers.remove(buffer); + } + + [[nodiscard]] bool addedUniqueIdToCell(gc::Cell* cell) { + MOZ_ASSERT(IsInsideNursery(cell)); + MOZ_ASSERT(isEnabled()); + return cellsWithUid_.append(cell); + } + + size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const; + + // Wasm "trailer" (C++-heap-allocated) blocks. + // + // All involved blocks are allocated/deallocated via this nursery's + // `mallocedBlockCache_`. Hence we must store both the block address and + // its freelist ID, wrapped up in a PointerAndUint7. + // + // Trailer blocks registered here are added to `trailersAdded_`. Those that + // are later deregistered as a result of `obj_moved` calls that indicate + // tenuring, should be added to `trailersRemoved_`. + // + // Unfortunately ::unregisterTrailer cannot be allowed to OOM. To get + // around this we rely on the observation that all deregistered blocks + // should previously have been registered, so the deregistered set can never + // be larger than the registered set. Hence ::registerTrailer effectively + // preallocates space in `trailersRemoved_` so as to ensure that, in the + // worst case, all registered blocks can be handed to ::unregisterTrailer + // without needing to resize `trailersRemoved_` in ::unregisterTrailer. + // + // The downside is that most of the space in `trailersRemoved_` is wasted in + // the case where there are few blocks deregistered. This is unfortunate + // but it's hard to see how to avoid it. + // + // At the end of a minor collection, all blocks in the set `trailersAdded_ - + // trailersRemoved_[0 .. trailersRemovedUsed_ - 1]` are handed back to the + // `mallocedBlockCache_`. + [[nodiscard]] bool registerTrailer(PointerAndUint7 blockAndListID, + size_t nBytes) { + MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length()); + MOZ_ASSERT(nBytes > 0); + if (MOZ_UNLIKELY(!trailersAdded_.append(blockAndListID))) { + return false; + } + if (MOZ_UNLIKELY(!trailersRemoved_.append(nullptr))) { + trailersAdded_.popBack(); + return false; + } + + // This is a clone of the logic in ::registerMallocedBuffer. It may be + // that some other heuristic is better, once we know more about the + // typical behaviour of wasm-GC applications. + trailerBytes_ += nBytes; + if (MOZ_UNLIKELY(trailerBytes_ > capacity() * 8)) { + requestMinorGC(JS::GCReason::NURSERY_TRAILERS); + } + return true; + } + + void unregisterTrailer(void* block) { + MOZ_ASSERT(trailersRemovedUsed_ < trailersRemoved_.length()); + trailersRemoved_[trailersRemovedUsed_] = block; + trailersRemovedUsed_++; + } + + size_t sizeOfTrailerBlockSets(mozilla::MallocSizeOf mallocSizeOf) const; + + // The number of bytes from the start position to the end of the nursery. + // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit() + // to calculate the nursery size, current lazy-allocated size or nursery + // limit respectively. + size_t spaceToEnd(unsigned chunkCount) const; + + size_t capacity() const { return capacity_; } + size_t committed() const { return spaceToEnd(allocatedChunkCount()); } + + // Used and free space both include chunk headers for that part of the + // nursery. + // + // usedSpace() + freeSpace() == capacity() + // + MOZ_ALWAYS_INLINE size_t usedSpace() const { + return capacity() - freeSpace(); + } + MOZ_ALWAYS_INLINE size_t freeSpace() const { + MOZ_ASSERT(isEnabled()); + MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); + MOZ_ASSERT(currentChunk_ < maxChunkCount()); + return (currentEnd_ - position_) + + (maxChunkCount() - currentChunk_ - 1) * gc::ChunkSize; + } + +#ifdef JS_GC_ZEAL + void enterZealMode(); + void leaveZealMode(); +#endif + + // Write profile time JSON on JSONPrinter. + void renderProfileJSON(JSONPrinter& json) const; + + // Print header line for profile times. + void printProfileHeader(); + + // Print total profile times on shutdown. + void printTotalProfileTimes(); + + void* addressOfPosition() const { return (void**)&position_; } + static constexpr int32_t offsetOfCurrentEndFromPosition() { + return offsetof(Nursery, currentEnd_) - offsetof(Nursery, position_); + } + + void* addressOfNurseryAllocatedSites() { + return pretenuringNursery.addressOfAllocatedSites(); + } + + void requestMinorGC(JS::GCReason reason) const; + + bool minorGCRequested() const { + return minorGCTriggerReason_ != JS::GCReason::NO_REASON; + } + JS::GCReason minorGCTriggerReason() const { return minorGCTriggerReason_; } + void clearMinorGCRequest() { + minorGCTriggerReason_ = JS::GCReason::NO_REASON; + } + + bool shouldCollect() const; + bool isNearlyFull() const; + bool isUnderused() const; + + bool enableProfiling() const { return enableProfiling_; } + + bool addMapWithNurseryMemory(MapObject* obj) { + MOZ_ASSERT_IF(!mapsWithNurseryMemory_.empty(), + mapsWithNurseryMemory_.back() != obj); + return mapsWithNurseryMemory_.append(obj); + } + bool addSetWithNurseryMemory(SetObject* obj) { + MOZ_ASSERT_IF(!setsWithNurseryMemory_.empty(), + setsWithNurseryMemory_.back() != obj); + return setsWithNurseryMemory_.append(obj); + } + + // The amount of space in the mapped nursery available to allocations. + static const size_t NurseryChunkUsableSize = + gc::ChunkSize - sizeof(gc::ChunkBase); + + void joinDecommitTask(); + + mozilla::TimeStamp collectionStartTime() { + return startTimes_[ProfileKey::Total]; + } + + bool canCreateAllocSite() { return pretenuringNursery.canCreateAllocSite(); } + void noteAllocSiteCreated() { pretenuringNursery.noteAllocSiteCreated(); } + bool reportPretenuring() const { return reportPretenuring_; } + void maybeStopPretenuring(gc::GCRuntime* gc) { + pretenuringNursery.maybeStopPretenuring(gc); + } + + void setAllocFlagsForZone(JS::Zone* zone); + + // Round a size in bytes to the nearest valid nursery size. + static size_t roundSize(size_t size); + + // The malloc'd block cache. + gc::MallocedBlockCache& mallocedBlockCache() { return mallocedBlockCache_; } + size_t sizeOfMallocedBlockCache(mozilla::MallocSizeOf mallocSizeOf) const { + return mallocedBlockCache_.sizeOfExcludingThis(mallocSizeOf); + } + + private: + // Fields used during allocation fast path are grouped first: + + // Pointer to the first unallocated byte in the nursery. + uintptr_t position_; + + // Pointer to the last byte of space in the current chunk. + uintptr_t currentEnd_; + + // Other fields not necessarily used during allocation follow: + + gc::GCRuntime* const gc; + + // Vector of allocated chunks to allocate from. + Vector chunks_; + + // The index of the chunk that is currently being allocated from. + uint32_t currentChunk_; + + // These fields refer to the beginning of the nursery. They're normally 0 + // and chunk(0).start() respectively. Except when a generational GC zeal + // mode is active, then they may be arbitrary (see Nursery::clear()). + uint32_t currentStartChunk_; + uintptr_t currentStartPosition_; + + // The current nursery capacity measured in bytes. It may grow up to this + // value without a collection, allocating chunks on demand. This limit may be + // changed by maybeResizeNursery() each collection. It includes chunk headers. + size_t capacity_; + + gc::PretenuringNursery pretenuringNursery; + + mozilla::TimeDuration timeInChunkAlloc_; + + // Report minor collections taking at least this long, if enabled. + bool enableProfiling_; + bool profileWorkers_; + mozilla::TimeDuration profileThreshold_; + + // Whether we will nursery-allocate strings. + bool canAllocateStrings_; + + // Whether we will nursery-allocate BigInts. + bool canAllocateBigInts_; + + // Report how many strings were deduplicated. + bool reportDeduplications_; + + // Whether to report information on pretenuring, and if so the allocation + // threshold at which to report details of each allocation site. + bool reportPretenuring_; + size_t reportPretenuringThreshold_; + + // Whether and why a collection of this nursery has been requested. This is + // mutable as it is set by the store buffer, which otherwise cannot modify + // anything in the nursery. + mutable JS::GCReason minorGCTriggerReason_; + + // Profiling data. + + enum class ProfileKey { +#define DEFINE_TIME_KEY(name, text) name, + FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY) +#undef DEFINE_TIME_KEY + KeyCount + }; + + using ProfileTimes = + mozilla::EnumeratedArray; + using ProfileDurations = + mozilla::EnumeratedArray; + + ProfileTimes startTimes_; + ProfileDurations profileDurations_; + ProfileDurations totalDurations_; + + // Data about the previous collection. + struct PreviousGC { + JS::GCReason reason = JS::GCReason::NO_REASON; + size_t nurseryCapacity = 0; + size_t nurseryCommitted = 0; + size_t nurseryUsedBytes = 0; + size_t nurseryUsedChunkCount = 0; + size_t tenuredBytes = 0; + size_t tenuredCells = 0; + mozilla::TimeStamp endTime; + }; + PreviousGC previousGC; + + bool hasRecentGrowthData; + double smoothedTargetSize; + + // Calculate the promotion rate of the most recent minor GC. + // The valid_for_tenuring parameter is used to return whether this + // promotion rate is accurate enough (the nursery was full enough) to be + // used for tenuring and other decisions. + // + // Must only be called if the previousGC data is initialised. + double calcPromotionRate(bool* validForTenuring) const; + + // The set of externally malloced buffers potentially kept live by objects + // stored in the nursery. Any external buffers that do not belong to a + // tenured thing at the end of a minor GC must be freed. + using BufferRelocationOverlay = void*; + using BufferSet = HashSet, SystemAllocPolicy>; + BufferSet mallocedBuffers; + size_t mallocedBufferBytes = 0; + + // Wasm "trailer" (C++-heap-allocated) blocks. See comments above on + // ::registerTrailer and ::unregisterTrailer. + Vector trailersAdded_; + Vector trailersRemoved_; + size_t trailersRemovedUsed_ = 0; + size_t trailerBytes_ = 0; + + void freeTrailerBlocks(); + + // During a collection most hoisted slot and element buffers indicate their + // new location with a forwarding pointer at the base. This does not work + // for buffers whose length is less than pointer width, or when different + // buffers might overlap each other. For these, an entry in the following + // table is used. + typedef HashMap, SystemAllocPolicy> + ForwardedBufferMap; + ForwardedBufferMap forwardedBuffers; + + // When we assign a unique id to cell in the nursery, that almost always + // means that the cell will be in a hash table, and thus, held live, + // automatically moving the uid from the nursery to its new home in + // tenured. It is possible, if rare, for an object that acquired a uid to + // be dead before the next collection, in which case we need to know to + // remove it when we sweep. + // + // Note: we store the pointers as Cell* here, resulting in an ugly cast in + // sweep. This is because this structure is used to help implement + // stable object hashing and we have to break the cycle somehow. + using CellsWithUniqueIdVector = Vector; + CellsWithUniqueIdVector cellsWithUid_; + + // Lists of map and set objects allocated in the nursery or with iterators + // allocated there. Such objects need to be swept after minor GC. + Vector mapsWithNurseryMemory_; + Vector setsWithNurseryMemory_; + + UniquePtr decommitTask; + + // A cache of small C++-heap allocated blocks associated with this Nursery. + // This provided so as to provide cheap allocation/deallocation of + // out-of-line storage areas as used by WasmStructObject and + // WasmArrayObject, although the mechanism is general and not specific to + // these object types. Regarding lifetimes, because the cache holds only + // blocks that are not currently in use, it can be flushed at any point with + // no correctness impact, only a performance impact. + gc::MallocedBlockCache mallocedBlockCache_; + + NurseryChunk& chunk(unsigned index) const { return *chunks_[index]; } + + // Set the current chunk. This updates the currentChunk_, position_ and + // currentEnd_ values as appropriate. It'll also poison the chunk, either a + // portion of the chunk if it is already the current chunk, or the whole chunk + // if fullPoison is true or it is not the current chunk. + void setCurrentChunk(unsigned chunkno); + + bool initFirstChunk(AutoLockGCBgAlloc& lock); + + // extent is advisory, it will be ignored in sub-chunk and generational zeal + // modes. It will be clamped to Min(NurseryChunkUsableSize, capacity_). + void poisonAndInitCurrentChunk(size_t extent = gc::ChunkSize); + + void setCurrentEnd(); + void setStartPosition(); + + // Allocate the next chunk, or the first chunk for initialization. + // Callers will probably want to call setCurrentChunk(0) next. + [[nodiscard]] bool allocateNextChunk(unsigned chunkno, + AutoLockGCBgAlloc& lock); + + MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; + + uintptr_t position() const { return position_; } + + MOZ_ALWAYS_INLINE bool isSubChunkMode() const; + + JSRuntime* runtime() const; + gcstats::Statistics& stats() const; + + const js::gc::GCSchedulingTunables& tunables() const; + + void getAllocFlagsForZone(JS::Zone* zone, bool* allocObjectsOut, + bool* allocStringsOut, bool* allocBigIntsOut); + void updateAllZoneAllocFlags(); + void updateAllocFlagsForZone(JS::Zone* zone); + void discardCodeAndSetJitFlagsForZone(JS::Zone* zone); + + // Common internal allocator function. + void* allocate(size_t size); + + void* moveToNextChunkAndAllocate(size_t size); + + struct CollectionResult { + size_t tenuredBytes; + size_t tenuredCells; + }; + CollectionResult doCollection(gc::AutoGCSession& session, + JS::GCOptions options, JS::GCReason reason); + void traceRoots(gc::AutoGCSession& session, gc::TenuringTracer& mover); + + size_t doPretenuring(JSRuntime* rt, JS::GCReason reason, + bool validPromotionRate, double promotionRate); + + // Handle relocation of slots/elements pointers stored in Ion frames. + inline void setForwardingPointer(void* oldData, void* newData, bool direct); + + inline void setDirectForwardingPointer(void* oldData, void* newData); + void setIndirectForwardingPointer(void* oldData, void* newData); + + inline void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, + uint32_t nslots); + inline void setElementsForwardingPointer(ObjectElements* oldHeader, + ObjectElements* newHeader, + uint32_t capacity); + +#ifdef DEBUG + bool checkForwardingPointerLocation(void* ptr, bool expectedInside); +#endif + + // Updates pointers to nursery objects that have been tenured and discards + // pointers to objects that have been freed. + void sweep(); + + // Reset the current chunk and position after a minor collection. Also poison + // the nursery on debug & nightly builds. + void clear(); + + void sweepMapAndSetObjects(); + + // Change the allocable space provided by the nursery. + void maybeResizeNursery(JS::GCOptions options, JS::GCReason reason); + size_t targetSize(JS::GCOptions options, JS::GCReason reason); + void clearRecentGrowthData(); + void growAllocableSpace(size_t newCapacity); + void shrinkAllocableSpace(size_t newCapacity); + void minimizeAllocableSpace(); + + // Free the chunks starting at firstFreeChunk until the end of the chunks + // vector. Shrinks the vector but does not update maxChunkCount(). + void freeChunksFrom(unsigned firstFreeChunk); + + void sendTelemetry(JS::GCReason reason, mozilla::TimeDuration totalTime, + bool wasEmpty, double promotionRate, + size_t sitesPretenured); + + void printCollectionProfile(JS::GCReason reason, double promotionRate); + void printDeduplicationData(js::StringStats& prev, js::StringStats& curr); + + // Profile recording and printing. + void maybeClearProfileDurations(); + void startProfile(ProfileKey key); + void endProfile(ProfileKey key); + static bool printProfileDurations(const ProfileDurations& times, + Sprinter& sprinter); + + mozilla::TimeStamp collectionStartTime() const; + mozilla::TimeStamp lastCollectionEndTime() const; + + friend class gc::GCRuntime; + friend class gc::TenuringTracer; + friend struct NurseryChunk; +}; + +} // namespace js + +#endif // gc_Nursery_h diff --git a/js/src/gc/NurseryAwareHashMap.h b/js/src/gc/NurseryAwareHashMap.h new file mode 100644 index 0000000000..35c2bebcea --- /dev/null +++ b/js/src/gc/NurseryAwareHashMap.h @@ -0,0 +1,209 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_NurseryAwareHashMap_h +#define gc_NurseryAwareHashMap_h + +#include "gc/Barrier.h" +#include "gc/Tracer.h" +#include "js/GCHashTable.h" +#include "js/GCPolicyAPI.h" +#include "js/HashTable.h" + +namespace js { + +namespace detail { + +// This class only handles the incremental case and does not deal with nursery +// pointers. The only users should be for NurseryAwareHashMap; it is defined +// externally because we need a GCPolicy for its use in the contained map. +template +class UnsafeBareWeakHeapPtr : public ReadBarriered { + public: + UnsafeBareWeakHeapPtr() + : ReadBarriered(JS::SafelyInitialized::create()) {} + MOZ_IMPLICIT UnsafeBareWeakHeapPtr(const T& v) : ReadBarriered(v) {} + explicit UnsafeBareWeakHeapPtr(const UnsafeBareWeakHeapPtr& v) + : ReadBarriered(v) {} + UnsafeBareWeakHeapPtr(UnsafeBareWeakHeapPtr&& v) + : ReadBarriered(std::move(v)) {} + + UnsafeBareWeakHeapPtr& operator=(const UnsafeBareWeakHeapPtr& v) { + this->value = v.value; + return *this; + } + + UnsafeBareWeakHeapPtr& operator=(const T& v) { + this->value = v; + return *this; + } + + const T get() const { + if (!InternalBarrierMethods::isMarkable(this->value)) { + return JS::SafelyInitialized::create(); + } + this->read(); + return this->value; + } + + explicit operator bool() const { return bool(this->value); } + + const T unbarrieredGet() const { return this->value; } + T* unsafeGet() { return &this->value; } + T const* unsafeGet() const { return &this->value; } +}; +} // namespace detail + +enum : bool { DuplicatesNotPossible, DuplicatesPossible }; + +// The "nursery aware" hash map is a special case of GCHashMap that is able to +// treat nursery allocated members weakly during a minor GC: e.g. it allows for +// nursery allocated objects to be collected during nursery GC where a normal +// hash table treats such edges strongly. +// +// Doing this requires some strong constraints on what can be stored in this +// table and how it can be accessed. At the moment, this table assumes that all +// values contain a strong reference to the key. This limits its usefulness to +// the CrossCompartmentMap at the moment, but might serve as a useful base for +// other tables in future. +template +class NurseryAwareHashMap { + using MapKey = UnsafeBarePtr; + using MapValue = detail::UnsafeBareWeakHeapPtr; + using HashPolicy = DefaultHasher; + using MapType = GCRekeyableHashMap; + MapType map; + + // Keep a list of all keys for which key->isTenured() is false. This lets us + // avoid a full traversal of the map on each minor GC, keeping the minor GC + // times proportional to the nursery heap size. + Vector nurseryEntries; + + public: + using Lookup = typename MapType::Lookup; + using Ptr = typename MapType::Ptr; + using Range = typename MapType::Range; + using Entry = typename MapType::Entry; + + explicit NurseryAwareHashMap(AllocPolicy a = AllocPolicy()) + : map(a), nurseryEntries(std::move(a)) {} + explicit NurseryAwareHashMap(size_t length) : map(length) {} + NurseryAwareHashMap(AllocPolicy a, size_t length) + : map(a, length), nurseryEntries(std::move(a)) {} + + bool empty() const { return map.empty(); } + Ptr lookup(const Lookup& l) const { return map.lookup(l); } + void remove(Ptr p) { map.remove(p); } + Range all() const { return map.all(); } + struct Enum : public MapType::Enum { + explicit Enum(NurseryAwareHashMap& namap) : MapType::Enum(namap.map) {} + }; + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return map.shallowSizeOfExcludingThis(mallocSizeOf) + + nurseryEntries.sizeOfExcludingThis(mallocSizeOf); + } + size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return map.shallowSizeOfIncludingThis(mallocSizeOf) + + nurseryEntries.sizeOfIncludingThis(mallocSizeOf); + } + + [[nodiscard]] bool put(const Key& key, const Value& value) { + if ((!key->isTenured() || !value->isTenured()) && + !nurseryEntries.append(key)) { + return false; + } + + auto p = map.lookupForAdd(key); + if (p) { + p->value() = value; + return true; + } + + return map.add(p, key, value); + } + + void sweepAfterMinorGC(JSTracer* trc) { + for (auto& key : nurseryEntries) { + auto p = map.lookup(key); + if (!p) { + continue; + } + + // Drop the entry if the value is not marked. + if (!JS::GCPolicy::traceWeak(trc, &p->value())) { + map.remove(p); + continue; + } + + // Update and relocate the key, if the value is still needed. + // + // Non-string Values will contain a strong reference to Key, as per its + // use in the CrossCompartmentWrapperMap, so the key will never be dying + // here. Strings do *not* have any sort of pointer from wrapper to + // wrappee, as they are just copies. The wrapper map entry is merely used + // as a cache to avoid re-copying the string, and currently that entire + // cache is flushed on major GC. + MapKey copy(key); + if (!JS::GCPolicy::traceWeak(trc, ©)) { + map.remove(p); + continue; + } + + if (AllowDuplicates) { + // Drop duplicated keys. + // + // A key can be forwarded to another place. In this case, rekey the + // item. If two or more different keys are forwarded to the same new + // key, simply drop the later ones. + if (key == copy) { + // No rekey needed. + } else if (map.has(copy)) { + // Key was forwarded to the same place that another key was already + // forwarded to. + map.remove(p); + } else { + map.rekeyAs(key, copy, copy); + } + } else { + MOZ_ASSERT(key == copy || !map.has(copy)); + map.rekeyIfMoved(key, copy); + } + } + nurseryEntries.clear(); + } + + void traceWeak(JSTracer* trc) { map.traceWeak(trc); } + + void clear() { + map.clear(); + nurseryEntries.clear(); + } + + bool hasNurseryEntries() const { return !nurseryEntries.empty(); } +}; + +} // namespace js + +namespace JS { + +template +struct GCPolicy> { + static void trace(JSTracer* trc, js::detail::UnsafeBareWeakHeapPtr* thingp, + const char* name) { + js::TraceEdge(trc, thingp, name); + } + static bool traceWeak(JSTracer* trc, + js::detail::UnsafeBareWeakHeapPtr* thingp) { + return js::TraceWeakEdge(trc, thingp, "UnsafeBareWeakHeapPtr"); + } +}; + +} // namespace JS + +namespace mozilla {} // namespace mozilla + +#endif // gc_NurseryAwareHashMap_h diff --git a/js/src/gc/ObjectKind-inl.h b/js/src/gc/ObjectKind-inl.h new file mode 100644 index 0000000000..a03b183447 --- /dev/null +++ b/js/src/gc/ObjectKind-inl.h @@ -0,0 +1,162 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal helper functions for getting the AllocKind used to allocate a + * JSObject and related information. + */ + +#ifndef gc_ObjectKind_inl_h +#define gc_ObjectKind_inl_h + +#include "util/Memory.h" +#include "vm/NativeObject.h" + +namespace js { +namespace gc { + +/* Capacity for slotsToThingKind */ +const size_t SLOTS_TO_THING_KIND_LIMIT = 17; + +extern const AllocKind slotsToThingKind[]; + +/* Get the best kind to use when making an object with the given slot count. */ +static inline AllocKind GetGCObjectKind(size_t numSlots) { + if (numSlots >= SLOTS_TO_THING_KIND_LIMIT) { + return AllocKind::OBJECT16; + } + return slotsToThingKind[numSlots]; +} + +static inline AllocKind GetGCObjectKind(const JSClass* clasp) { + MOZ_ASSERT(!clasp->isProxyObject(), + "Proxies should use GetProxyGCObjectKind"); + MOZ_ASSERT(!clasp->isJSFunction()); + + uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp); + return GetGCObjectKind(nslots); +} + +static constexpr bool CanUseFixedElementsForArray(size_t numElements) { + if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT) { + return false; + } + size_t numSlots = numElements + ObjectElements::VALUES_PER_HEADER; + return numSlots < SLOTS_TO_THING_KIND_LIMIT; +} + +/* As for GetGCObjectKind, but for dense array allocation. */ +static inline AllocKind GetGCArrayKind(size_t numElements) { + /* + * Dense arrays can use their fixed slots to hold their elements array + * (less two Values worth of ObjectElements header), but if more than the + * maximum number of fixed slots is needed then the fixed slots will be + * unused. + */ + static_assert(ObjectElements::VALUES_PER_HEADER == 2); + if (!CanUseFixedElementsForArray(numElements)) { + return AllocKind::OBJECT2; + } + return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER]; +} + +static inline AllocKind GetGCObjectFixedSlotsKind(size_t numFixedSlots) { + MOZ_ASSERT(numFixedSlots < SLOTS_TO_THING_KIND_LIMIT); + return slotsToThingKind[numFixedSlots]; +} + +// Get the best kind to use when allocating an object that needs a specific +// number of bytes. +static inline AllocKind GetGCObjectKindForBytes(size_t nbytes) { + MOZ_ASSERT(nbytes <= JSObject::MAX_BYTE_SIZE); + + if (nbytes <= sizeof(NativeObject)) { + return AllocKind::OBJECT0; + } + nbytes -= sizeof(NativeObject); + + size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); + MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); + return GetGCObjectKind(dataSlots); +} + +/* Get the number of fixed slots and initial capacity associated with a kind. */ +static constexpr inline size_t GetGCKindSlots(AllocKind thingKind) { + // Using a switch in hopes that thingKind will usually be a compile-time + // constant. + switch (thingKind) { + case AllocKind::OBJECT0: + case AllocKind::OBJECT0_BACKGROUND: + return 0; + case AllocKind::OBJECT2: + case AllocKind::OBJECT2_BACKGROUND: + return 2; + case AllocKind::FUNCTION: + case AllocKind::OBJECT4: + case AllocKind::OBJECT4_BACKGROUND: + return 4; + case AllocKind::FUNCTION_EXTENDED: + return 6; + case AllocKind::OBJECT8: + case AllocKind::OBJECT8_BACKGROUND: + return 8; + case AllocKind::OBJECT12: + case AllocKind::OBJECT12_BACKGROUND: + return 12; + case AllocKind::OBJECT16: + case AllocKind::OBJECT16_BACKGROUND: + return 16; + default: + MOZ_CRASH("Bad object alloc kind"); + } +} + +static inline size_t GetGCKindBytes(AllocKind thingKind) { + return sizeof(JSObject_Slots0) + GetGCKindSlots(thingKind) * sizeof(Value); +} + +static inline bool CanUseBackgroundAllocKind(const JSClass* clasp) { + return !clasp->hasFinalize() || (clasp->flags & JSCLASS_BACKGROUND_FINALIZE); +} + +static inline bool CanChangeToBackgroundAllocKind(AllocKind kind, + const JSClass* clasp) { + // If a foreground alloc kind is specified but the class has no finalizer or a + // finalizer that is safe to call on a different thread, we can change the + // alloc kind to one which is finalized on a background thread. + // + // For example, AllocKind::OBJECT0 calls the finalizer on the main thread, and + // AllocKind::OBJECT0_BACKGROUND calls the finalizer on the a helper thread. + + MOZ_ASSERT(IsObjectAllocKind(kind)); + + if (IsBackgroundFinalized(kind)) { + return false; // This kind is already a background finalized kind. + } + + return CanUseBackgroundAllocKind(clasp); +} + +static inline AllocKind ForegroundToBackgroundAllocKind(AllocKind fgKind) { + MOZ_ASSERT(IsObjectAllocKind(fgKind)); + MOZ_ASSERT(IsForegroundFinalized(fgKind)); + + // For objects, each background alloc kind is defined just after the + // corresponding foreground alloc kind so we can convert between them by + // incrementing or decrementing as appropriate. + AllocKind bgKind = AllocKind(size_t(fgKind) + 1); + + MOZ_ASSERT(IsObjectAllocKind(bgKind)); + MOZ_ASSERT(IsBackgroundFinalized(bgKind)); + MOZ_ASSERT(GetGCKindSlots(bgKind) == GetGCKindSlots(fgKind)); + + return bgKind; +} + +} // namespace gc +} // namespace js + +#endif // gc_ObjectKind_inl_h diff --git a/js/src/gc/ParallelMarking.cpp b/js/src/gc/ParallelMarking.cpp new file mode 100644 index 0000000000..67c29c02d7 --- /dev/null +++ b/js/src/gc/ParallelMarking.cpp @@ -0,0 +1,359 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/ParallelMarking.h" + +#include "gc/GCLock.h" +#include "gc/ParallelWork.h" +#include "vm/GeckoProfiler.h" +#include "vm/HelperThreadState.h" +#include "vm/Runtime.h" + +using namespace js; +using namespace js::gc; + +using mozilla::Maybe; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +class AutoAddTimeDuration { + TimeStamp start; + TimeDuration& result; + + public: + explicit AutoAddTimeDuration(TimeDuration& result) + : start(TimeStamp::Now()), result(result) {} + ~AutoAddTimeDuration() { result += TimeSince(start); } +}; + +ParallelMarker::ParallelMarker(GCRuntime* gc) : gc(gc) {} + +size_t ParallelMarker::workerCount() const { return gc->markers.length(); } + +bool ParallelMarker::mark(SliceBudget& sliceBudget) { +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(workerCount() <= HelperThreadState().maxGCParallelThreads(lock)); + + // TODO: Even if the thread limits checked above are correct, there may not + // be enough threads available to start our mark tasks immediately due to + // other runtimes in the same process running GC. + } +#endif + + if (markOneColor(MarkColor::Black, sliceBudget) == NotFinished) { + return false; + } + MOZ_ASSERT(!hasWork(MarkColor::Black)); + + if (markOneColor(MarkColor::Gray, sliceBudget) == NotFinished) { + return false; + } + MOZ_ASSERT(!hasWork(MarkColor::Gray)); + + // Handle any delayed marking, which is not performed in parallel. + if (gc->hasDelayedMarking()) { + gc->markAllDelayedChildren(ReportMarkTime); + } + + return true; +} + +bool ParallelMarker::markOneColor(MarkColor color, SliceBudget& sliceBudget) { + // Run a marking slice and return whether the stack is now empty. + + if (!hasWork(color)) { + return true; + } + + gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::PARALLEL_MARK); + + MOZ_ASSERT(workerCount() <= MaxParallelWorkers); + mozilla::Maybe tasks[MaxParallelWorkers]; + + for (size_t i = 0; i < workerCount(); i++) { + GCMarker* marker = gc->markers[i].get(); + tasks[i].emplace(this, marker, color, sliceBudget); + + // Attempt to populate empty mark stacks. + // + // TODO: When tuning for more than two markers we may need to adopt a more + // sophisticated approach. + if (!marker->hasEntriesForCurrentColor() && gc->marker().canDonateWork()) { + GCMarker::moveWork(marker, &gc->marker()); + } + } + + { + AutoLockGC lock(gc); + + activeTasks = 0; + for (size_t i = 0; i < workerCount(); i++) { + ParallelMarkTask& task = *tasks[i]; + if (task.hasWork()) { + incActiveTasks(&task, lock); + } + } + } + + { + AutoLockHelperThreadState lock; + + // There should always be enough parallel tasks to run our marking work. + MOZ_RELEASE_ASSERT(HelperThreadState().getGCParallelThreadCount(lock) >= + workerCount()); + + for (size_t i = 0; i < workerCount(); i++) { + gc->startTask(*tasks[i], lock); + } + + for (size_t i = 0; i < workerCount(); i++) { + gc->joinTask(*tasks[i], lock); + } + } + +#ifdef DEBUG + { + AutoLockGC lock(gc); + MOZ_ASSERT(waitingTasks.ref().isEmpty()); + MOZ_ASSERT(waitingTaskCount == 0); + MOZ_ASSERT(activeTasks == 0); + } +#endif + + return !hasWork(color); +} + +bool ParallelMarker::hasWork(MarkColor color) const { + for (const auto& marker : gc->markers) { + if (marker->hasEntries(color)) { + return true; + } + } + + return false; +} + +ParallelMarkTask::ParallelMarkTask(ParallelMarker* pm, GCMarker* marker, + MarkColor color, const SliceBudget& budget) + : GCParallelTask(pm->gc, gcstats::PhaseKind::PARALLEL_MARK, GCUse::Marking), + pm(pm), + marker(marker), + color(*marker, color), + budget(budget) { + marker->enterParallelMarkingMode(pm); +} + +ParallelMarkTask::~ParallelMarkTask() { + MOZ_ASSERT(!isWaiting.refNoCheck()); + marker->leaveParallelMarkingMode(); +} + +bool ParallelMarkTask::hasWork() const { + return marker->hasEntriesForCurrentColor(); +} + +void ParallelMarkTask::recordDuration() { + gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK, + duration()); + gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK_MARK, + markTime.ref()); + gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK_WAIT, + waitTime.ref()); +} + +void ParallelMarkTask::run(AutoLockHelperThreadState& lock) { + AutoUnlockHelperThreadState unlock(lock); + + AutoLockGC gcLock(pm->gc); + + markOrRequestWork(gcLock); + + MOZ_ASSERT(!isWaiting); +} + +void ParallelMarkTask::markOrRequestWork(AutoLockGC& lock) { + for (;;) { + if (hasWork()) { + if (!tryMarking(lock)) { + return; + } + } else { + if (!requestWork(lock)) { + return; + } + } + } +} + +bool ParallelMarkTask::tryMarking(AutoLockGC& lock) { + MOZ_ASSERT(hasWork()); + MOZ_ASSERT(marker->isParallelMarking()); + + // Mark until budget exceeded or we run out of work. + bool finished; + { + AutoUnlockGC unlock(lock); + + AutoAddTimeDuration time(markTime.ref()); + finished = marker->markCurrentColorInParallel(budget); + } + + MOZ_ASSERT_IF(finished, !hasWork()); + pm->decActiveTasks(this, lock); + + return finished; +} + +bool ParallelMarkTask::requestWork(AutoLockGC& lock) { + MOZ_ASSERT(!hasWork()); + + if (!pm->hasActiveTasks(lock)) { + return false; // All other tasks are empty. We're finished. + } + + budget.stepAndForceCheck(); + if (budget.isOverBudget()) { + return false; // Over budget or interrupted. + } + + // Add ourselves to the waiting list and wait for another task to give us + // work. The task with work calls ParallelMarker::donateWorkFrom. + waitUntilResumed(lock); + + return true; +} + +void ParallelMarkTask::waitUntilResumed(AutoLockGC& lock) { + GeckoProfilerRuntime& profiler = gc->rt->geckoProfiler(); + if (profiler.enabled()) { + profiler.markEvent("Parallel marking wait start", ""); + } + + pm->addTaskToWaitingList(this, lock); + + // Set isWaiting flag and wait for another thread to clear it and resume us. + MOZ_ASSERT(!isWaiting); + isWaiting = true; + + AutoAddTimeDuration time(waitTime.ref()); + + do { + MOZ_ASSERT(pm->hasActiveTasks(lock)); + resumed.wait(lock.guard()); + } while (isWaiting); + + MOZ_ASSERT(!pm->isTaskInWaitingList(this, lock)); + + if (profiler.enabled()) { + profiler.markEvent("Parallel marking wait end", ""); + } +} + +void ParallelMarkTask::resume() { + { + AutoLockGC lock(gc); + MOZ_ASSERT(isWaiting); + + isWaiting = false; + + // Increment the active task count before donateWorkFrom() returns so this + // can't reach zero before the waiting task runs again. + if (hasWork()) { + pm->incActiveTasks(this, lock); + } + } + + resumed.notify_all(); +} + +void ParallelMarkTask::resumeOnFinish(const AutoLockGC& lock) { + MOZ_ASSERT(isWaiting); + MOZ_ASSERT(!hasWork()); + + isWaiting = false; + resumed.notify_all(); +} + +void ParallelMarker::addTaskToWaitingList(ParallelMarkTask* task, + const AutoLockGC& lock) { + MOZ_ASSERT(!task->hasWork()); + MOZ_ASSERT(hasActiveTasks(lock)); + MOZ_ASSERT(!isTaskInWaitingList(task, lock)); + MOZ_ASSERT(waitingTaskCount < workerCount() - 1); + + waitingTasks.ref().pushBack(task); + waitingTaskCount++; +} + +#ifdef DEBUG +bool ParallelMarker::isTaskInWaitingList(const ParallelMarkTask* task, + const AutoLockGC& lock) const { + // The const cast is because ElementProbablyInList is not const. + return const_cast(waitingTasks.ref()) + .ElementProbablyInList(const_cast(task)); +} +#endif + +void ParallelMarker::incActiveTasks(ParallelMarkTask* task, + const AutoLockGC& lock) { + MOZ_ASSERT(task->hasWork()); + MOZ_ASSERT(activeTasks < workerCount()); + + activeTasks++; +} + +void ParallelMarker::decActiveTasks(ParallelMarkTask* task, + const AutoLockGC& lock) { + MOZ_ASSERT(activeTasks != 0); + + activeTasks--; + + if (activeTasks == 0) { + while (!waitingTasks.ref().isEmpty()) { + ParallelMarkTask* task = waitingTasks.ref().popFront(); + MOZ_ASSERT(waitingTaskCount != 0); + waitingTaskCount--; + task->resumeOnFinish(lock); + } + } +} + +void ParallelMarker::donateWorkFrom(GCMarker* src) { + if (!gc->tryLockGC()) { + return; + } + + // Check there are tasks waiting for work while holding the lock. + if (waitingTaskCount == 0) { + gc->unlockGC(); + return; + } + + // Take the first waiting task off the list. + ParallelMarkTask* waitingTask = waitingTasks.ref().popFront(); + waitingTaskCount--; + + // |task| is not running so it's safe to move work to it. + MOZ_ASSERT(waitingTask->isWaiting); + + gc->unlockGC(); + + // Move some work from this thread's mark stack to the waiting task. + MOZ_ASSERT(!waitingTask->hasWork()); + GCMarker::moveWork(waitingTask->marker, src); + + gc->stats().count(gcstats::COUNT_PARALLEL_MARK_INTERRUPTIONS); + + GeckoProfilerRuntime& profiler = gc->rt->geckoProfiler(); + if (profiler.enabled()) { + profiler.markEvent("Parallel marking donated work", ""); + } + + // Resume waiting task. + waitingTask->resume(); +} diff --git a/js/src/gc/ParallelMarking.h b/js/src/gc/ParallelMarking.h new file mode 100644 index 0000000000..46d8f381c6 --- /dev/null +++ b/js/src/gc/ParallelMarking.h @@ -0,0 +1,123 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_ParallelMarking_h +#define gc_ParallelMarking_h + +#include "mozilla/Atomics.h" +#include "mozilla/DoublyLinkedList.h" +#include "mozilla/TimeStamp.h" + +#include "gc/GCMarker.h" +#include "gc/GCParallelTask.h" +#include "js/HeapAPI.h" +#include "js/SliceBudget.h" +#include "threading/ConditionVariable.h" +#include "threading/ProtectedData.h" + +namespace js { + +class AutoLockGC; +class AutoLockHelperThreadState; +class AutoUnlockGC; + +namespace gc { + +class ParallelMarkTask; + +// Per-runtime parallel marking state. +// +// This class is used on the main thread and coordinates parallel marking using +// several helper threads running ParallelMarkTasks. +// +// This uses a work-requesting approach. Threads mark until they run out of +// work and then add themselves to a list of waiting tasks and block. Running +// tasks with enough work may donate work to a waiting task and resume it. +class MOZ_STACK_CLASS ParallelMarker { + public: + explicit ParallelMarker(GCRuntime* gc); + + bool mark(SliceBudget& sliceBudget); + + using AtomicCount = mozilla::Atomic; + AtomicCount& waitingTaskCountRef() { return waitingTaskCount; } + bool hasWaitingTasks() { return waitingTaskCount != 0; } + void donateWorkFrom(GCMarker* src); + + private: + bool markOneColor(MarkColor color, SliceBudget& sliceBudget); + + bool hasWork(MarkColor color) const; + + void addTask(ParallelMarkTask* task, const AutoLockGC& lock); + + void addTaskToWaitingList(ParallelMarkTask* task, const AutoLockGC& lock); +#ifdef DEBUG + bool isTaskInWaitingList(const ParallelMarkTask* task, + const AutoLockGC& lock) const; +#endif + + bool hasActiveTasks(const AutoLockGC& lock) const { return activeTasks; } + void incActiveTasks(ParallelMarkTask* task, const AutoLockGC& lock); + void decActiveTasks(ParallelMarkTask* task, const AutoLockGC& lock); + + size_t workerCount() const; + + friend class ParallelMarkTask; + + GCRuntime* const gc; + + using ParallelMarkTaskList = mozilla::DoublyLinkedList; + GCLockData waitingTasks; + AtomicCount waitingTaskCount; + + GCLockData activeTasks; +}; + +// A helper thread task that performs parallel marking. +class alignas(TypicalCacheLineSize) ParallelMarkTask + : public GCParallelTask, + public mozilla::DoublyLinkedListElement { + public: + friend class ParallelMarker; + + ParallelMarkTask(ParallelMarker* pm, GCMarker* marker, MarkColor color, + const SliceBudget& budget); + ~ParallelMarkTask(); + + void run(AutoLockHelperThreadState& lock) override; + + void recordDuration() override; + + private: + void markOrRequestWork(AutoLockGC& lock); + bool tryMarking(AutoLockGC& lock); + bool requestWork(AutoLockGC& lock); + + void waitUntilResumed(AutoLockGC& lock); + void resume(); + void resumeOnFinish(const AutoLockGC& lock); + + bool hasWork() const; + + // The following fields are only accessed by the marker thread: + ParallelMarker* const pm; + GCMarker* const marker; + AutoSetMarkColor color; + SliceBudget budget; + ConditionVariable resumed; + + GCLockData isWaiting; + + // Length of time this task spent blocked waiting for work. + MainThreadOrGCTaskData markTime; + MainThreadOrGCTaskData waitTime; +}; + +} // namespace gc +} // namespace js + +#endif /* gc_ParallelMarking_h */ diff --git a/js/src/gc/ParallelWork.h b/js/src/gc/ParallelWork.h new file mode 100644 index 0000000000..db97c75009 --- /dev/null +++ b/js/src/gc/ParallelWork.h @@ -0,0 +1,136 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_ParallelWork_h +#define gc_ParallelWork_h + +#include "mozilla/Maybe.h" + +#include + +#include "gc/GCParallelTask.h" +#include "gc/GCRuntime.h" +#include "js/SliceBudget.h" +#include "vm/HelperThreads.h" + +namespace js { + +namespace gcstats { +enum class PhaseKind : uint8_t; +} + +namespace gc { + +template +using ParallelWorkFunc = size_t (*)(GCRuntime*, const WorkItem&); + +// A GCParallelTask task that executes WorkItems from a WorkItemIterator. +// +// The WorkItemIterator class must supply done(), next() and get() methods. The +// get() method must return WorkItems objects. +template +class ParallelWorker : public GCParallelTask { + public: + using WorkFunc = ParallelWorkFunc; + + ParallelWorker(GCRuntime* gc, gcstats::PhaseKind phaseKind, GCUse use, + WorkFunc func, WorkItemIterator& work, + const SliceBudget& budget, AutoLockHelperThreadState& lock) + : GCParallelTask(gc, phaseKind, use), + func_(func), + work_(work), + budget_(budget), + item_(work.get()) { + // Consume a work item on creation so that we can stop creating workers if + // the number of workers exceeds the number of work items. + work.next(); + } + + void run(AutoLockHelperThreadState& lock) { + AutoUnlockHelperThreadState unlock(lock); + + for (;;) { + size_t steps = func_(gc, item_); + budget_.step(std::max(steps, size_t(1))); + if (budget_.isOverBudget()) { + break; + } + + AutoLockHelperThreadState lock; + if (work().done()) { + break; + } + + item_ = work().get(); + work().next(); + } + } + + private: + WorkItemIterator& work() { return work_.ref(); } + + // A function to execute work items on the helper thread. + WorkFunc func_; + + // An iterator which produces work items to execute. + HelperThreadLockData work_; + + // The budget that determines how long to run for. + SliceBudget budget_; + + // The next work item to process. + WorkItem item_; +}; + +static constexpr size_t MaxParallelWorkers = 8; + +// An RAII class that starts a number of ParallelWorkers and waits for them to +// finish. +template +class MOZ_RAII AutoRunParallelWork { + public: + using Worker = ParallelWorker; + using WorkFunc = ParallelWorkFunc; + + AutoRunParallelWork(GCRuntime* gc, WorkFunc func, + gcstats::PhaseKind phaseKind, GCUse use, + WorkItemIterator& work, const SliceBudget& budget, + AutoLockHelperThreadState& lock) + : gc(gc), phaseKind(phaseKind), lock(lock), tasksStarted(0) { + size_t workerCount = gc->parallelWorkerCount(); + MOZ_ASSERT(workerCount <= MaxParallelWorkers); + MOZ_ASSERT_IF(workerCount == 0, work.done()); + + for (size_t i = 0; i < workerCount && !work.done(); i++) { + tasks[i].emplace(gc, phaseKind, use, func, work, budget, lock); + gc->startTask(*tasks[i], lock); + tasksStarted++; + } + } + + ~AutoRunParallelWork() { + gHelperThreadLock.assertOwnedByCurrentThread(); + + for (size_t i = 0; i < tasksStarted; i++) { + gc->joinTask(*tasks[i], lock); + } + for (size_t i = tasksStarted; i < MaxParallelWorkers; i++) { + MOZ_ASSERT(tasks[i].isNothing()); + } + } + + private: + GCRuntime* gc; + gcstats::PhaseKind phaseKind; + AutoLockHelperThreadState& lock; + size_t tasksStarted; + mozilla::Maybe tasks[MaxParallelWorkers]; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_ParallelWork_h */ diff --git a/js/src/gc/Policy.h b/js/src/gc/Policy.h new file mode 100644 index 0000000000..290e4ed177 --- /dev/null +++ b/js/src/gc/Policy.h @@ -0,0 +1,101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* JS Garbage Collector. */ + +#ifndef gc_Policy_h +#define gc_Policy_h + +#include + +#include "gc/Barrier.h" +#include "gc/Tracer.h" +#include "js/GCPolicyAPI.h" + +namespace js { + +// Define the GCPolicy for all internal pointers. +template +struct InternalGCPointerPolicy : public JS::GCPointerPolicy { + using Type = std::remove_pointer_t; + +#define IS_BASE_OF_OR(_1, BaseType, _2, _3) std::is_base_of_v || + static_assert( + JS_FOR_EACH_TRACEKIND(IS_BASE_OF_OR) false, + "InternalGCPointerPolicy must only be used for GC thing pointers"); +#undef IS_BASE_OF_OR + + static void trace(JSTracer* trc, T* vp, const char* name) { + // It's not safe to trace unbarriered pointers except as part of root + // marking. If you get an assertion here you probably need to add a barrier, + // e.g. HeapPtr. + TraceNullableRoot(trc, vp, name); + } +}; + +} // namespace js + +namespace JS { + +// Internally, all pointer types are treated as pointers to GC things by +// default. +template +struct GCPolicy : public js::InternalGCPointerPolicy {}; +template +struct GCPolicy : public js::InternalGCPointerPolicy {}; + +template +struct GCPolicy> { + static void trace(JSTracer* trc, js::HeapPtr* thingp, const char* name) { + js::TraceNullableEdge(trc, thingp, name); + } + static bool traceWeak(JSTracer* trc, js::HeapPtr* thingp) { + return js::TraceWeakEdge(trc, thingp, "HeapPtr"); + } +}; + +template +struct GCPolicy> { + static void trace(JSTracer* trc, js::PreBarriered* thingp, + const char* name) { + js::TraceNullableEdge(trc, thingp, name); + } +}; + +template +struct GCPolicy> { + static void trace(JSTracer* trc, js::WeakHeapPtr* thingp, + const char* name) { + js::TraceEdge(trc, thingp, name); + } + static bool traceWeak(JSTracer* trc, js::WeakHeapPtr* thingp) { + return js::TraceWeakEdge(trc, thingp, "traceWeak"); + } +}; + +template +struct GCPolicy> { + static bool traceWeak(JSTracer* trc, js::UnsafeBarePtr* vp) { + if (*vp) { + return js::TraceManuallyBarrieredWeakEdge(trc, vp->unbarrieredAddress(), + "UnsafeBarePtr"); + } + return true; + } +}; + +template <> +struct GCPolicy { + static void trace(JSTracer* trc, JS::GCCellPtr* thingp, const char* name) { + // It's not safe to trace unbarriered pointers except as part of root + // marking. + js::TraceGCCellPtrRoot(trc, thingp, name); + } +}; + +} // namespace JS + +#endif // gc_Policy_h diff --git a/js/src/gc/Pretenuring.cpp b/js/src/gc/Pretenuring.cpp new file mode 100644 index 0000000000..5703663e47 --- /dev/null +++ b/js/src/gc/Pretenuring.cpp @@ -0,0 +1,459 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sw=2 et tw=80: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Pretenuring.h" + +#include "mozilla/Sprintf.h" + +#include "gc/GCInternals.h" +#include "gc/PublicIterators.h" +#include "jit/Invalidation.h" + +#include "gc/PrivateIterators-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::gc; + +// The number of nursery allocations at which to pay attention to an allocation +// site. This must be large enough to ensure we have enough information to infer +// the lifetime and also large enough to avoid pretenuring low volume allocation +// sites. +static constexpr size_t AllocSiteAttentionThreshold = 500; + +// The maximum number of alloc sites to create between each minor +// collection. Stop tracking allocation after this limit is reached. This +// prevents unbounded time traversing the list during minor GC. +static constexpr size_t MaxAllocSitesPerMinorGC = 500; + +// The maximum number of times to invalidate JIT code for a site. After this we +// leave the site's state as Unknown and don't pretenure allocations. +// Note we use 4 bits to store the invalidation count. +static constexpr size_t MaxInvalidationCount = 5; + +// The minimum number of allocated cells needed to determine the survival rate +// of cells in newly created arenas. +static constexpr size_t MinCellsRequiredForSurvivalRate = 100; + +// The young survival rate below which a major collection is determined to have +// a low young survival rate. +static constexpr double LowYoungSurvivalThreshold = 0.05; + +// The number of consecutive major collections with a low young survival rate +// that must occur before recovery is attempted. +static constexpr size_t LowYoungSurvivalCountBeforeRecovery = 2; + +// The proportion of the nursery that must be tenured above which a minor +// collection may be determined to have a high nursery survival rate. +static constexpr double HighNurserySurvivalPromotionThreshold = 0.6; + +// The number of nursery allocations made by optimized JIT code that must be +// tenured above which a minor collection may be determined to have a high +// nursery survival rate. +static constexpr size_t HighNurserySurvivalOptimizedAllocThreshold = 10000; + +// The number of consecutive minor collections with a high nursery survival rate +// that must occur before recovery is attempted. +static constexpr size_t HighNurserySurvivalCountBeforeRecovery = 2; + +AllocSite* const AllocSite::EndSentinel = reinterpret_cast(1); +JSScript* const AllocSite::WasmScript = + reinterpret_cast(AllocSite::STATE_MASK + 1); + +static bool SiteBasedPretenuringEnabled = true; + +JS_PUBLIC_API void JS::SetSiteBasedPretenuringEnabled(bool enable) { + SiteBasedPretenuringEnabled = enable; +} + +bool PretenuringNursery::canCreateAllocSite() { + MOZ_ASSERT(allocSitesCreated <= MaxAllocSitesPerMinorGC); + return SiteBasedPretenuringEnabled && + allocSitesCreated < MaxAllocSitesPerMinorGC; +} + +size_t PretenuringNursery::doPretenuring(GCRuntime* gc, JS::GCReason reason, + bool validPromotionRate, + double promotionRate, bool reportInfo, + size_t reportThreshold) { + size_t sitesActive = 0; + size_t sitesPretenured = 0; + size_t sitesInvalidated = 0; + size_t zonesWithHighNurserySurvival = 0; + + // Zero allocation counts. + totalAllocCount_ = 0; + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + for (auto& count : zone->pretenuring.nurseryAllocCounts) { + count = 0; + } + } + + // Check whether previously optimized code has changed its behaviour and + // needs to be recompiled so that it can pretenure its allocations. + if (validPromotionRate) { + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + bool highNurserySurvivalRate = + promotionRate > HighNurserySurvivalPromotionThreshold && + zone->optimizedAllocSite()->nurseryTenuredCount >= + HighNurserySurvivalOptimizedAllocThreshold; + zone->pretenuring.noteHighNurserySurvivalRate(highNurserySurvivalRate); + if (highNurserySurvivalRate) { + zonesWithHighNurserySurvival++; + } + } + } + + if (reportInfo) { + AllocSite::printInfoHeader(reason, promotionRate); + } + + AllocSite* site = allocatedSites; + allocatedSites = AllocSite::EndSentinel; + while (site != AllocSite::EndSentinel) { + AllocSite* next = site->nextNurseryAllocated; + site->nextNurseryAllocated = nullptr; + + MOZ_ASSERT_IF(site->isNormal(), + site->nurseryAllocCount >= site->nurseryTenuredCount); + + if (site->isNormal()) { + processSite(gc, site, sitesActive, sitesPretenured, sitesInvalidated, + reportInfo, reportThreshold); + } + + site = next; + } + + // Catch-all sites don't end up on the list if they are only used from + // optimized JIT code, so process them here. + for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) { + for (auto& site : zone->pretenuring.unknownAllocSites) { + processCatchAllSite(&site, reportInfo, reportThreshold); + } + processCatchAllSite(zone->optimizedAllocSite(), reportInfo, + reportThreshold); + } + + if (reportInfo) { + AllocSite::printInfoFooter(allocSitesCreated, sitesActive, sitesPretenured, + sitesInvalidated); + if (zonesWithHighNurserySurvival) { + fprintf(stderr, " %zu zones with high nursery survival rate\n", + zonesWithHighNurserySurvival); + } + } + + allocSitesCreated = 0; + + return sitesPretenured; +} + +void PretenuringNursery::processSite(GCRuntime* gc, AllocSite* site, + size_t& sitesActive, + size_t& sitesPretenured, + size_t& sitesInvalidated, bool reportInfo, + size_t reportThreshold) { + sitesActive++; + + updateAllocCounts(site); + + bool hasPromotionRate = false; + double promotionRate = 0.0; + bool wasInvalidated = false; + if (site->nurseryAllocCount > AllocSiteAttentionThreshold) { + promotionRate = + double(site->nurseryTenuredCount) / double(site->nurseryAllocCount); + hasPromotionRate = true; + + AllocSite::State prevState = site->state(); + site->updateStateOnMinorGC(promotionRate); + AllocSite::State newState = site->state(); + + if (prevState == AllocSite::State::Unknown && + newState == AllocSite::State::LongLived) { + sitesPretenured++; + + // We can optimize JIT code before we realise that a site should be + // pretenured. Make sure we invalidate any existing optimized code. + if (site->hasScript()) { + wasInvalidated = site->invalidateScript(gc); + if (wasInvalidated) { + sitesInvalidated++; + } + } + } + } + + if (reportInfo && site->allocCount() >= reportThreshold) { + site->printInfo(hasPromotionRate, promotionRate, wasInvalidated); + } + + site->resetNurseryAllocations(); +} + +void PretenuringNursery::processCatchAllSite(AllocSite* site, bool reportInfo, + size_t reportThreshold) { + if (!site->hasNurseryAllocations()) { + return; + } + + updateAllocCounts(site); + + if (reportInfo && site->allocCount() >= reportThreshold) { + site->printInfo(false, 0.0, false); + } + + site->resetNurseryAllocations(); +} + +void PretenuringNursery::updateAllocCounts(AllocSite* site) { + JS::TraceKind kind = site->traceKind(); + totalAllocCount_ += site->nurseryAllocCount; + PretenuringZone& zone = site->zone()->pretenuring; + zone.nurseryAllocCount(kind) += site->nurseryAllocCount; +} + +bool AllocSite::invalidateScript(GCRuntime* gc) { + CancelOffThreadIonCompile(script()); + + if (!script()->hasIonScript()) { + return false; + } + + if (invalidationLimitReached()) { + MOZ_ASSERT(state() == State::Unknown); + return false; + } + + invalidationCount++; + if (invalidationLimitReached()) { + setState(State::Unknown); + } + + JSContext* cx = gc->rt->mainContextFromOwnThread(); + jit::Invalidate(cx, script(), + /* resetUses = */ false, + /* cancelOffThread = */ true); + return true; +} + +bool AllocSite::invalidationLimitReached() const { + MOZ_ASSERT(invalidationCount <= MaxInvalidationCount); + return invalidationCount == MaxInvalidationCount; +} + +void PretenuringNursery::maybeStopPretenuring(GCRuntime* gc) { + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + double rate; + if (zone->pretenuring.calculateYoungTenuredSurvivalRate(&rate)) { + bool lowYoungSurvivalRate = rate < LowYoungSurvivalThreshold; + zone->pretenuring.noteLowYoungTenuredSurvivalRate(lowYoungSurvivalRate); + } + } +} + +AllocSite::Kind AllocSite::kind() const { + if (isNormal()) { + return Kind::Normal; + } + + if (this == zone()->optimizedAllocSite()) { + return Kind::Optimized; + } + + MOZ_ASSERT(this == zone()->unknownAllocSite(traceKind())); + return Kind::Unknown; +} + +void AllocSite::updateStateOnMinorGC(double promotionRate) { + // The state changes based on whether the promotion rate is deemed high + // (greater that 90%): + // + // high high + // ------------------> ------------------> + // ShortLived Unknown LongLived + // <------------------ <------------------ + // !high !high + // + // The nursery is used to allocate if the site's state is Unknown or + // ShortLived. There are no direct transition between ShortLived and LongLived + // to avoid pretenuring sites that we've recently observed being short-lived. + + if (invalidationLimitReached()) { + MOZ_ASSERT(state() == State::Unknown); + return; + } + + bool highPromotionRate = promotionRate >= 0.9; + + switch (state()) { + case State::Unknown: + if (highPromotionRate) { + setState(State::LongLived); + } else { + setState(State::ShortLived); + } + break; + + case State::ShortLived: { + if (highPromotionRate) { + setState(State::Unknown); + } + break; + } + + case State::LongLived: { + if (!highPromotionRate) { + setState(State::Unknown); + } + break; + } + } +} + +bool AllocSite::maybeResetState() { + if (invalidationLimitReached()) { + MOZ_ASSERT(state() == State::Unknown); + return false; + } + + invalidationCount++; + setState(State::Unknown); + return true; +} + +void AllocSite::trace(JSTracer* trc) { + if (JSScript* s = script()) { + TraceManuallyBarrieredEdge(trc, &s, "AllocSite script"); + if (s != script()) { + setScript(s); + } + } +} + +bool PretenuringZone::calculateYoungTenuredSurvivalRate(double* rateOut) { + MOZ_ASSERT(allocCountInNewlyCreatedArenas >= + survivorCountInNewlyCreatedArenas); + if (allocCountInNewlyCreatedArenas < MinCellsRequiredForSurvivalRate) { + return false; + } + + *rateOut = double(survivorCountInNewlyCreatedArenas) / + double(allocCountInNewlyCreatedArenas); + return true; +} + +void PretenuringZone::noteLowYoungTenuredSurvivalRate( + bool lowYoungSurvivalRate) { + if (lowYoungSurvivalRate) { + lowYoungTenuredSurvivalCount++; + } else { + lowYoungTenuredSurvivalCount = 0; + } +} + +void PretenuringZone::noteHighNurserySurvivalRate( + bool highNurserySurvivalRate) { + if (highNurserySurvivalRate) { + highNurserySurvivalCount++; + } else { + highNurserySurvivalCount = 0; + } +} + +bool PretenuringZone::shouldResetNurseryAllocSites() { + bool shouldReset = + highNurserySurvivalCount >= HighNurserySurvivalCountBeforeRecovery; + if (shouldReset) { + highNurserySurvivalCount = 0; + } + return shouldReset; +} + +bool PretenuringZone::shouldResetPretenuredAllocSites() { + bool shouldReset = + lowYoungTenuredSurvivalCount >= LowYoungSurvivalCountBeforeRecovery; + if (shouldReset) { + lowYoungTenuredSurvivalCount = 0; + } + return shouldReset; +} + +/* static */ +void AllocSite::printInfoHeader(JS::GCReason reason, double promotionRate) { + fprintf(stderr, + "Pretenuring info after %s minor GC with %4.1f%% promotion rate:\n", + ExplainGCReason(reason), promotionRate * 100.0); +} + +/* static */ +void AllocSite::printInfoFooter(size_t sitesCreated, size_t sitesActive, + size_t sitesPretenured, + size_t sitesInvalidated) { + fprintf(stderr, + " %zu alloc sites created, %zu active, %zu pretenured, %zu " + "invalidated\n", + sitesCreated, sitesActive, sitesPretenured, sitesInvalidated); +} + +void AllocSite::printInfo(bool hasPromotionRate, double promotionRate, + bool wasInvalidated) const { + // Zone. + fprintf(stderr, " %p %p", this, zone()); + + // Script, or which kind of catch-all site this is. + if (!hasScript()) { + fprintf(stderr, " %16s", + kind() == Kind::Optimized + ? "optimized" + : (kind() == Kind::Normal ? "normal" : "unknown")); + } else { + fprintf(stderr, " %16p", script()); + } + + // Nursery allocation count, missing for optimized sites. + char buffer[16] = {'\0'}; + if (kind() != Kind::Optimized) { + SprintfLiteral(buffer, "%8" PRIu32, nurseryAllocCount); + } + fprintf(stderr, " %8s", buffer); + + // Nursery tenure count. + fprintf(stderr, " %8" PRIu32, nurseryTenuredCount); + + // Promotion rate, if there were enough allocations. + buffer[0] = '\0'; + if (hasPromotionRate) { + SprintfLiteral(buffer, "%5.1f%%", std::min(1.0, promotionRate) * 100); + } + fprintf(stderr, " %6s", buffer); + + // Current state for sites associated with a script. + const char* state = isNormal() ? stateName() : ""; + fprintf(stderr, " %10s", state); + + // Whether the associated script was invalidated. + if (wasInvalidated) { + fprintf(stderr, " invalidated"); + } + + fprintf(stderr, "\n"); +} + +const char* AllocSite::stateName() const { + switch (state()) { + case State::ShortLived: + return "ShortLived"; + case State::Unknown: + return "Unknown"; + case State::LongLived: + return "LongLived"; + } + + MOZ_CRASH("Unknown state"); +} diff --git a/js/src/gc/Pretenuring.h b/js/src/gc/Pretenuring.h new file mode 100644 index 0000000000..ed156cbbd0 --- /dev/null +++ b/js/src/gc/Pretenuring.h @@ -0,0 +1,348 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Pretenuring. + * + * Some kinds of GC cells can be allocated in either the nursery or the tenured + * heap. The pretenuring system decides where to allocate such cells based on + * their expected lifetime with the aim of minimising total collection time. + * + * Lifetime is predicted based on data gathered about the cells' allocation + * site. This data is gathered in the middle JIT tiers, after code has stopped + * executing in the interpreter and before we generate fully optimized code. + */ + +#ifndef gc_Pretenuring_h +#define gc_Pretenuring_h + +#include + +#include "gc/AllocKind.h" +#include "js/TypeDecls.h" + +class JS_PUBLIC_API JSTracer; + +namespace JS { +enum class GCReason; +} // namespace JS + +namespace js { +namespace gc { + +class GCRuntime; +class PretenuringNursery; + +// Number of trace kinds supportd by the nursery. These are arranged at the +// start of JS::TraceKind. +static constexpr size_t NurseryTraceKinds = 3; + +enum class CatchAllAllocSite { Unknown, Optimized }; + +// Information about an allocation site. +// +// Nursery cells contain a pointer to one of these in their cell header (stored +// before the cell). The site can relate to either a specific JS bytecode +// instruction, a specific WebAssembly type, or can be a catch-all instance for +// unknown sites or JS JIT optimized code. +class AllocSite { + public: + enum class State : uint32_t { ShortLived = 0, Unknown = 1, LongLived = 2 }; + + // The JIT depends on being able to tell the states apart by checking a single + // bit. + static constexpr int32_t LONG_LIVED_BIT = int32_t(State::LongLived); + static_assert((LONG_LIVED_BIT & int32_t(State::Unknown)) == 0); + static_assert((AllocSite::LONG_LIVED_BIT & int32_t(State::ShortLived)) == 0); + + private: + JS::Zone* zone_ = nullptr; + + // Word storing JSScript pointer and site state. + // + // The script pointer is the script that owns this allocation site, a special + // sentinel script for wasm sites, or null for unknown sites. This is used + // when we need to invalidate the script. + uintptr_t scriptAndState = uintptr_t(State::Unknown); + static constexpr uintptr_t STATE_MASK = BitMask(2); + + // Next pointer forming a linked list of sites at which nursery allocation + // happened since the last nursery collection. + AllocSite* nextNurseryAllocated = nullptr; + + // Number of nursery allocations at this site since last nursery collection. + uint32_t nurseryAllocCount = 0; + + // Number of nursery allocations that survived. Used during collection. + uint32_t nurseryTenuredCount : 24; + + // Number of times the script has been invalidated. + uint32_t invalidationCount : 4; + + // The trace kind of the allocation. Only kinds up to NurseryTraceKinds are + // allowed. + uint32_t traceKind_ : 4; + + static AllocSite* const EndSentinel; + + // Sentinel script for wasm sites. + static JSScript* const WasmScript; + + friend class PretenuringZone; + friend class PretenuringNursery; + + uintptr_t rawScript() const { return scriptAndState & ~STATE_MASK; } + + public: + AllocSite() : nurseryTenuredCount(0), invalidationCount(0), traceKind_(0) {} + + // Create a dummy site to use for unknown allocations. + explicit AllocSite(JS::Zone* zone, JS::TraceKind kind) + : zone_(zone), + nurseryTenuredCount(0), + invalidationCount(0), + traceKind_(uint32_t(kind)) { + MOZ_ASSERT(traceKind_ < NurseryTraceKinds); + } + + // Create a site for an opcode in the given script. + AllocSite(JS::Zone* zone, JSScript* script, JS::TraceKind kind) + : AllocSite(zone, kind) { + MOZ_ASSERT(script != WasmScript); + setScript(script); + } + + void initUnknownSite(JS::Zone* zone, JS::TraceKind kind) { + MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown)); + zone_ = zone; + nurseryTenuredCount = 0; + invalidationCount = 0; + traceKind_ = uint32_t(kind); + MOZ_ASSERT(traceKind_ < NurseryTraceKinds); + } + + // Initialize a site to be a wasm site. + void initWasm(JS::Zone* zone) { + MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown)); + zone_ = zone; + setScript(WasmScript); + nurseryTenuredCount = 0; + invalidationCount = 0; + traceKind_ = uint32_t(JS::TraceKind::Object); + } + + JS::Zone* zone() const { return zone_; } + + JS::TraceKind traceKind() const { return JS::TraceKind(traceKind_); } + + State state() const { return State(scriptAndState & STATE_MASK); } + + // Whether this site has a script associated with it. This is not true if + // this site is for a wasm site. + bool hasScript() const { return rawScript() != uintptr_t(WasmScript); } + JSScript* script() const { + MOZ_ASSERT(hasScript()); + return reinterpret_cast(rawScript()); + } + + // Whether this site is not an unknown or optimized site. + bool isNormal() const { return rawScript() != 0; } + + enum class Kind : uint32_t { Normal, Unknown, Optimized }; + Kind kind() const; + + bool isInAllocatedList() const { return nextNurseryAllocated; } + + // Whether allocations at this site should be allocated in the nursery or the + // tenured heap. + Heap initialHeap() const { + return state() == State::LongLived ? Heap::Tenured : Heap::Default; + } + + bool hasNurseryAllocations() const { + return nurseryAllocCount != 0 || nurseryTenuredCount != 0; + } + void resetNurseryAllocations() { + nurseryAllocCount = 0; + nurseryTenuredCount = 0; + } + + uint32_t incAllocCount() { return ++nurseryAllocCount; } + uint32_t* nurseryAllocCountAddress() { return &nurseryAllocCount; } + + void incTenuredCount() { + // The nursery is not large enough for this to overflow. + nurseryTenuredCount++; + MOZ_ASSERT(nurseryTenuredCount != 0); + } + + size_t allocCount() const { + return std::max(nurseryAllocCount, nurseryTenuredCount); + } + + void updateStateOnMinorGC(double promotionRate); + + // Reset the state to 'Unknown' unless we have reached the invalidation limit + // for this site. Return whether the state was reset. + bool maybeResetState(); + + bool invalidationLimitReached() const; + bool invalidateScript(GCRuntime* gc); + + void trace(JSTracer* trc); + + static void printInfoHeader(JS::GCReason reason, double promotionRate); + static void printInfoFooter(size_t sitesCreated, size_t sitesActive, + size_t sitesPretenured, size_t sitesInvalidated); + void printInfo(bool hasPromotionRate, double promotionRate, + bool wasInvalidated) const; + + static constexpr size_t offsetOfScriptAndState() { + return offsetof(AllocSite, scriptAndState); + } + static constexpr size_t offsetOfNurseryAllocCount() { + return offsetof(AllocSite, nurseryAllocCount); + } + static constexpr size_t offsetOfNextNurseryAllocated() { + return offsetof(AllocSite, nextNurseryAllocated); + } + + private: + void setScript(JSScript* newScript) { + MOZ_ASSERT((uintptr_t(newScript) & STATE_MASK) == 0); + scriptAndState = uintptr_t(newScript) | uintptr_t(state()); + } + + void setState(State newState) { + MOZ_ASSERT((uintptr_t(newState) & ~STATE_MASK) == 0); + scriptAndState = rawScript() | uintptr_t(newState); + } + + const char* stateName() const; +}; + +// Pretenuring information stored per zone. +class PretenuringZone { + public: + // Catch-all allocation site instance used when the actual site is unknown, or + // when optimized JIT code allocates a GC thing that's not handled by the + // pretenuring system. + AllocSite unknownAllocSites[NurseryTraceKinds]; + + // Catch-all allocation instance used by optimized JIT code when allocating GC + // things that are handled by the pretenuring system. Allocation counts are + // not recorded by optimized JIT code. + AllocSite optimizedAllocSite; + + // Count of tenured cell allocations made between each major collection and + // how many survived. + uint32_t allocCountInNewlyCreatedArenas = 0; + uint32_t survivorCountInNewlyCreatedArenas = 0; + + // Count of successive collections that had a low young tenured survival + // rate. Used to discard optimized code if we get the pretenuring decision + // wrong. + uint32_t lowYoungTenuredSurvivalCount = 0; + + // Count of successive nursery collections that had a high survival rate for + // objects allocated by optimized code. Used to discard optimized code if we + // get the pretenuring decision wrong. + uint32_t highNurserySurvivalCount = 0; + + // Total allocation count by trace kind (ignoring optimized + // allocations). Calculated during nursery collection. + uint32_t nurseryAllocCounts[NurseryTraceKinds] = {0}; + + explicit PretenuringZone(JS::Zone* zone) + : optimizedAllocSite(zone, JS::TraceKind::Object) { + for (uint32_t i = 0; i < NurseryTraceKinds; i++) { + unknownAllocSites[i].initUnknownSite(zone, JS::TraceKind(i)); + } + } + + AllocSite& unknownAllocSite(JS::TraceKind kind) { + size_t i = size_t(kind); + MOZ_ASSERT(i < NurseryTraceKinds); + return unknownAllocSites[i]; + } + + void clearCellCountsInNewlyCreatedArenas() { + allocCountInNewlyCreatedArenas = 0; + survivorCountInNewlyCreatedArenas = 0; + } + void updateCellCountsInNewlyCreatedArenas(uint32_t allocCount, + uint32_t survivorCount) { + allocCountInNewlyCreatedArenas += allocCount; + survivorCountInNewlyCreatedArenas += survivorCount; + } + + bool calculateYoungTenuredSurvivalRate(double* rateOut); + + void noteLowYoungTenuredSurvivalRate(bool lowYoungSurvivalRate); + void noteHighNurserySurvivalRate(bool highNurserySurvivalRate); + + // Recovery: if code behaviour change we may need to reset allocation site + // state and invalidate JIT code. + bool shouldResetNurseryAllocSites(); + bool shouldResetPretenuredAllocSites(); + + uint32_t& nurseryAllocCount(JS::TraceKind kind) { + size_t i = size_t(kind); + MOZ_ASSERT(i < NurseryTraceKinds); + return nurseryAllocCounts[i]; + } + uint32_t nurseryAllocCount(JS::TraceKind kind) const { + return const_cast(this)->nurseryAllocCount(kind); + } +}; + +// Pretenuring information stored as part of the the GC nursery. +class PretenuringNursery { + gc::AllocSite* allocatedSites; + + size_t allocSitesCreated = 0; + + uint32_t totalAllocCount_ = 0; + + public: + PretenuringNursery() : allocatedSites(AllocSite::EndSentinel) {} + + bool hasAllocatedSites() const { + return allocatedSites != AllocSite::EndSentinel; + } + + bool canCreateAllocSite(); + void noteAllocSiteCreated() { allocSitesCreated++; } + + void insertIntoAllocatedList(AllocSite* site) { + MOZ_ASSERT(!site->isInAllocatedList()); + site->nextNurseryAllocated = allocatedSites; + allocatedSites = site; + } + + size_t doPretenuring(GCRuntime* gc, JS::GCReason reason, + bool validPromotionRate, double promotionRate, + bool reportInfo, size_t reportThreshold); + + void maybeStopPretenuring(GCRuntime* gc); + + uint32_t totalAllocCount() const { return totalAllocCount_; } + + void* addressOfAllocatedSites() { return &allocatedSites; } + + private: + void processSite(GCRuntime* gc, AllocSite* site, size_t& sitesActive, + size_t& sitesPretenured, size_t& sitesInvalidated, + bool reportInfo, size_t reportThreshold); + void processCatchAllSite(AllocSite* site, bool reportInfo, + size_t reportThreshold); + void updateAllocCounts(AllocSite* site); +}; + +} // namespace gc +} // namespace js + +#endif /* gc_Pretenuring_h */ diff --git a/js/src/gc/PrivateIterators-inl.h b/js/src/gc/PrivateIterators-inl.h new file mode 100644 index 0000000000..359e6b22b7 --- /dev/null +++ b/js/src/gc/PrivateIterators-inl.h @@ -0,0 +1,164 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal iterators for various data structures. + */ + +#ifndef gc_PrivateIterators_inl_h +#define gc_PrivateIterators_inl_h + +#include "gc/PublicIterators.h" + +#include "gc/GC-inl.h" + +namespace js { +namespace gc { + +class ArenaCellIterUnderGC : public ArenaCellIter { + public: + explicit ArenaCellIterUnderGC(Arena* arena) : ArenaCellIter(arena) { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + } +}; + +class ArenaCellIterUnderFinalize : public ArenaCellIter { + public: + explicit ArenaCellIterUnderFinalize(Arena* arena) : ArenaCellIter(arena) { + MOZ_ASSERT(CurrentThreadIsGCFinalizing()); + } +}; + +class GrayObjectIter : public ZoneAllCellIter { + public: + explicit GrayObjectIter(JS::Zone* zone, AllocKind kind) + : ZoneAllCellIter() { + initForTenuredIteration(zone, kind); + } + + JSObject* get() const { + return ZoneAllCellIter::get(); + } + operator JSObject*() const { return get(); } + JSObject* operator->() const { return get(); } +}; + +class GCZonesIter { + AllZonesIter zone; + + public: + explicit GCZonesIter(GCRuntime* gc) : zone(gc) { + MOZ_ASSERT(gc->heapState() != JS::HeapState::Idle); + if (!done() && !zone->wasGCStarted()) { + next(); + } + } + explicit GCZonesIter(JSRuntime* rt) : GCZonesIter(&rt->gc) {} + + bool done() const { return zone.done(); } + + void next() { + MOZ_ASSERT(!done()); + do { + zone.next(); + } while (!zone.done() && !zone->wasGCStarted()); + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return zone; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +using GCCompartmentsIter = + CompartmentsOrRealmsIterT; +using GCRealmsIter = CompartmentsOrRealmsIterT; + +/* Iterates over all zones in the current sweep group. */ +class SweepGroupZonesIter { + JS::Zone* current; + + public: + explicit SweepGroupZonesIter(GCRuntime* gc) + : current(gc->getCurrentSweepGroup()) { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + } + explicit SweepGroupZonesIter(JSRuntime* rt) : SweepGroupZonesIter(&rt->gc) {} + + bool done() const { return !current; } + + void next() { + MOZ_ASSERT(!done()); + current = current->nextNodeInGroup(); + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return current; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +using SweepGroupCompartmentsIter = + CompartmentsOrRealmsIterT; +using SweepGroupRealmsIter = + CompartmentsOrRealmsIterT; + +// Iterate the free cells in an arena. See also ArenaCellIter which iterates +// the allocated cells. +class ArenaFreeCellIter { + Arena* arena; + size_t thingSize; + FreeSpan span; + uint_fast16_t thing; + + public: + explicit ArenaFreeCellIter(Arena* arena) + : arena(arena), + thingSize(arena->getThingSize()), + span(*arena->getFirstFreeSpan()), + thing(span.first) { + MOZ_ASSERT(arena); + MOZ_ASSERT(thing < ArenaSize); + } + + bool done() const { + MOZ_ASSERT(thing < ArenaSize); + return !thing; + } + + TenuredCell* get() const { + MOZ_ASSERT(!done()); + return reinterpret_cast(uintptr_t(arena) + thing); + } + + void next() { + MOZ_ASSERT(!done()); + MOZ_ASSERT(thing >= span.first && thing <= span.last); + + if (thing == span.last) { + span = *span.nextSpan(arena); + thing = span.first; + } else { + thing += thingSize; + } + + MOZ_ASSERT(thing < ArenaSize); + } + + operator TenuredCell*() const { return get(); } + TenuredCell* operator->() const { return get(); } +}; + +} // namespace gc +} // namespace js + +#endif // gc_PrivateIterators_inl_h diff --git a/js/src/gc/PublicIterators.cpp b/js/src/gc/PublicIterators.cpp new file mode 100644 index 0000000000..582a21ddf3 --- /dev/null +++ b/js/src/gc/PublicIterators.cpp @@ -0,0 +1,272 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "vm/Realm.h" +#include "vm/Runtime.h" + +#include "gc/PrivateIterators-inl.h" + +using namespace js; +using namespace js::gc; + +static void IterateRealmsArenasCellsUnbarriered( + JSContext* cx, Zone* zone, void* data, + JS::IterateRealmCallback realmCallback, IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback, const JS::AutoRequireNoGC& nogc) { + { + Rooted realm(cx); + for (RealmsInZoneIter r(zone); !r.done(); r.next()) { + realm = r; + (*realmCallback)(cx, data, realm, nogc); + } + } + + for (auto thingKind : AllAllocKinds()) { + JS::TraceKind traceKind = MapAllocToTraceKind(thingKind); + size_t thingSize = Arena::thingSize(thingKind); + + for (ArenaIter aiter(zone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + (*arenaCallback)(cx->runtime(), data, arena, traceKind, thingSize, nogc); + for (ArenaCellIter cell(arena); !cell.done(); cell.next()) { + (*cellCallback)(cx->runtime(), data, JS::GCCellPtr(cell, traceKind), + thingSize, nogc); + } + } + } +} + +void js::IterateHeapUnbarriered(JSContext* cx, void* data, + IterateZoneCallback zoneCallback, + JS::IterateRealmCallback realmCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback) { + AutoPrepareForTracing prep(cx); + JS::AutoSuppressGCAnalysis nogc(cx); + + auto iterateZone = [&](Zone* zone) -> void { + (*zoneCallback)(cx->runtime(), data, zone, nogc); + IterateRealmsArenasCellsUnbarriered(cx, zone, data, realmCallback, + arenaCallback, cellCallback, nogc); + }; + + // Include the shared atoms zone if present. + if (Zone* zone = cx->runtime()->gc.maybeSharedAtomsZone()) { + iterateZone(zone); + } + + for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) { + iterateZone(zone); + } +} + +void js::IterateHeapUnbarrieredForZone(JSContext* cx, Zone* zone, void* data, + IterateZoneCallback zoneCallback, + JS::IterateRealmCallback realmCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback) { + AutoPrepareForTracing prep(cx); + JS::AutoSuppressGCAnalysis nogc(cx); + + (*zoneCallback)(cx->runtime(), data, zone, nogc); + IterateRealmsArenasCellsUnbarriered(cx, zone, data, realmCallback, + arenaCallback, cellCallback, nogc); +} + +void js::IterateChunks(JSContext* cx, void* data, + IterateChunkCallback chunkCallback) { + AutoPrepareForTracing prep(cx); + AutoLockGC lock(cx->runtime()); + JS::AutoSuppressGCAnalysis nogc(cx); + + for (auto chunk = cx->runtime()->gc.allNonEmptyChunks(lock); !chunk.done(); + chunk.next()) { + chunkCallback(cx->runtime(), data, chunk, nogc); + } +} + +static void TraverseInnerLazyScriptsForLazyScript( + JSContext* cx, void* data, BaseScript* enclosingScript, + IterateScriptCallback lazyScriptCallback, const JS::AutoRequireNoGC& nogc) { + for (JS::GCCellPtr gcThing : enclosingScript->gcthings()) { + if (!gcThing.is()) { + continue; + } + JSObject* obj = &gcThing.as(); + + MOZ_ASSERT(obj->is(), + "All objects in lazy scripts should be functions"); + JSFunction* fun = &obj->as(); + + if (!fun->hasBaseScript()) { + // Ignore asm.js. + continue; + } + MOZ_ASSERT(fun->baseScript()); + if (!fun->baseScript()) { + // If the function doesn't have script, ignore it. + continue; + } + + if (fun->hasBytecode()) { + // Ignore non lazy function. + continue; + } + + // If the function is "ghost", we shouldn't expose it to the debugger. + // + // See GHOST_FUNCTION in FunctionFlags.h for more details. + if (fun->isGhost()) { + continue; + } + + BaseScript* script = fun->baseScript(); + MOZ_ASSERT_IF(script->hasEnclosingScript(), + script->enclosingScript() == enclosingScript); + + lazyScriptCallback(cx->runtime(), data, script, nogc); + + TraverseInnerLazyScriptsForLazyScript(cx, data, script, lazyScriptCallback, + nogc); + } +} + +static inline void DoScriptCallback(JSContext* cx, void* data, + BaseScript* script, + IterateScriptCallback callback, + const JS::AutoRequireNoGC& nogc) { + // Exclude any scripts that may be the result of a failed compile. Check that + // script either has bytecode or is ready to delazify. + // + // This excludes lazy scripts that do not have an enclosing scope because we + // cannot distinguish a failed compile fragment from a lazy script with a lazy + // parent. + if (!script->hasBytecode() && !script->isReadyForDelazification()) { + return; + } + + // Invoke callback. + callback(cx->runtime(), data, script, nogc); + + // The check above excluded lazy scripts with lazy parents, so explicitly + // visit inner scripts now if we are lazy with a successfully compiled parent. + if (!script->hasBytecode()) { + TraverseInnerLazyScriptsForLazyScript(cx, data, script, callback, nogc); + } +} + +void js::IterateScripts(JSContext* cx, Realm* realm, void* data, + IterateScriptCallback scriptCallback) { + MOZ_ASSERT(!cx->suppressGC); + AutoEmptyNurseryAndPrepareForTracing prep(cx); + JS::AutoSuppressGCAnalysis nogc; + + if (realm) { + Zone* zone = realm->zone(); + for (auto iter = zone->cellIter(prep); !iter.done(); + iter.next()) { + if (iter->realm() != realm) { + continue; + } + DoScriptCallback(cx, data, iter.get(), scriptCallback, nogc); + } + } else { + for (ZonesIter zone(cx->runtime(), SkipAtoms); !zone.done(); zone.next()) { + for (auto iter = zone->cellIter(prep); !iter.done(); + iter.next()) { + DoScriptCallback(cx, data, iter.get(), scriptCallback, nogc); + } + } + } +} + +void js::IterateGrayObjects(Zone* zone, IterateGCThingCallback cellCallback, + void* data) { + MOZ_ASSERT(!JS::RuntimeHeapIsBusy()); + + JSContext* cx = TlsContext.get(); + AutoPrepareForTracing prep(cx); + JS::AutoSuppressGCAnalysis nogc(cx); + + for (auto kind : ObjectAllocKinds()) { + for (GrayObjectIter obj(zone, kind); !obj.done(); obj.next()) { + if (obj->asTenured().isMarkedGray()) { + cellCallback(data, JS::GCCellPtr(obj.get()), nogc); + } + } + } +} + +JS_PUBLIC_API void JS_IterateCompartments( + JSContext* cx, void* data, + JSIterateCompartmentCallback compartmentCallback) { + AutoTraceSession session(cx->runtime()); + + for (CompartmentsIter c(cx->runtime()); !c.done(); c.next()) { + if ((*compartmentCallback)(cx, data, c) == + JS::CompartmentIterResult::Stop) { + break; + } + } +} + +JS_PUBLIC_API void JS_IterateCompartmentsInZone( + JSContext* cx, JS::Zone* zone, void* data, + JSIterateCompartmentCallback compartmentCallback) { + AutoTraceSession session(cx->runtime()); + + for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) { + if ((*compartmentCallback)(cx, data, c) == + JS::CompartmentIterResult::Stop) { + break; + } + } +} + +JS_PUBLIC_API void JS::IterateRealms(JSContext* cx, void* data, + JS::IterateRealmCallback realmCallback) { + AutoTraceSession session(cx->runtime()); + JS::AutoSuppressGCAnalysis nogc(cx); + + Rooted realm(cx); + for (RealmsIter r(cx->runtime()); !r.done(); r.next()) { + realm = r; + (*realmCallback)(cx, data, realm, nogc); + } +} + +JS_PUBLIC_API void JS::IterateRealmsWithPrincipals( + JSContext* cx, JSPrincipals* principals, void* data, + JS::IterateRealmCallback realmCallback) { + MOZ_ASSERT(principals); + + AutoTraceSession session(cx->runtime()); + JS::AutoSuppressGCAnalysis nogc(cx); + + Rooted realm(cx); + for (RealmsIter r(cx->runtime()); !r.done(); r.next()) { + if (r->principals() != principals) { + continue; + } + realm = r; + (*realmCallback)(cx, data, realm, nogc); + } +} + +JS_PUBLIC_API void JS::IterateRealmsInCompartment( + JSContext* cx, JS::Compartment* compartment, void* data, + JS::IterateRealmCallback realmCallback) { + AutoTraceSession session(cx->runtime()); + JS::AutoSuppressGCAnalysis nogc(cx); + + Rooted realm(cx); + for (RealmsInCompartmentIter r(compartment); !r.done(); r.next()) { + realm = r; + (*realmCallback)(cx, data, realm, nogc); + } +} diff --git a/js/src/gc/PublicIterators.h b/js/src/gc/PublicIterators.h new file mode 100644 index 0000000000..d1072cfe98 --- /dev/null +++ b/js/src/gc/PublicIterators.h @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Iterators for various data structures. + */ + +#ifndef gc_PublicIterators_h +#define gc_PublicIterators_h + +#include "jstypes.h" +#include "gc/GCRuntime.h" +#include "gc/IteratorUtils.h" +#include "gc/Zone.h" +#include "vm/Compartment.h" +#include "vm/Runtime.h" + +namespace JS { +class JS_PUBLIC_API Realm; +} + +namespace js { + +enum ZoneSelector { WithAtoms, SkipAtoms }; + +// Iterate over all zones in the runtime. May or may not include the atoms zone. +class ZonesIter { + gc::AutoEnterIteration iterMarker; + JS::Zone** it; + JS::Zone** const end; + + public: + ZonesIter(gc::GCRuntime* gc, ZoneSelector selector) + : iterMarker(gc), it(gc->zones().begin()), end(gc->zones().end()) { + if (selector == SkipAtoms) { + MOZ_ASSERT(get()->isAtomsZone()); + next(); + } + } + ZonesIter(JSRuntime* rt, ZoneSelector selector) + : ZonesIter(&rt->gc, selector) {} + + bool done() const { return it == end; } + + void next() { + MOZ_ASSERT(!done()); + it++; + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return *it; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +// Iterate over all zones in the runtime apart from the atoms zone. +class NonAtomZonesIter : public ZonesIter { + public: + explicit NonAtomZonesIter(gc::GCRuntime* gc) : ZonesIter(gc, SkipAtoms) {} + explicit NonAtomZonesIter(JSRuntime* rt) : NonAtomZonesIter(&rt->gc) {} +}; + +// Iterate over all zones in the runtime, except those which may be in use by +// parse threads. +class AllZonesIter : public ZonesIter { + public: + explicit AllZonesIter(gc::GCRuntime* gc) : ZonesIter(gc, WithAtoms) {} + explicit AllZonesIter(JSRuntime* rt) : AllZonesIter(&rt->gc) {} +}; + +struct CompartmentsInZoneIter { + explicit CompartmentsInZoneIter(JS::Zone* zone) : zone(zone) { + it = zone->compartments().begin(); + } + + bool done() const { + MOZ_ASSERT(it); + return it < zone->compartments().begin() || + it >= zone->compartments().end(); + } + void next() { + MOZ_ASSERT(!done()); + it++; + } + + JS::Compartment* get() const { + MOZ_ASSERT(it); + return *it; + } + + operator JS::Compartment*() const { return get(); } + JS::Compartment* operator->() const { return get(); } + + private: + JS::Zone* zone; + JS::Compartment** it; +}; + +class RealmsInCompartmentIter { + JS::Compartment* comp; + JS::Realm** it; + + public: + explicit RealmsInCompartmentIter(JS::Compartment* comp) : comp(comp) { + it = comp->realms().begin(); + MOZ_ASSERT(!done(), "Compartments must have at least one realm"); + } + + bool done() const { + MOZ_ASSERT(it); + return it < comp->realms().begin() || it >= comp->realms().end(); + } + void next() { + MOZ_ASSERT(!done()); + it++; + } + + JS::Realm* get() const { + MOZ_ASSERT(!done()); + return *it; + } + + operator JS::Realm*() const { return get(); } + JS::Realm* operator->() const { return get(); } +}; + +using RealmsInZoneIter = + NestedIterator; + +// This iterator iterates over all the compartments or realms in a given set of +// zones. The set of zones is determined by iterating ZoneIterT. The set of +// compartments or realms is determined by InnerIterT. +template +class CompartmentsOrRealmsIterT + : public NestedIterator { + gc::AutoEnterIteration iterMarker; + + public: + explicit CompartmentsOrRealmsIterT(gc::GCRuntime* gc) + : NestedIterator(gc), iterMarker(gc) {} + explicit CompartmentsOrRealmsIterT(JSRuntime* rt) + : CompartmentsOrRealmsIterT(&rt->gc) {} +}; + +using CompartmentsIter = + CompartmentsOrRealmsIterT; +using RealmsIter = + CompartmentsOrRealmsIterT; + +} // namespace js + +#endif // gc_PublicIterators_h diff --git a/js/src/gc/RelocationOverlay.h b/js/src/gc/RelocationOverlay.h new file mode 100644 index 0000000000..047e763d44 --- /dev/null +++ b/js/src/gc/RelocationOverlay.h @@ -0,0 +1,66 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definition of relocation overlay used while moving cells. + */ + +#ifndef gc_RelocationOverlay_h +#define gc_RelocationOverlay_h + +#include "mozilla/Assertions.h" + +#include + +#include "gc/Cell.h" + +namespace js { +namespace gc { + +/* + * This structure overlays a Cell that has been moved and provides a way to find + * its new location. It's used during generational and compacting GC. + */ +class RelocationOverlay : public Cell { + public: + /* The location the cell has been moved to, stored in the cell header. */ + Cell* forwardingAddress() const { + MOZ_ASSERT(isForwarded()); + return reinterpret_cast(header_.getForwardingAddress()); + } + + protected: + /* A list entry to track all relocated things. */ + RelocationOverlay* next_; + + explicit RelocationOverlay(Cell* dst); + + public: + static const RelocationOverlay* fromCell(const Cell* cell) { + return static_cast(cell); + } + + static RelocationOverlay* fromCell(Cell* cell) { + return static_cast(cell); + } + + static RelocationOverlay* forwardCell(Cell* src, Cell* dst); + + void setNext(RelocationOverlay* next) { + MOZ_ASSERT(isForwarded()); + next_ = next; + } + + RelocationOverlay* next() const { + MOZ_ASSERT(isForwarded()); + return next_; + } +}; + +} // namespace gc +} // namespace js + +#endif /* gc_RelocationOverlay_h */ diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp new file mode 100644 index 0000000000..c397711933 --- /dev/null +++ b/js/src/gc/RootMarking.cpp @@ -0,0 +1,467 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_VALGRIND +# include +#endif + +#include "jstypes.h" + +#include "debugger/DebugAPI.h" +#include "gc/ClearEdgesTracer.h" +#include "gc/GCInternals.h" +#include "gc/PublicIterators.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "js/ValueArray.h" +#include "vm/BigIntType.h" +#include "vm/Compartment.h" +#include "vm/HelperThreadState.h" +#include "vm/JSContext.h" + +using namespace js; +using namespace js::gc; + +using mozilla::LinkedList; + +using JS::AutoGCRooter; + +using RootRange = RootedValueMap::Range; +using RootEntry = RootedValueMap::Entry; +using RootEnum = RootedValueMap::Enum; + +template +inline void TypedRootedGCThingBase::trace(JSTracer* trc, + const char* name) { + auto* self = this->template derived(); + TraceNullableRoot(trc, self->address(), name); +} + +template +static inline void TraceExactStackRootList(JSTracer* trc, + StackRootedBase* listHead, + const char* name) { + // Check size of Rooted does not increase. + static_assert(sizeof(Rooted) == sizeof(T) + 2 * sizeof(uintptr_t)); + + for (StackRootedBase* root = listHead; root; root = root->previous()) { + static_cast*>(root)->trace(trc, name); + } +} + +static inline void TraceExactStackRootTraceableList(JSTracer* trc, + StackRootedBase* listHead, + const char* name) { + for (StackRootedBase* root = listHead; root; root = root->previous()) { + static_cast(root)->trace(trc, name); + } +} + +static inline void TraceStackRoots(JSTracer* trc, + JS::RootedListHeads& stackRoots) { +#define TRACE_ROOTS(name, type, _, _1) \ + TraceExactStackRootList(trc, stackRoots[JS::RootKind::name], \ + "exact-" #name); + JS_FOR_EACH_TRACEKIND(TRACE_ROOTS) +#undef TRACE_ROOTS + TraceExactStackRootList(trc, stackRoots[JS::RootKind::Id], "exact-id"); + TraceExactStackRootList(trc, stackRoots[JS::RootKind::Value], + "exact-value"); + + // RootedTraceable uses virtual dispatch. + JS::AutoSuppressGCAnalysis nogc; + + TraceExactStackRootTraceableList(trc, stackRoots[JS::RootKind::Traceable], + "Traceable"); +} + +void JS::RootingContext::traceStackRoots(JSTracer* trc) { + TraceStackRoots(trc, stackRoots_); +} + +static void TraceExactStackRoots(JSContext* cx, JSTracer* trc) { + cx->traceStackRoots(trc); +} + +template +static inline void TracePersistentRootedList( + JSTracer* trc, LinkedList& list, const char* name) { + for (PersistentRootedBase* root : list) { + static_cast*>(root)->trace(trc, name); + } +} + +static inline void TracePersistentRootedTraceableList( + JSTracer* trc, LinkedList& list, const char* name) { + for (PersistentRootedBase* root : list) { + static_cast(root)->trace(trc, name); + } +} + +void JSRuntime::tracePersistentRoots(JSTracer* trc) { +#define TRACE_ROOTS(name, type, _, _1) \ + TracePersistentRootedList(trc, heapRoots.ref()[JS::RootKind::name], \ + "persistent-" #name); + JS_FOR_EACH_TRACEKIND(TRACE_ROOTS) +#undef TRACE_ROOTS + TracePersistentRootedList(trc, heapRoots.ref()[JS::RootKind::Id], + "persistent-id"); + TracePersistentRootedList(trc, heapRoots.ref()[JS::RootKind::Value], + "persistent-value"); + + // RootedTraceable uses virtual dispatch. + JS::AutoSuppressGCAnalysis nogc; + + TracePersistentRootedTraceableList( + trc, heapRoots.ref()[JS::RootKind::Traceable], "persistent-traceable"); +} + +static void TracePersistentRooted(JSRuntime* rt, JSTracer* trc) { + rt->tracePersistentRoots(trc); +} + +template +static void FinishPersistentRootedChain( + LinkedList& list) { + while (!list.isEmpty()) { + static_cast*>(list.getFirst())->reset(); + } +} + +void JSRuntime::finishPersistentRoots() { +#define FINISH_ROOT_LIST(name, type, _, _1) \ + FinishPersistentRootedChain(heapRoots.ref()[JS::RootKind::name]); + JS_FOR_EACH_TRACEKIND(FINISH_ROOT_LIST) +#undef FINISH_ROOT_LIST + FinishPersistentRootedChain(heapRoots.ref()[JS::RootKind::Id]); + FinishPersistentRootedChain(heapRoots.ref()[JS::RootKind::Value]); + + // Note that we do not finalize the Traceable list as we do not know how to + // safely clear members. We instead assert that none escape the RootLists. + // See the comment on RootLists::~RootLists for details. +} + +JS_PUBLIC_API void js::TraceValueArray(JSTracer* trc, size_t length, + Value* elements) { + TraceRootRange(trc, length, elements, "JS::RootedValueArray"); +} + +void AutoGCRooter::trace(JSTracer* trc) { + switch (kind_) { + case Kind::Wrapper: + static_cast(this)->trace(trc); + break; + + case Kind::WrapperVector: + static_cast(this)->trace(trc); + break; + + case Kind::Custom: + static_cast(this)->trace(trc); + break; + + default: + MOZ_CRASH("Bad AutoGCRooter::Kind"); + break; + } +} + +void AutoWrapperRooter::trace(JSTracer* trc) { + /* + * We need to use TraceManuallyBarrieredEdge here because we trace wrapper + * roots in every slice. This is because of some rule-breaking in + * RemapAllWrappersForObject; see comment there. + */ + TraceManuallyBarrieredEdge(trc, &value.get(), "js::AutoWrapperRooter.value"); +} + +void AutoWrapperVector::trace(JSTracer* trc) { + /* + * We need to use TraceManuallyBarrieredEdge here because we trace wrapper + * roots in every slice. This is because of some rule-breaking in + * RemapAllWrappersForObject; see comment there. + */ + for (WrapperValue& value : *this) { + TraceManuallyBarrieredEdge(trc, &value.get(), + "js::AutoWrapperVector.vector"); + } +} + +void JS::RootingContext::traceAllGCRooters(JSTracer* trc) { + for (AutoGCRooter* list : autoGCRooters_) { + traceGCRooterList(trc, list); + } +} + +void JS::RootingContext::traceWrapperGCRooters(JSTracer* trc) { + traceGCRooterList(trc, autoGCRooters_[AutoGCRooter::Kind::Wrapper]); + traceGCRooterList(trc, autoGCRooters_[AutoGCRooter::Kind::WrapperVector]); +} + +/* static */ +inline void JS::RootingContext::traceGCRooterList(JSTracer* trc, + AutoGCRooter* head) { + for (AutoGCRooter* rooter = head; rooter; rooter = rooter->down) { + rooter->trace(trc); + } +} + +void PropertyDescriptor::trace(JSTracer* trc) { + TraceRoot(trc, &value_, "Descriptor::value"); + if (getter_) { + TraceRoot(trc, &getter_, "Descriptor::getter"); + } + if (setter_) { + TraceRoot(trc, &setter_, "Descriptor::setter"); + } +} + +void js::gc::GCRuntime::traceRuntimeForMajorGC(JSTracer* trc, + AutoGCSession& session) { + MOZ_ASSERT(!TlsContext.get()->suppressGC); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS); + + // We only need to trace atoms when we're marking; atoms are never moved by + // compacting GC. + if (atomsZone()->isGCMarking()) { + traceRuntimeAtoms(trc); + } + + { + // Trace incoming cross compartment edges from uncollected compartments, + // skipping gray edges which are traced later. + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_CCWS); + Compartment::traceIncomingCrossCompartmentEdgesForZoneGC( + trc, Compartment::NonGrayEdges); + } + + traceRuntimeCommon(trc, MarkRuntime); +} + +void js::gc::GCRuntime::traceRuntimeForMinorGC(JSTracer* trc, + AutoGCSession& session) { + MOZ_ASSERT(!TlsContext.get()->suppressGC); + + // Note that we *must* trace the runtime during the SHUTDOWN_GC's minor GC + // despite having called FinishRoots already. This is because FinishRoots + // does not clear the crossCompartmentWrapper map. It cannot do this + // because Proxy's trace for CrossCompartmentWrappers asserts presence in + // the map. And we can reach its trace function despite having finished the + // roots via the edges stored by the pre-barrier verifier when we finish + // the verifier for the last time. + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS); + + traceRuntimeCommon(trc, TraceRuntime); +} + +void js::TraceRuntime(JSTracer* trc) { + MOZ_ASSERT(!trc->isMarkingTracer()); + + JSRuntime* rt = trc->runtime(); + AutoEmptyNurseryAndPrepareForTracing prep(rt->mainContextFromOwnThread()); + gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP); + rt->gc.traceRuntime(trc, prep); +} + +void js::TraceRuntimeWithoutEviction(JSTracer* trc) { + MOZ_ASSERT(!trc->isMarkingTracer()); + + JSRuntime* rt = trc->runtime(); + AutoTraceSession session(rt); + gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP); + rt->gc.traceRuntime(trc, session); +} + +void js::gc::GCRuntime::traceRuntime(JSTracer* trc, AutoTraceSession& session) { + MOZ_ASSERT(!rt->isBeingDestroyed()); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS); + + traceRuntimeAtoms(trc); + traceRuntimeCommon(trc, TraceRuntime); +} + +void js::gc::GCRuntime::traceRuntimeAtoms(JSTracer* trc) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_RUNTIME_DATA); + TraceAtoms(trc); + jit::JitRuntime::TraceAtomZoneRoots(trc); +} + +void js::gc::GCRuntime::traceRuntimeCommon(JSTracer* trc, + TraceOrMarkRuntime traceOrMark) { + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_STACK); + + JSContext* cx = rt->mainContextFromOwnThread(); + + // Trace active interpreter and JIT stack roots. + TraceInterpreterActivations(cx, trc); + jit::TraceJitActivations(cx, trc); + + // Trace legacy C stack roots. + cx->traceAllGCRooters(trc); + + // Trace C stack roots. + TraceExactStackRoots(cx, trc); + + for (RootRange r = rootsHash.ref().all(); !r.empty(); r.popFront()) { + const RootEntry& entry = r.front(); + TraceRoot(trc, entry.key(), entry.value()); + } + } + + // Trace runtime global roots. + TracePersistentRooted(rt, trc); + +#ifdef JS_HAS_INTL_API + // Trace the shared Intl data. + rt->traceSharedIntlData(trc); +#endif + + // Trace the JSContext. + rt->mainContextFromOwnThread()->trace(trc); + + // Trace all realm roots, but not the realm itself; it is traced via the + // parent pointer if traceRoots actually traces anything. + for (RealmsIter r(rt); !r.done(); r.next()) { + r->traceRoots(trc, traceOrMark); + } + + if (!JS::RuntimeHeapIsMinorCollecting()) { + // Trace the self-hosting stencil. The contents of this are always tenured. + rt->traceSelfHostingStencil(trc); + + for (ZonesIter zone(this, ZoneSelector::SkipAtoms); !zone.done(); + zone.next()) { + zone->traceRootsInMajorGC(trc); + } + + // Trace interpreter entry code generated with --emit-interpreter-entry + if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) { + rt->jitRuntime()->getInterpreterEntryMap()->traceTrampolineCode(trc); + } + } + + // Trace helper thread roots. + HelperThreadState().trace(trc); + + // Trace Debugger.Frames that have live hooks, since dropping them would be + // observable. In effect, they are rooted by the stack frames. + DebugAPI::traceFramesWithLiveHooks(trc); + + // Trace the embedding's black and gray roots. + if (!JS::RuntimeHeapIsMinorCollecting()) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_EMBEDDING); + + /* + * The embedding can register additional roots here. + * + * We don't need to trace these in a minor GC because all pointers into + * the nursery should be in the store buffer, and we want to avoid the + * time taken to trace all these roots. + */ + traceEmbeddingBlackRoots(trc); + + /* During GC, we don't trace gray roots at this stage. */ + if (traceOrMark == TraceRuntime) { + traceEmbeddingGrayRoots(trc); + } + } + + traceKeptObjects(trc); +} + +void GCRuntime::traceEmbeddingBlackRoots(JSTracer* trc) { + // The analysis doesn't like the function pointer below. + JS::AutoSuppressGCAnalysis nogc; + + for (size_t i = 0; i < blackRootTracers.ref().length(); i++) { + const Callback& e = blackRootTracers.ref()[i]; + (*e.op)(trc, e.data); + } +} + +void GCRuntime::traceEmbeddingGrayRoots(JSTracer* trc) { + SliceBudget budget = SliceBudget::unlimited(); + MOZ_ALWAYS_TRUE(traceEmbeddingGrayRoots(trc, budget) == Finished); +} + +IncrementalProgress GCRuntime::traceEmbeddingGrayRoots(JSTracer* trc, + SliceBudget& budget) { + // The analysis doesn't like the function pointer below. + JS::AutoSuppressGCAnalysis nogc; + + const auto& callback = grayRootTracer.ref(); + if (!callback.op) { + return Finished; + } + + return callback.op(trc, budget, callback.data) ? Finished : NotFinished; +} + +#ifdef DEBUG +class AssertNoRootsTracer final : public JS::CallbackTracer { + void onChild(JS::GCCellPtr thing, const char* name) override { + MOZ_CRASH("There should not be any roots during runtime shutdown"); + } + + public: + // This skips tracking WeakMap entries because they are not roots. + explicit AssertNoRootsTracer(JSRuntime* rt) + : JS::CallbackTracer(rt, JS::TracerKind::Callback, + JS::WeakMapTraceAction::Skip) {} +}; +#endif // DEBUG + +void js::gc::GCRuntime::finishRoots() { + AutoNoteSingleThreadedRegion anstr; + + rt->finishAtoms(); + restoreSharedAtomsZone(); + + rootsHash.ref().clear(); + + rt->finishPersistentRoots(); + + rt->finishSelfHosting(); + + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) { + zone->finishRoots(); + } + +#ifdef JS_GC_ZEAL + clearSelectedForMarking(); +#endif + + // Clear out the interpreter entry map before the final gc. + ClearInterpreterEntryMap(rt); + + // Clear any remaining roots from the embedding (as otherwise they will be + // left dangling after we shut down) and remove the callbacks. + ClearEdgesTracer trc(rt); + traceEmbeddingBlackRoots(&trc); + traceEmbeddingGrayRoots(&trc); + clearBlackAndGrayRootTracers(); +} + +void js::gc::GCRuntime::checkNoRuntimeRoots(AutoGCSession& session) { +#ifdef DEBUG + AssertNoRootsTracer trc(rt); + traceRuntimeForMajorGC(&trc, session); +#endif // DEBUG +} + +JS_PUBLIC_API void JS::AddPersistentRoot(JS::RootingContext* cx, RootKind kind, + PersistentRootedBase* root) { + JSRuntime* rt = static_cast(cx)->runtime(); + rt->heapRoots.ref()[kind].insertBack(root); +} + +JS_PUBLIC_API void JS::AddPersistentRoot(JSRuntime* rt, RootKind kind, + PersistentRootedBase* root) { + rt->heapRoots.ref()[kind].insertBack(root); +} diff --git a/js/src/gc/Scheduling.cpp b/js/src/gc/Scheduling.cpp new file mode 100644 index 0000000000..ec03c85f8d --- /dev/null +++ b/js/src/gc/Scheduling.cpp @@ -0,0 +1,873 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Scheduling.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/TimeStamp.h" + +#include +#include + +#include "gc/Memory.h" +#include "gc/Nursery.h" +#include "gc/RelocationOverlay.h" +#include "gc/ZoneAllocator.h" +#include "util/DifferentialTesting.h" +#include "vm/MutexIDs.h" + +using namespace js; +using namespace js::gc; + +using mozilla::CheckedInt; +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Some; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +/* + * We may start to collect a zone before its trigger threshold is reached if + * GCRuntime::maybeGC() is called for that zone or we start collecting other + * zones. These eager threshold factors are not configurable. + */ +static constexpr double HighFrequencyEagerAllocTriggerFactor = 0.85; +static constexpr double LowFrequencyEagerAllocTriggerFactor = 0.9; + +/* + * Don't allow heap growth factors to be set so low that eager collections could + * reduce the trigger threshold. + */ +static constexpr double MinHeapGrowthFactor = + 1.0f / std::min(HighFrequencyEagerAllocTriggerFactor, + LowFrequencyEagerAllocTriggerFactor); + +// Limit various parameters to reasonable levels to catch errors. +static constexpr double MaxHeapGrowthFactor = 100; +static constexpr size_t MaxNurseryBytesParam = 128 * 1024 * 1024; + +namespace { + +// Helper classes to marshal GC parameter values to/from uint32_t. + +template +struct ConvertGeneric { + static uint32_t toUint32(T value) { + static_assert(std::is_arithmetic_v); + if constexpr (std::is_signed_v) { + MOZ_ASSERT(value >= 0); + } + if constexpr (!std::is_same_v && + std::numeric_limits::max() > + std::numeric_limits::max()) { + MOZ_ASSERT(value <= UINT32_MAX); + } + return uint32_t(value); + } + static Maybe fromUint32(uint32_t param) { + // Currently we use explicit conversion and don't range check. + return Some(T(param)); + } +}; + +using ConvertBool = ConvertGeneric; +using ConvertSize = ConvertGeneric; +using ConvertDouble = ConvertGeneric; + +struct ConvertTimes100 { + static uint32_t toUint32(double value) { return uint32_t(value * 100.0); } + static Maybe fromUint32(uint32_t param) { + return Some(double(param) / 100.0); + } +}; + +struct ConvertNurseryBytes : ConvertSize { + static Maybe fromUint32(uint32_t param) { + return Some(Nursery::roundSize(param)); + } +}; + +struct ConvertKB { + static uint32_t toUint32(size_t value) { return value / 1024; } + static Maybe fromUint32(uint32_t param) { + // Parameters which represent heap sizes in bytes are restricted to values + // which can be represented on 32 bit platforms. + CheckedInt size = CheckedInt(param) * 1024; + return size.isValid() ? Some(size_t(size.value())) : Nothing(); + } +}; + +struct ConvertMB { + static uint32_t toUint32(size_t value) { return value / (1024 * 1024); } + static Maybe fromUint32(uint32_t param) { + // Parameters which represent heap sizes in bytes are restricted to values + // which can be represented on 32 bit platforms. + CheckedInt size = CheckedInt(param) * 1024 * 1024; + return size.isValid() ? Some(size_t(size.value())) : Nothing(); + } +}; + +struct ConvertMillis { + static uint32_t toUint32(TimeDuration value) { + return uint32_t(value.ToMilliseconds()); + } + static Maybe fromUint32(uint32_t param) { + return Some(TimeDuration::FromMilliseconds(param)); + } +}; + +struct ConvertSeconds { + static uint32_t toUint32(TimeDuration value) { + return uint32_t(value.ToSeconds()); + } + static Maybe fromUint32(uint32_t param) { + return Some(TimeDuration::FromSeconds(param)); + } +}; + +} // anonymous namespace + +// Helper functions to check GC parameter values + +template +static bool NoCheck(T value) { + return true; +} + +template +static bool CheckNonZero(T value) { + return value != 0; +} + +static bool CheckNurserySize(size_t bytes) { + return bytes >= SystemPageSize() && bytes <= MaxNurseryBytesParam; +} + +static bool CheckHeapGrowth(double growth) { + return growth >= MinHeapGrowthFactor && growth <= MaxHeapGrowthFactor; +} + +static bool CheckIncrementalLimit(double factor) { + return factor >= 1.0 && factor <= MaxHeapGrowthFactor; +} + +static bool CheckNonZeroUnitRange(double value) { + return value > 0.0 && value <= 100.0; +} + +GCSchedulingTunables::GCSchedulingTunables() { +#define INIT_TUNABLE_FIELD(key, type, name, convert, check, default) \ + name##_ = default; \ + MOZ_ASSERT(check(name##_)); + FOR_EACH_GC_TUNABLE(INIT_TUNABLE_FIELD) +#undef INIT_TUNABLE_FIELD + + checkInvariants(); +} + +uint32_t GCSchedulingTunables::getParameter(JSGCParamKey key) { + switch (key) { +#define GET_TUNABLE_FIELD(key, type, name, convert, check, default) \ + case key: \ + return convert::toUint32(name##_); + FOR_EACH_GC_TUNABLE(GET_TUNABLE_FIELD) +#undef GET_TUNABLE_FIELD + + default: + MOZ_CRASH("Unknown parameter key"); + } +} + +bool GCSchedulingTunables::setParameter(JSGCParamKey key, uint32_t value) { + auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); }); + + switch (key) { +#define SET_TUNABLE_FIELD(key, type, name, convert, check, default) \ + case key: { \ + Maybe converted = convert::fromUint32(value); \ + if (!converted || !check(converted.value())) { \ + return false; \ + } \ + name##_ = converted.value(); \ + break; \ + } + FOR_EACH_GC_TUNABLE(SET_TUNABLE_FIELD) +#undef SET_TUNABLE_FIELD + + default: + MOZ_CRASH("Unknown GC parameter."); + } + + maintainInvariantsAfterUpdate(key); + return true; +} + +void GCSchedulingTunables::resetParameter(JSGCParamKey key) { + auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); }); + + switch (key) { +#define RESET_TUNABLE_FIELD(key, type, name, convert, check, default) \ + case key: \ + name##_ = default; \ + MOZ_ASSERT(check(name##_)); \ + break; + FOR_EACH_GC_TUNABLE(RESET_TUNABLE_FIELD) +#undef RESET_TUNABLE_FIELD + + default: + MOZ_CRASH("Unknown GC parameter."); + } + + maintainInvariantsAfterUpdate(key); +} + +void GCSchedulingTunables::maintainInvariantsAfterUpdate(JSGCParamKey updated) { + switch (updated) { + case JSGC_MIN_NURSERY_BYTES: + if (gcMaxNurseryBytes_ < gcMinNurseryBytes_) { + gcMaxNurseryBytes_ = gcMinNurseryBytes_; + } + break; + case JSGC_MAX_NURSERY_BYTES: + if (gcMinNurseryBytes_ > gcMaxNurseryBytes_) { + gcMinNurseryBytes_ = gcMaxNurseryBytes_; + } + break; + case JSGC_SMALL_HEAP_SIZE_MAX: + if (smallHeapSizeMaxBytes_ >= largeHeapSizeMinBytes_) { + largeHeapSizeMinBytes_ = smallHeapSizeMaxBytes_ + 1; + } + break; + case JSGC_LARGE_HEAP_SIZE_MIN: + if (largeHeapSizeMinBytes_ <= smallHeapSizeMaxBytes_) { + smallHeapSizeMaxBytes_ = largeHeapSizeMinBytes_ - 1; + } + break; + case JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH: + if (highFrequencySmallHeapGrowth_ < highFrequencyLargeHeapGrowth_) { + highFrequencyLargeHeapGrowth_ = highFrequencySmallHeapGrowth_; + } + break; + case JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH: + if (highFrequencyLargeHeapGrowth_ > highFrequencySmallHeapGrowth_) { + highFrequencySmallHeapGrowth_ = highFrequencyLargeHeapGrowth_; + } + break; + default: + break; + } +} + +void GCSchedulingTunables::checkInvariants() { + MOZ_ASSERT(gcMinNurseryBytes_ == Nursery::roundSize(gcMinNurseryBytes_)); + MOZ_ASSERT(gcMaxNurseryBytes_ == Nursery::roundSize(gcMaxNurseryBytes_)); + MOZ_ASSERT(gcMinNurseryBytes_ <= gcMaxNurseryBytes_); + MOZ_ASSERT(gcMinNurseryBytes_ >= SystemPageSize()); + MOZ_ASSERT(gcMaxNurseryBytes_ <= MaxNurseryBytesParam); + + MOZ_ASSERT(largeHeapSizeMinBytes_ > smallHeapSizeMaxBytes_); + + MOZ_ASSERT(lowFrequencyHeapGrowth_ >= MinHeapGrowthFactor); + MOZ_ASSERT(lowFrequencyHeapGrowth_ <= MaxHeapGrowthFactor); + + MOZ_ASSERT(highFrequencySmallHeapGrowth_ >= MinHeapGrowthFactor); + MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor); + MOZ_ASSERT(highFrequencyLargeHeapGrowth_ <= highFrequencySmallHeapGrowth_); + MOZ_ASSERT(highFrequencyLargeHeapGrowth_ >= MinHeapGrowthFactor); + MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor); +} + +void GCSchedulingState::updateHighFrequencyMode( + const mozilla::TimeStamp& lastGCTime, const mozilla::TimeStamp& currentTime, + const GCSchedulingTunables& tunables) { + if (js::SupportDifferentialTesting()) { + return; + } + + inHighFrequencyGCMode_ = + !lastGCTime.IsNull() && + lastGCTime + tunables.highFrequencyThreshold() > currentTime; +} + +void GCSchedulingState::updateHighFrequencyModeForReason(JS::GCReason reason) { + // These reason indicate that the embedding isn't triggering GC slices often + // enough and allocation rate is high. + if (reason == JS::GCReason::ALLOC_TRIGGER || + reason == JS::GCReason::TOO_MUCH_MALLOC) { + inHighFrequencyGCMode_ = true; + } +} + +static constexpr size_t BytesPerMB = 1024 * 1024; +static constexpr double CollectionRateSmoothingFactor = 0.5; +static constexpr double AllocationRateSmoothingFactor = 0.5; + +static double ExponentialMovingAverage(double prevAverage, double newData, + double smoothingFactor) { + MOZ_ASSERT(smoothingFactor > 0.0 && smoothingFactor <= 1.0); + return smoothingFactor * newData + (1.0 - smoothingFactor) * prevAverage; +} + +void js::ZoneAllocator::updateCollectionRate( + mozilla::TimeDuration mainThreadGCTime, size_t initialBytesForAllZones) { + MOZ_ASSERT(initialBytesForAllZones != 0); + MOZ_ASSERT(gcHeapSize.initialBytes() <= initialBytesForAllZones); + + double zoneFraction = + double(gcHeapSize.initialBytes()) / double(initialBytesForAllZones); + double zoneDuration = mainThreadGCTime.ToSeconds() * zoneFraction + + perZoneGCTime.ref().ToSeconds(); + double collectionRate = + double(gcHeapSize.initialBytes()) / (zoneDuration * BytesPerMB); + + if (!smoothedCollectionRate.ref()) { + smoothedCollectionRate = Some(collectionRate); + } else { + double prevRate = smoothedCollectionRate.ref().value(); + smoothedCollectionRate = Some(ExponentialMovingAverage( + prevRate, collectionRate, CollectionRateSmoothingFactor)); + } +} + +void js::ZoneAllocator::updateAllocationRate(TimeDuration mutatorTime) { + // To get the total size allocated since the last collection we have to + // take account of how much memory got freed in the meantime. + size_t freedBytes = gcHeapSize.freedBytes(); + + size_t sizeIncludingFreedBytes = gcHeapSize.bytes() + freedBytes; + + MOZ_ASSERT(prevGCHeapSize <= sizeIncludingFreedBytes); + size_t allocatedBytes = sizeIncludingFreedBytes - prevGCHeapSize; + + double allocationRate = + double(allocatedBytes) / (mutatorTime.ToSeconds() * BytesPerMB); + + if (!smoothedAllocationRate.ref()) { + smoothedAllocationRate = Some(allocationRate); + } else { + double prevRate = smoothedAllocationRate.ref().value(); + smoothedAllocationRate = Some(ExponentialMovingAverage( + prevRate, allocationRate, AllocationRateSmoothingFactor)); + } + + gcHeapSize.clearFreedBytes(); + prevGCHeapSize = gcHeapSize.bytes(); +} + +// GC thresholds may exceed the range of size_t on 32-bit platforms, so these +// are calculated using 64-bit integers and clamped. +static inline size_t ToClampedSize(uint64_t bytes) { + return std::min(bytes, uint64_t(SIZE_MAX)); +} + +void HeapThreshold::setIncrementalLimitFromStartBytes( + size_t retainedBytes, const GCSchedulingTunables& tunables) { + // Calculate the incremental limit for a heap based on its size and start + // threshold. + // + // This effectively classifies the heap size into small, medium or large, and + // uses the small heap incremental limit paramer, the large heap incremental + // limit parameter or an interpolation between them. + // + // The incremental limit is always set greater than the start threshold by at + // least the maximum nursery size to reduce the chance that tenuring a full + // nursery will send us straight into non-incremental collection. + + MOZ_ASSERT(tunables.smallHeapIncrementalLimit() >= + tunables.largeHeapIncrementalLimit()); + + double factor = LinearInterpolate( + retainedBytes, tunables.smallHeapSizeMaxBytes(), + tunables.smallHeapIncrementalLimit(), tunables.largeHeapSizeMinBytes(), + tunables.largeHeapIncrementalLimit()); + + uint64_t bytes = + std::max(uint64_t(double(startBytes_) * factor), + uint64_t(startBytes_) + tunables.gcMaxNurseryBytes()); + incrementalLimitBytes_ = ToClampedSize(bytes); + MOZ_ASSERT(incrementalLimitBytes_ >= startBytes_); + + // Maintain the invariant that the slice threshold is always less than the + // incremental limit when adjusting GC parameters. + if (hasSliceThreshold() && sliceBytes() > incrementalLimitBytes()) { + sliceBytes_ = incrementalLimitBytes(); + } +} + +double HeapThreshold::eagerAllocTrigger(bool highFrequencyGC) const { + double eagerTriggerFactor = highFrequencyGC + ? HighFrequencyEagerAllocTriggerFactor + : LowFrequencyEagerAllocTriggerFactor; + return eagerTriggerFactor * startBytes(); +} + +void HeapThreshold::setSliceThreshold(ZoneAllocator* zone, + const HeapSize& heapSize, + const GCSchedulingTunables& tunables, + bool waitingOnBGTask) { + // Set the allocation threshold at which to trigger the a GC slice in an + // ongoing incremental collection. This is used to ensure progress in + // allocation heavy code that may not return to the main event loop. + // + // The threshold is based on the JSGC_ZONE_ALLOC_DELAY_KB parameter, but this + // is reduced to increase the slice frequency as we approach the incremental + // limit, in the hope that we never reach it. If collector is waiting for a + // background task to complete, don't trigger any slices until we reach the + // urgent threshold. + + size_t bytesRemaining = incrementalBytesRemaining(heapSize); + bool isUrgent = bytesRemaining < tunables.urgentThresholdBytes(); + + size_t delayBeforeNextSlice = tunables.zoneAllocDelayBytes(); + if (isUrgent) { + double fractionRemaining = + double(bytesRemaining) / double(tunables.urgentThresholdBytes()); + delayBeforeNextSlice = + size_t(double(delayBeforeNextSlice) * fractionRemaining); + MOZ_ASSERT(delayBeforeNextSlice <= tunables.zoneAllocDelayBytes()); + } else if (waitingOnBGTask) { + delayBeforeNextSlice = bytesRemaining - tunables.urgentThresholdBytes(); + } + + sliceBytes_ = ToClampedSize( + std::min(uint64_t(heapSize.bytes()) + uint64_t(delayBeforeNextSlice), + uint64_t(incrementalLimitBytes_))); +} + +size_t HeapThreshold::incrementalBytesRemaining( + const HeapSize& heapSize) const { + if (heapSize.bytes() >= incrementalLimitBytes_) { + return 0; + } + + return incrementalLimitBytes_ - heapSize.bytes(); +} + +/* static */ +double HeapThreshold::computeZoneHeapGrowthFactorForHeapSize( + size_t lastBytes, const GCSchedulingTunables& tunables, + const GCSchedulingState& state) { + // For small zones, our collection heuristics do not matter much: favor + // something simple in this case. + if (lastBytes < 1 * 1024 * 1024) { + return tunables.lowFrequencyHeapGrowth(); + } + + // The heap growth factor depends on the heap size after a GC and the GC + // frequency. If GC's are not triggering in rapid succession, use a lower + // threshold so that we will collect garbage sooner. + if (!state.inHighFrequencyGCMode()) { + return tunables.lowFrequencyHeapGrowth(); + } + + // For high frequency GCs we let the heap grow depending on whether we + // classify the heap as small, medium or large. There are parameters for small + // and large heap sizes and linear interpolation is used between them for + // medium sized heaps. + + MOZ_ASSERT(tunables.smallHeapSizeMaxBytes() <= + tunables.largeHeapSizeMinBytes()); + MOZ_ASSERT(tunables.highFrequencyLargeHeapGrowth() <= + tunables.highFrequencySmallHeapGrowth()); + + return LinearInterpolate(lastBytes, tunables.smallHeapSizeMaxBytes(), + tunables.highFrequencySmallHeapGrowth(), + tunables.largeHeapSizeMinBytes(), + tunables.highFrequencyLargeHeapGrowth()); +} + +/* static */ +size_t GCHeapThreshold::computeZoneTriggerBytes( + double growthFactor, size_t lastBytes, + const GCSchedulingTunables& tunables) { + size_t base = std::max(lastBytes, tunables.gcZoneAllocThresholdBase()); + double trigger = double(base) * growthFactor; + double triggerMax = + double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit(); + return ToClampedSize(std::min(triggerMax, trigger)); +} + +// Parameters for balanced heap limits computation. + +// The W0 parameter. How much memory can be traversed in the minimum collection +// time. +static constexpr double BalancedHeapBaseMB = 5.0; + +// The minimum heap limit. Do not constrain the heap to any less than this size. +static constexpr double MinBalancedHeapLimitMB = 10.0; + +// The minimum amount of additional space to allow beyond the retained size. +static constexpr double MinBalancedHeadroomMB = 3.0; + +// The maximum factor by which to expand the heap beyond the retained size. +static constexpr double MaxHeapGrowth = 3.0; + +// The default allocation rate in MB/s allocated by the mutator to use before we +// have an estimate. Used to set the heap limit for zones that have not yet been +// collected. +static constexpr double DefaultAllocationRate = 0.0; + +// The s0 parameter. The default collection rate in MB/s to use before we have +// an estimate. Used to set the heap limit for zones that have not yet been +// collected. +static constexpr double DefaultCollectionRate = 200.0; + +double GCHeapThreshold::computeBalancedHeapLimit( + size_t lastBytes, double allocationRate, double collectionRate, + const GCSchedulingTunables& tunables) { + MOZ_ASSERT(tunables.balancedHeapLimitsEnabled()); + + // Optimal heap limits as described in https://arxiv.org/abs/2204.10455 + + double W = double(lastBytes) / BytesPerMB; // Retained size / MB. + double W0 = BalancedHeapBaseMB; + double d = tunables.heapGrowthFactor(); // Rearranged constant 'c'. + double g = allocationRate; + double s = collectionRate; + double f = d * sqrt((W + W0) * (g / s)); + double M = W + std::min(f, MaxHeapGrowth * W); + M = std::max({MinBalancedHeapLimitMB, W + MinBalancedHeadroomMB, M}); + + return M * double(BytesPerMB); +} + +void GCHeapThreshold::updateStartThreshold( + size_t lastBytes, mozilla::Maybe allocationRate, + mozilla::Maybe collectionRate, const GCSchedulingTunables& tunables, + const GCSchedulingState& state, bool isAtomsZone) { + if (!tunables.balancedHeapLimitsEnabled()) { + double growthFactor = + computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state); + + startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes, tunables); + } else { + double threshold = computeBalancedHeapLimit( + lastBytes, allocationRate.valueOr(DefaultAllocationRate), + collectionRate.valueOr(DefaultCollectionRate), tunables); + + double triggerMax = + double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit(); + + startBytes_ = ToClampedSize(uint64_t(std::min(triggerMax, threshold))); + } + + setIncrementalLimitFromStartBytes(lastBytes, tunables); +} + +/* static */ +size_t MallocHeapThreshold::computeZoneTriggerBytes(double growthFactor, + size_t lastBytes, + size_t baseBytes) { + return ToClampedSize(double(std::max(lastBytes, baseBytes)) * growthFactor); +} + +void MallocHeapThreshold::updateStartThreshold( + size_t lastBytes, const GCSchedulingTunables& tunables, + const GCSchedulingState& state) { + double growthFactor = + computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state); + + startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes, + tunables.mallocThresholdBase()); + + setIncrementalLimitFromStartBytes(lastBytes, tunables); +} + +#ifdef DEBUG + +static const char* MemoryUseName(MemoryUse use) { + switch (use) { +# define DEFINE_CASE(Name) \ + case MemoryUse::Name: \ + return #Name; + JS_FOR_EACH_MEMORY_USE(DEFINE_CASE) +# undef DEFINE_CASE + } + + MOZ_CRASH("Unknown memory use"); +} + +MemoryTracker::MemoryTracker() : mutex(mutexid::MemoryTracker) {} + +void MemoryTracker::checkEmptyOnDestroy() { + bool ok = true; + + if (!gcMap.empty()) { + ok = false; + fprintf(stderr, "Missing calls to JS::RemoveAssociatedMemory:\n"); + for (auto r = gcMap.all(); !r.empty(); r.popFront()) { + fprintf(stderr, " %p 0x%zx %s\n", r.front().key().ptr(), + r.front().value(), MemoryUseName(r.front().key().use())); + } + } + + if (!nonGCMap.empty()) { + ok = false; + fprintf(stderr, "Missing calls to Zone::decNonGCMemory:\n"); + for (auto r = nonGCMap.all(); !r.empty(); r.popFront()) { + fprintf(stderr, " %p 0x%zx\n", r.front().key().ptr(), r.front().value()); + } + } + + MOZ_ASSERT(ok); +} + +/* static */ +inline bool MemoryTracker::isGCMemoryUse(MemoryUse use) { + // Most memory uses are for memory associated with GC things but some are for + // memory associated with non-GC thing pointers. + return !isNonGCMemoryUse(use); +} + +/* static */ +inline bool MemoryTracker::isNonGCMemoryUse(MemoryUse use) { + return use == MemoryUse::TrackedAllocPolicy; +} + +/* static */ +inline bool MemoryTracker::allowMultipleAssociations(MemoryUse use) { + // For most uses only one association is possible for each GC thing. Allow a + // one-to-many relationship only where necessary. + return isNonGCMemoryUse(use) || use == MemoryUse::RegExpSharedBytecode || + use == MemoryUse::BreakpointSite || use == MemoryUse::Breakpoint || + use == MemoryUse::ForOfPICStub || use == MemoryUse::ICUObject; +} + +void MemoryTracker::trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) { + MOZ_ASSERT(cell->isTenured()); + MOZ_ASSERT(isGCMemoryUse(use)); + + LockGuard lock(mutex); + + Key key{cell, use}; + AutoEnterOOMUnsafeRegion oomUnsafe; + auto ptr = gcMap.lookupForAdd(key); + if (ptr) { + if (!allowMultipleAssociations(use)) { + MOZ_CRASH_UNSAFE_PRINTF("Association already present: %p 0x%zx %s", cell, + nbytes, MemoryUseName(use)); + } + ptr->value() += nbytes; + return; + } + + if (!gcMap.add(ptr, key, nbytes)) { + oomUnsafe.crash("MemoryTracker::trackGCMemory"); + } +} + +void MemoryTracker::untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) { + MOZ_ASSERT(cell->isTenured()); + + LockGuard lock(mutex); + + Key key{cell, use}; + auto ptr = gcMap.lookup(key); + if (!ptr) { + MOZ_CRASH_UNSAFE_PRINTF("Association not found: %p 0x%zx %s", cell, nbytes, + MemoryUseName(use)); + } + + if (!allowMultipleAssociations(use) && ptr->value() != nbytes) { + MOZ_CRASH_UNSAFE_PRINTF( + "Association for %p %s has different size: " + "expected 0x%zx but got 0x%zx", + cell, MemoryUseName(use), ptr->value(), nbytes); + } + + if (nbytes > ptr->value()) { + MOZ_CRASH_UNSAFE_PRINTF( + "Association for %p %s size is too large: " + "expected at most 0x%zx but got 0x%zx", + cell, MemoryUseName(use), ptr->value(), nbytes); + } + + ptr->value() -= nbytes; + + if (ptr->value() == 0) { + gcMap.remove(ptr); + } +} + +void MemoryTracker::swapGCMemory(Cell* a, Cell* b, MemoryUse use) { + Key ka{a, use}; + Key kb{b, use}; + + LockGuard lock(mutex); + + size_t sa = getAndRemoveEntry(ka, lock); + size_t sb = getAndRemoveEntry(kb, lock); + + AutoEnterOOMUnsafeRegion oomUnsafe; + + if ((sa && b->isTenured() && !gcMap.put(kb, sa)) || + (sb && a->isTenured() && !gcMap.put(ka, sb))) { + oomUnsafe.crash("MemoryTracker::swapGCMemory"); + } +} + +size_t MemoryTracker::getAndRemoveEntry(const Key& key, + LockGuard& lock) { + auto ptr = gcMap.lookup(key); + if (!ptr) { + return 0; + } + + size_t size = ptr->value(); + gcMap.remove(ptr); + return size; +} + +void MemoryTracker::registerNonGCMemory(void* mem, MemoryUse use) { + LockGuard lock(mutex); + + Key key{mem, use}; + auto ptr = nonGCMap.lookupForAdd(key); + if (ptr) { + MOZ_CRASH_UNSAFE_PRINTF("%s assocaition %p already registered", + MemoryUseName(use), mem); + } + + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!nonGCMap.add(ptr, key, 0)) { + oomUnsafe.crash("MemoryTracker::registerNonGCMemory"); + } +} + +void MemoryTracker::unregisterNonGCMemory(void* mem, MemoryUse use) { + LockGuard lock(mutex); + + Key key{mem, use}; + auto ptr = nonGCMap.lookup(key); + if (!ptr) { + MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use), + mem); + } + + if (ptr->value() != 0) { + MOZ_CRASH_UNSAFE_PRINTF( + "%s association %p still has 0x%zx bytes associated", + MemoryUseName(use), mem, ptr->value()); + } + + nonGCMap.remove(ptr); +} + +void MemoryTracker::moveNonGCMemory(void* dst, void* src, MemoryUse use) { + LockGuard lock(mutex); + + Key srcKey{src, use}; + auto srcPtr = nonGCMap.lookup(srcKey); + if (!srcPtr) { + MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use), + src); + } + + size_t nbytes = srcPtr->value(); + nonGCMap.remove(srcPtr); + + Key dstKey{dst, use}; + auto dstPtr = nonGCMap.lookupForAdd(dstKey); + if (dstPtr) { + MOZ_CRASH_UNSAFE_PRINTF("%s %p already registered", MemoryUseName(use), + dst); + } + + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!nonGCMap.add(dstPtr, dstKey, nbytes)) { + oomUnsafe.crash("MemoryTracker::moveNonGCMemory"); + } +} + +void MemoryTracker::incNonGCMemory(void* mem, size_t nbytes, MemoryUse use) { + MOZ_ASSERT(isNonGCMemoryUse(use)); + + LockGuard lock(mutex); + + Key key{mem, use}; + auto ptr = nonGCMap.lookup(key); + if (!ptr) { + MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use), + mem); + } + + ptr->value() += nbytes; +} + +void MemoryTracker::decNonGCMemory(void* mem, size_t nbytes, MemoryUse use) { + MOZ_ASSERT(isNonGCMemoryUse(use)); + + LockGuard lock(mutex); + + Key key{mem, use}; + auto ptr = nonGCMap.lookup(key); + if (!ptr) { + MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use), + mem); + } + + size_t& value = ptr->value(); + if (nbytes > value) { + MOZ_CRASH_UNSAFE_PRINTF( + "%s allocation %p is too large: " + "expected at most 0x%zx but got 0x%zx bytes", + MemoryUseName(use), mem, value, nbytes); + } + + value -= nbytes; +} + +void MemoryTracker::fixupAfterMovingGC() { + // Update the table after we move GC things. We don't use StableCellHasher + // because that would create a difference between debug and release builds. + for (GCMap::Enum e(gcMap); !e.empty(); e.popFront()) { + const auto& key = e.front().key(); + Cell* cell = key.ptr(); + if (cell->isForwarded()) { + cell = gc::RelocationOverlay::fromCell(cell)->forwardingAddress(); + e.rekeyFront(Key{cell, key.use()}); + } + } +} + +template +inline MemoryTracker::Key::Key(Ptr* ptr, MemoryUse use) + : ptr_(uint64_t(ptr)), use_(uint64_t(use)) { +# ifdef JS_64BIT + static_assert(sizeof(Key) == 8, + "MemoryTracker::Key should be packed into 8 bytes"); +# endif + MOZ_ASSERT(this->ptr() == ptr); + MOZ_ASSERT(this->use() == use); +} + +template +inline Ptr* MemoryTracker::Key::ptr() const { + return reinterpret_cast(ptr_); +} +template +inline MemoryUse MemoryTracker::Key::use() const { + return static_cast(use_); +} + +template +inline HashNumber MemoryTracker::Hasher::hash(const Lookup& l) { + return mozilla::HashGeneric(DefaultHasher::hash(l.ptr()), + DefaultHasher::hash(unsigned(l.use()))); +} + +template +inline bool MemoryTracker::Hasher::match(const KeyT& k, const Lookup& l) { + return k.ptr() == l.ptr() && k.use() == l.use(); +} + +template +inline void MemoryTracker::Hasher::rekey(KeyT& k, const KeyT& newKey) { + k = newKey; +} + +#endif // DEBUG diff --git a/js/src/gc/Scheduling.h b/js/src/gc/Scheduling.h new file mode 100644 index 0000000000..94b06bc786 --- /dev/null +++ b/js/src/gc/Scheduling.h @@ -0,0 +1,917 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * [SMDOC] GC Scheduling + * + * GC Scheduling Overview + * ====================== + * + * See also GC scheduling from Firefox's perspective here: + * https://searchfox.org/mozilla-central/source/dom/base/CCGCScheduler.cpp + * + * Scheduling GC's in SpiderMonkey/Firefox is tremendously complicated because + * of the large number of subtle, cross-cutting, and widely dispersed factors + * that must be taken into account. A summary of some of the more important + * factors follows. + * + * Cost factors: + * + * * GC too soon and we'll revisit an object graph almost identical to the + * one we just visited; since we are unlikely to find new garbage, the + * traversal will be largely overhead. We rely heavily on external factors + * to signal us that we are likely to find lots of garbage: e.g. "a tab + * just got closed". + * + * * GC too late and we'll run out of memory to allocate (e.g. Out-Of-Memory, + * hereafter simply abbreviated to OOM). If this happens inside + * SpiderMonkey we may be able to recover, but most embedder allocations + * will simply crash on OOM, even if the GC has plenty of free memory it + * could surrender. + * + * * Memory fragmentation: if we fill the process with GC allocations, a + * request for a large block of contiguous memory may fail because no + * contiguous block is free, despite having enough memory available to + * service the request. + * + * * Management overhead: if our GC heap becomes large, we create extra + * overhead when managing the GC's structures, even if the allocations are + * mostly unused. + * + * Heap Management Factors: + * + * * GC memory: The GC has its own allocator that it uses to make fixed size + * allocations for GC managed things. In cases where the GC thing requires + * larger or variable sized memory to implement itself, it is responsible + * for using the system heap. + * + * * C Heap Memory: Rather than allowing for large or variable allocations, + * the SpiderMonkey GC allows GC things to hold pointers to C heap memory. + * It is the responsibility of the thing to free this memory with a custom + * finalizer (with the sole exception of NativeObject, which knows about + * slots and elements for performance reasons). C heap memory has different + * performance and overhead tradeoffs than GC internal memory, which need + * to be considered with scheduling a GC. + * + * Application Factors: + * + * * Most applications allocate heavily at startup, then enter a processing + * stage where memory utilization remains roughly fixed with a slower + * allocation rate. This is not always the case, however, so while we may + * optimize for this pattern, we must be able to handle arbitrary + * allocation patterns. + * + * Other factors: + * + * * Other memory: This is memory allocated outside the purview of the GC. + * Data mapped by the system for code libraries, data allocated by those + * libraries, data in the JSRuntime that is used to manage the engine, + * memory used by the embedding that is not attached to a GC thing, memory + * used by unrelated processes running on the hardware that use space we + * could otherwise use for allocation, etc. While we don't have to manage + * it, we do have to take it into account when scheduling since it affects + * when we will OOM. + * + * * Physical Reality: All real machines have limits on the number of bits + * that they are physically able to store. While modern operating systems + * can generally make additional space available with swapping, at some + * point there are simply no more bits to allocate. There is also the + * factor of address space limitations, particularly on 32bit machines. + * + * * Platform Factors: Each OS makes use of wildly different memory + * management techniques. These differences result in different performance + * tradeoffs, different fragmentation patterns, and different hard limits + * on the amount of physical and/or virtual memory that we can use before + * OOMing. + * + * + * Reasons for scheduling GC + * ------------------------- + * + * While code generally takes the above factors into account in only an ad-hoc + * fashion, the API forces the user to pick a "reason" for the GC. We have a + * bunch of JS::GCReason reasons in GCAPI.h. These fall into a few categories + * that generally coincide with one or more of the above factors. + * + * Embedding reasons: + * + * 1) Do a GC now because the embedding knows something useful about the + * zone's memory retention state. These are GCReasons like LOAD_END, + * PAGE_HIDE, SET_NEW_DOCUMENT, DOM_UTILS. Mostly, Gecko uses these to + * indicate that a significant fraction of the scheduled zone's memory is + * probably reclaimable. + * + * 2) Do some known amount of GC work now because the embedding knows now is + * a good time to do a long, unblockable operation of a known duration. + * These are INTER_SLICE_GC and REFRESH_FRAME. + * + * Correctness reasons: + * + * 3) Do a GC now because correctness depends on some GC property. For + * example, CC_FORCED is where the embedding requires the mark bits to be + * set correctly. Also, EVICT_NURSERY where we need to work on the tenured + * heap. + * + * 4) Do a GC because we are shutting down: e.g. SHUTDOWN_CC or DESTROY_*. + * + * 5) Do a GC because a compartment was accessed between GC slices when we + * would have otherwise discarded it. We have to do a second GC to clean + * it up: e.g. COMPARTMENT_REVIVED. + * + * Emergency Reasons: + * + * 6) Do an all-zones, non-incremental GC now because the embedding knows it + * cannot wait: e.g. MEM_PRESSURE. + * + * 7) OOM when fetching a new Chunk results in a LAST_DITCH GC. + * + * Heap Size Limitation Reasons: + * + * 8) Do an incremental, zonal GC with reason MAYBEGC when we discover that + * the gc's allocated size is approaching the current trigger. This is + * called MAYBEGC because we make this check in the MaybeGC function. + * MaybeGC gets called at the top of the main event loop. Normally, it is + * expected that this callback will keep the heap size limited. It is + * relatively inexpensive, because it is invoked with no JS running and + * thus few stack roots to scan. For this reason, the GC's "trigger" bytes + * is less than the GC's "max" bytes as used by the trigger below. + * + * 9) Do an incremental, zonal GC with reason MAYBEGC when we go to allocate + * a new GC thing and find that the GC heap size has grown beyond the + * configured maximum (JSGC_MAX_BYTES). We trigger this GC by returning + * nullptr and then calling maybeGC at the top level of the allocator. + * This is then guaranteed to fail the "size greater than trigger" check + * above, since trigger is always less than max. After performing the GC, + * the allocator unconditionally returns nullptr to force an OOM exception + * is raised by the script. + * + * Note that this differs from a LAST_DITCH GC where we actually run out + * of memory (i.e., a call to a system allocator fails) when trying to + * allocate. Unlike above, LAST_DITCH GC only happens when we are really + * out of memory, not just when we cross an arbitrary trigger; despite + * this, it may still return an allocation at the end and allow the script + * to continue, if the LAST_DITCH GC was able to free up enough memory. + * + * 10) Do a GC under reason ALLOC_TRIGGER when we are over the GC heap trigger + * limit, but in the allocator rather than in a random call to maybeGC. + * This occurs if we allocate too much before returning to the event loop + * and calling maybeGC; this is extremely common in benchmarks and + * long-running Worker computations. Note that this uses a wildly + * different mechanism from the above in that it sets the interrupt flag + * and does the GC at the next loop head, before the next alloc, or + * maybeGC. The reason for this is that this check is made after the + * allocation and we cannot GC with an uninitialized thing in the heap. + * + * 11) Do an incremental, zonal GC with reason TOO_MUCH_MALLOC when the total + * amount of malloced memory is greater than the malloc trigger limit for the + * zone. + * + * + * Size Limitation Triggers Explanation + * ------------------------------------ + * + * The GC internally is entirely unaware of the context of the execution of + * the mutator. It sees only: + * + * A) Allocated size: this is the amount of memory currently requested by the + * mutator. This quantity is monotonically increasing: i.e. the allocation + * rate is always >= 0. It is also easy for the system to track. + * + * B) Retained size: this is the amount of memory that the mutator can + * currently reach. Said another way, it is the size of the heap + * immediately after a GC (modulo background sweeping). This size is very + * costly to know exactly and also extremely hard to estimate with any + * fidelity. + * + * For reference, a common allocated vs. retained graph might look like: + * + * | ** ** + * | ** ** * ** + * | ** * ** * ** + * | * ** * ** * ** + * | ** ** * ** * ** + * s| * * ** ** + + ** + * i| * * * + + + + + + * z| * * * + + + + + + * e| * **+ + * | * + + * | * + + * | * + + * | * + + * | * + + * |*+ + * +-------------------------------------------------- + * time + * *** = allocated + * +++ = retained + * + * Note that this is a bit of a simplification + * because in reality we track malloc and GC heap + * sizes separately and have a different level of + * granularity and accuracy on each heap. + * + * This presents some obvious implications for Mark-and-Sweep collectors. + * Namely: + * -> t[marking] ~= size[retained] + * -> t[sweeping] ~= size[allocated] - size[retained] + * + * In a non-incremental collector, maintaining low latency and high + * responsiveness requires that total GC times be as low as possible. Thus, + * in order to stay responsive when we did not have a fully incremental + * collector, our GC triggers were focused on minimizing collection time. + * Furthermore, since size[retained] is not under control of the GC, all the + * GC could do to control collection times was reduce sweep times by + * minimizing size[allocated], per the equation above. + * + * The result of the above is GC triggers that focus on size[allocated] to + * the exclusion of other important factors and default heuristics that are + * not optimal for a fully incremental collector. On the other hand, this is + * not all bad: minimizing size[allocated] also minimizes the chance of OOM + * and sweeping remains one of the hardest areas to further incrementalize. + * + * EAGER_ALLOC_TRIGGER + * ------------------- + * Occurs when we return to the event loop and find our heap is getting + * largish, but before t[marking] OR t[sweeping] is too large for a + * responsive non-incremental GC. This is intended to be the common case + * in normal web applications: e.g. we just finished an event handler and + * the few objects we allocated when computing the new whatzitz have + * pushed us slightly over the limit. After this GC we rescale the new + * EAGER_ALLOC_TRIGGER trigger to 150% of size[retained] so that our + * non-incremental GC times will always be proportional to this size + * rather than being dominated by sweeping. + * + * As a concession to mutators that allocate heavily during their startup + * phase, we have a highFrequencyGCMode that ups the growth rate to 300% + * of the current size[retained] so that we'll do fewer longer GCs at the + * end of the mutator startup rather than more, smaller GCs. + * + * Assumptions: + * -> Responsiveness is proportional to t[marking] + t[sweeping]. + * -> size[retained] is proportional only to GC allocations. + * + * ALLOC_TRIGGER (non-incremental) + * ------------------------------- + * If we do not return to the event loop before getting all the way to our + * gc trigger bytes then MAYBEGC will never fire. To avoid OOMing, we + * succeed the current allocation and set the script interrupt so that we + * will (hopefully) do a GC before we overflow our max and have to raise + * an OOM exception for the script. + * + * Assumptions: + * -> Common web scripts will return to the event loop before using + * 10% of the current triggerBytes worth of GC memory. + * + * ALLOC_TRIGGER (incremental) + * --------------------------- + * In practice the above trigger is rough: if a website is just on the + * cusp, sometimes it will trigger a non-incremental GC moments before + * returning to the event loop, where it could have done an incremental + * GC. Thus, we recently added an incremental version of the above with a + * substantially lower threshold, so that we have a soft limit here. If + * IGC can collect faster than the allocator generates garbage, even if + * the allocator does not return to the event loop frequently, we should + * not have to fall back to a non-incremental GC. + * + * INCREMENTAL_TOO_SLOW + * -------------------- + * Do a full, non-incremental GC if we overflow ALLOC_TRIGGER during an + * incremental GC. When in the middle of an incremental GC, we suppress + * our other triggers, so we need a way to backstop the IGC if the + * mutator allocates faster than the IGC can clean things up. + * + * TOO_MUCH_MALLOC + * --------------- + * Performs a GC before size[allocated] - size[retained] gets too large + * for non-incremental sweeping to be fast in the case that we have + * significantly more malloc allocation than GC allocation. This is meant + * to complement MAYBEGC triggers. We track this by counting malloced + * bytes; the counter gets reset at every GC since we do not always have a + * size at the time we call free. Because of this, the malloc heuristic + * is, unfortunately, not usefully able to augment our other GC heap + * triggers and is limited to this singular heuristic. + * + * Assumptions: + * -> EITHER size[allocated_by_malloc] ~= size[allocated_by_GC] + * OR time[sweeping] ~= size[allocated_by_malloc] + * -> size[retained] @ t0 ~= size[retained] @ t1 + * i.e. That the mutator is in steady-state operation. + * + * LAST_DITCH_GC + * ------------- + * Does a GC because we are out of memory. + * + * Assumptions: + * -> size[retained] < size[available_memory] + */ + +#ifndef gc_Scheduling_h +#define gc_Scheduling_h + +#include "mozilla/Atomics.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "gc/GCEnum.h" +#include "js/AllocPolicy.h" +#include "js/GCAPI.h" +#include "js/HashTable.h" +#include "js/HeapAPI.h" +#include "threading/ProtectedData.h" + +// Macro to define scheduling tunables for GC parameters. Expands its argument +// repeatedly with the following arguments: +// - key: the JSGCParamKey value for this parameter +// - type: the storage type +// - name: the name of GCSchedulingTunables getter method +// - convert: a helper class defined in Scheduling.cpp that provides +// conversion methods +// - check: a helper function defined in Scheduling.cppto check the value is +// valid +// - default: the initial value and that assigned by resetParameter +#define FOR_EACH_GC_TUNABLE(_) \ + /* \ + * JSGC_MAX_BYTES \ + * \ + * Maximum nominal heap before last ditch GC. \ + */ \ + _(JSGC_MAX_BYTES, size_t, gcMaxBytes, ConvertSize, NoCheck, 0xffffffff) \ + \ + /* \ + * JSGC_MIN_NURSERY_BYTES \ + * JSGC_MAX_NURSERY_BYTES \ + * \ + * Minimum and maximum nursery size for each runtime. \ + */ \ + _(JSGC_MIN_NURSERY_BYTES, size_t, gcMinNurseryBytes, ConvertNurseryBytes, \ + CheckNurserySize, 256 * 1024) \ + _(JSGC_MAX_NURSERY_BYTES, size_t, gcMaxNurseryBytes, ConvertNurseryBytes, \ + CheckNurserySize, JS::DefaultNurseryMaxBytes) \ + \ + /* \ + * JSGC_ALLOCATION_THRESHOLD \ + * \ + * \ + * The base value used to compute zone->threshold.bytes(). When \ + * gcHeapSize.bytes() exceeds threshold.bytes() for a zone, the zone may be \ + * scheduled for a GC, depending on the exact circumstances. \ + */ \ + _(JSGC_ALLOCATION_THRESHOLD, size_t, gcZoneAllocThresholdBase, ConvertMB, \ + NoCheck, 27 * 1024 * 1024) \ + \ + /* \ + * JSGC_SMALL_HEAP_SIZE_MAX \ + * JSGC_LARGE_HEAP_SIZE_MIN \ + * \ + * Used to classify heap sizes into one of small, medium or large. This \ + * affects the calcuation of the incremental GC trigger and the heap growth \ + * factor in high frequency GC mode. \ + */ \ + _(JSGC_SMALL_HEAP_SIZE_MAX, size_t, smallHeapSizeMaxBytes, ConvertMB, \ + NoCheck, 100 * 1024 * 1024) \ + _(JSGC_LARGE_HEAP_SIZE_MIN, size_t, largeHeapSizeMinBytes, ConvertMB, \ + CheckNonZero, 500 * 1024 * 1024) \ + \ + /* \ + * JSGC_SMALL_HEAP_INCREMENTAL_LIMIT \ + * JSGC_LARGE_HEAP_INCREMENTAL_LIMIT \ + * \ + * Multiple of threshold.bytes() which triggers a non-incremental GC. \ + * \ + * The small heap limit must be greater than 1.3 to maintain performance on \ + * splay-latency. \ + */ \ + _(JSGC_SMALL_HEAP_INCREMENTAL_LIMIT, double, smallHeapIncrementalLimit, \ + ConvertTimes100, CheckIncrementalLimit, 1.50) \ + _(JSGC_LARGE_HEAP_INCREMENTAL_LIMIT, double, largeHeapIncrementalLimit, \ + ConvertTimes100, CheckIncrementalLimit, 1.10) \ + \ + /* \ + * JSGC_HIGH_FREQUENCY_TIME_LIMIT \ + * \ + * We enter high-frequency mode if we GC a twice within this many \ + * millisconds. \ + */ \ + _(JSGC_HIGH_FREQUENCY_TIME_LIMIT, mozilla::TimeDuration, \ + highFrequencyThreshold, ConvertMillis, NoCheck, \ + mozilla::TimeDuration::FromSeconds(1)) \ + \ + /* \ + * JSGC_LOW_FREQUENCY_HEAP_GROWTH \ + * \ + * When not in |highFrequencyGC| mode, this is the global (stored per-zone) \ + * "HeapGrowthFactor". \ + */ \ + _(JSGC_LOW_FREQUENCY_HEAP_GROWTH, double, lowFrequencyHeapGrowth, \ + ConvertTimes100, CheckHeapGrowth, 1.5) \ + \ + /* \ + * JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH \ + * JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH \ + * \ + * When in the |highFrequencyGC| mode, these parameterize the per-zone \ + * "HeapGrowthFactor" computation. \ + */ \ + _(JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH, double, \ + highFrequencySmallHeapGrowth, ConvertTimes100, CheckHeapGrowth, 3.0) \ + _(JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH, double, \ + highFrequencyLargeHeapGrowth, ConvertTimes100, CheckHeapGrowth, 1.5) \ + \ + /* \ + * JSGC_MALLOC_THRESHOLD_BASE \ + * \ + * The base value used to compute the GC trigger for malloc allocated \ + * memory. \ + */ \ + _(JSGC_MALLOC_THRESHOLD_BASE, size_t, mallocThresholdBase, ConvertMB, \ + NoCheck, 38 * 1024 * 1024) \ + \ + /* \ + * Number of bytes to allocate between incremental slices in GCs triggered \ + * by the zone allocation threshold. \ + */ \ + _(JSGC_ZONE_ALLOC_DELAY_KB, size_t, zoneAllocDelayBytes, ConvertKB, \ + CheckNonZero, 1024 * 1024) \ + \ + /* \ + * JSGC_URGENT_THRESHOLD_MB \ + * \ + * The point before reaching the non-incremental limit at which to start \ + * increasing the slice budget and frequency of allocation triggered slices. \ + */ \ + _(JSGC_URGENT_THRESHOLD_MB, size_t, urgentThresholdBytes, ConvertMB, \ + NoCheck, 16 * 1024 * 1024) \ + \ + /* \ + * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION \ + * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_FRACTION \ + * JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS \ + * \ + * Attempt to run a minor GC in the idle time if the free space falls below \ + * this threshold or if it hasn't been collected for too long. The absolute \ + * threshold is used when the nursery is large and the percentage when it is \ + * small. See Nursery::shouldCollect(). \ + */ \ + _(JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION, size_t, \ + nurseryFreeThresholdForIdleCollection, ConvertSize, NoCheck, \ + ChunkSize / 4) \ + _(JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_PERCENT, double, \ + nurseryFreeThresholdForIdleCollectionFraction, ConvertTimes100, \ + CheckNonZeroUnitRange, 0.25) \ + _(JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS, mozilla::TimeDuration, \ + nurseryTimeoutForIdleCollection, ConvertMillis, NoCheck, \ + mozilla::TimeDuration::FromSeconds(5)) \ + \ + /* \ + * JSGC_BALANCED_HEAP_LIMITS_ENABLED \ + * JSGC_HEAP_GROWTH_FACTOR \ + */ \ + _(JSGC_BALANCED_HEAP_LIMITS_ENABLED, bool, balancedHeapLimitsEnabled, \ + ConvertBool, NoCheck, false) \ + _(JSGC_HEAP_GROWTH_FACTOR, double, heapGrowthFactor, ConvertDouble, NoCheck, \ + 50.0) \ + \ + /* \ + * JSGC_PRETENURE_THRESHOLD \ + * \ + * Fraction of objects tenured to trigger pretenuring (between 0 and 1). If \ + * this fraction is met, the GC proceeds to calculate which objects will be \ + * tenured. If this is 1.0f (actually if it is not < 1.0f) then pretenuring \ + * is disabled. \ + */ \ + _(JSGC_PRETENURE_THRESHOLD, double, pretenureThreshold, ConvertTimes100, \ + CheckNonZeroUnitRange, 0.6) \ + \ + /* \ + * JSGC_PRETENURE_STRING_THRESHOLD \ + * \ + * If the percentage of the tenured strings exceeds this threshold, string \ + * will be allocated in tenured heap instead. (Default is allocated in \ + * nursery.) \ + */ \ + _(JSGC_PRETENURE_STRING_THRESHOLD, double, pretenureStringThreshold, \ + ConvertTimes100, CheckNonZeroUnitRange, 0.55) \ + \ + /* \ + * JSGC_STOP_PRETENURE_STRING_THRESHOLD \ + * \ + * If the finalization rate of the tenured strings exceeds this threshold, \ + * string will be allocated in nursery. \ + */ \ + _(JSGC_STOP_PRETENURE_STRING_THRESHOLD, double, \ + stopPretenureStringThreshold, ConvertTimes100, CheckNonZeroUnitRange, 0.9) \ + \ + /* \ + * JSGC_MIN_LAST_DITCH_GC_PERIOD \ + * \ + * Last ditch GC is skipped if allocation failure occurs less than this many \ + * seconds from the previous one. \ + */ \ + _(JSGC_MIN_LAST_DITCH_GC_PERIOD, mozilla::TimeDuration, \ + minLastDitchGCPeriod, ConvertSeconds, NoCheck, \ + TimeDuration::FromSeconds(60)) \ + \ + /* \ + * JSGC_PARALLEL_MARKING_THRESHOLD_KB \ + */ \ + _(JSGC_PARALLEL_MARKING_THRESHOLD_KB, size_t, parallelMarkingThresholdBytes, \ + ConvertKB, NoCheck, 10 * 1024 * 1024) + +namespace js { + +class ZoneAllocator; + +namespace gc { + +struct Cell; + +/* + * Default settings for tuning the GC. Some of these can be set at runtime, + * This list is not complete, some tuning parameters are not listed here. + * + * If you change the values here, please also consider changing them in + * modules/libpref/init/all.js where they are duplicated for the Firefox + * preferences. + */ +namespace TuningDefaults { + +/* JSGC_MIN_EMPTY_CHUNK_COUNT */ +static const uint32_t MinEmptyChunkCount = 1; + +/* JSGC_MAX_EMPTY_CHUNK_COUNT */ +static const uint32_t MaxEmptyChunkCount = 30; + +/* JSGC_SLICE_TIME_BUDGET_MS */ +static const int64_t DefaultTimeBudgetMS = 0; // Unlimited by default. + +/* JSGC_INCREMENTAL_GC_ENABLED */ +static const bool IncrementalGCEnabled = false; + +/* JSGC_PER_ZONE_GC_ENABLED */ +static const bool PerZoneGCEnabled = false; + +/* JSGC_COMPACTING_ENABLED */ +static const bool CompactingEnabled = true; + +/* JSGC_PARALLEL_MARKING_ENABLED */ +static const bool ParallelMarkingEnabled = false; + +/* JSGC_INCREMENTAL_WEAKMAP_ENABLED */ +static const bool IncrementalWeakMapMarkingEnabled = true; + +/* JSGC_HELPER_THREAD_RATIO */ +static const double HelperThreadRatio = 0.5; + +/* JSGC_MAX_HELPER_THREADS */ +static const size_t MaxHelperThreads = 8; + +} // namespace TuningDefaults + +/* + * Encapsulates all of the GC tunables. These are effectively constant and + * should only be modified by setParameter. + */ +class GCSchedulingTunables { +#define DEFINE_TUNABLE_FIELD(key, type, name, convert, check, default) \ + MainThreadOrGCTaskData name##_; + FOR_EACH_GC_TUNABLE(DEFINE_TUNABLE_FIELD) +#undef DEFINE_TUNABLE_FIELD + + public: + GCSchedulingTunables(); + +#define DEFINE_TUNABLE_ACCESSOR(key, type, name, convert, check, default) \ + type name() const { return name##_; } + FOR_EACH_GC_TUNABLE(DEFINE_TUNABLE_ACCESSOR) +#undef DEFINE_TUNABLE_ACCESSOR + + bool attemptPretenuring() const { return pretenureThreshold_ < 1.0; } + + uint32_t getParameter(JSGCParamKey key); + [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value); + void resetParameter(JSGCParamKey key); + + private: + void maintainInvariantsAfterUpdate(JSGCParamKey updated); + void checkInvariants(); +}; + +class GCSchedulingState { + /* + * Influences how we schedule and run GC's in several subtle ways. The most + * important factor is in how it controls the "HeapGrowthFactor". The + * growth factor is a measure of how large (as a percentage of the last GC) + * the heap is allowed to grow before we try to schedule another GC. + */ + mozilla::Atomic inHighFrequencyGCMode_; + + public: + GCSchedulingState() : inHighFrequencyGCMode_(false) {} + + bool inHighFrequencyGCMode() const { return inHighFrequencyGCMode_; } + + void updateHighFrequencyMode(const mozilla::TimeStamp& lastGCTime, + const mozilla::TimeStamp& currentTime, + const GCSchedulingTunables& tunables); + void updateHighFrequencyModeForReason(JS::GCReason reason); +}; + +struct TriggerResult { + bool shouldTrigger; + size_t usedBytes; + size_t thresholdBytes; +}; + +using AtomicByteCount = mozilla::Atomic; + +/* + * Tracks the size of allocated data. This is used for both GC and malloc data. + * It automatically maintains the memory usage relationship between parent and + * child instances, i.e. between those in a GCRuntime and its Zones. + */ +class HeapSize { + /* + * The number of bytes in use. For GC heaps this is approximate to the nearest + * ArenaSize. It is atomic because it is updated by both the active and GC + * helper threads. + */ + AtomicByteCount bytes_; + + /* + * The number of bytes in use at the start of the last collection. + */ + MainThreadData initialBytes_; + + /* + * The number of bytes retained after the last collection. This is updated + * dynamically during incremental GC. It does not include allocations that + * happen during a GC. + */ + AtomicByteCount retainedBytes_; + + public: + explicit HeapSize() { + MOZ_ASSERT(bytes_ == 0); + MOZ_ASSERT(retainedBytes_ == 0); + } + + size_t bytes() const { return bytes_; } + size_t initialBytes() const { return initialBytes_; } + size_t retainedBytes() const { return retainedBytes_; } + + void updateOnGCStart() { retainedBytes_ = initialBytes_ = bytes(); } + + void addGCArena() { addBytes(ArenaSize); } + void removeGCArena() { + MOZ_ASSERT(retainedBytes_ >= ArenaSize); + removeBytes(ArenaSize, true /* only sweeping removes arenas */); + MOZ_ASSERT(retainedBytes_ <= bytes_); + } + + void addBytes(size_t nbytes) { + mozilla::DebugOnly initialBytes(bytes_); + MOZ_ASSERT(initialBytes + nbytes > initialBytes); + bytes_ += nbytes; + } + void removeBytes(size_t nbytes, bool updateRetainedSize) { + if (updateRetainedSize) { + MOZ_ASSERT(retainedBytes_ >= nbytes); + retainedBytes_ -= nbytes; + } + MOZ_ASSERT(bytes_ >= nbytes); + bytes_ -= nbytes; + } +}; + +/* + * Like HeapSize, but also updates a 'parent' HeapSize. Used for per-zone heap + * size data that also updates a runtime-wide parent. + */ +class HeapSizeChild : public HeapSize { + public: + void addGCArena(HeapSize& parent) { + HeapSize::addGCArena(); + parent.addGCArena(); + } + + void removeGCArena(HeapSize& parent) { + HeapSize::removeGCArena(); + parent.removeGCArena(); + } + + void addBytes(size_t nbytes, HeapSize& parent) { + HeapSize::addBytes(nbytes); + parent.addBytes(nbytes); + } + + void removeBytes(size_t nbytes, bool updateRetainedSize, HeapSize& parent) { + HeapSize::removeBytes(nbytes, updateRetainedSize); + parent.removeBytes(nbytes, updateRetainedSize); + } +}; + +class PerZoneGCHeapSize : public HeapSizeChild { + public: + size_t freedBytes() const { return freedBytes_; } + void clearFreedBytes() { freedBytes_ = 0; } + + void removeGCArena(HeapSize& parent) { + HeapSizeChild::removeGCArena(parent); + freedBytes_ += ArenaSize; + } + + void removeBytes(size_t nbytes, bool updateRetainedSize, HeapSize& parent) { + HeapSizeChild::removeBytes(nbytes, updateRetainedSize, parent); + freedBytes_ += nbytes; + } + + private: + AtomicByteCount freedBytes_; +}; + +// Heap size thresholds used to trigger GC. This is an abstract base class for +// GC heap and malloc thresholds defined below. +class HeapThreshold { + protected: + HeapThreshold() + : startBytes_(SIZE_MAX), + incrementalLimitBytes_(SIZE_MAX), + sliceBytes_(SIZE_MAX) {} + + // The threshold at which to start a new incremental collection. + // + // This can be read off main thread during collection, for example by sweep + // tasks that resize tables. + MainThreadOrGCTaskData startBytes_; + + // The threshold at which start a new non-incremental collection or finish an + // ongoing collection non-incrementally. + MainThreadData incrementalLimitBytes_; + + // The threshold at which to trigger a slice during an ongoing incremental + // collection. + MainThreadData sliceBytes_; + + public: + size_t startBytes() const { return startBytes_; } + size_t sliceBytes() const { return sliceBytes_; } + size_t incrementalLimitBytes() const { return incrementalLimitBytes_; } + double eagerAllocTrigger(bool highFrequencyGC) const; + size_t incrementalBytesRemaining(const HeapSize& heapSize) const; + + void setSliceThreshold(ZoneAllocator* zone, const HeapSize& heapSize, + const GCSchedulingTunables& tunables, + bool waitingOnBGTask); + void clearSliceThreshold() { sliceBytes_ = SIZE_MAX; } + bool hasSliceThreshold() const { return sliceBytes_ != SIZE_MAX; } + + protected: + static double computeZoneHeapGrowthFactorForHeapSize( + size_t lastBytes, const GCSchedulingTunables& tunables, + const GCSchedulingState& state); + + void setIncrementalLimitFromStartBytes(size_t retainedBytes, + const GCSchedulingTunables& tunables); +}; + +// A heap threshold that is based on a multiple of the retained size after the +// last collection adjusted based on collection frequency and retained +// size. This is used to determine when to do a zone GC based on GC heap size. +class GCHeapThreshold : public HeapThreshold { + public: + void updateStartThreshold(size_t lastBytes, + mozilla::Maybe allocationRate, + mozilla::Maybe collectionRate, + const GCSchedulingTunables& tunables, + const GCSchedulingState& state, bool isAtomsZone); + + private: + // This is our original algorithm for calculating heap limits. + static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes, + const GCSchedulingTunables& tunables); + + // This is the algorithm described in the optimal heap limits paper. + static double computeBalancedHeapLimit(size_t lastBytes, + double allocationRate, + double collectionRate, + const GCSchedulingTunables& tunables); +}; + +// A heap threshold that is calculated as a constant multiple of the retained +// size after the last collection. This is used to determines when to do a zone +// GC based on malloc data. +class MallocHeapThreshold : public HeapThreshold { + public: + void updateStartThreshold(size_t lastBytes, + const GCSchedulingTunables& tunables, + const GCSchedulingState& state); + + private: + static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes, + size_t baseBytes); +}; + +// A fixed threshold that's used to determine when we need to do a zone GC based +// on allocated JIT code. +class JitHeapThreshold : public HeapThreshold { + public: + explicit JitHeapThreshold(size_t bytes) { startBytes_ = bytes; } +}; + +#ifdef DEBUG + +// Counts memory associated with GC things in a zone. +// +// This records details of the cell (or non-cell pointer) the memory allocation +// is associated with to check the correctness of the information provided. This +// is not present in opt builds. +class MemoryTracker { + public: + MemoryTracker(); + void fixupAfterMovingGC(); + void checkEmptyOnDestroy(); + + // Track memory by associated GC thing pointer. + void trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use); + void untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use); + void swapGCMemory(Cell* a, Cell* b, MemoryUse use); + + // Track memory by associated non-GC thing pointer. + void registerNonGCMemory(void* ptr, MemoryUse use); + void unregisterNonGCMemory(void* ptr, MemoryUse use); + void moveNonGCMemory(void* dst, void* src, MemoryUse use); + void incNonGCMemory(void* ptr, size_t nbytes, MemoryUse use); + void decNonGCMemory(void* ptr, size_t nbytes, MemoryUse use); + + private: + template + struct Key { + Key(Ptr* ptr, MemoryUse use); + Ptr* ptr() const; + MemoryUse use() const; + + private: +# ifdef JS_64BIT + // Pack this into a single word on 64 bit platforms. + uintptr_t ptr_ : 56; + uintptr_t use_ : 8; +# else + uintptr_t ptr_ : 32; + uintptr_t use_ : 8; +# endif + }; + + template + struct Hasher { + using KeyT = Key; + using Lookup = KeyT; + static HashNumber hash(const Lookup& l); + static bool match(const KeyT& key, const Lookup& l); + static void rekey(KeyT& k, const KeyT& newKey); + }; + + template + using Map = HashMap, size_t, Hasher, SystemAllocPolicy>; + using GCMap = Map; + using NonGCMap = Map; + + static bool isGCMemoryUse(MemoryUse use); + static bool isNonGCMemoryUse(MemoryUse use); + static bool allowMultipleAssociations(MemoryUse use); + + size_t getAndRemoveEntry(const Key& key, LockGuard& lock); + + Mutex mutex MOZ_UNANNOTATED; + + // Map containing the allocated size associated with (cell, use) pairs. + GCMap gcMap; + + // Map containing the allocated size associated (non-cell pointer, use) pairs. + NonGCMap nonGCMap; +}; + +#endif // DEBUG + +static inline double LinearInterpolate(double x, double x0, double y0, + double x1, double y1) { + MOZ_ASSERT(x0 < x1); + + if (x < x0) { + return y0; + } + + if (x < x1) { + return y0 + (y1 - y0) * ((x - x0) / (x1 - x0)); + } + + return y1; +} + +} // namespace gc +} // namespace js + +#endif // gc_Scheduling_h diff --git a/js/src/gc/StableCellHasher-inl.h b/js/src/gc/StableCellHasher-inl.h new file mode 100644 index 0000000000..f76748afef --- /dev/null +++ b/js/src/gc/StableCellHasher-inl.h @@ -0,0 +1,245 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StableCellHasher_inl_h +#define gc_StableCellHasher_inl_h + +#include "gc/StableCellHasher.h" + +#include "mozilla/HashFunctions.h" + +#include "gc/Cell.h" +#include "gc/Marking.h" +#include "gc/Zone.h" +#include "vm/JSObject.h" +#include "vm/NativeObject.h" +#include "vm/Runtime.h" + +namespace js { +namespace gc { + +extern uint64_t NextCellUniqueId(JSRuntime* rt); + +inline bool MaybeGetUniqueId(Cell* cell, uint64_t* uidp) { + MOZ_ASSERT(uidp); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) || + CurrentThreadIsPerformingGC()); + + if (cell->is()) { + JSObject* obj = cell->as(); + if (obj->is()) { + auto* nobj = &obj->as(); + if (!nobj->hasUniqueId()) { + return false; + } + + *uidp = nobj->uniqueId(); + return true; + } + } + + // Get an existing uid, if one has been set. + auto p = cell->zone()->uniqueIds().readonlyThreadsafeLookup(cell); + if (!p) { + return false; + } + + *uidp = p->value(); + + return true; +} + +extern bool CreateUniqueIdForNativeObject(NativeObject* obj, uint64_t* uidp); +extern bool CreateUniqueIdForNonNativeObject(Cell* cell, UniqueIdMap::AddPtr, + uint64_t* uidp); + +inline bool GetOrCreateUniqueId(Cell* cell, uint64_t* uidp) { + MOZ_ASSERT(uidp); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) || + CurrentThreadIsPerformingGC()); + + if (cell->is()) { + JSObject* obj = cell->as(); + if (obj->is()) { + auto* nobj = &obj->as(); + if (nobj->hasUniqueId()) { + *uidp = nobj->uniqueId(); + return true; + } + + return CreateUniqueIdForNativeObject(nobj, uidp); + } + } + + // Get an existing uid, if one has been set. + auto p = cell->zone()->uniqueIds().lookupForAdd(cell); + if (p) { + *uidp = p->value(); + return true; + } + + return CreateUniqueIdForNonNativeObject(cell, p, uidp); +} + +inline bool SetOrUpdateUniqueId(JSContext* cx, Cell* cell, uint64_t uid) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread())); + + if (cell->is()) { + JSObject* obj = cell->as(); + if (obj->is()) { + auto* nobj = &obj->as(); + return nobj->setOrUpdateUniqueId(cx, uid); + } + } + + // If the cell was in the nursery, hopefully unlikely, then we need to + // tell the nursery about it so that it can sweep the uid if the thing + // does not get tenured. + JSRuntime* runtime = cell->runtimeFromMainThread(); + if (IsInsideNursery(cell) && + !runtime->gc.nursery().addedUniqueIdToCell(cell)) { + return false; + } + + return cell->zone()->uniqueIds().put(cell, uid); +} + +inline uint64_t GetUniqueIdInfallible(Cell* cell) { + uint64_t uid; + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!GetOrCreateUniqueId(cell, &uid)) { + oomUnsafe.crash("failed to allocate uid"); + } + return uid; +} + +inline bool HasUniqueId(Cell* cell) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) || + CurrentThreadIsPerformingGC()); + + if (cell->is()) { + JSObject* obj = cell->as(); + if (obj->is()) { + return obj->as().hasUniqueId(); + } + } + + return cell->zone()->uniqueIds().has(cell); +} + +inline void TransferUniqueId(Cell* tgt, Cell* src) { + MOZ_ASSERT(src != tgt); + MOZ_ASSERT(!IsInsideNursery(tgt)); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(tgt->runtimeFromAnyThread())); + MOZ_ASSERT(src->zone() == tgt->zone()); + + Zone* zone = tgt->zone(); + MOZ_ASSERT(!zone->uniqueIds().has(tgt)); + zone->uniqueIds().rekeyIfMoved(src, tgt); +} + +inline void RemoveUniqueId(Cell* cell) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread())); + // The cell may no longer be in the hash table if it was swapped with a + // NativeObject. + cell->zone()->uniqueIds().remove(cell); +} + +} // namespace gc + +static inline js::HashNumber UniqueIdToHash(uint64_t uid) { + // This uses the default hasher which returns the lower 32 bits of 64 bit + // integers as the hash code. This is OK because he result will be scrambled + // later by ScrambleHashCode(). + return DefaultHasher::hash(uid); +} + +template +/* static */ bool StableCellHasher::maybeGetHash(const Lookup& l, + HashNumber* hashOut) { + if (!l) { + *hashOut = 0; + return true; + } + + uint64_t uid; + if (!gc::MaybeGetUniqueId(l, &uid)) { + return false; + } + + *hashOut = UniqueIdToHash(uid); + return true; +} + +template +/* static */ bool StableCellHasher::ensureHash(const Lookup& l, + HashNumber* hashOut) { + if (!l) { + *hashOut = 0; + return true; + } + + uint64_t uid; + if (!gc::GetOrCreateUniqueId(l, &uid)) { + return false; + } + + *hashOut = UniqueIdToHash(uid); + return true; +} + +template +/* static */ HashNumber StableCellHasher::hash(const Lookup& l) { + if (!l) { + return 0; + } + + // We have to access the zone from-any-thread here: a worker thread may be + // cloning a self-hosted object from the main runtime's self- hosting zone + // into another runtime. The zone's uid lock will protect against multiple + // workers doing this simultaneously. + MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) || + CurrentThreadIsPerformingGC()); + + return UniqueIdToHash(gc::GetUniqueIdInfallible(l)); +} + +template +/* static */ bool StableCellHasher::match(const Key& k, const Lookup& l) { + if (k == l) { + return true; + } + + if (!k || !l) { + return false; + } + + MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) || + CurrentThreadIsPerformingGC()); + +#ifdef DEBUG + // Incremental table sweeping means that existing table entries may no + // longer have unique IDs. We fail the match in that case and the entry is + // removed from the table later on. + if (!gc::HasUniqueId(k)) { + Key key = k; + MOZ_ASSERT(IsAboutToBeFinalizedUnbarriered(key)); + } + MOZ_ASSERT(gc::HasUniqueId(l)); +#endif + + uint64_t keyId; + if (!gc::MaybeGetUniqueId(k, &keyId)) { + // Key is dead and cannot match lookup which must be live. + return false; + } + + return keyId == gc::GetUniqueIdInfallible(l); +} + +} // namespace js + +#endif // gc_StableCellHasher_inl_h diff --git a/js/src/gc/StableCellHasher.h b/js/src/gc/StableCellHasher.h new file mode 100644 index 0000000000..80b1856976 --- /dev/null +++ b/js/src/gc/StableCellHasher.h @@ -0,0 +1,46 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StableCellHasher_h +#define gc_StableCellHasher_h + +#include "js/RootingAPI.h" +#include "js/TypeDecls.h" + +namespace js { + +// StableCellHasher itself is defined in js/public/RootingAPI.h. + +namespace gc { + +struct Cell; + +// Gets an existing UID in |uidp| if one exists. +[[nodiscard]] bool MaybeGetUniqueId(Cell* cell, uint64_t* uidp); + +// Puts an existing UID in |uidp|, or creates a new UID for this Cell and +// puts that into |uidp|. Returns false on OOM. +[[nodiscard]] bool GetOrCreateUniqueId(Cell* cell, uint64_t* uidp); + +uint64_t GetUniqueIdInfallible(Cell* cell); + +// Return true if this cell has a UID associated with it. +[[nodiscard]] bool HasUniqueId(Cell* cell); + +// Transfer an id from another cell. This must only be called on behalf of a +// moving GC. This method is infallible. +void TransferUniqueId(Cell* tgt, Cell* src); + +// Remove any unique id associated with this Cell. +void RemoveUniqueId(Cell* cell); + +// Used to restore unique ID after JSObject::swap. +bool SetOrUpdateUniqueId(JSContext* cx, Cell* cell, uint64_t uid); + +} // namespace gc +} // namespace js + +#endif // gc_StableCellHasher_h diff --git a/js/src/gc/Statistics.cpp b/js/src/gc/Statistics.cpp new file mode 100644 index 0000000000..66b1c40673 --- /dev/null +++ b/js/src/gc/Statistics.cpp @@ -0,0 +1,1811 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Statistics.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Sprintf.h" +#include "mozilla/TimeStamp.h" + +#include +#include +#include +#include + +#include "gc/GC.h" +#include "gc/GCInternals.h" +#include "gc/Memory.h" +#include "js/Printer.h" +#include "util/GetPidProvider.h" +#include "util/Text.h" +#include "vm/JSONPrinter.h" +#include "vm/Runtime.h" +#include "vm/Time.h" + +#include "gc/PrivateIterators-inl.h" + +using namespace js; +using namespace js::gc; +using namespace js::gcstats; + +using mozilla::DebugOnly; +using mozilla::EnumeratedArray; +using mozilla::Maybe; +using mozilla::TimeDuration; +using mozilla::TimeStamp; + +static const size_t BYTES_PER_MB = 1024 * 1024; + +/* + * If this fails, then you can either delete this assertion and allow all + * larger-numbered reasons to pile up in the last telemetry bucket, or switch + * to GC_REASON_3 and bump the max value. + */ +static_assert(JS::GCReason::NUM_TELEMETRY_REASONS >= JS::GCReason::NUM_REASONS); + +static inline auto AllPhaseKinds() { + return mozilla::MakeEnumeratedRange(PhaseKind::FIRST, PhaseKind::LIMIT); +} + +static inline auto MajorGCPhaseKinds() { + return mozilla::MakeEnumeratedRange(PhaseKind::GC_BEGIN, + PhaseKind(size_t(PhaseKind::GC_END) + 1)); +} + +static const char* ExplainGCOptions(JS::GCOptions options) { + switch (options) { + case JS::GCOptions::Normal: + return "Normal"; + case JS::GCOptions::Shrink: + return "Shrink"; + case JS::GCOptions::Shutdown: + return "Shutdown"; + } + + MOZ_CRASH("Unexpected GCOptions value"); +} + +JS_PUBLIC_API const char* JS::ExplainGCReason(JS::GCReason reason) { + switch (reason) { +#define SWITCH_REASON(name, _) \ + case JS::GCReason::name: \ + return #name; + GCREASONS(SWITCH_REASON) +#undef SWITCH_REASON + + case JS::GCReason::NO_REASON: + return "NO_REASON"; + + default: + MOZ_CRASH("bad GC reason"); + } +} + +JS_PUBLIC_API bool JS::InternalGCReason(JS::GCReason reason) { + return reason < JS::GCReason::FIRST_FIREFOX_REASON; +} + +const char* js::gcstats::ExplainAbortReason(GCAbortReason reason) { + switch (reason) { +#define SWITCH_REASON(name, _) \ + case GCAbortReason::name: \ + return #name; + GC_ABORT_REASONS(SWITCH_REASON) + + default: + MOZ_CRASH("bad GC abort reason"); +#undef SWITCH_REASON + } +} + +static FILE* MaybeOpenFileFromEnv(const char* env, + FILE* defaultFile = nullptr) { + const char* value = getenv(env); + if (!value) { + return defaultFile; + } + + FILE* file; + if (strcmp(value, "none") == 0) { + file = nullptr; + } else if (strcmp(value, "stdout") == 0) { + file = stdout; + } else if (strcmp(value, "stderr") == 0) { + file = stderr; + } else { + char path[300]; + if (value[0] != '/') { + const char* dir = getenv("MOZ_UPLOAD_DIR"); + if (dir) { + SprintfLiteral(path, "%s/%s", dir, value); + value = path; + } + } + + file = fopen(value, "a"); + if (!file || setvbuf(file, nullptr, _IOLBF, 256) != 0) { + perror("Error opening log file"); + MOZ_CRASH("Failed to open log file."); + } + } + + return file; +} + +struct PhaseKindInfo { + Phase firstPhase; + uint8_t telemetryBucket; + const char* name; +}; + +// PhaseInfo objects form a tree. +struct PhaseInfo { + Phase parent; + Phase firstChild; + Phase nextSibling; + Phase nextWithPhaseKind; + PhaseKind phaseKind; + uint8_t depth; + const char* name; + const char* path; +}; + +// A table of PhaseInfo indexed by Phase. +using PhaseTable = EnumeratedArray; + +// A table of PhaseKindInfo indexed by PhaseKind. +using PhaseKindTable = + EnumeratedArray; + +#include "gc/StatsPhasesGenerated.inc" + +// Iterate the phases in a phase kind. +class PhaseIter { + Phase phase; + + public: + explicit PhaseIter(PhaseKind kind) : phase(phaseKinds[kind].firstPhase) {} + bool done() const { return phase == Phase::NONE; } + void next() { phase = phases[phase].nextWithPhaseKind; } + Phase get() const { return phase; } + operator Phase() const { return phase; } +}; + +static double t(TimeDuration duration) { return duration.ToMilliseconds(); } + +inline JSContext* Statistics::context() { + return gc->rt->mainContextFromOwnThread(); +} + +inline Phase Statistics::currentPhase() const { + return phaseStack.empty() ? Phase::NONE : phaseStack.back(); +} + +PhaseKind Statistics::currentPhaseKind() const { + // Public API to get the current phase kind, suppressing the synthetic + // PhaseKind::MUTATOR phase. + + Phase phase = currentPhase(); + MOZ_ASSERT_IF(phase == Phase::MUTATOR, phaseStack.length() == 1); + if (phase == Phase::NONE || phase == Phase::MUTATOR) { + return PhaseKind::NONE; + } + + return phases[phase].phaseKind; +} + +static Phase LookupPhaseWithParent(PhaseKind phaseKind, Phase parentPhase) { + for (PhaseIter phase(phaseKind); !phase.done(); phase.next()) { + if (phases[phase].parent == parentPhase) { + return phase; + } + } + + return Phase::NONE; +} + +static const char* PhaseKindName(PhaseKind kind) { + if (kind == PhaseKind::NONE) { + return "NONE"; + } + + return phaseKinds[kind].name; +} + +Phase Statistics::lookupChildPhase(PhaseKind phaseKind) const { + if (phaseKind == PhaseKind::IMPLICIT_SUSPENSION) { + return Phase::IMPLICIT_SUSPENSION; + } + if (phaseKind == PhaseKind::EXPLICIT_SUSPENSION) { + return Phase::EXPLICIT_SUSPENSION; + } + + MOZ_ASSERT(phaseKind < PhaseKind::LIMIT); + + // Search all expanded phases that correspond to the required + // phase to find the one whose parent is the current expanded phase. + Phase phase = LookupPhaseWithParent(phaseKind, currentPhase()); + + if (phase == Phase::NONE) { + MOZ_CRASH_UNSAFE_PRINTF( + "Child phase kind %s not found under current phase kind %s", + PhaseKindName(phaseKind), PhaseKindName(currentPhaseKind())); + } + + return phase; +} + +inline auto AllPhases() { + return mozilla::MakeEnumeratedRange(Phase::FIRST, Phase::LIMIT); +} + +void Statistics::gcDuration(TimeDuration* total, TimeDuration* maxPause) const { + *total = *maxPause = 0; + for (auto& slice : slices_) { + *total += slice.duration(); + if (slice.duration() > *maxPause) { + *maxPause = slice.duration(); + } + } + if (*maxPause > maxPauseInInterval) { + maxPauseInInterval = *maxPause; + } +} + +void Statistics::sccDurations(TimeDuration* total, + TimeDuration* maxPause) const { + *total = *maxPause = 0; + for (size_t i = 0; i < sccTimes.length(); i++) { + *total += sccTimes[i]; + *maxPause = std::max(*maxPause, sccTimes[i]); + } +} + +typedef Vector FragmentVector; + +static UniqueChars Join(const FragmentVector& fragments, + const char* separator = "") { + const size_t separatorLength = strlen(separator); + size_t length = 0; + for (size_t i = 0; i < fragments.length(); ++i) { + length += fragments[i] ? strlen(fragments[i].get()) : 0; + if (i < (fragments.length() - 1)) { + length += separatorLength; + } + } + + char* joined = js_pod_malloc(length + 1); + if (!joined) { + return UniqueChars(); + } + + joined[length] = '\0'; + char* cursor = joined; + for (size_t i = 0; i < fragments.length(); ++i) { + if (fragments[i]) { + strcpy(cursor, fragments[i].get()); + } + cursor += fragments[i] ? strlen(fragments[i].get()) : 0; + if (i < (fragments.length() - 1)) { + if (separatorLength) { + strcpy(cursor, separator); + } + cursor += separatorLength; + } + } + + return UniqueChars(joined); +} + +static TimeDuration SumChildTimes(Phase phase, + const Statistics::PhaseTimes& phaseTimes) { + TimeDuration total = 0; + for (phase = phases[phase].firstChild; phase != Phase::NONE; + phase = phases[phase].nextSibling) { + total += phaseTimes[phase]; + } + return total; +} + +UniqueChars Statistics::formatCompactSliceMessage() const { + // Skip if we OOM'ed. + if (slices_.length() == 0) { + return UniqueChars(nullptr); + } + + const size_t index = slices_.length() - 1; + const SliceData& slice = slices_.back(); + + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + + const char* format = + "GC Slice %u - Pause: %.3fms of %s budget (@ %.3fms); Reason: %s; Reset: " + "%s%s; Times: "; + char buffer[1024]; + SprintfLiteral(buffer, format, index, t(slice.duration()), budgetDescription, + t(slice.start - slices_[0].start), + ExplainGCReason(slice.reason), + slice.wasReset() ? "yes - " : "no", + slice.wasReset() ? ExplainAbortReason(slice.resetReason) : ""); + + FragmentVector fragments; + if (!fragments.append(DuplicateString(buffer)) || + !fragments.append( + formatCompactSlicePhaseTimes(slices_[index].phaseTimes))) { + return UniqueChars(nullptr); + } + return Join(fragments); +} + +UniqueChars Statistics::formatCompactSummaryMessage() const { + FragmentVector fragments; + if (!fragments.append(DuplicateString("Summary - "))) { + return UniqueChars(nullptr); + } + + TimeDuration total, longest; + gcDuration(&total, &longest); + + const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20)); + const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50)); + + char buffer[1024]; + if (!nonincremental()) { + SprintfLiteral(buffer, + "Max Pause: %.3fms; MMU 20ms: %.1f%%; MMU 50ms: %.1f%%; " + "Total: %.3fms; ", + t(longest), mmu20 * 100., mmu50 * 100., t(total)); + } else { + SprintfLiteral(buffer, "Non-Incremental: %.3fms (%s); ", t(total), + ExplainAbortReason(nonincrementalReason_)); + } + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + + SprintfLiteral(buffer, + "Zones: %d of %d (-%d); Compartments: %d of %d (-%d); " + "HeapSize: %.3f MiB; " + "HeapChange (abs): %+d (%u); ", + zoneStats.collectedZoneCount, zoneStats.zoneCount, + zoneStats.sweptZoneCount, zoneStats.collectedCompartmentCount, + zoneStats.compartmentCount, zoneStats.sweptCompartmentCount, + double(preTotalHeapBytes) / BYTES_PER_MB, + int32_t(counts[COUNT_NEW_CHUNK] - counts[COUNT_DESTROY_CHUNK]), + counts[COUNT_NEW_CHUNK] + counts[COUNT_DESTROY_CHUNK]); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + + MOZ_ASSERT_IF(counts[COUNT_ARENA_RELOCATED], + gcOptions == JS::GCOptions::Shrink); + if (gcOptions == JS::GCOptions::Shrink) { + SprintfLiteral( + buffer, "Kind: %s; Relocated: %.3f MiB; ", ExplainGCOptions(gcOptions), + double(ArenaSize * counts[COUNT_ARENA_RELOCATED]) / BYTES_PER_MB); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + } + + return Join(fragments); +} + +UniqueChars Statistics::formatCompactSlicePhaseTimes( + const PhaseTimes& phaseTimes) const { + static const TimeDuration MaxUnaccountedTime = + TimeDuration::FromMicroseconds(100); + + FragmentVector fragments; + char buffer[128]; + for (auto phase : AllPhases()) { + DebugOnly level = phases[phase].depth; + MOZ_ASSERT(level < 4); + + TimeDuration ownTime = phaseTimes[phase]; + TimeDuration childTime = SumChildTimes(phase, phaseTimes); + if (ownTime > MaxUnaccountedTime) { + SprintfLiteral(buffer, "%s: %.3fms", phases[phase].name, t(ownTime)); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + + if (childTime && (ownTime - childTime) > MaxUnaccountedTime) { + MOZ_ASSERT(level < 3); + SprintfLiteral(buffer, "%s: %.3fms", "Other", t(ownTime - childTime)); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + } + } + } + return Join(fragments, ", "); +} + +UniqueChars Statistics::formatDetailedMessage() const { + FragmentVector fragments; + + if (!fragments.append(formatDetailedDescription())) { + return UniqueChars(nullptr); + } + + if (!slices_.empty()) { + for (unsigned i = 0; i < slices_.length(); i++) { + if (!fragments.append(formatDetailedSliceDescription(i, slices_[i]))) { + return UniqueChars(nullptr); + } + if (!fragments.append(formatDetailedPhaseTimes(slices_[i].phaseTimes))) { + return UniqueChars(nullptr); + } + } + } + if (!fragments.append(formatDetailedTotals())) { + return UniqueChars(nullptr); + } + if (!fragments.append(formatDetailedPhaseTimes(phaseTimes))) { + return UniqueChars(nullptr); + } + + return Join(fragments); +} + +UniqueChars Statistics::formatDetailedDescription() const { + TimeDuration sccTotal, sccLongest; + sccDurations(&sccTotal, &sccLongest); + + const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20)); + const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50)); + + const char* format = + "=================================================================\n\ + Invocation Kind: %s\n\ + Reason: %s\n\ + Incremental: %s%s\n\ + Zones Collected: %d of %d (-%d)\n\ + Compartments Collected: %d of %d (-%d)\n\ + MinorGCs since last GC: %d\n\ + Store Buffer Overflows: %d\n\ + MMU 20ms:%.1f%%; 50ms:%.1f%%\n\ + SCC Sweep Total (MaxPause): %.3fms (%.3fms)\n\ + HeapSize: %.3f MiB\n\ + Chunk Delta (magnitude): %+d (%d)\n\ + Arenas Relocated: %.3f MiB\n\ +"; + + char buffer[1024]; + SprintfLiteral( + buffer, format, ExplainGCOptions(gcOptions), + ExplainGCReason(slices_[0].reason), nonincremental() ? "no - " : "yes", + nonincremental() ? ExplainAbortReason(nonincrementalReason_) : "", + zoneStats.collectedZoneCount, zoneStats.zoneCount, + zoneStats.sweptZoneCount, zoneStats.collectedCompartmentCount, + zoneStats.compartmentCount, zoneStats.sweptCompartmentCount, + getCount(COUNT_MINOR_GC), getCount(COUNT_STOREBUFFER_OVERFLOW), + mmu20 * 100., mmu50 * 100., t(sccTotal), t(sccLongest), + double(preTotalHeapBytes) / BYTES_PER_MB, + getCount(COUNT_NEW_CHUNK) - getCount(COUNT_DESTROY_CHUNK), + getCount(COUNT_NEW_CHUNK) + getCount(COUNT_DESTROY_CHUNK), + double(ArenaSize * getCount(COUNT_ARENA_RELOCATED)) / BYTES_PER_MB); + + return DuplicateString(buffer); +} + +UniqueChars Statistics::formatDetailedSliceDescription( + unsigned i, const SliceData& slice) const { + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + + const char* format = + "\ + ---- Slice %u ----\n\ + Reason: %s\n\ + Trigger: %s\n\ + Reset: %s%s\n\ + State: %s -> %s\n\ + Page Faults: %" PRIu64 + "\n\ + Pause: %.3fms of %s budget (@ %.3fms)\n\ +"; + + char triggerBuffer[100] = "n/a"; + if (slice.trigger) { + Trigger trigger = slice.trigger.value(); + SprintfLiteral(triggerBuffer, "%.3f MiB of %.3f MiB threshold\n", + double(trigger.amount) / BYTES_PER_MB, + double(trigger.threshold) / BYTES_PER_MB); + } + + char buffer[1024]; + SprintfLiteral( + buffer, format, i, ExplainGCReason(slice.reason), triggerBuffer, + slice.wasReset() ? "yes - " : "no", + slice.wasReset() ? ExplainAbortReason(slice.resetReason) : "", + gc::StateName(slice.initialState), gc::StateName(slice.finalState), + uint64_t(slice.endFaults - slice.startFaults), t(slice.duration()), + budgetDescription, t(slice.start - slices_[0].start)); + return DuplicateString(buffer); +} + +static bool IncludePhase(TimeDuration duration) { + // Don't include durations that will print as "0.000ms". + return duration.ToMilliseconds() >= 0.001; +} + +UniqueChars Statistics::formatDetailedPhaseTimes( + const PhaseTimes& phaseTimes) const { + static const TimeDuration MaxUnaccountedChildTime = + TimeDuration::FromMicroseconds(50); + + FragmentVector fragments; + char buffer[128]; + for (auto phase : AllPhases()) { + uint8_t level = phases[phase].depth; + TimeDuration ownTime = phaseTimes[phase]; + TimeDuration childTime = SumChildTimes(phase, phaseTimes); + if (IncludePhase(ownTime)) { + SprintfLiteral(buffer, " %*s%s: %.3fms\n", level * 2, "", + phases[phase].name, t(ownTime)); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + + if (childTime && (ownTime - childTime) > MaxUnaccountedChildTime) { + SprintfLiteral(buffer, " %*s%s: %.3fms\n", (level + 1) * 2, "", + "Other", t(ownTime - childTime)); + if (!fragments.append(DuplicateString(buffer))) { + return UniqueChars(nullptr); + } + } + } + } + return Join(fragments); +} + +UniqueChars Statistics::formatDetailedTotals() const { + TimeDuration total, longest; + gcDuration(&total, &longest); + + const char* format = + "\ + ---- Totals ----\n\ + Total Time: %.3fms\n\ + Max Pause: %.3fms\n\ +"; + char buffer[1024]; + SprintfLiteral(buffer, format, t(total), t(longest)); + return DuplicateString(buffer); +} + +void Statistics::formatJsonSlice(size_t sliceNum, JSONPrinter& json) const { + /* + * We number each of the slice properties to keep the code in + * GCTelemetry.jsm in sync. See MAX_SLICE_KEYS. + */ + json.beginObject(); + formatJsonSliceDescription(sliceNum, slices_[sliceNum], json); // # 1-11 + + json.beginObjectProperty("times"); // # 12 + formatJsonPhaseTimes(slices_[sliceNum].phaseTimes, json); + json.endObject(); + + json.endObject(); +} + +UniqueChars Statistics::renderJsonSlice(size_t sliceNum) const { + Sprinter printer(nullptr, false); + if (!printer.init()) { + return UniqueChars(nullptr); + } + JSONPrinter json(printer, false); + + formatJsonSlice(sliceNum, json); + return printer.release(); +} + +UniqueChars Statistics::renderNurseryJson() const { + Sprinter printer(nullptr, false); + if (!printer.init()) { + return UniqueChars(nullptr); + } + JSONPrinter json(printer, false); + gc->nursery().renderProfileJSON(json); + return printer.release(); +} + +#ifdef DEBUG +void Statistics::log(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + if (gcDebugFile) { + TimeDuration sinceStart = TimeStamp::Now() - TimeStamp::FirstTimeStamp(); + fprintf(gcDebugFile, "%12.3f: ", sinceStart.ToMicroseconds()); + vfprintf(gcDebugFile, fmt, args); + fprintf(gcDebugFile, "\n"); + fflush(gcDebugFile); + } + va_end(args); +} +#endif + +UniqueChars Statistics::renderJsonMessage() const { + /* + * The format of the JSON message is specified by the GCMajorMarkerPayload + * type in profiler.firefox.com + * https://github.com/firefox-devtools/profiler/blob/master/src/types/markers.js#L62 + * + * All the properties listed here are created within the timings property + * of the GCMajor marker. + */ + if (aborted) { + return DuplicateString("{status:\"aborted\"}"); // May return nullptr + } + + Sprinter printer(nullptr, false); + if (!printer.init()) { + return UniqueChars(nullptr); + } + JSONPrinter json(printer, false); + + json.beginObject(); + json.property("status", "completed"); + formatJsonDescription(json); + + json.beginObjectProperty("totals"); + formatJsonPhaseTimes(phaseTimes, json); + json.endObject(); + + json.endObject(); + + return printer.release(); +} + +void Statistics::formatJsonDescription(JSONPrinter& json) const { + // If you change JSON properties here, please update: + // Firefox Profiler: + // https://github.com/firefox-devtools/profiler + + TimeDuration total, longest; + gcDuration(&total, &longest); + json.property("max_pause", longest, JSONPrinter::MILLISECONDS); + json.property("total_time", total, JSONPrinter::MILLISECONDS); + // We might be able to omit reason if profiler.firefox.com was able to retrive + // it from the first slice. But it doesn't do this yet. + json.property("reason", ExplainGCReason(slices_[0].reason)); + json.property("zones_collected", zoneStats.collectedZoneCount); + json.property("total_zones", zoneStats.zoneCount); + json.property("total_compartments", zoneStats.compartmentCount); + json.property("minor_gcs", getCount(COUNT_MINOR_GC)); + json.property("minor_gc_number", gc->minorGCCount()); + json.property("major_gc_number", gc->majorGCCount()); + uint32_t storebufferOverflows = getCount(COUNT_STOREBUFFER_OVERFLOW); + if (storebufferOverflows) { + json.property("store_buffer_overflows", storebufferOverflows); + } + json.property("slices", slices_.length()); + + const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20)); + const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50)); + json.property("mmu_20ms", int(mmu20 * 100)); + json.property("mmu_50ms", int(mmu50 * 100)); + + TimeDuration sccTotal, sccLongest; + sccDurations(&sccTotal, &sccLongest); + json.property("scc_sweep_total", sccTotal, JSONPrinter::MILLISECONDS); + json.property("scc_sweep_max_pause", sccLongest, JSONPrinter::MILLISECONDS); + + if (nonincrementalReason_ != GCAbortReason::None) { + json.property("nonincremental_reason", + ExplainAbortReason(nonincrementalReason_)); + } + json.property("allocated_bytes", preTotalHeapBytes); + json.property("post_heap_size", postTotalHeapBytes); + + uint32_t addedChunks = getCount(COUNT_NEW_CHUNK); + if (addedChunks) { + json.property("added_chunks", addedChunks); + } + uint32_t removedChunks = getCount(COUNT_DESTROY_CHUNK); + if (removedChunks) { + json.property("removed_chunks", removedChunks); + } + json.property("major_gc_number", startingMajorGCNumber); + json.property("minor_gc_number", startingMinorGCNumber); + json.property("slice_number", startingSliceNumber); +} + +void Statistics::formatJsonSliceDescription(unsigned i, const SliceData& slice, + JSONPrinter& json) const { + // If you change JSON properties here, please update: + // Firefox Profiler: + // https://github.com/firefox-devtools/profiler + // + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + TimeStamp originTime = TimeStamp::ProcessCreation(); + + json.property("slice", i); + json.property("pause", slice.duration(), JSONPrinter::MILLISECONDS); + json.property("reason", ExplainGCReason(slice.reason)); + json.property("initial_state", gc::StateName(slice.initialState)); + json.property("final_state", gc::StateName(slice.finalState)); + json.property("budget", budgetDescription); + json.property("major_gc_number", startingMajorGCNumber); + if (slice.trigger) { + Trigger trigger = slice.trigger.value(); + json.property("trigger_amount", trigger.amount); + json.property("trigger_threshold", trigger.threshold); + } + int64_t numFaults = slice.endFaults - slice.startFaults; + if (numFaults != 0) { + json.property("page_faults", numFaults); + } + json.property("start_timestamp", slice.start - originTime, + JSONPrinter::SECONDS); +} + +void Statistics::formatJsonPhaseTimes(const PhaseTimes& phaseTimes, + JSONPrinter& json) const { + for (auto phase : AllPhases()) { + TimeDuration ownTime = phaseTimes[phase]; + if (!ownTime.IsZero()) { + json.property(phases[phase].path, ownTime, JSONPrinter::MILLISECONDS); + } + } +} + +Statistics::Statistics(GCRuntime* gc) + : gc(gc), + gcTimerFile(nullptr), + gcDebugFile(nullptr), + nonincrementalReason_(GCAbortReason::None), + creationTime_(TimeStamp::Now()), + tenuredAllocsSinceMinorGC(0), + preTotalHeapBytes(0), + postTotalHeapBytes(0), + preCollectedHeapBytes(0), + startingMinorGCNumber(0), + startingMajorGCNumber(0), + startingSliceNumber(0), + maxPauseInInterval(0), + sliceCallback(nullptr), + nurseryCollectionCallback(nullptr), + aborted(false), + enableProfiling_(false), + sliceCount_(0) { + for (auto& count : counts) { + count = 0; + } + + for (auto& stat : stats) { + stat = 0; + } + +#ifdef DEBUG + for (const auto& duration : totalTimes_) { + using ElementType = std::remove_reference_t; + static_assert(!std::is_trivially_constructible_v, + "Statistics::Statistics will only initialize " + "totalTimes_'s elements if their default constructor is " + "non-trivial"); + MOZ_ASSERT(duration.IsZero(), + "totalTimes_ default-initialization should have " + "default-initialized every element of totalTimes_ to zero"); + } +#endif + + MOZ_ALWAYS_TRUE(phaseStack.reserve(MAX_PHASE_NESTING)); + MOZ_ALWAYS_TRUE(suspendedPhases.reserve(MAX_SUSPENDED_PHASES)); + + gcTimerFile = MaybeOpenFileFromEnv("MOZ_GCTIMER"); + gcDebugFile = MaybeOpenFileFromEnv("JS_GC_DEBUG"); + gcProfileFile = MaybeOpenFileFromEnv("JS_GC_PROFILE_FILE", stderr); + + gc::ReadProfileEnv("JS_GC_PROFILE", + "Report major GCs taking more than N milliseconds for " + "all or just the main runtime\n", + &enableProfiling_, &profileWorkers_, &profileThreshold_); +} + +Statistics::~Statistics() { + if (gcTimerFile && gcTimerFile != stdout && gcTimerFile != stderr) { + fclose(gcTimerFile); + } + if (gcDebugFile && gcDebugFile != stdout && gcDebugFile != stderr) { + fclose(gcDebugFile); + } +} + +/* static */ +bool Statistics::initialize() { +#ifdef DEBUG + // Sanity check generated tables. + for (auto i : AllPhases()) { + auto parent = phases[i].parent; + if (parent != Phase::NONE) { + MOZ_ASSERT(phases[i].depth == phases[parent].depth + 1); + } + auto firstChild = phases[i].firstChild; + if (firstChild != Phase::NONE) { + MOZ_ASSERT(i == phases[firstChild].parent); + MOZ_ASSERT(phases[i].depth == phases[firstChild].depth - 1); + } + auto nextSibling = phases[i].nextSibling; + if (nextSibling != Phase::NONE) { + MOZ_ASSERT(parent == phases[nextSibling].parent); + MOZ_ASSERT(phases[i].depth == phases[nextSibling].depth); + } + auto nextWithPhaseKind = phases[i].nextWithPhaseKind; + if (nextWithPhaseKind != Phase::NONE) { + MOZ_ASSERT(phases[i].phaseKind == phases[nextWithPhaseKind].phaseKind); + MOZ_ASSERT(parent != phases[nextWithPhaseKind].parent); + } + } + for (auto i : AllPhaseKinds()) { + MOZ_ASSERT(phases[phaseKinds[i].firstPhase].phaseKind == i); + for (auto j : AllPhaseKinds()) { + MOZ_ASSERT_IF(i != j, phaseKinds[i].telemetryBucket != + phaseKinds[j].telemetryBucket); + } + } +#endif + + return true; +} + +JS::GCSliceCallback Statistics::setSliceCallback( + JS::GCSliceCallback newCallback) { + JS::GCSliceCallback oldCallback = sliceCallback; + sliceCallback = newCallback; + return oldCallback; +} + +JS::GCNurseryCollectionCallback Statistics::setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback newCallback) { + auto oldCallback = nurseryCollectionCallback; + nurseryCollectionCallback = newCallback; + return oldCallback; +} + +TimeDuration Statistics::clearMaxGCPauseAccumulator() { + TimeDuration prior = maxPauseInInterval; + maxPauseInInterval = 0; + return prior; +} + +TimeDuration Statistics::getMaxGCPauseSinceClear() { + return maxPauseInInterval; +} + +// Sum up the time for a phase, including instances of the phase with different +// parents. +static TimeDuration SumPhase(PhaseKind phaseKind, + const Statistics::PhaseTimes& times) { + TimeDuration sum; + for (PhaseIter phase(phaseKind); !phase.done(); phase.next()) { + sum += times[phase]; + } + return sum; +} + +static bool CheckSelfTime(Phase parent, Phase child, + const Statistics::PhaseTimes& times, + const Statistics::PhaseTimes& selfTimes, + TimeDuration childTime) { + if (selfTimes[parent] < childTime) { + fprintf( + stderr, + "Parent %s time = %.3fms with %.3fms remaining, child %s time %.3fms\n", + phases[parent].name, times[parent].ToMilliseconds(), + selfTimes[parent].ToMilliseconds(), phases[child].name, + childTime.ToMilliseconds()); + fflush(stderr); + return false; + } + + return true; +} + +static PhaseKind FindLongestPhaseKind(const Statistics::PhaseKindTimes& times) { + TimeDuration longestTime; + PhaseKind phaseKind = PhaseKind::NONE; + for (auto i : MajorGCPhaseKinds()) { + if (times[i] > longestTime) { + longestTime = times[i]; + phaseKind = i; + } + } + + return phaseKind; +} + +static PhaseKind LongestPhaseSelfTimeInMajorGC( + const Statistics::PhaseTimes& times) { + // Start with total times per expanded phase, including children's times. + Statistics::PhaseTimes selfTimes(times); + + // We have the total time spent in each phase, including descendant times. + // Loop over the children and subtract their times from their parent's self + // time. + for (auto i : AllPhases()) { + Phase parent = phases[i].parent; + if (parent != Phase::NONE) { + bool ok = CheckSelfTime(parent, i, times, selfTimes, times[i]); + + // This happens very occasionally in release builds and frequently + // in Windows debug builds. Skip collecting longest phase telemetry + // if it does. +#ifndef XP_WIN + MOZ_ASSERT(ok, "Inconsistent time data; see bug 1400153"); +#endif + if (!ok) { + return PhaseKind::NONE; + } + + selfTimes[parent] -= times[i]; + } + } + + // Sum expanded phases corresponding to the same phase. + Statistics::PhaseKindTimes phaseKindTimes; + for (auto i : AllPhaseKinds()) { + phaseKindTimes[i] = SumPhase(i, selfTimes); + } + + return FindLongestPhaseKind(phaseKindTimes); +} + +void Statistics::printStats() { + if (aborted) { + fprintf(gcTimerFile, + "OOM during GC statistics collection. The report is unavailable " + "for this GC.\n"); + } else { + UniqueChars msg = formatDetailedMessage(); + if (msg) { + double secSinceStart = + (slices_[0].start - TimeStamp::ProcessCreation()).ToSeconds(); + fprintf(gcTimerFile, "GC(T+%.3fs) %s\n", secSinceStart, msg.get()); + } + } + fflush(gcTimerFile); +} + +void Statistics::beginGC(JS::GCOptions options, const TimeStamp& currentTime) { + slices_.clearAndFree(); + sccTimes.clearAndFree(); + gcOptions = options; + nonincrementalReason_ = GCAbortReason::None; + + preTotalHeapBytes = gc->heapSize.bytes(); + + preCollectedHeapBytes = 0; + + startingMajorGCNumber = gc->majorGCCount(); + startingSliceNumber = gc->gcNumber(); + + if (gc->lastGCEndTime()) { + timeSinceLastGC = currentTime - gc->lastGCEndTime(); + } + + totalGCTime_ = TimeDuration(); +} + +void Statistics::measureInitialHeapSize() { + MOZ_ASSERT(preCollectedHeapBytes == 0); + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + preCollectedHeapBytes += zone->gcHeapSize.bytes(); + } +} + +void Statistics::endGC() { + postTotalHeapBytes = gc->heapSize.bytes(); + + sendGCTelemetry(); +} + +TimeDuration Statistics::sumTotalParallelTime(PhaseKind phaseKind) const { + TimeDuration total; + for (const SliceData& slice : slices_) { + total += slice.totalParallelTimes[phaseKind]; + } + return total; +} + +void Statistics::sendGCTelemetry() { + JSRuntime* runtime = gc->rt; + // NOTE: "Compartmental" is term that was deprecated with the + // introduction of zone-based GC, but the old telemetry probe + // continues to be used. + runtime->metrics().GC_IS_COMPARTMENTAL(!gc->fullGCRequested); + runtime->metrics().GC_ZONE_COUNT(zoneStats.zoneCount); + runtime->metrics().GC_ZONES_COLLECTED(zoneStats.collectedZoneCount); + + TimeDuration prepareTotal = phaseTimes[Phase::PREPARE]; + TimeDuration markTotal = SumPhase(PhaseKind::MARK, phaseTimes); + TimeDuration markRootsTotal = SumPhase(PhaseKind::MARK_ROOTS, phaseTimes); + + // Gray and weak marking time is counted under MARK_WEAK and not MARK_GRAY. + TimeDuration markWeakTotal = SumPhase(PhaseKind::MARK_WEAK, phaseTimes); + TimeDuration markGrayNotWeak = + SumPhase(PhaseKind::MARK_GRAY, phaseTimes) + + SumPhase(PhaseKind::MARK_INCOMING_GRAY, phaseTimes); + TimeDuration markGrayWeak = SumPhase(PhaseKind::MARK_GRAY_WEAK, phaseTimes); + TimeDuration markGrayTotal = markGrayNotWeak + markGrayWeak; + TimeDuration markNotGrayOrWeak = markTotal - markGrayNotWeak - markWeakTotal; + if (markNotGrayOrWeak < TimeDuration::FromMilliseconds(0)) { + markNotGrayOrWeak = TimeDuration(); + } + + size_t markCount = getCount(COUNT_CELLS_MARKED); + + runtime->metrics().GC_PREPARE_MS(prepareTotal); + runtime->metrics().GC_MARK_MS(markNotGrayOrWeak); + if (markTotal >= TimeDuration::FromMicroseconds(1)) { + double markRate = double(markCount) / t(markTotal); + runtime->metrics().GC_MARK_RATE_2(uint32_t(markRate)); + } + runtime->metrics().GC_SWEEP_MS(phaseTimes[Phase::SWEEP]); + if (gc->didCompactZones()) { + runtime->metrics().GC_COMPACT_MS(phaseTimes[Phase::COMPACT]); + } + runtime->metrics().GC_MARK_ROOTS_US(markRootsTotal); + runtime->metrics().GC_MARK_GRAY_MS_2(markGrayTotal); + runtime->metrics().GC_MARK_WEAK_MS(markWeakTotal); + runtime->metrics().GC_NON_INCREMENTAL(nonincremental()); + if (nonincremental()) { + runtime->metrics().GC_NON_INCREMENTAL_REASON( + uint32_t(nonincrementalReason_)); + } + +#ifdef DEBUG + // Reset happens non-incrementally, so only the last slice can be reset. + for (size_t i = 0; i < slices_.length() - 1; i++) { + MOZ_ASSERT(!slices_[i].wasReset()); + } +#endif + const auto& lastSlice = slices_.back(); + runtime->metrics().GC_RESET(lastSlice.wasReset()); + if (lastSlice.wasReset()) { + runtime->metrics().GC_RESET_REASON(uint32_t(lastSlice.resetReason)); + } + + TimeDuration total, longest; + gcDuration(&total, &longest); + + runtime->metrics().GC_MS(total); + runtime->metrics().GC_MAX_PAUSE_MS_2(longest); + + const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50)); + runtime->metrics().GC_MMU_50(mmu50 * 100); + + // Record scheduling telemetry for the main runtime but not for workers, which + // are scheduled differently. + if (!runtime->parentRuntime && timeSinceLastGC) { + runtime->metrics().GC_TIME_BETWEEN_S(timeSinceLastGC); + if (!nonincremental()) { + runtime->metrics().GC_SLICE_COUNT(slices_.length()); + } + } + + if (!lastSlice.wasReset() && preCollectedHeapBytes != 0) { + size_t bytesSurvived = 0; + for (ZonesIter zone(runtime, WithAtoms); !zone.done(); zone.next()) { + if (zone->wasCollected()) { + bytesSurvived += zone->gcHeapSize.retainedBytes(); + } + } + + MOZ_ASSERT(preCollectedHeapBytes >= bytesSurvived); + double survivalRate = + 100.0 * double(bytesSurvived) / double(preCollectedHeapBytes); + runtime->metrics().GC_TENURED_SURVIVAL_RATE(uint32_t(survivalRate)); + + // Calculate 'effectiveness' in MB / second, on main thread only for now. + if (!runtime->parentRuntime) { + size_t bytesFreed = preCollectedHeapBytes - bytesSurvived; + TimeDuration clampedTotal = + TimeDuration::Max(total, TimeDuration::FromMilliseconds(1)); + double effectiveness = + (double(bytesFreed) / BYTES_PER_MB) / clampedTotal.ToSeconds(); + runtime->metrics().GC_EFFECTIVENESS(uint32_t(effectiveness)); + } + } + + // Parallel marking stats. + if (gc->isParallelMarkingEnabled()) { + TimeDuration wallTime = SumPhase(PhaseKind::PARALLEL_MARK, phaseTimes); + TimeDuration parallelRunTime = + sumTotalParallelTime(PhaseKind::PARALLEL_MARK) - + sumTotalParallelTime(PhaseKind::PARALLEL_MARK_WAIT); + TimeDuration parallelMarkTime = + sumTotalParallelTime(PhaseKind::PARALLEL_MARK_MARK); + if (wallTime && parallelMarkTime) { + uint32_t threadCount = gc->markers.length(); + double speedup = parallelMarkTime / wallTime; + double utilization = parallelRunTime / (wallTime * threadCount); + runtime->metrics().GC_PARALLEL_MARK_SPEEDUP(uint32_t(speedup * 100.0)); + runtime->metrics().GC_PARALLEL_MARK_UTILIZATION( + std::clamp(utilization * 100.0, 0, 100)); + runtime->metrics().GC_PARALLEL_MARK_INTERRUPTIONS( + getCount(COUNT_PARALLEL_MARK_INTERRUPTIONS)); + } + } +} + +void Statistics::beginNurseryCollection(JS::GCReason reason) { + count(COUNT_MINOR_GC); + startingMinorGCNumber = gc->minorGCCount(); + if (nurseryCollectionCallback) { + (*nurseryCollectionCallback)( + context(), JS::GCNurseryProgress::GC_NURSERY_COLLECTION_START, reason); + } +} + +void Statistics::endNurseryCollection(JS::GCReason reason) { + if (nurseryCollectionCallback) { + (*nurseryCollectionCallback)( + context(), JS::GCNurseryProgress::GC_NURSERY_COLLECTION_END, reason); + } + + tenuredAllocsSinceMinorGC = 0; +} + +Statistics::SliceData::SliceData(const SliceBudget& budget, + Maybe trigger, JS::GCReason reason, + TimeStamp start, size_t startFaults, + gc::State initialState) + : budget(budget), + reason(reason), + trigger(trigger), + initialState(initialState), + start(start), + startFaults(startFaults) {} + +void Statistics::beginSlice(const ZoneGCStats& zoneStats, JS::GCOptions options, + const SliceBudget& budget, JS::GCReason reason, + bool budgetWasIncreased) { + MOZ_ASSERT(phaseStack.empty() || + (phaseStack.length() == 1 && phaseStack[0] == Phase::MUTATOR)); + + this->zoneStats = zoneStats; + + TimeStamp currentTime = TimeStamp::Now(); + + bool first = !gc->isIncrementalGCInProgress(); + if (first) { + beginGC(options, currentTime); + } + + JSRuntime* runtime = gc->rt; + if (!runtime->parentRuntime && !slices_.empty()) { + TimeDuration timeSinceLastSlice = currentTime - slices_.back().end; + runtime->metrics().GC_TIME_BETWEEN_SLICES_MS(timeSinceLastSlice); + } + + Maybe trigger = recordedTrigger; + recordedTrigger.reset(); + + if (!slices_.emplaceBack(budget, trigger, reason, currentTime, + GetPageFaultCount(), gc->state())) { + // If we are OOM, set a flag to indicate we have missing slice data. + aborted = true; + return; + } + + runtime->metrics().GC_REASON_2(uint32_t(reason)); + runtime->metrics().GC_BUDGET_WAS_INCREASED(budgetWasIncreased); + + // Slice callbacks should only fire for the outermost level. + if (sliceCallback) { + JSContext* cx = context(); + JS::GCDescription desc(!gc->fullGCRequested, false, options, reason); + if (first) { + (*sliceCallback)(cx, JS::GC_CYCLE_BEGIN, desc); + } + (*sliceCallback)(cx, JS::GC_SLICE_BEGIN, desc); + } + + log("begin slice"); +} + +void Statistics::endSlice() { + MOZ_ASSERT(phaseStack.empty() || + (phaseStack.length() == 1 && phaseStack[0] == Phase::MUTATOR)); + + if (!aborted) { + auto& slice = slices_.back(); + slice.end = TimeStamp::Now(); + slice.endFaults = GetPageFaultCount(); + slice.finalState = gc->state(); + + log("end slice"); + + sendSliceTelemetry(slice); + + sliceCount_++; + + totalGCTime_ += slice.end - slice.start; + } + + bool last = !gc->isIncrementalGCInProgress(); + if (last) { + if (gcTimerFile) { + printStats(); + } + + if (!aborted) { + endGC(); + } + } + + if (!aborted && + ShouldPrintProfile(gc->rt, enableProfiling_, profileWorkers_, + profileThreshold_, slices_.back().duration())) { + printSliceProfile(); + } + + // Slice callbacks should only fire for the outermost level. + if (!aborted) { + if (sliceCallback) { + JSContext* cx = context(); + JS::GCDescription desc(!gc->fullGCRequested, last, gcOptions, + slices_.back().reason); + (*sliceCallback)(cx, JS::GC_SLICE_END, desc); + if (last) { + (*sliceCallback)(cx, JS::GC_CYCLE_END, desc); + } + } + } + + // Do this after the slice callback since it uses these values. + if (last) { + for (auto& count : counts) { + count = 0; + } + + // Clear the timers at the end of a GC, preserving the data for + // PhaseKind::MUTATOR. + auto mutatorStartTime = phaseStartTimes[Phase::MUTATOR]; + auto mutatorTime = phaseTimes[Phase::MUTATOR]; + + phaseStartTimes = PhaseTimeStamps(); +#ifdef DEBUG + phaseEndTimes = PhaseTimeStamps(); +#endif + phaseTimes = PhaseTimes(); + + phaseStartTimes[Phase::MUTATOR] = mutatorStartTime; + phaseTimes[Phase::MUTATOR] = mutatorTime; + } + + aborted = false; +} + +void Statistics::sendSliceTelemetry(const SliceData& slice) { + JSRuntime* runtime = gc->rt; + TimeDuration sliceTime = slice.end - slice.start; + runtime->metrics().GC_SLICE_MS(sliceTime); + + if (slice.budget.isTimeBudget()) { + TimeDuration budgetDuration = slice.budget.timeBudgetDuration(); + runtime->metrics().GC_BUDGET_MS_2(budgetDuration); + + if (IsCurrentlyAnimating(runtime->lastAnimationTime, slice.end)) { + runtime->metrics().GC_ANIMATION_MS(sliceTime); + } + + bool wasLongSlice = false; + if (sliceTime > budgetDuration) { + // Record how long we went over budget. + TimeDuration overrun = sliceTime - budgetDuration; + runtime->metrics().GC_BUDGET_OVERRUN(overrun); + + // Long GC slices are those that go 50% or 5ms over their budget. + wasLongSlice = (overrun > TimeDuration::FromMilliseconds(5)) || + (overrun > (budgetDuration / int64_t(2))); + + // Record the longest phase in any long slice. + if (wasLongSlice) { + PhaseKind longest = LongestPhaseSelfTimeInMajorGC(slice.phaseTimes); + reportLongestPhaseInMajorGC(longest, [runtime](auto sample) { + runtime->metrics().GC_SLOW_PHASE(sample); + }); + + // If the longest phase was waiting for parallel tasks then record the + // longest task. + if (longest == PhaseKind::JOIN_PARALLEL_TASKS) { + PhaseKind longestParallel = + FindLongestPhaseKind(slice.maxParallelTimes); + reportLongestPhaseInMajorGC(longestParallel, [runtime](auto sample) { + runtime->metrics().GC_SLOW_TASK(sample); + }); + } + } + } + + // Record `wasLongSlice` for all TimeBudget slices. + runtime->metrics().GC_SLICE_WAS_LONG(wasLongSlice); + } +} + +template +void Statistics::reportLongestPhaseInMajorGC(PhaseKind longest, Fn reportFn) { + if (longest != PhaseKind::NONE) { + uint8_t bucket = phaseKinds[longest].telemetryBucket; + reportFn(bucket); + } +} + +bool Statistics::startTimingMutator() { + if (phaseStack.length() != 0) { + // Should only be called from outside of GC. + MOZ_ASSERT(phaseStack.length() == 1); + MOZ_ASSERT(phaseStack[0] == Phase::MUTATOR); + return false; + } + + MOZ_ASSERT(suspendedPhases.empty()); + + timedGCTime = 0; + phaseStartTimes[Phase::MUTATOR] = TimeStamp(); + phaseTimes[Phase::MUTATOR] = 0; + timedGCStart = TimeStamp(); + + beginPhase(PhaseKind::MUTATOR); + return true; +} + +bool Statistics::stopTimingMutator(double& mutator_ms, double& gc_ms) { + // This should only be called from outside of GC, while timing the mutator. + if (phaseStack.length() != 1 || phaseStack[0] != Phase::MUTATOR) { + return false; + } + + endPhase(PhaseKind::MUTATOR); + mutator_ms = t(phaseTimes[Phase::MUTATOR]); + gc_ms = t(timedGCTime); + + return true; +} + +void Statistics::suspendPhases(PhaseKind suspension) { + MOZ_ASSERT(suspension == PhaseKind::EXPLICIT_SUSPENSION || + suspension == PhaseKind::IMPLICIT_SUSPENSION); + while (!phaseStack.empty()) { + MOZ_ASSERT(suspendedPhases.length() < MAX_SUSPENDED_PHASES); + Phase parent = phaseStack.back(); + suspendedPhases.infallibleAppend(parent); + recordPhaseEnd(parent); + } + suspendedPhases.infallibleAppend(lookupChildPhase(suspension)); +} + +void Statistics::resumePhases() { + MOZ_ASSERT(suspendedPhases.back() == Phase::EXPLICIT_SUSPENSION || + suspendedPhases.back() == Phase::IMPLICIT_SUSPENSION); + suspendedPhases.popBack(); + + while (!suspendedPhases.empty() && + suspendedPhases.back() != Phase::EXPLICIT_SUSPENSION && + suspendedPhases.back() != Phase::IMPLICIT_SUSPENSION) { + Phase resumePhase = suspendedPhases.popCopy(); + if (resumePhase == Phase::MUTATOR) { + timedGCTime += TimeStamp::Now() - timedGCStart; + } + recordPhaseBegin(resumePhase); + } +} + +void Statistics::beginPhase(PhaseKind phaseKind) { + // No longer timing these phases. We should never see these. + MOZ_ASSERT(phaseKind != PhaseKind::GC_BEGIN && + phaseKind != PhaseKind::GC_END); + + // PhaseKind::MUTATOR is suspended while performing GC. + if (currentPhase() == Phase::MUTATOR) { + suspendPhases(PhaseKind::IMPLICIT_SUSPENSION); + } + + recordPhaseBegin(lookupChildPhase(phaseKind)); +} + +void Statistics::recordPhaseBegin(Phase phase) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt)); + + // Guard against any other re-entry. + MOZ_ASSERT(!phaseStartTimes[phase]); + + MOZ_ASSERT(phaseStack.length() < MAX_PHASE_NESTING); + + Phase current = currentPhase(); + MOZ_ASSERT(phases[phase].parent == current); + + TimeStamp now = TimeStamp::Now(); + + if (current != Phase::NONE) { + MOZ_ASSERT(now >= phaseStartTimes[currentPhase()], + "Inconsistent time data; see bug 1400153"); + if (now < phaseStartTimes[currentPhase()]) { + now = phaseStartTimes[currentPhase()]; + aborted = true; + } + } + + phaseStack.infallibleAppend(phase); + phaseStartTimes[phase] = now; + log("begin: %s", phases[phase].path); +} + +void Statistics::recordPhaseEnd(Phase phase) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt)); + + MOZ_ASSERT(phaseStartTimes[phase]); + + TimeStamp now = TimeStamp::Now(); + + // Make sure this phase ends after it starts. + MOZ_ASSERT(now >= phaseStartTimes[phase], + "Inconsistent time data; see bug 1400153"); + +#ifdef DEBUG + // Make sure this phase ends after all of its children. Note that some + // children might not have run in this instance, in which case they will + // have run in a previous instance of this parent or not at all. + for (Phase kid = phases[phase].firstChild; kid != Phase::NONE; + kid = phases[kid].nextSibling) { + if (phaseEndTimes[kid].IsNull()) { + continue; + } + if (phaseEndTimes[kid] > now) { + fprintf(stderr, + "Parent %s ended at %.3fms, before child %s ended at %.3fms?\n", + phases[phase].name, t(now - TimeStamp::FirstTimeStamp()), + phases[kid].name, + t(phaseEndTimes[kid] - TimeStamp::FirstTimeStamp())); + } + MOZ_ASSERT(phaseEndTimes[kid] <= now, + "Inconsistent time data; see bug 1400153"); + } +#endif + + if (now < phaseStartTimes[phase]) { + now = phaseStartTimes[phase]; + aborted = true; + } + + if (phase == Phase::MUTATOR) { + timedGCStart = now; + } + + phaseStack.popBack(); + + TimeDuration t = now - phaseStartTimes[phase]; + if (!slices_.empty()) { + slices_.back().phaseTimes[phase] += t; + } + phaseTimes[phase] += t; + phaseStartTimes[phase] = TimeStamp(); + +#ifdef DEBUG + phaseEndTimes[phase] = now; + log("end: %s", phases[phase].path); +#endif +} + +void Statistics::endPhase(PhaseKind phaseKind) { + Phase phase = currentPhase(); + MOZ_ASSERT(phase != Phase::NONE); + MOZ_ASSERT(phases[phase].phaseKind == phaseKind); + + recordPhaseEnd(phase); + + // When emptying the stack, we may need to return to timing the mutator + // (PhaseKind::MUTATOR). + if (phaseStack.empty() && !suspendedPhases.empty() && + suspendedPhases.back() == Phase::IMPLICIT_SUSPENSION) { + resumePhases(); + } +} + +void Statistics::recordParallelPhase(PhaseKind phaseKind, + TimeDuration duration) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt)); + + if (aborted) { + return; + } + + slices_.back().totalParallelTimes[phaseKind] += duration; + + // Also record the maximum task time for each phase. Don't record times for + // parent phases. + TimeDuration& maxTime = slices_.back().maxParallelTimes[phaseKind]; + maxTime = std::max(maxTime, duration); +} + +TimeStamp Statistics::beginSCC() { return TimeStamp::Now(); } + +void Statistics::endSCC(unsigned scc, TimeStamp start) { + if (scc >= sccTimes.length() && !sccTimes.resize(scc + 1)) { + return; + } + + sccTimes[scc] += TimeStamp::Now() - start; +} + +/* + * MMU (minimum mutator utilization) is a measure of how much garbage collection + * is affecting the responsiveness of the system. MMU measurements are given + * with respect to a certain window size. If we report MMU(50ms) = 80%, then + * that means that, for any 50ms window of time, at least 80% of the window is + * devoted to the mutator. In other words, the GC is running for at most 20% of + * the window, or 10ms. The GC can run multiple slices during the 50ms window + * as long as the total time it spends is at most 10ms. + */ +double Statistics::computeMMU(TimeDuration window) const { + MOZ_ASSERT(!slices_.empty()); + + TimeDuration gc = slices_[0].end - slices_[0].start; + TimeDuration gcMax = gc; + + if (gc >= window) { + return 0.0; + } + + int startIndex = 0; + for (size_t endIndex = 1; endIndex < slices_.length(); endIndex++) { + auto* startSlice = &slices_[startIndex]; + auto& endSlice = slices_[endIndex]; + gc += endSlice.end - endSlice.start; + + while (endSlice.end - startSlice->end >= window) { + gc -= startSlice->end - startSlice->start; + startSlice = &slices_[++startIndex]; + } + + TimeDuration cur = gc; + if (endSlice.end - startSlice->start > window) { + cur -= (endSlice.end - startSlice->start - window); + } + if (cur > gcMax) { + gcMax = cur; + } + } + + return double((window - gcMax) / window); +} + +void Statistics::maybePrintProfileHeaders() { + static int printedHeader = 0; + if ((printedHeader++ % 200) == 0) { + printProfileHeader(); + if (gc->nursery().enableProfiling()) { + gc->nursery().printProfileHeader(); + } + } +} + +// The following macros define GC profile metadata fields that are printed +// before the timing information defined by FOR_EACH_GC_PROFILE_TIME. + +#define FOR_EACH_GC_PROFILE_COMMON_METADATA(_) \ + _("PID", 7, "%7zu", pid) \ + _("Runtime", 14, "0x%12p", runtime) + +#define FOR_EACH_GC_PROFILE_SLICE_METADATA(_) \ + _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \ + _("Reason", 20, "%-20.20s", reason) \ + _("States", 6, "%6s", formatGCStates(slice)) \ + _("FSNR", 4, "%4s", formatGCFlags(slice)) \ + _("SizeKB", 8, "%8zu", sizeKB) \ + _("Budget", 6, "%6s", formatBudget(slice)) + +#define FOR_EACH_GC_PROFILE_METADATA(_) \ + FOR_EACH_GC_PROFILE_COMMON_METADATA(_) \ + FOR_EACH_GC_PROFILE_SLICE_METADATA(_) + +void Statistics::printProfileHeader() { + if (!enableProfiling_) { + return; + } + + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) { + return; + } + +#define PRINT_METADATA_NAME(name, width, _1, _2) \ + if (!sprinter.jsprintf(" %-*s", width, name)) { \ + return; \ + } + FOR_EACH_GC_PROFILE_METADATA(PRINT_METADATA_NAME) +#undef PRINT_METADATA_NAME + +#define PRINT_PROFILE_NAME(_1, text, _2) \ + if (!sprinter.jsprintf(" %-6.6s", text)) { \ + return; \ + } + FOR_EACH_GC_PROFILE_TIME(PRINT_PROFILE_NAME) +#undef PRINT_PROFILE_NAME + + if (!sprinter.put("\n")) { + return; + } + + fputs(sprinter.string(), profileFile()); +} + +static TimeDuration SumAllPhaseKinds(const Statistics::PhaseKindTimes& times) { + TimeDuration sum; + for (PhaseKind kind : AllPhaseKinds()) { + sum += times[kind]; + } + return sum; +} + +void Statistics::printSliceProfile() { + maybePrintProfileHeaders(); + + const SliceData& slice = slices_.back(); + ProfileDurations times = getProfileTimes(slice); + updateTotalProfileTimes(times); + + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) { + return; + } + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + TimeDuration timestamp = slice.end - creationTime(); + const char* reason = ExplainGCReason(slice.reason); + size_t sizeKB = gc->heapSize.bytes() / 1024; + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + if (!sprinter.jsprintf(" " format, value)) { \ + return; \ + } + FOR_EACH_GC_PROFILE_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + if (!printProfileTimes(times, sprinter)) { + return; + } + + fputs(sprinter.string(), profileFile()); +} + +Statistics::ProfileDurations Statistics::getProfileTimes( + const SliceData& slice) const { + ProfileDurations times; + + times[ProfileKey::Total] = slice.duration(); + times[ProfileKey::Background] = SumAllPhaseKinds(slice.totalParallelTimes); + +#define GET_PROFILE_TIME(name, text, phase) \ + if (phase != PhaseKind::NONE) { \ + times[ProfileKey::name] = SumPhase(phase, slice.phaseTimes); \ + } + FOR_EACH_GC_PROFILE_TIME(GET_PROFILE_TIME) +#undef GET_PROFILE_TIME + + return times; +} + +void Statistics::updateTotalProfileTimes(const ProfileDurations& times) { +#define UPDATE_PROFILE_TIME(name, _, phase) \ + totalTimes_[ProfileKey::name] += times[ProfileKey::name]; + FOR_EACH_GC_PROFILE_TIME(UPDATE_PROFILE_TIME) +#undef UPDATE_PROFILE_TIME +} + +const char* Statistics::formatGCStates(const SliceData& slice) { + DebugOnly r = + SprintfLiteral(formatBuffer_, "%1d -> %1d", int(slice.initialState), + int(slice.finalState)); + MOZ_ASSERT(r > 0 && r < FormatBufferLength); + return formatBuffer_; +} + +const char* Statistics::formatGCFlags(const SliceData& slice) { + bool fullGC = gc->fullGCRequested; + bool shrinkingGC = gcOptions == JS::GCOptions::Shrink; + bool nonIncrementalGC = nonincrementalReason_ != GCAbortReason::None; + bool wasReset = slice.resetReason != GCAbortReason::None; + + MOZ_ASSERT(FormatBufferLength >= 5); + formatBuffer_[0] = fullGC ? 'F' : ' '; + formatBuffer_[1] = shrinkingGC ? 'S' : ' '; + formatBuffer_[2] = nonIncrementalGC ? 'N' : ' '; + formatBuffer_[3] = wasReset ? 'R' : ' '; + formatBuffer_[4] = '\0'; + + return formatBuffer_; +} + +const char* Statistics::formatBudget(const SliceData& slice) { + if (nonincrementalReason_ != GCAbortReason::None || + !slice.budget.isTimeBudget()) { + formatBuffer_[0] = '\0'; + return formatBuffer_; + } + + DebugOnly r = + SprintfLiteral(formatBuffer_, " %6" PRIi64, slice.budget.timeBudget()); + MOZ_ASSERT(r > 0 && r < FormatBufferLength); + return formatBuffer_; +} + +/* static */ +bool Statistics::printProfileTimes(const ProfileDurations& times, + Sprinter& sprinter) { + for (auto time : times) { + int64_t millis = int64_t(time.ToMilliseconds()); + if (!sprinter.jsprintf(" %6" PRIi64, millis)) { + return false; + } + } + + return sprinter.put("\n"); +} + +constexpr size_t SliceMetadataFormatWidth() { + size_t fieldCount = 0; + size_t totalWidth = 0; + +#define UPDATE_COUNT_AND_WIDTH(_1, width, _2, _3) \ + fieldCount++; \ + totalWidth += width; + FOR_EACH_GC_PROFILE_SLICE_METADATA(UPDATE_COUNT_AND_WIDTH) +#undef UPDATE_COUNT_AND_WIDTH + + // Add padding between fields. + totalWidth += fieldCount - 1; + + return totalWidth; +} + +void Statistics::printTotalProfileTimes() { + if (!enableProfiling_) { + return; + } + + Sprinter sprinter; + if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) { + return; + } + + size_t pid = getpid(); + JSRuntime* runtime = gc->rt; + +#define PRINT_FIELD_VALUE(_1, _2, format, value) \ + if (!sprinter.jsprintf(" " format, value)) { \ + return; \ + } + FOR_EACH_GC_PROFILE_COMMON_METADATA(PRINT_FIELD_VALUE) +#undef PRINT_FIELD_VALUE + + // Use whole width of per-slice metadata to print total slices so the profile + // totals that follow line up. + size_t width = SliceMetadataFormatWidth(); + if (!sprinter.jsprintf(" %-*s", int(width), formatTotalSlices())) { + return; + } + + if (!printProfileTimes(totalTimes_, sprinter)) { + return; + } + + fputs(sprinter.string(), profileFile()); +} + +const char* Statistics::formatTotalSlices() { + DebugOnly r = SprintfLiteral( + formatBuffer_, "TOTALS: %7" PRIu64 " slices:", sliceCount_); + MOZ_ASSERT(r > 0 && r < FormatBufferLength); + return formatBuffer_; +} diff --git a/js/src/gc/Statistics.h b/js/src/gc/Statistics.h new file mode 100644 index 0000000000..0e45cd651a --- /dev/null +++ b/js/src/gc/Statistics.h @@ -0,0 +1,606 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Statistics_h +#define gc_Statistics_h + +#include "mozilla/Array.h" +#include "mozilla/Atomics.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" + +#include "jspubtd.h" +#include "NamespaceImports.h" + +#include "gc/GCEnum.h" +#include "js/AllocPolicy.h" +#include "js/GCAPI.h" +#include "js/SliceBudget.h" +#include "js/Vector.h" + +namespace js { + +class JS_PUBLIC_API Sprinter; +class JSONPrinter; + +namespace gcstats { + +// Phase data is generated by a script. If you need to add phases, edit +// js/src/gc/GenerateStatsPhases.py + +#include "gc/StatsPhasesGenerated.h" + +// Counts can be incremented with Statistics::count(). They're reset at the end +// of a Major GC. +enum Count { + COUNT_NEW_CHUNK, + COUNT_DESTROY_CHUNK, + COUNT_MINOR_GC, + + // Number of times a 'put' into a storebuffer overflowed, triggering a + // compaction + COUNT_STOREBUFFER_OVERFLOW, + + // Number of arenas relocated by compacting GC. + COUNT_ARENA_RELOCATED, + + // Number of cells marked during the marking phase. Excludes atoms marked when + // not collecting the atoms zone. + COUNT_CELLS_MARKED, + + // Number of times work was donated to a requesting thread during parallel + // marking. + COUNT_PARALLEL_MARK_INTERRUPTIONS, + + COUNT_LIMIT +}; + +// Stats can be set with Statistics::setStat(). They're not reset automatically. +enum Stat { + // Number of strings tenured. + STAT_STRINGS_TENURED, + + // Number of strings deduplicated. + STAT_STRINGS_DEDUPLICATED, + + // Number of BigInts tenured. + STAT_BIGINTS_TENURED, + + STAT_LIMIT +}; + +struct ZoneGCStats { + /* Number of zones collected in this GC. */ + int collectedZoneCount = 0; + + /* Total number of zones in the Runtime at the start of this GC. */ + int zoneCount = 0; + + /* Number of zones swept in this GC. */ + int sweptZoneCount = 0; + + /* Total number of compartments in all zones collected. */ + int collectedCompartmentCount = 0; + + /* Total number of compartments in the Runtime at the start of this GC. */ + int compartmentCount = 0; + + /* Total number of compartments swept by this GC. */ + int sweptCompartmentCount = 0; + + ZoneGCStats() = default; +}; + +struct Trigger { + size_t amount = 0; + size_t threshold = 0; +}; + +#define FOR_EACH_GC_PROFILE_TIME(_) \ + _(Total, "total", PhaseKind::NONE) \ + _(Background, "bgwrk", PhaseKind::NONE) \ + _(BeginCallback, "bgnCB", PhaseKind::GC_BEGIN) \ + _(MinorForMajor, "evct4m", PhaseKind::EVICT_NURSERY_FOR_MAJOR_GC) \ + _(WaitBgThread, "waitBG", PhaseKind::WAIT_BACKGROUND_THREAD) \ + _(Prepare, "prep", PhaseKind::PREPARE) \ + _(Mark, "mark", PhaseKind::MARK) \ + _(Sweep, "sweep", PhaseKind::SWEEP) \ + _(Compact, "cmpct", PhaseKind::COMPACT) \ + _(EndCallback, "endCB", PhaseKind::GC_END) \ + _(MinorGC, "minor", PhaseKind::MINOR_GC) \ + _(EvictNursery, "evict", PhaseKind::EVICT_NURSERY) + +static const char* const MajorGCProfilePrefix = "MajorGC:"; +static const char* const MinorGCProfilePrefix = "MinorGC:"; + +const char* ExplainAbortReason(GCAbortReason reason); + +/* + * Struct for collecting timing statistics on a "phase tree". The tree is + * specified as a limited DAG, but the timings are collected for the whole tree + * that you would get by expanding out the DAG by duplicating subtrees rooted + * at nodes with multiple parents. + * + * During execution, a child phase can be activated multiple times, and the + * total time will be accumulated. (So for example, you can start and end + * PhaseKind::MARK_ROOTS multiple times before completing the parent phase.) + * + * Incremental GC is represented by recording separate timing results for each + * slice within the overall GC. + */ +struct Statistics { + template + using Array = mozilla::Array; + + template + using EnumeratedArray = + mozilla::EnumeratedArray; + + using TimeDuration = mozilla::TimeDuration; + using TimeStamp = mozilla::TimeStamp; + + // Create types for tables of times, by phase and phase kind. + using PhaseTimes = EnumeratedArray; + using PhaseKindTimes = + EnumeratedArray; + + using PhaseTimeStamps = EnumeratedArray; + + [[nodiscard]] static bool initialize(); + + explicit Statistics(gc::GCRuntime* gc); + ~Statistics(); + + Statistics(const Statistics&) = delete; + Statistics& operator=(const Statistics&) = delete; + + void beginPhase(PhaseKind phaseKind); + void endPhase(PhaseKind phaseKind); + void recordParallelPhase(PhaseKind phaseKind, TimeDuration duration); + + // Occasionally, we may be in the middle of something that is tracked by + // this class, and we need to do something unusual (eg evict the nursery) + // that doesn't normally nest within the current phase. Suspend the + // currently tracked phase stack, at which time the caller is free to do + // other tracked operations. + // + // This also happens internally with the PhaseKind::MUTATOR "phase". While in + // this phase, any beginPhase will automatically suspend the non-GC phase, + // until that inner stack is complete, at which time it will automatically + // resume the non-GC phase. Explicit suspensions do not get auto-resumed. + void suspendPhases(PhaseKind suspension = PhaseKind::EXPLICIT_SUSPENSION); + + // Resume a suspended stack of phases. + void resumePhases(); + + void beginSlice(const ZoneGCStats& zoneStats, JS::GCOptions options, + const SliceBudget& budget, JS::GCReason reason, + bool budgetWasIncreased); + void endSlice(); + + [[nodiscard]] bool startTimingMutator(); + [[nodiscard]] bool stopTimingMutator(double& mutator_ms, double& gc_ms); + + // Note when we sweep a zone or compartment. + void sweptZone() { ++zoneStats.sweptZoneCount; } + void sweptCompartment() { ++zoneStats.sweptCompartmentCount; } + + void reset(GCAbortReason reason) { + MOZ_ASSERT(reason != GCAbortReason::None); + if (!aborted) { + slices_.back().resetReason = reason; + } + } + + void measureInitialHeapSize(); + + void nonincremental(GCAbortReason reason) { + MOZ_ASSERT(reason != GCAbortReason::None); + nonincrementalReason_ = reason; + log("Non-incremental reason: %s", nonincrementalReason()); + } + + bool nonincremental() const { + return nonincrementalReason_ != GCAbortReason::None; + } + + const char* nonincrementalReason() const { + return ExplainAbortReason(nonincrementalReason_); + } + + void count(Count s) { counts[s]++; } + void addCount(Count s, uint32_t count) { counts[s] += count; } + + uint32_t getCount(Count s) const { return uint32_t(counts[s]); } + + void setStat(Stat s, uint32_t value) { stats[s] = value; } + + uint32_t getStat(Stat s) const { return stats[s]; } + + void recordTrigger(size_t amount, size_t threshold) { + recordedTrigger = mozilla::Some(Trigger{amount, threshold}); + } + bool hasTrigger() const { return recordedTrigger.isSome(); } + + // tenured allocs don't include nursery evictions. + void setAllocsSinceMinorGCTenured(uint32_t allocs) { + tenuredAllocsSinceMinorGC = allocs; + } + + uint32_t allocsSinceMinorGCTenured() { return tenuredAllocsSinceMinorGC; } + + void beginNurseryCollection(JS::GCReason reason); + void endNurseryCollection(JS::GCReason reason); + + TimeStamp beginSCC(); + void endSCC(unsigned scc, TimeStamp start); + + UniqueChars formatCompactSliceMessage() const; + UniqueChars formatCompactSummaryMessage() const; + UniqueChars formatDetailedMessage() const; + + JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); + JS::GCNurseryCollectionCallback setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback callback); + + TimeDuration clearMaxGCPauseAccumulator(); + TimeDuration getMaxGCPauseSinceClear(); + + PhaseKind currentPhaseKind() const; + + static const size_t MAX_SUSPENDED_PHASES = MAX_PHASE_NESTING * 3; + + struct SliceData { + SliceData(const SliceBudget& budget, mozilla::Maybe trigger, + JS::GCReason reason, TimeStamp start, size_t startFaults, + gc::State initialState); + + SliceBudget budget; + JS::GCReason reason = JS::GCReason::NO_REASON; + mozilla::Maybe trigger; + gc::State initialState = gc::State::NotActive; + gc::State finalState = gc::State::NotActive; + GCAbortReason resetReason = GCAbortReason::None; + TimeStamp start; + TimeStamp end; + size_t startFaults = 0; + size_t endFaults = 0; + PhaseTimes phaseTimes; + PhaseKindTimes totalParallelTimes; + PhaseKindTimes maxParallelTimes; + + TimeDuration duration() const { return end - start; } + bool wasReset() const { return resetReason != GCAbortReason::None; } + }; + + typedef Vector SliceDataVector; + + const SliceDataVector& slices() const { return slices_; } + + const SliceData* lastSlice() const { + if (slices_.length() == 0) { + return nullptr; + } + + return &slices_.back(); + } + + TimeStamp start() const { return slices_[0].start; } + + TimeStamp end() const { return slices_.back().end; } + + TimeStamp creationTime() const { return creationTime_; } + + TimeDuration totalGCTime() const { return totalGCTime_; } + size_t initialCollectedBytes() const { return preCollectedHeapBytes; } + + // File to write profiling information to, either stderr or file specified + // with JS_GC_PROFILE_FILE. + FILE* profileFile() const { return gcProfileFile; } + + // Occasionally print header lines for profiling information. + void maybePrintProfileHeaders(); + + // Print header line for profile times. + void printProfileHeader(); + + // Print total profile times on shutdown. + void printTotalProfileTimes(); + + // These JSON strings are used by the firefox profiler to display the GC + // markers. + + // Return JSON for a whole major GC + UniqueChars renderJsonMessage() const; + + // Return JSON for the timings of just the given slice. + UniqueChars renderJsonSlice(size_t sliceNum) const; + + // Return JSON for the previous nursery collection. + UniqueChars renderNurseryJson() const; + +#ifdef DEBUG + // Print a logging message. + void log(const char* fmt, ...); +#else + void log(const char* fmt, ...){}; +#endif + + private: + gc::GCRuntime* const gc; + + /* File used for MOZ_GCTIMER output. */ + FILE* gcTimerFile; + + /* File used for JS_GC_DEBUG output. */ + FILE* gcDebugFile; + + /* File used for JS_GC_PROFILE output. */ + FILE* gcProfileFile; + + ZoneGCStats zoneStats; + + JS::GCOptions gcOptions; + + GCAbortReason nonincrementalReason_; + + SliceDataVector slices_; + + /* Most recent time when the given phase started. */ + PhaseTimeStamps phaseStartTimes; + +#ifdef DEBUG + /* Most recent time when the given phase ended. */ + PhaseTimeStamps phaseEndTimes; +#endif + + TimeStamp creationTime_; + + /* Bookkeeping for GC timings when timingMutator is true */ + TimeStamp timedGCStart; + TimeDuration timedGCTime; + + /* Total main thread time in a given phase for this GC. */ + PhaseTimes phaseTimes; + + /* Total main thread time for this GC. */ + TimeDuration totalGCTime_; + + /* Number of events of this type for this GC. */ + EnumeratedArray> + counts; + + /* Other GC statistics. */ + EnumeratedArray stats; + + /* + * These events cannot be kept in the above array, we need to take their + * address. + */ + uint32_t tenuredAllocsSinceMinorGC; + + /* Total GC heap size before and after the GC ran. */ + size_t preTotalHeapBytes; + size_t postTotalHeapBytes; + + /* GC heap size for collected zones before GC ran. */ + size_t preCollectedHeapBytes; + + /* + * If a GC slice was triggered by exceeding some threshold, record the + * threshold and the value that exceeded it. This happens before the slice + * starts so this is recorded here first and then transferred to SliceData. + */ + mozilla::Maybe recordedTrigger; + + /* GC numbers as of the beginning of the collection. */ + uint64_t startingMinorGCNumber; + uint64_t startingMajorGCNumber; + uint64_t startingSliceNumber; + + /* Records the maximum GC pause in an API-controlled interval. */ + mutable TimeDuration maxPauseInInterval; + + /* Phases that are currently on stack. */ + Vector phaseStack; + + /* + * Certain phases can interrupt the phase stack, eg callback phases. When + * this happens, we move the suspended phases over to a sepearate list, + * terminated by a dummy PhaseKind::SUSPENSION phase (so that we can nest + * suspensions by suspending multiple stacks with a PhaseKind::SUSPENSION in + * between). + */ + Vector suspendedPhases; + + /* Sweep times for SCCs of compartments. */ + Vector sccTimes; + + TimeDuration timeSinceLastGC; + + JS::GCSliceCallback sliceCallback; + JS::GCNurseryCollectionCallback nurseryCollectionCallback; + + /* + * True if we saw an OOM while allocating slices or we saw an impossible + * timestamp. The statistics for this GC will be invalid. + */ + bool aborted; + + /* Profiling data. */ + + enum class ProfileKey { +#define DEFINE_PROFILE_KEY(name, _1, _2) name, + FOR_EACH_GC_PROFILE_TIME(DEFINE_PROFILE_KEY) +#undef DEFINE_PROFILE_KEY + KeyCount + }; + + using ProfileDurations = + EnumeratedArray; + + bool enableProfiling_; + bool profileWorkers_; + TimeDuration profileThreshold_; + ProfileDurations totalTimes_; + uint64_t sliceCount_; + + char formatBuffer_[32]; + static constexpr int FormatBufferLength = sizeof(formatBuffer_); + + JSContext* context(); + + Phase currentPhase() const; + Phase lookupChildPhase(PhaseKind phaseKind) const; + + void beginGC(JS::GCOptions options, const TimeStamp& currentTime); + void endGC(); + + void sendGCTelemetry(); + void sendSliceTelemetry(const SliceData& slice); + + TimeDuration sumTotalParallelTime(PhaseKind phaseKind) const; + + void recordPhaseBegin(Phase phase); + void recordPhaseEnd(Phase phase); + + void gcDuration(TimeDuration* total, TimeDuration* maxPause) const; + void sccDurations(TimeDuration* total, TimeDuration* maxPause) const; + void printStats(); + + template + void reportLongestPhaseInMajorGC(PhaseKind longest, Fn reportFn); + + UniqueChars formatCompactSlicePhaseTimes(const PhaseTimes& phaseTimes) const; + + UniqueChars formatDetailedDescription() const; + UniqueChars formatDetailedSliceDescription(unsigned i, + const SliceData& slice) const; + UniqueChars formatDetailedPhaseTimes(const PhaseTimes& phaseTimes) const; + UniqueChars formatDetailedTotals() const; + + void formatJsonDescription(JSONPrinter&) const; + void formatJsonSliceDescription(unsigned i, const SliceData& slice, + JSONPrinter&) const; + void formatJsonPhaseTimes(const PhaseTimes& phaseTimes, JSONPrinter&) const; + void formatJsonSlice(size_t sliceNum, JSONPrinter&) const; + + double computeMMU(TimeDuration resolution) const; + + void printSliceProfile(); + ProfileDurations getProfileTimes(const SliceData& slice) const; + void updateTotalProfileTimes(const ProfileDurations& times); + const char* formatGCStates(const SliceData& slice); + const char* formatGCFlags(const SliceData& slice); + const char* formatBudget(const SliceData& slice); + const char* formatTotalSlices(); + static bool printProfileTimes(const ProfileDurations& times, + Sprinter& sprinter); +}; + +struct MOZ_RAII AutoGCSlice { + AutoGCSlice(Statistics& stats, const ZoneGCStats& zoneStats, + JS::GCOptions options, const SliceBudget& budget, + JS::GCReason reason, bool budgetWasIncreased) + : stats(stats) { + stats.beginSlice(zoneStats, options, budget, reason, budgetWasIncreased); + } + ~AutoGCSlice() { stats.endSlice(); } + + Statistics& stats; +}; + +struct MOZ_RAII AutoPhase { + AutoPhase(Statistics& stats, PhaseKind phaseKind) + : stats(stats), phaseKind(phaseKind), enabled(true) { + stats.beginPhase(phaseKind); + } + + AutoPhase(Statistics& stats, bool condition, PhaseKind phaseKind) + : stats(stats), phaseKind(phaseKind), enabled(condition) { + if (enabled) { + stats.beginPhase(phaseKind); + } + } + + ~AutoPhase() { + if (enabled) { + stats.endPhase(phaseKind); + } + } + + Statistics& stats; + PhaseKind phaseKind; + bool enabled; +}; + +struct MOZ_RAII AutoSCC { + AutoSCC(Statistics& stats, unsigned scc) : stats(stats), scc(scc) { + start = stats.beginSCC(); + } + ~AutoSCC() { stats.endSCC(scc, start); } + + Statistics& stats; + unsigned scc; + mozilla::TimeStamp start; +}; + +void ReadProfileEnv(const char* envName, const char* helpText, bool* enableOut, + bool* workersOut, mozilla::TimeDuration* thresholdOut); + +} /* namespace gcstats */ + +struct StringStats { + // number of strings that were deduplicated, and their sizes in characters + // and bytes + uint64_t deduplicatedStrings = 0; + uint64_t deduplicatedChars = 0; + uint64_t deduplicatedBytes = 0; + + // number of live nursery strings at the start of a nursery collection + uint64_t liveNurseryStrings = 0; + + // number of new strings added to the tenured heap + uint64_t tenuredStrings = 0; + + // Currently, liveNurseryStrings = tenuredStrings + deduplicatedStrings (but + // in the future we may do more transformation during tenuring, eg + // atomizing.) + + // number of malloced bytes associated with tenured strings (the actual + // malloc will have happened when the strings were allocated in the nursery; + // the ownership of the bytes will be transferred to the tenured strings) + uint64_t tenuredBytes = 0; + + StringStats& operator+=(const StringStats& other) { + deduplicatedStrings += other.deduplicatedStrings; + deduplicatedChars += other.deduplicatedChars; + deduplicatedBytes += other.deduplicatedBytes; + liveNurseryStrings += other.liveNurseryStrings; + tenuredStrings += other.tenuredStrings; + tenuredBytes += other.tenuredBytes; + return *this; + } + + void noteTenured(size_t mallocBytes) { + liveNurseryStrings++; + tenuredStrings++; + tenuredBytes += mallocBytes; + } + + void noteDeduplicated(size_t numChars, size_t mallocBytes) { + liveNurseryStrings++; + deduplicatedStrings++; + deduplicatedChars += numChars; + deduplicatedBytes += mallocBytes; + } +}; + +} /* namespace js */ + +#endif /* gc_Statistics_h */ diff --git a/js/src/gc/StoreBuffer-inl.h b/js/src/gc/StoreBuffer-inl.h new file mode 100644 index 0000000000..1c97654761 --- /dev/null +++ b/js/src/gc/StoreBuffer-inl.h @@ -0,0 +1,97 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StoreBuffer_inl_h +#define gc_StoreBuffer_inl_h + +#include "gc/StoreBuffer.h" + +#include "gc/Cell.h" +#include "gc/Heap.h" + +#include "gc/Heap-inl.h" + +namespace js { +namespace gc { + +inline /* static */ size_t ArenaCellSet::getCellIndex(const TenuredCell* cell) { + uintptr_t cellOffset = uintptr_t(cell) & ArenaMask; + MOZ_ASSERT(cellOffset % ArenaCellIndexBytes == 0); + return cellOffset / ArenaCellIndexBytes; +} + +inline /* static */ void ArenaCellSet::getWordIndexAndMask(size_t cellIndex, + size_t* wordp, + uint32_t* maskp) { + BitArray::getIndexAndMask(cellIndex, wordp, maskp); +} + +inline bool ArenaCellSet::hasCell(size_t cellIndex) const { + MOZ_ASSERT(cellIndex < MaxArenaCellIndex); + return bits.get(cellIndex); +} + +inline void ArenaCellSet::putCell(size_t cellIndex) { + MOZ_ASSERT(cellIndex < MaxArenaCellIndex); + MOZ_ASSERT(arena); + + bits.set(cellIndex); + check(); +} + +inline void ArenaCellSet::check() const { +#ifdef DEBUG + bool bitsZero = bits.isAllClear(); + MOZ_ASSERT(isEmpty() == bitsZero); + MOZ_ASSERT(isEmpty() == !arena); + if (!isEmpty()) { + MOZ_ASSERT(IsCellPointerValid(arena)); + MOZ_ASSERT(arena->bufferedCells() == this); + JSRuntime* runtime = arena->zone->runtimeFromMainThread(); + MOZ_ASSERT(runtime->gc.minorGCCount() == minorGCNumberAtCreation); + } +#endif +} + +inline void StoreBuffer::WholeCellBuffer::put(const Cell* cell) { + if (cell != last_) { + putDontCheckLast(cell); + } +} + +inline void StoreBuffer::WholeCellBuffer::putDontCheckLast(const Cell* cell) { + // This can still be called when |cell == last_| if the caller didn't check + // and that's OK. + + MOZ_ASSERT(cell->isTenured()); + + // BigInts don't have any children, so shouldn't show up here. + MOZ_ASSERT(cell->getTraceKind() != JS::TraceKind::BigInt); + + Arena* arena = cell->asTenured().arena(); + ArenaCellSet* cells = arena->bufferedCells(); + if (cells->isEmpty()) { + cells = allocateCellSet(arena); + if (!cells) { + return; + } + } + + cells->putCell(&cell->asTenured()); + cells->check(); + + last_ = cell; +} + +inline void StoreBuffer::putWholeCell(Cell* cell) { bufferWholeCell.put(cell); } +inline void StoreBuffer::putWholeCellDontCheckLast(Cell* cell) { + bufferWholeCell.putDontCheckLast(cell); +} + +} // namespace gc +} // namespace js + +#endif // gc_StoreBuffer_inl_h diff --git a/js/src/gc/StoreBuffer.cpp b/js/src/gc/StoreBuffer.cpp new file mode 100644 index 0000000000..4f6240d48b --- /dev/null +++ b/js/src/gc/StoreBuffer.cpp @@ -0,0 +1,255 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/StoreBuffer-inl.h" + +#include "mozilla/Assertions.h" + +#include "gc/Statistics.h" +#include "vm/MutexIDs.h" +#include "vm/Runtime.h" + +using namespace js; +using namespace js::gc; + +JS_PUBLIC_API void js::gc::LockStoreBuffer(StoreBuffer* sb) { + MOZ_ASSERT(sb); + sb->lock(); +} + +JS_PUBLIC_API void js::gc::UnlockStoreBuffer(StoreBuffer* sb) { + MOZ_ASSERT(sb); + sb->unlock(); +} + +#ifdef DEBUG +void StoreBuffer::checkAccess() const { + // The GC runs tasks that may access the storebuffer in parallel and so must + // take a lock. The mutator may only access the storebuffer from the main + // thread. + if (runtime_->heapState() != JS::HeapState::Idle) { + MOZ_ASSERT(!CurrentThreadIsGCMarking()); + lock_.assertOwnedByCurrentThread(); + } else { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_)); + } +} +#endif + +bool StoreBuffer::WholeCellBuffer::init() { + MOZ_ASSERT(!stringHead_); + MOZ_ASSERT(!nonStringHead_); + if (!storage_) { + storage_ = MakeUnique(LifoAllocBlockSize); + // This prevents LifoAlloc::Enum from crashing with a release + // assertion if we ever allocate one entry larger than + // LifoAllocBlockSize. + if (storage_) { + storage_->disableOversize(); + } + } + clear(); + return bool(storage_); +} + +bool StoreBuffer::GenericBuffer::init() { + if (!storage_) { + storage_ = MakeUnique(LifoAllocBlockSize); + } + clear(); + return bool(storage_); +} + +void StoreBuffer::GenericBuffer::trace(JSTracer* trc) { + mozilla::ReentrancyGuard g(*owner_); + MOZ_ASSERT(owner_->isEnabled()); + if (!storage_) { + return; + } + + for (LifoAlloc::Enum e(*storage_); !e.empty();) { + unsigned size = *e.read(); + BufferableRef* edge = e.read(size); + edge->trace(trc); + } +} + +StoreBuffer::StoreBuffer(JSRuntime* rt, const Nursery& nursery) + : lock_(mutexid::StoreBuffer), + bufferVal(this, JS::GCReason::FULL_VALUE_BUFFER), + bufStrCell(this, JS::GCReason::FULL_CELL_PTR_STR_BUFFER), + bufBigIntCell(this, JS::GCReason::FULL_CELL_PTR_BIGINT_BUFFER), + bufObjCell(this, JS::GCReason::FULL_CELL_PTR_OBJ_BUFFER), + bufferSlot(this, JS::GCReason::FULL_SLOT_BUFFER), + bufferWholeCell(this), + bufferGeneric(this), + runtime_(rt), + nursery_(nursery), + aboutToOverflow_(false), + enabled_(false), + mayHavePointersToDeadCells_(false) +#ifdef DEBUG + , + mEntered(false), + markingNondeduplicatable(false) +#endif +{ +} + +void StoreBuffer::checkEmpty() const { MOZ_ASSERT(isEmpty()); } + +bool StoreBuffer::isEmpty() const { + return bufferVal.isEmpty() && bufStrCell.isEmpty() && + bufBigIntCell.isEmpty() && bufObjCell.isEmpty() && + bufferSlot.isEmpty() && bufferWholeCell.isEmpty() && + bufferGeneric.isEmpty(); +} + +bool StoreBuffer::enable() { + if (enabled_) { + return true; + } + + checkEmpty(); + + if (!bufferWholeCell.init() || !bufferGeneric.init()) { + return false; + } + + enabled_ = true; + return true; +} + +void StoreBuffer::disable() { + checkEmpty(); + + if (!enabled_) { + return; + } + + aboutToOverflow_ = false; + + enabled_ = false; +} + +void StoreBuffer::clear() { + if (!enabled_) { + return; + } + + aboutToOverflow_ = false; + mayHavePointersToDeadCells_ = false; + + bufferVal.clear(); + bufStrCell.clear(); + bufBigIntCell.clear(); + bufObjCell.clear(); + bufferSlot.clear(); + bufferWholeCell.clear(); + bufferGeneric.clear(); +} + +void StoreBuffer::setAboutToOverflow(JS::GCReason reason) { + if (!aboutToOverflow_) { + aboutToOverflow_ = true; + runtime_->gc.stats().count(gcstats::COUNT_STOREBUFFER_OVERFLOW); + } + nursery_.requestMinorGC(reason); +} + +void StoreBuffer::addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, + JS::GCSizes* sizes) { + sizes->storeBufferVals += bufferVal.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferCells += bufStrCell.sizeOfExcludingThis(mallocSizeOf) + + bufBigIntCell.sizeOfExcludingThis(mallocSizeOf) + + bufObjCell.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferSlots += bufferSlot.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferWholeCells += + bufferWholeCell.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferGenerics += bufferGeneric.sizeOfExcludingThis(mallocSizeOf); +} + +ArenaCellSet ArenaCellSet::Empty; + +ArenaCellSet::ArenaCellSet(Arena* arena, ArenaCellSet* next) + : arena(arena), + next(next) +#ifdef DEBUG + , + minorGCNumberAtCreation( + arena->zone->runtimeFromMainThread()->gc.minorGCCount()) +#endif +{ + MOZ_ASSERT(arena); + MOZ_ASSERT(bits.isAllClear()); +} + +ArenaCellSet* StoreBuffer::WholeCellBuffer::allocateCellSet(Arena* arena) { + Zone* zone = arena->zone; + JSRuntime* rt = zone->runtimeFromMainThread(); + if (!rt->gc.nursery().isEnabled()) { + return nullptr; + } + + // Maintain separate lists for strings and non-strings, so that all buffered + // string whole cells will be processed before anything else (to prevent them + // from being deduplicated when their chars are used by a tenured string.) + bool isString = + MapAllocToTraceKind(arena->getAllocKind()) == JS::TraceKind::String; + + AutoEnterOOMUnsafeRegion oomUnsafe; + ArenaCellSet*& head = isString ? stringHead_ : nonStringHead_; + auto cells = storage_->new_(arena, head); + if (!cells) { + oomUnsafe.crash("Failed to allocate ArenaCellSet"); + } + + arena->bufferedCells() = cells; + head = cells; + + if (isAboutToOverflow()) { + rt->gc.storeBuffer().setAboutToOverflow( + JS::GCReason::FULL_WHOLE_CELL_BUFFER); + } + + return cells; +} + +void gc::CellHeaderPostWriteBarrier(JSObject** ptr, JSObject* prev, + JSObject* next) { + InternalBarrierMethods::postBarrier(ptr, prev, next); +} + +void StoreBuffer::WholeCellBuffer::clear() { + for (auto** headPtr : {&stringHead_, &nonStringHead_}) { + for (auto* set = *headPtr; set; set = set->next) { + set->arena->bufferedCells() = &ArenaCellSet::Empty; + } + *headPtr = nullptr; + } + + if (storage_) { + storage_->used() ? storage_->releaseAll() : storage_->freeAll(); + } + + last_ = nullptr; +} + +template struct StoreBuffer::MonoTypeBuffer; +template struct StoreBuffer::MonoTypeBuffer; + +void js::gc::PostWriteBarrierCell(Cell* cell, Cell* prev, Cell* next) { + if (!next || !cell->isTenured()) { + return; + } + + StoreBuffer* buffer = next->storeBuffer(); + if (!buffer || (prev && prev->storeBuffer())) { + return; + } + + buffer->putWholeCell(cell); +} diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h new file mode 100644 index 0000000000..4700392074 --- /dev/null +++ b/js/src/gc/StoreBuffer.h @@ -0,0 +1,662 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StoreBuffer_h +#define gc_StoreBuffer_h + +#include "mozilla/Attributes.h" +#include "mozilla/HashFunctions.h" +#include "mozilla/ReentrancyGuard.h" + +#include + +#include "ds/BitArray.h" +#include "ds/LifoAlloc.h" +#include "gc/Cell.h" +#include "gc/Nursery.h" +#include "gc/TraceKind.h" +#include "js/AllocPolicy.h" +#include "js/UniquePtr.h" +#include "threading/Mutex.h" + +namespace JS { +struct GCSizes; +} + +namespace js { + +class NativeObject; + +#ifdef DEBUG +extern bool CurrentThreadIsGCMarking(); +#endif + +namespace gc { + +class Arena; +class ArenaCellSet; + +#ifdef DEBUG +extern bool CurrentThreadHasLockedGC(); +#endif + +/* + * BufferableRef represents an abstract reference for use in the generational + * GC's remembered set. Entries in the store buffer that cannot be represented + * with the simple pointer-to-a-pointer scheme must derive from this class and + * use the generic store buffer interface. + * + * A single BufferableRef entry in the generic buffer can represent many entries + * in the remembered set. For example js::OrderedHashTableRef represents all + * the incoming edges corresponding to keys in an ordered hash table. + */ +class BufferableRef { + public: + virtual void trace(JSTracer* trc) = 0; + bool maybeInRememberedSet(const Nursery&) const { return true; } +}; + +typedef HashSet, SystemAllocPolicy> EdgeSet; + +/* The size of a single block of store buffer storage space. */ +static const size_t LifoAllocBlockSize = 8 * 1024; + +/* + * The StoreBuffer observes all writes that occur in the system and performs + * efficient filtering of them to derive a remembered set for nursery GC. + */ +class StoreBuffer { + friend class mozilla::ReentrancyGuard; + + /* The size at which a block is about to overflow for the generic buffer. */ + static const size_t GenericBufferLowAvailableThreshold = + LifoAllocBlockSize / 2; + + /* The size at which other store buffers are about to overflow. */ + static const size_t BufferOverflowThresholdBytes = 128 * 1024; + + /* + * This buffer holds only a single type of edge. Using this buffer is more + * efficient than the generic buffer when many writes will be to the same + * type of edge: e.g. Value or Cell*. + */ + template + struct MonoTypeBuffer { + /* The canonical set of stores. */ + typedef HashSet StoreSet; + StoreSet stores_; + + /* + * A one element cache in front of the canonical set to speed up + * temporary instances of HeapPtr. + */ + T last_; + + StoreBuffer* owner_; + + JS::GCReason gcReason_; + + /* Maximum number of entries before we request a minor GC. */ + const static size_t MaxEntries = BufferOverflowThresholdBytes / sizeof(T); + + explicit MonoTypeBuffer(StoreBuffer* owner, JS::GCReason reason) + : last_(T()), owner_(owner), gcReason_(reason) {} + + void clear() { + last_ = T(); + stores_.clear(); + } + + /* Add one item to the buffer. */ + void put(const T& t) { + sinkStore(); + last_ = t; + } + + /* Remove an item from the store buffer. */ + void unput(const T& v) { + // Fast, hashless remove of last put. + if (last_ == v) { + last_ = T(); + return; + } + stores_.remove(v); + } + + /* Move any buffered stores to the canonical store set. */ + void sinkStore() { + if (last_) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!stores_.put(last_)) { + oomUnsafe.crash("Failed to allocate for MonoTypeBuffer::put."); + } + } + last_ = T(); + + if (MOZ_UNLIKELY(stores_.count() > MaxEntries)) { + owner_->setAboutToOverflow(gcReason_); + } + } + + /* Trace the source of all edges in the store buffer. */ + void trace(TenuringTracer& mover); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return stores_.shallowSizeOfExcludingThis(mallocSizeOf); + } + + bool isEmpty() const { return last_ == T() && stores_.empty(); } + + private: + MonoTypeBuffer(const MonoTypeBuffer& other) = delete; + MonoTypeBuffer& operator=(const MonoTypeBuffer& other) = delete; + }; + + struct WholeCellBuffer { + UniquePtr storage_; + ArenaCellSet* stringHead_ = nullptr; + ArenaCellSet* nonStringHead_ = nullptr; + const Cell* last_ = nullptr; + StoreBuffer* owner_; + + explicit WholeCellBuffer(StoreBuffer* owner) : owner_(owner) {} + + [[nodiscard]] bool init(); + + void clear(); + + bool isAboutToOverflow() const { + return !storage_->isEmpty() && + storage_->used() > BufferOverflowThresholdBytes; + } + + void trace(TenuringTracer& mover); + + inline void put(const Cell* cell); + inline void putDontCheckLast(const Cell* cell); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return storage_ ? storage_->sizeOfIncludingThis(mallocSizeOf) : 0; + } + + bool isEmpty() const { + MOZ_ASSERT_IF(!stringHead_ && !nonStringHead_, + !storage_ || storage_->isEmpty()); + return !stringHead_ && !nonStringHead_; + } + + const Cell** lastBufferedPtr() { return &last_; } + + private: + ArenaCellSet* allocateCellSet(Arena* arena); + + WholeCellBuffer(const WholeCellBuffer& other) = delete; + WholeCellBuffer& operator=(const WholeCellBuffer& other) = delete; + }; + + struct GenericBuffer { + UniquePtr storage_; + StoreBuffer* owner_; + + explicit GenericBuffer(StoreBuffer* owner) + : storage_(nullptr), owner_(owner) {} + + [[nodiscard]] bool init(); + + void clear() { + if (storage_) { + storage_->used() ? storage_->releaseAll() : storage_->freeAll(); + } + } + + bool isAboutToOverflow() const { + return !storage_->isEmpty() && storage_->availableInCurrentChunk() < + GenericBufferLowAvailableThreshold; + } + + /* Trace all generic edges. */ + void trace(JSTracer* trc); + + template + void put(const T& t) { + MOZ_ASSERT(storage_); + + /* Ensure T is derived from BufferableRef. */ + (void)static_cast(&t); + + AutoEnterOOMUnsafeRegion oomUnsafe; + unsigned size = sizeof(T); + unsigned* sizep = storage_->pod_malloc(); + if (!sizep) { + oomUnsafe.crash("Failed to allocate for GenericBuffer::put."); + } + *sizep = size; + + T* tp = storage_->new_(t); + if (!tp) { + oomUnsafe.crash("Failed to allocate for GenericBuffer::put."); + } + + if (isAboutToOverflow()) { + owner_->setAboutToOverflow(JS::GCReason::FULL_GENERIC_BUFFER); + } + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return storage_ ? storage_->sizeOfIncludingThis(mallocSizeOf) : 0; + } + + bool isEmpty() const { return !storage_ || storage_->isEmpty(); } + + private: + GenericBuffer(const GenericBuffer& other) = delete; + GenericBuffer& operator=(const GenericBuffer& other) = delete; + }; + + template + struct PointerEdgeHasher { + using Lookup = Edge; + static HashNumber hash(const Lookup& l) { + return mozilla::HashGeneric(l.edge); + } + static bool match(const Edge& k, const Lookup& l) { return k == l; } + }; + + template + struct CellPtrEdge { + T** edge = nullptr; + + CellPtrEdge() = default; + explicit CellPtrEdge(T** v) : edge(v) {} + bool operator==(const CellPtrEdge& other) const { + return edge == other.edge; + } + bool operator!=(const CellPtrEdge& other) const { + return edge != other.edge; + } + + bool maybeInRememberedSet(const Nursery& nursery) const { + MOZ_ASSERT(IsInsideNursery(*edge)); + return !nursery.isInside(edge); + } + + void trace(TenuringTracer& mover) const; + + explicit operator bool() const { return edge != nullptr; } + + using Hasher = PointerEdgeHasher>; + }; + + using ObjectPtrEdge = CellPtrEdge; + using StringPtrEdge = CellPtrEdge; + using BigIntPtrEdge = CellPtrEdge; + + struct ValueEdge { + JS::Value* edge; + + ValueEdge() : edge(nullptr) {} + explicit ValueEdge(JS::Value* v) : edge(v) {} + bool operator==(const ValueEdge& other) const { return edge == other.edge; } + bool operator!=(const ValueEdge& other) const { return edge != other.edge; } + + Cell* deref() const { + return edge->isGCThing() ? static_cast(edge->toGCThing()) + : nullptr; + } + + bool maybeInRememberedSet(const Nursery& nursery) const { + MOZ_ASSERT(IsInsideNursery(deref())); + return !nursery.isInside(edge); + } + + void trace(TenuringTracer& mover) const; + + explicit operator bool() const { return edge != nullptr; } + + using Hasher = PointerEdgeHasher; + }; + + struct SlotsEdge { + // These definitions must match those in HeapSlot::Kind. + const static int SlotKind = 0; + const static int ElementKind = 1; + + uintptr_t objectAndKind_; // NativeObject* | Kind + uint32_t start_; + uint32_t count_; + + SlotsEdge() : objectAndKind_(0), start_(0), count_(0) {} + SlotsEdge(NativeObject* object, int kind, uint32_t start, uint32_t count) + : objectAndKind_(uintptr_t(object) | kind), + start_(start), + count_(count) { + MOZ_ASSERT((uintptr_t(object) & 1) == 0); + MOZ_ASSERT(kind <= 1); + MOZ_ASSERT(count > 0); + MOZ_ASSERT(start + count > start); + } + + NativeObject* object() const { + return reinterpret_cast(objectAndKind_ & ~1); + } + int kind() const { return (int)(objectAndKind_ & 1); } + + bool operator==(const SlotsEdge& other) const { + return objectAndKind_ == other.objectAndKind_ && start_ == other.start_ && + count_ == other.count_; + } + + bool operator!=(const SlotsEdge& other) const { return !(*this == other); } + + // True if this SlotsEdge range overlaps with the other SlotsEdge range, + // false if they do not overlap. + bool overlaps(const SlotsEdge& other) const { + if (objectAndKind_ != other.objectAndKind_) { + return false; + } + + // Widen our range by one on each side so that we consider + // adjacent-but-not-actually-overlapping ranges as overlapping. This + // is particularly useful for coalescing a series of increasing or + // decreasing single index writes 0, 1, 2, ..., N into a SlotsEdge + // range of elements [0, N]. + uint32_t end = start_ + count_ + 1; + uint32_t start = start_ > 0 ? start_ - 1 : 0; + MOZ_ASSERT(start < end); + + uint32_t otherEnd = other.start_ + other.count_; + MOZ_ASSERT(other.start_ <= otherEnd); + return (start <= other.start_ && other.start_ <= end) || + (start <= otherEnd && otherEnd <= end); + } + + // Destructively make this SlotsEdge range the union of the other + // SlotsEdge range and this one. A precondition is that the ranges must + // overlap. + void merge(const SlotsEdge& other) { + MOZ_ASSERT(overlaps(other)); + uint32_t end = std::max(start_ + count_, other.start_ + other.count_); + start_ = std::min(start_, other.start_); + count_ = end - start_; + } + + bool maybeInRememberedSet(const Nursery& n) const { + return !IsInsideNursery(reinterpret_cast(object())); + } + + void trace(TenuringTracer& mover) const; + + explicit operator bool() const { return objectAndKind_ != 0; } + + typedef struct Hasher { + using Lookup = SlotsEdge; + static HashNumber hash(const Lookup& l) { + return mozilla::HashGeneric(l.objectAndKind_, l.start_, l.count_); + } + static bool match(const SlotsEdge& k, const Lookup& l) { return k == l; } + } Hasher; + }; + +#ifdef DEBUG + void checkAccess() const; +#else + void checkAccess() const {} +#endif + + template + void unput(Buffer& buffer, const Edge& edge) { + checkAccess(); + if (!isEnabled()) { + return; + } + mozilla::ReentrancyGuard g(*this); + buffer.unput(edge); + } + + template + void put(Buffer& buffer, const Edge& edge) { + checkAccess(); + if (!isEnabled()) { + return; + } + mozilla::ReentrancyGuard g(*this); + if (edge.maybeInRememberedSet(nursery_)) { + buffer.put(edge); + } + } + + Mutex lock_ MOZ_UNANNOTATED; + + MonoTypeBuffer bufferVal; + MonoTypeBuffer bufStrCell; + MonoTypeBuffer bufBigIntCell; + MonoTypeBuffer bufObjCell; + MonoTypeBuffer bufferSlot; + WholeCellBuffer bufferWholeCell; + GenericBuffer bufferGeneric; + + JSRuntime* runtime_; + const Nursery& nursery_; + + bool aboutToOverflow_; + bool enabled_; + bool mayHavePointersToDeadCells_; +#ifdef DEBUG + bool mEntered; /* For ReentrancyGuard. */ +#endif + + public: +#ifdef DEBUG + bool markingNondeduplicatable; +#endif + + explicit StoreBuffer(JSRuntime* rt, const Nursery& nursery); + [[nodiscard]] bool enable(); + + void disable(); + bool isEnabled() const { return enabled_; } + + bool isEmpty() const; + void clear(); + + const Nursery& nursery() const { return nursery_; } + + /* Get the overflowed status. */ + bool isAboutToOverflow() const { return aboutToOverflow_; } + + /* + * Brain transplants may add whole cell buffer entires for dead cells. We must + * evict the nursery prior to sweeping arenas if any such entries are present. + */ + bool mayHavePointersToDeadCells() const { + return mayHavePointersToDeadCells_; + } + + /* Insert a single edge into the buffer/remembered set. */ + void putValue(JS::Value* vp) { put(bufferVal, ValueEdge(vp)); } + void unputValue(JS::Value* vp) { unput(bufferVal, ValueEdge(vp)); } + + void putCell(JSString** strp) { put(bufStrCell, StringPtrEdge(strp)); } + void unputCell(JSString** strp) { unput(bufStrCell, StringPtrEdge(strp)); } + + void putCell(JS::BigInt** bip) { put(bufBigIntCell, BigIntPtrEdge(bip)); } + void unputCell(JS::BigInt** bip) { unput(bufBigIntCell, BigIntPtrEdge(bip)); } + + void putCell(JSObject** strp) { put(bufObjCell, ObjectPtrEdge(strp)); } + void unputCell(JSObject** strp) { unput(bufObjCell, ObjectPtrEdge(strp)); } + + void putSlot(NativeObject* obj, int kind, uint32_t start, uint32_t count) { + SlotsEdge edge(obj, kind, start, count); + if (bufferSlot.last_.overlaps(edge)) { + bufferSlot.last_.merge(edge); + } else { + put(bufferSlot, edge); + } + } + + inline void putWholeCell(Cell* cell); + inline void putWholeCellDontCheckLast(Cell* cell); + const void* addressOfLastBufferedWholeCell() { + return bufferWholeCell.lastBufferedPtr(); + } + + /* Insert an entry into the generic buffer. */ + template + void putGeneric(const T& t) { + put(bufferGeneric, t); + } + + void setMayHavePointersToDeadCells() { mayHavePointersToDeadCells_ = true; } + + /* Methods to trace the source of all edges in the store buffer. */ + void traceValues(TenuringTracer& mover) { bufferVal.trace(mover); } + void traceCells(TenuringTracer& mover) { + bufStrCell.trace(mover); + bufBigIntCell.trace(mover); + bufObjCell.trace(mover); + } + void traceSlots(TenuringTracer& mover) { bufferSlot.trace(mover); } + void traceWholeCells(TenuringTracer& mover) { bufferWholeCell.trace(mover); } + void traceGenericEntries(JSTracer* trc) { bufferGeneric.trace(trc); } + + /* For use by our owned buffers and for testing. */ + void setAboutToOverflow(JS::GCReason); + + void addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, + JS::GCSizes* sizes); + + void checkEmpty() const; + + // For use by the GC only. + void lock() { lock_.lock(); } + void unlock() { lock_.unlock(); } +}; + +// A set of cells in an arena used to implement the whole cell store buffer. +class ArenaCellSet { + friend class StoreBuffer; + + using ArenaCellBits = BitArray; + + // The arena this relates to. + Arena* arena; + + // Pointer to next set forming a linked list. + ArenaCellSet* next; + + // Bit vector for each possible cell start position. + ArenaCellBits bits; + +#ifdef DEBUG + // The minor GC number when this was created. This object should not survive + // past the next minor collection. + const uint64_t minorGCNumberAtCreation; +#endif + + // Construct the empty sentinel object. + constexpr ArenaCellSet() + : arena(nullptr), + next(nullptr) +#ifdef DEBUG + , + minorGCNumberAtCreation(0) +#endif + { + } + + public: + using WordT = ArenaCellBits::WordT; + const size_t BitsPerWord = ArenaCellBits::bitsPerElement; + const size_t NumWords = ArenaCellBits::numSlots; + + ArenaCellSet(Arena* arena, ArenaCellSet* next); + + bool hasCell(const TenuredCell* cell) const { + return hasCell(getCellIndex(cell)); + } + + void putCell(const TenuredCell* cell) { putCell(getCellIndex(cell)); } + + bool isEmpty() const { return this == &Empty; } + + bool hasCell(size_t cellIndex) const; + + void putCell(size_t cellIndex); + + void check() const; + + WordT getWord(size_t wordIndex) const { return bits.getWord(wordIndex); } + + void trace(TenuringTracer& mover); + + // Sentinel object used for all empty sets. + // + // We use a sentinel because it simplifies the JIT code slightly as we can + // assume all arenas have a cell set. + static ArenaCellSet Empty; + + static size_t getCellIndex(const TenuredCell* cell); + static void getWordIndexAndMask(size_t cellIndex, size_t* wordp, + uint32_t* maskp); + + // Attempt to trigger a minor GC if free space in the nursery (where these + // objects are allocated) falls below this threshold. + static const size_t NurseryFreeThresholdBytes = 64 * 1024; + + static size_t offsetOfArena() { return offsetof(ArenaCellSet, arena); } + static size_t offsetOfBits() { return offsetof(ArenaCellSet, bits); } +}; + +// Post-write barrier implementation for GC cells. + +// Implement the post-write barrier for nursery allocateable cell type |T|. Call +// this from |T::postWriteBarrier|. +template +MOZ_ALWAYS_INLINE void PostWriteBarrierImpl(void* cellp, T* prev, T* next) { + MOZ_ASSERT(cellp); + + // If the target needs an entry, add it. + StoreBuffer* buffer; + if (next && (buffer = next->storeBuffer())) { + // If we know that the prev has already inserted an entry, we can skip + // doing the lookup to add the new entry. Note that we cannot safely + // assert the presence of the entry because it may have been added + // via a different store buffer. + if (prev && prev->storeBuffer()) { + return; + } + buffer->putCell(static_cast(cellp)); + return; + } + + // Remove the prev entry if the new value does not need it. There will only + // be a prev entry if the prev value was in the nursery. + if (prev && (buffer = prev->storeBuffer())) { + buffer->unputCell(static_cast(cellp)); + } +} + +template +MOZ_ALWAYS_INLINE void PostWriteBarrier(T** vp, T* prev, T* next) { + static_assert(std::is_base_of_v); + static_assert(!std::is_same_v && !std::is_same_v); + + if constexpr (!GCTypeIsTenured()) { + using BaseT = typename BaseGCType::type; + PostWriteBarrierImpl(vp, prev, next); + return; + } + + MOZ_ASSERT_IF(next, !IsInsideNursery(next)); +} + +// Used when we don't have a specific edge to put in the store buffer. +void PostWriteBarrierCell(Cell* cell, Cell* prev, Cell* next); + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_StoreBuffer_h */ diff --git a/js/src/gc/Sweeping.cpp b/js/src/gc/Sweeping.cpp new file mode 100644 index 0000000000..80f4a20e74 --- /dev/null +++ b/js/src/gc/Sweeping.cpp @@ -0,0 +1,2383 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Implementation of GC sweeping. + * + * In the SpiderMonkey GC, 'sweeping' is used to mean two things: + * - updating data structures to remove pointers to dead GC things and updating + * pointers to moved GC things + * - finalizing dead GC things + * + * Furthermore, the GC carries out gray and weak marking after the start of the + * sweep phase. This is also implemented in this file. + */ + +#include "mozilla/Maybe.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/TimeStamp.h" + +#include "builtin/FinalizationRegistryObject.h" +#include "builtin/WeakRefObject.h" +#include "debugger/DebugAPI.h" +#include "gc/AllocKind.h" +#include "gc/FinalizationObservers.h" +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/GCProbes.h" +#include "gc/GCRuntime.h" +#include "gc/ParallelWork.h" +#include "gc/Statistics.h" +#include "gc/TraceKind.h" +#include "gc/WeakMap.h" +#include "gc/Zone.h" +#include "jit/JitRuntime.h" +#include "jit/JitZone.h" +#include "proxy/DeadObjectProxy.h" +#include "vm/BigIntType.h" +#include "vm/HelperThreads.h" +#include "vm/JSContext.h" +#include "vm/Time.h" +#include "vm/WrapperObject.h" + +#include "gc/PrivateIterators-inl.h" +#include "vm/GeckoProfiler-inl.h" +#include "vm/JSObject-inl.h" +#include "vm/PropMap-inl.h" +#include "vm/Shape-inl.h" +#include "vm/StringType-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::TimeStamp; + +struct js::gc::FinalizePhase { + gcstats::PhaseKind statsPhase; + AllocKinds kinds; +}; + +/* + * Finalization order for objects swept incrementally on the main thread. + */ +static constexpr FinalizePhase ForegroundObjectFinalizePhase = { + gcstats::PhaseKind::FINALIZE_OBJECT, + {AllocKind::OBJECT0, AllocKind::OBJECT2, AllocKind::OBJECT4, + AllocKind::OBJECT8, AllocKind::OBJECT12, AllocKind::OBJECT16}}; + +/* + * Finalization order for GC things swept incrementally on the main thread. + */ +static constexpr FinalizePhase ForegroundNonObjectFinalizePhase = { + gcstats::PhaseKind::FINALIZE_NON_OBJECT, + {AllocKind::SCRIPT, AllocKind::JITCODE}}; + +/* + * Finalization order for GC things swept on the background thread. + */ +static constexpr FinalizePhase BackgroundFinalizePhases[] = { + {gcstats::PhaseKind::FINALIZE_OBJECT, + {AllocKind::FUNCTION, AllocKind::FUNCTION_EXTENDED, + AllocKind::OBJECT0_BACKGROUND, AllocKind::OBJECT2_BACKGROUND, + AllocKind::ARRAYBUFFER4, AllocKind::OBJECT4_BACKGROUND, + AllocKind::ARRAYBUFFER8, AllocKind::OBJECT8_BACKGROUND, + AllocKind::ARRAYBUFFER12, AllocKind::OBJECT12_BACKGROUND, + AllocKind::ARRAYBUFFER16, AllocKind::OBJECT16_BACKGROUND}}, + {gcstats::PhaseKind::FINALIZE_NON_OBJECT, + {AllocKind::SCOPE, AllocKind::REGEXP_SHARED, AllocKind::FAT_INLINE_STRING, + AllocKind::STRING, AllocKind::EXTERNAL_STRING, AllocKind::FAT_INLINE_ATOM, + AllocKind::ATOM, AllocKind::SYMBOL, AllocKind::BIGINT, AllocKind::SHAPE, + AllocKind::BASE_SHAPE, AllocKind::GETTER_SETTER, + AllocKind::COMPACT_PROP_MAP, AllocKind::NORMAL_PROP_MAP, + AllocKind::DICT_PROP_MAP}}}; + +template +inline size_t Arena::finalize(JS::GCContext* gcx, AllocKind thingKind, + size_t thingSize) { + /* Enforce requirements on size of T. */ + MOZ_ASSERT(thingSize % CellAlignBytes == 0); + MOZ_ASSERT(thingSize >= MinCellSize); + MOZ_ASSERT(thingSize <= 255); + + MOZ_ASSERT(allocated()); + MOZ_ASSERT(thingKind == getAllocKind()); + MOZ_ASSERT(thingSize == getThingSize()); + MOZ_ASSERT(!onDelayedMarkingList_); + + uint_fast16_t firstThing = firstThingOffset(thingKind); + uint_fast16_t firstThingOrSuccessorOfLastMarkedThing = firstThing; + uint_fast16_t lastThing = ArenaSize - thingSize; + + FreeSpan newListHead; + FreeSpan* newListTail = &newListHead; + size_t nmarked = 0, nfinalized = 0; + + for (ArenaCellIterUnderFinalize cell(this); !cell.done(); cell.next()) { + T* t = cell.as(); + if (TenuredThingIsMarkedAny(t)) { + uint_fast16_t thing = uintptr_t(t) & ArenaMask; + if (thing != firstThingOrSuccessorOfLastMarkedThing) { + // We just finished passing over one or more free things, + // so record a new FreeSpan. + newListTail->initBounds(firstThingOrSuccessorOfLastMarkedThing, + thing - thingSize, this); + newListTail = newListTail->nextSpanUnchecked(this); + } + firstThingOrSuccessorOfLastMarkedThing = thing + thingSize; + nmarked++; + } else { + t->finalize(gcx); + AlwaysPoison(t, JS_SWEPT_TENURED_PATTERN, thingSize, + MemCheckKind::MakeUndefined); + gcprobes::TenuredFinalize(t); + nfinalized++; + } + } + + if constexpr (std::is_same_v) { + if (isNewlyCreated_) { + zone->pretenuring.updateCellCountsInNewlyCreatedArenas( + nmarked + nfinalized, nmarked); + } + } + isNewlyCreated_ = 0; + + if (thingKind == AllocKind::STRING || + thingKind == AllocKind::FAT_INLINE_STRING) { + zone->markedStrings += nmarked; + zone->finalizedStrings += nfinalized; + } + + if (nmarked == 0) { + // Do nothing. The caller will update the arena appropriately. + MOZ_ASSERT(newListTail == &newListHead); + DebugOnlyPoison(data, JS_SWEPT_TENURED_PATTERN, sizeof(data), + MemCheckKind::MakeUndefined); + return nmarked; + } + + MOZ_ASSERT(firstThingOrSuccessorOfLastMarkedThing != firstThing); + uint_fast16_t lastMarkedThing = + firstThingOrSuccessorOfLastMarkedThing - thingSize; + if (lastThing == lastMarkedThing) { + // If the last thing was marked, we will have already set the bounds of + // the final span, and we just need to terminate the list. + newListTail->initAsEmpty(); + } else { + // Otherwise, end the list with a span that covers the final stretch of free + // things. + newListTail->initFinal(firstThingOrSuccessorOfLastMarkedThing, lastThing, + this); + } + + firstFreeSpan = newListHead; +#ifdef DEBUG + size_t nfree = numFreeThings(thingSize); + MOZ_ASSERT(nfree + nmarked == thingsPerArena(thingKind)); +#endif + return nmarked; +} + +// Finalize arenas from src list, releasing empty arenas if keepArenas wasn't +// specified and inserting the others into the appropriate destination size +// bins. +template +static inline bool FinalizeTypedArenas(JS::GCContext* gcx, ArenaList& src, + SortedArenaList& dest, + AllocKind thingKind, + SliceBudget& budget) { + MOZ_ASSERT(gcx->isFinalizing()); + + size_t thingSize = Arena::thingSize(thingKind); + size_t thingsPerArena = Arena::thingsPerArena(thingKind); + size_t markCount = 0; + + auto updateMarkCount = mozilla::MakeScopeExit([&] { + GCRuntime* gc = &gcx->runtimeFromAnyThread()->gc; + gc->stats().addCount(gcstats::COUNT_CELLS_MARKED, markCount); + }); + + while (Arena* arena = src.takeFirstArena()) { + size_t nmarked = arena->finalize(gcx, thingKind, thingSize); + size_t nfree = thingsPerArena - nmarked; + + markCount += nmarked; + + if (nmarked) { + dest.insertAt(arena, nfree); + } else { + arena->chunk()->recycleArena(arena, dest, thingsPerArena); + } + + budget.step(thingsPerArena); + if (budget.isOverBudget()) { + return false; + } + } + + return true; +} + +/* + * Finalize the list of areans. + */ +static bool FinalizeArenas(JS::GCContext* gcx, ArenaList& src, + SortedArenaList& dest, AllocKind thingKind, + SliceBudget& budget) { + switch (thingKind) { +#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery, \ + compact) \ + case AllocKind::allocKind: \ + return FinalizeTypedArenas(gcx, src, dest, thingKind, budget); + FOR_EACH_ALLOCKIND(EXPAND_CASE) +#undef EXPAND_CASE + + default: + MOZ_CRASH("Invalid alloc kind"); + } +} + +void GCRuntime::initBackgroundSweep(Zone* zone, JS::GCContext* gcx, + const FinalizePhase& phase) { + gcstats::AutoPhase ap(stats(), phase.statsPhase); + for (auto kind : phase.kinds) { + zone->arenas.initBackgroundSweep(kind); + } +} + +void ArenaLists::initBackgroundSweep(AllocKind thingKind) { + MOZ_ASSERT(IsBackgroundFinalized(thingKind)); + MOZ_ASSERT(concurrentUse(thingKind) == ConcurrentUse::None); + + if (!collectingArenaList(thingKind).isEmpty()) { + concurrentUse(thingKind) = ConcurrentUse::BackgroundFinalize; + } +} + +void GCRuntime::backgroundFinalize(JS::GCContext* gcx, Zone* zone, + AllocKind kind, Arena** empty) { + MOZ_ASSERT(empty); + + ArenaLists* lists = &zone->arenas; + ArenaList& arenas = lists->collectingArenaList(kind); + if (arenas.isEmpty()) { + MOZ_ASSERT(lists->concurrentUse(kind) == ArenaLists::ConcurrentUse::None); + return; + } + + SortedArenaList finalizedSorted(Arena::thingsPerArena(kind)); + + auto unlimited = SliceBudget::unlimited(); + FinalizeArenas(gcx, arenas, finalizedSorted, kind, unlimited); + MOZ_ASSERT(arenas.isEmpty()); + + finalizedSorted.extractEmpty(empty); + + // When marking begins, all arenas are moved from arenaLists to + // collectingArenaLists. When the mutator runs, new arenas are allocated in + // arenaLists. Now that finalization is complete, we want to merge these lists + // back together. + + // We must take the GC lock to be able to safely modify the ArenaList; + // however, this does not by itself make the changes visible to all threads, + // as not all threads take the GC lock to read the ArenaLists. + // That safety is provided by the ReleaseAcquire memory ordering of the + // background finalize state, which we explicitly set as the final step. + { + AutoLockGC lock(rt); + MOZ_ASSERT(lists->concurrentUse(kind) == + ArenaLists::ConcurrentUse::BackgroundFinalize); + lists->mergeFinalizedArenas(kind, finalizedSorted); + } + + lists->concurrentUse(kind) = ArenaLists::ConcurrentUse::None; +} + +// After finalizing arenas, merge the following to get the final state of an +// arena list: +// - arenas allocated during marking +// - arenas allocated during sweeping +// - finalized arenas +void ArenaLists::mergeFinalizedArenas(AllocKind kind, + SortedArenaList& finalizedArenas) { +#ifdef DEBUG + // Updating arena lists off-thread requires taking the GC lock because the + // main thread uses these when allocating. + if (IsBackgroundFinalized(kind)) { + runtimeFromAnyThread()->gc.assertCurrentThreadHasLockedGC(); + } +#endif + + ArenaList& arenas = arenaList(kind); + + ArenaList allocatedDuringCollection = std::move(arenas); + arenas = finalizedArenas.toArenaList(); + arenas.insertListWithCursorAtEnd(allocatedDuringCollection); + + collectingArenaList(kind).clear(); +} + +void ArenaLists::queueForegroundThingsForSweep() { + gcCompactPropMapArenasToUpdate = + collectingArenaList(AllocKind::COMPACT_PROP_MAP).head(); + gcNormalPropMapArenasToUpdate = + collectingArenaList(AllocKind::NORMAL_PROP_MAP).head(); +} + +void GCRuntime::sweepBackgroundThings(ZoneList& zones) { + if (zones.isEmpty()) { + return; + } + + JS::GCContext* gcx = TlsGCContext.get(); + MOZ_ASSERT(gcx->isFinalizing()); + + // Sweep zones in order. The atoms zone must be finalized last as other + // zones may have direct pointers into it. + while (!zones.isEmpty()) { + Zone* zone = zones.removeFront(); + MOZ_ASSERT(zone->isGCFinished()); + + TimeStamp startTime = TimeStamp::Now(); + + Arena* emptyArenas = zone->arenas.takeSweptEmptyArenas(); + + // We must finalize thing kinds in the order specified by + // BackgroundFinalizePhases. + for (auto phase : BackgroundFinalizePhases) { + for (auto kind : phase.kinds) { + backgroundFinalize(gcx, zone, kind, &emptyArenas); + } + } + + // Release any arenas that are now empty. + // + // Empty arenas are only released after everything has been finalized so + // that it's still possible to get a thing's zone after the thing has been + // finalized. The HeapPtr destructor depends on this, and this allows + // HeapPtrs between things of different alloc kind regardless of + // finalization order. + // + // Periodically drop and reaquire the GC lock every so often to avoid + // blocking the main thread from allocating chunks. + static const size_t LockReleasePeriod = 32; + + while (emptyArenas) { + AutoLockGC lock(this); + for (size_t i = 0; i < LockReleasePeriod && emptyArenas; i++) { + Arena* arena = emptyArenas; + emptyArenas = emptyArenas->next; + releaseArena(arena, lock); + } + } + + // Record time spent sweeping this zone. + TimeStamp endTime = TimeStamp::Now(); + zone->perZoneGCTime += endTime - startTime; + } +} + +void GCRuntime::assertBackgroundSweepingFinished() { +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(backgroundSweepZones.ref().isEmpty()); + } + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + for (auto kind : AllAllocKinds()) { + MOZ_ASSERT_IF(state() != State::Prepare && state() != State::Mark && + state() != State::Sweep, + zone->arenas.collectingArenaList(kind).isEmpty()); + MOZ_ASSERT(zone->arenas.doneBackgroundFinalize(kind)); + } + } +#endif +} + +void GCRuntime::queueZonesAndStartBackgroundSweep(ZoneList&& zones) { + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(!requestSliceAfterBackgroundTask); + backgroundSweepZones.ref().appendList(std::move(zones)); + if (useBackgroundThreads) { + sweepTask.startOrRunIfIdle(lock); + } + } + if (!useBackgroundThreads) { + sweepTask.join(); + sweepTask.runFromMainThread(); + } +} + +BackgroundSweepTask::BackgroundSweepTask(GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::SWEEP, GCUse::Finalizing) {} + +void BackgroundSweepTask::run(AutoLockHelperThreadState& lock) { + gc->sweepFromBackgroundThread(lock); +} + +void GCRuntime::sweepFromBackgroundThread(AutoLockHelperThreadState& lock) { + do { + ZoneList zones; + zones.appendList(std::move(backgroundSweepZones.ref())); + + AutoUnlockHelperThreadState unlock(lock); + sweepBackgroundThings(zones); + + // The main thread may call queueZonesAndStartBackgroundSweep() while this + // is running so we must check there is no more work after releasing the + // lock. + } while (!backgroundSweepZones.ref().isEmpty()); + + maybeRequestGCAfterBackgroundTask(lock); +} + +void GCRuntime::waitBackgroundSweepEnd() { + sweepTask.join(); + if (state() != State::Sweep) { + assertBackgroundSweepingFinished(); + } +} + +void GCRuntime::startBackgroundFree() { + AutoLockHelperThreadState lock; + freeTask.startOrRunIfIdle(lock); +} + +BackgroundFreeTask::BackgroundFreeTask(GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::NONE) { + // This can occur outside GCs so doesn't have a stats phase. +} + +void BackgroundFreeTask::run(AutoLockHelperThreadState& lock) { + gc->freeFromBackgroundThread(lock); +} + +void GCRuntime::freeFromBackgroundThread(AutoLockHelperThreadState& lock) { + do { + LifoAlloc lifoBlocks(JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE); + lifoBlocks.transferFrom(&lifoBlocksToFree.ref()); + + Nursery::BufferSet buffers; + std::swap(buffers, buffersToFreeAfterMinorGC.ref()); + + AutoUnlockHelperThreadState unlock(lock); + + lifoBlocks.freeAll(); + + JS::GCContext* gcx = TlsGCContext.get(); + for (Nursery::BufferSet::Range r = buffers.all(); !r.empty(); + r.popFront()) { + // Malloc memory associated with nursery objects is not tracked as these + // are assumed to be short lived. + gcx->freeUntracked(r.front()); + } + } while (!lifoBlocksToFree.ref().isEmpty() || + !buffersToFreeAfterMinorGC.ref().empty()); +} + +void GCRuntime::waitBackgroundFreeEnd() { freeTask.join(); } + +template +IncrementalProgress GCRuntime::markWeakReferences( + SliceBudget& incrementalBudget) { + MOZ_ASSERT(!marker().isWeakMarking()); + + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::MARK_WEAK); + + auto unlimited = SliceBudget::unlimited(); + SliceBudget& budget = + marker().incrementalWeakMapMarkingEnabled ? incrementalBudget : unlimited; + + // Ensure we don't return to the mutator while we're still in weak marking + // mode. + auto leaveOnExit = + mozilla::MakeScopeExit([&] { marker().leaveWeakMarkingMode(); }); + + if (marker().enterWeakMarkingMode()) { + // If there was an 'enter-weak-marking-mode' token in the queue, then it + // and everything after it will still be in the queue so we can process + // them now. + while (processTestMarkQueue() == QueueYielded) { + }; + + // Do not rely on the information about not-yet-marked weak keys that have + // been collected by barriers. Clear out the gcEphemeronEdges entries and + // rebuild the full table. Note that this a cross-zone operation; delegate + // zone entries will be populated by map zone traversals, so everything + // needs to be cleared first, then populated. + if (!marker().incrementalWeakMapMarkingEnabled) { + for (ZoneIterT zone(this); !zone.done(); zone.next()) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing weak keys when entering weak marking mode"); + } + } + } + + for (ZoneIterT zone(this); !zone.done(); zone.next()) { + if (zone->enterWeakMarkingMode(&marker(), budget) == NotFinished) { + return NotFinished; + } + } + } + + bool markedAny = true; + while (markedAny) { + if (!marker().markUntilBudgetExhausted(budget)) { + MOZ_ASSERT(marker().incrementalWeakMapMarkingEnabled); + return NotFinished; + } + + markedAny = false; + + if (!marker().isWeakMarking()) { + for (ZoneIterT zone(this); !zone.done(); zone.next()) { + markedAny |= WeakMapBase::markZoneIteratively(zone, &marker()); + } + } + + markedAny |= jit::JitRuntime::MarkJitcodeGlobalTableIteratively(&marker()); + } + + assertNoMarkingWork(); + + return Finished; +} + +IncrementalProgress GCRuntime::markWeakReferencesInCurrentGroup( + SliceBudget& budget) { + return markWeakReferences(budget); +} + +template +IncrementalProgress GCRuntime::markGrayRoots(SliceBudget& budget, + gcstats::PhaseKind phase) { + MOZ_ASSERT(marker().markColor() == MarkColor::Gray); + + gcstats::AutoPhase ap(stats(), phase); + + AutoUpdateLiveCompartments updateLive(this); + marker().setRootMarkingMode(true); + auto guard = + mozilla::MakeScopeExit([this]() { marker().setRootMarkingMode(false); }); + + IncrementalProgress result = + traceEmbeddingGrayRoots(marker().tracer(), budget); + if (result == NotFinished) { + return NotFinished; + } + + Compartment::traceIncomingCrossCompartmentEdgesForZoneGC( + marker().tracer(), Compartment::GrayEdges); + + return Finished; +} + +IncrementalProgress GCRuntime::markAllWeakReferences() { + SliceBudget budget = SliceBudget::unlimited(); + return markWeakReferences(budget); +} + +void GCRuntime::markAllGrayReferences(gcstats::PhaseKind phase) { + SliceBudget budget = SliceBudget::unlimited(); + markGrayRoots(budget, phase); + drainMarkStack(); +} + +void GCRuntime::dropStringWrappers() { + /* + * String "wrappers" are dropped on GC because their presence would require + * us to sweep the wrappers in all compartments every time we sweep a + * compartment group. + */ + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->dropStringWrappersOnGC(); + } +} + +/* + * Group zones that must be swept at the same time. + * + * From the point of view of the mutator, groups of zones transition atomically + * from marking to sweeping. If compartment A has an edge to an unmarked object + * in compartment B, then we must not start sweeping A in a later slice than we + * start sweeping B. That's because a write barrier in A could lead to the + * unmarked object in B becoming marked. However, if we had already swept that + * object, we would be in trouble. + * + * If we consider these dependencies as a graph, then all the compartments in + * any strongly-connected component of this graph must start sweeping in the + * same slice. + * + * Tarjan's algorithm is used to calculate the components. + */ + +bool Compartment::findSweepGroupEdges() { + Zone* source = zone(); + for (WrappedObjectCompartmentEnum e(this); !e.empty(); e.popFront()) { + Compartment* targetComp = e.front(); + Zone* target = targetComp->zone(); + + if (!target->isGCMarking() || source->hasSweepGroupEdgeTo(target)) { + continue; + } + + for (ObjectWrapperEnum e(this, targetComp); !e.empty(); e.popFront()) { + JSObject* key = e.front().mutableKey(); + MOZ_ASSERT(key->zone() == target); + + // Add an edge to the wrapped object's zone to ensure that the wrapper + // zone is not still being marked when we start sweeping the wrapped zone. + // As an optimization, if the wrapped object is already marked black there + // is no danger of later marking and we can skip this. + if (key->isMarkedBlack()) { + continue; + } + + if (!source->addSweepGroupEdgeTo(target)) { + return false; + } + + // We don't need to consider any more wrappers for this target + // compartment since we already added an edge. + break; + } + } + + return true; +} + +bool Zone::findSweepGroupEdges(Zone* atomsZone) { + MOZ_ASSERT_IF(this != atomsZone, !isAtomsZone()); + +#ifdef DEBUG + if (FinalizationObservers* observers = finalizationObservers()) { + observers->checkTables(); + } +#endif + + // Any zone may have a pointer to an atom in the atoms zone, and these aren't + // in the cross compartment map. + if (atomsZone->wasGCStarted() && !addSweepGroupEdgeTo(atomsZone)) { + return false; + } + + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + if (!comp->findSweepGroupEdges()) { + return false; + } + } + + return WeakMapBase::findSweepGroupEdgesForZone(this); +} + +bool GCRuntime::addEdgesForMarkQueue() { +#ifdef DEBUG + // For testing only. + // + // Add edges between all objects mentioned in the test mark queue, since + // otherwise they will get marked in a different order than their sweep + // groups. Note that this is only done at the beginning of an incremental + // collection, so it is possible for objects to be added later that do not + // follow the sweep group ordering. These objects will wait until their sweep + // group comes up, or will be skipped if their sweep group is already past. + JS::Zone* prevZone = nullptr; + for (size_t i = 0; i < testMarkQueue.length(); i++) { + Value val = testMarkQueue[i].get(); + if (!val.isObject()) { + continue; + } + JSObject* obj = &val.toObject(); + JS::Zone* zone = obj->zone(); + if (!zone->isGCMarking()) { + continue; + } + if (prevZone && prevZone != zone) { + if (!prevZone->addSweepGroupEdgeTo(zone)) { + return false; + } + } + prevZone = zone; + } +#endif + return true; +} + +bool GCRuntime::findSweepGroupEdges() { + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + if (!zone->findSweepGroupEdges(atomsZone())) { + return false; + } + } + + if (!addEdgesForMarkQueue()) { + return false; + } + + return DebugAPI::findSweepGroupEdges(rt); +} + +void GCRuntime::groupZonesForSweeping(JS::GCReason reason) { +#ifdef DEBUG + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->gcSweepGroupEdges().empty()); + } +#endif + + JSContext* cx = rt->mainContextFromOwnThread(); + ZoneComponentFinder finder(cx); + if (!isIncremental || !findSweepGroupEdges()) { + finder.useOneComponent(); + } + + // Use one component for two-slice zeal modes. + if (useZeal && hasIncrementalTwoSliceZealMode()) { + finder.useOneComponent(); + } + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->isGCMarking()); + finder.addNode(zone); + } + sweepGroups = finder.getResultsList(); + currentSweepGroup = sweepGroups; + sweepGroupIndex = 1; + + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + zone->clearSweepGroupEdges(); + } + +#ifdef DEBUG + unsigned idx = sweepGroupIndex; + for (Zone* head = currentSweepGroup; head; head = head->nextGroup()) { + for (Zone* zone = head; zone; zone = zone->nextNodeInGroup()) { + MOZ_ASSERT(zone->isGCMarking()); + zone->gcSweepGroupIndex = idx; + } + idx++; + } + + MOZ_ASSERT_IF(!isIncremental, !currentSweepGroup->nextGroup()); + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->gcSweepGroupEdges().empty()); + } +#endif +} + +void GCRuntime::getNextSweepGroup() { + currentSweepGroup = currentSweepGroup->nextGroup(); + ++sweepGroupIndex; + if (!currentSweepGroup) { + abortSweepAfterCurrentGroup = false; + return; + } + + MOZ_ASSERT_IF(abortSweepAfterCurrentGroup, !isIncremental); + if (!isIncremental) { + ZoneComponentFinder::mergeGroups(currentSweepGroup); + } + + for (Zone* zone = currentSweepGroup; zone; zone = zone->nextNodeInGroup()) { + MOZ_ASSERT(zone->gcState() == zone->initialMarkingState()); + MOZ_ASSERT(!zone->isQueuedForBackgroundSweep()); + } + + if (abortSweepAfterCurrentGroup) { + markTask.join(); + + // Abort collection of subsequent sweep groups. + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + MOZ_ASSERT(!zone->gcNextGraphComponent); + zone->changeGCState(zone->initialMarkingState(), Zone::NoGC); + zone->arenas.unmarkPreMarkedFreeCells(); + zone->arenas.mergeArenasFromCollectingLists(); + zone->clearGCSliceThresholds(); + } + + for (SweepGroupCompartmentsIter comp(rt); !comp.done(); comp.next()) { + resetGrayList(comp); + } + + abortSweepAfterCurrentGroup = false; + currentSweepGroup = nullptr; + } +} + +/* + * Gray marking: + * + * At the end of collection, anything reachable from a gray root that has not + * otherwise been marked black must be marked gray. + * + * This means that when marking things gray we must not allow marking to leave + * the current compartment group, as that could result in things being marked + * gray when they might subsequently be marked black. To achieve this, when we + * find a cross compartment pointer we don't mark the referent but add it to a + * singly-linked list of incoming gray pointers that is stored with each + * compartment. + * + * The list head is stored in Compartment::gcIncomingGrayPointers and contains + * cross compartment wrapper objects. The next pointer is stored in the second + * extra slot of the cross compartment wrapper. + * + * The list is created during gray marking when one of the + * MarkCrossCompartmentXXX functions is called for a pointer that leaves the + * current compartent group. This calls DelayCrossCompartmentGrayMarking to + * push the referring object onto the list. + * + * The list is traversed and then unlinked in + * GCRuntime::markIncomingGrayCrossCompartmentPointers. + */ + +static bool IsGrayListObject(JSObject* obj) { + MOZ_ASSERT(obj); + return obj->is() && !IsDeadProxyObject(obj); +} + +/* static */ +unsigned ProxyObject::grayLinkReservedSlot(JSObject* obj) { + MOZ_ASSERT(IsGrayListObject(obj)); + return CrossCompartmentWrapperObject::GrayLinkReservedSlot; +} + +#ifdef DEBUG +static void AssertNotOnGrayList(JSObject* obj) { + MOZ_ASSERT_IF( + IsGrayListObject(obj), + GetProxyReservedSlot(obj, ProxyObject::grayLinkReservedSlot(obj)) + .isUndefined()); +} +#endif + +static void AssertNoWrappersInGrayList(JSRuntime* rt) { +#ifdef DEBUG + for (CompartmentsIter c(rt); !c.done(); c.next()) { + MOZ_ASSERT(!c->gcIncomingGrayPointers); + for (Compartment::ObjectWrapperEnum e(c); !e.empty(); e.popFront()) { + AssertNotOnGrayList(e.front().value().unbarrieredGet()); + } + } +#endif +} + +static JSObject* CrossCompartmentPointerReferent(JSObject* obj) { + MOZ_ASSERT(IsGrayListObject(obj)); + return &obj->as().private_().toObject(); +} + +static JSObject* NextIncomingCrossCompartmentPointer(JSObject* prev, + bool unlink) { + unsigned slot = ProxyObject::grayLinkReservedSlot(prev); + JSObject* next = GetProxyReservedSlot(prev, slot).toObjectOrNull(); + MOZ_ASSERT_IF(next, IsGrayListObject(next)); + + if (unlink) { + SetProxyReservedSlot(prev, slot, UndefinedValue()); + } + + return next; +} + +void js::gc::DelayCrossCompartmentGrayMarking(GCMarker* maybeMarker, + JSObject* src) { + MOZ_ASSERT_IF(!maybeMarker, !JS::RuntimeHeapIsBusy()); + MOZ_ASSERT(IsGrayListObject(src)); + MOZ_ASSERT(src->isMarkedGray()); + + AutoTouchingGrayThings tgt; + + mozilla::Maybe lock; + if (maybeMarker && maybeMarker->isParallelMarking()) { + // Synchronize access to JSCompartment::gcIncomingGrayPointers. + // + // TODO: Instead of building this list we could scan all incoming CCWs and + // mark through gray ones when marking gray roots for a sweep group. + lock.emplace(maybeMarker->runtime()); + } + + /* Called from MarkCrossCompartmentXXX functions. */ + unsigned slot = ProxyObject::grayLinkReservedSlot(src); + JSObject* dest = CrossCompartmentPointerReferent(src); + Compartment* comp = dest->compartment(); + + if (GetProxyReservedSlot(src, slot).isUndefined()) { + SetProxyReservedSlot(src, slot, + ObjectOrNullValue(comp->gcIncomingGrayPointers)); + comp->gcIncomingGrayPointers = src; + } else { + MOZ_ASSERT(GetProxyReservedSlot(src, slot).isObjectOrNull()); + } + +#ifdef DEBUG + /* + * Assert that the object is in our list, also walking the list to check its + * integrity. + */ + JSObject* obj = comp->gcIncomingGrayPointers; + bool found = false; + while (obj) { + if (obj == src) { + found = true; + } + obj = NextIncomingCrossCompartmentPointer(obj, false); + } + MOZ_ASSERT(found); +#endif +} + +void GCRuntime::markIncomingGrayCrossCompartmentPointers() { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_INCOMING_GRAY); + + for (SweepGroupCompartmentsIter c(rt); !c.done(); c.next()) { + MOZ_ASSERT(c->zone()->isGCMarkingBlackAndGray()); + MOZ_ASSERT_IF(c->gcIncomingGrayPointers, + IsGrayListObject(c->gcIncomingGrayPointers)); + + for (JSObject* src = c->gcIncomingGrayPointers; src; + src = NextIncomingCrossCompartmentPointer(src, true)) { + JSObject* dst = CrossCompartmentPointerReferent(src); + MOZ_ASSERT(dst->compartment() == c); + MOZ_ASSERT_IF(src->asTenured().isMarkedBlack(), + dst->asTenured().isMarkedBlack()); + + if (src->asTenured().isMarkedGray()) { + TraceManuallyBarrieredEdge(marker().tracer(), &dst, + "cross-compartment gray pointer"); + } + } + + c->gcIncomingGrayPointers = nullptr; + } +} + +static bool RemoveFromGrayList(JSObject* wrapper) { + AutoTouchingGrayThings tgt; + + if (!IsGrayListObject(wrapper)) { + return false; + } + + unsigned slot = ProxyObject::grayLinkReservedSlot(wrapper); + if (GetProxyReservedSlot(wrapper, slot).isUndefined()) { + return false; /* Not on our list. */ + } + + JSObject* tail = GetProxyReservedSlot(wrapper, slot).toObjectOrNull(); + SetProxyReservedSlot(wrapper, slot, UndefinedValue()); + + Compartment* comp = CrossCompartmentPointerReferent(wrapper)->compartment(); + JSObject* obj = comp->gcIncomingGrayPointers; + if (obj == wrapper) { + comp->gcIncomingGrayPointers = tail; + return true; + } + + while (obj) { + unsigned slot = ProxyObject::grayLinkReservedSlot(obj); + JSObject* next = GetProxyReservedSlot(obj, slot).toObjectOrNull(); + if (next == wrapper) { + js::detail::SetProxyReservedSlotUnchecked(obj, slot, + ObjectOrNullValue(tail)); + return true; + } + obj = next; + } + + MOZ_CRASH("object not found in gray link list"); +} + +void GCRuntime::resetGrayList(Compartment* comp) { + JSObject* src = comp->gcIncomingGrayPointers; + while (src) { + src = NextIncomingCrossCompartmentPointer(src, true); + } + comp->gcIncomingGrayPointers = nullptr; +} + +#ifdef DEBUG +static bool HasIncomingCrossCompartmentPointers(JSRuntime* rt) { + for (SweepGroupCompartmentsIter c(rt); !c.done(); c.next()) { + if (c->gcIncomingGrayPointers) { + return true; + } + } + + return false; +} +#endif + +void js::NotifyGCNukeWrapper(JSContext* cx, JSObject* wrapper) { + MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper)); + + /* + * References to target of wrapper are being removed, we no longer have to + * remember to mark it. + */ + RemoveFromGrayList(wrapper); + + /* + * Clean up WeakRef maps which might include this wrapper. + */ + JSObject* target = UncheckedUnwrapWithoutExpose(wrapper); + if (target->is()) { + WeakRefObject* weakRef = &target->as(); + if (weakRef->target()) { + cx->runtime()->gc.nukeWeakRefWrapper(wrapper, weakRef); + } + } + + /* + * Clean up FinalizationRecord record objects which might be the target of + * this wrapper. + */ + if (target->is()) { + auto* record = &target->as(); + cx->runtime()->gc.nukeFinalizationRecordWrapper(wrapper, record); + } +} + +enum { + JS_GC_SWAP_OBJECT_A_REMOVED = 1 << 0, + JS_GC_SWAP_OBJECT_B_REMOVED = 1 << 1 +}; + +unsigned js::NotifyGCPreSwap(JSObject* a, JSObject* b) { + /* + * Two objects in the same compartment are about to have had their contents + * swapped. If either of them are in our gray pointer list, then we remove + * them from the lists, returning a bitset indicating what happened. + */ + return (RemoveFromGrayList(a) ? JS_GC_SWAP_OBJECT_A_REMOVED : 0) | + (RemoveFromGrayList(b) ? JS_GC_SWAP_OBJECT_B_REMOVED : 0); +} + +void js::NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned removedFlags) { + /* + * Two objects in the same compartment have had their contents swapped. If + * either of them were in our gray pointer list, we re-add them again. + */ + if (removedFlags & JS_GC_SWAP_OBJECT_A_REMOVED) { + DelayCrossCompartmentGrayMarking(nullptr, b); + } + if (removedFlags & JS_GC_SWAP_OBJECT_B_REMOVED) { + DelayCrossCompartmentGrayMarking(nullptr, a); + } +} + +static inline void MaybeCheckWeakMapMarking(GCRuntime* gc) { +#if defined(JS_GC_ZEAL) || defined(DEBUG) + + bool shouldCheck; +# if defined(DEBUG) + shouldCheck = true; +# else + shouldCheck = gc->hasZealMode(ZealMode::CheckWeakMapMarking); +# endif + + if (shouldCheck) { + for (SweepGroupZonesIter zone(gc); !zone.done(); zone.next()) { + MOZ_RELEASE_ASSERT(WeakMapBase::checkMarkingForZone(zone)); + } + } + +#endif +} + +IncrementalProgress GCRuntime::beginMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget) { +#ifdef DEBUG + MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping); + assertNoMarkingWork(); + for (auto& marker : markers) { + MOZ_ASSERT(marker->markColor() == MarkColor::Black); + } + MOZ_ASSERT(cellsToAssertNotGray.ref().empty()); +#endif + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + + // Change state of current group to MarkBlackAndGray to restrict gray marking + // to this group. Note that there may be pointers to the atoms zone, and these + // will be marked through, as they are not marked with + // TraceCrossCompartmentEdge. + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + zone->changeGCState(zone->initialMarkingState(), Zone::MarkBlackAndGray); + } + + AutoSetMarkColor setColorGray(marker(), MarkColor::Gray); + + // Mark incoming gray pointers from previously swept compartments. + markIncomingGrayCrossCompartmentPointers(); + + return Finished; +} + +IncrementalProgress GCRuntime::markGrayRootsInCurrentGroup( + JS::GCContext* gcx, SliceBudget& budget) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + + AutoSetMarkColor setColorGray(marker(), MarkColor::Gray); + + return markGrayRoots(budget, + gcstats::PhaseKind::MARK_GRAY); +} + +IncrementalProgress GCRuntime::markGray(JS::GCContext* gcx, + SliceBudget& budget) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + + if (markUntilBudgetExhausted(budget, AllowParallelMarking) == NotFinished) { + return NotFinished; + } + + return Finished; +} + +IncrementalProgress GCRuntime::endMarkingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget) { +#ifdef DEBUG + MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping); + assertNoMarkingWork(); + for (auto& marker : markers) { + MOZ_ASSERT(marker->markColor() == MarkColor::Black); + } + MOZ_ASSERT(!HasIncomingCrossCompartmentPointers(rt)); +#endif + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + + if (markWeakReferencesInCurrentGroup(budget) == NotFinished) { + return NotFinished; + } + + AutoSetMarkColor setColorGray(marker(), MarkColor::Gray); + + // Mark transitively inside the current compartment group. + if (markWeakReferencesInCurrentGroup(budget) == NotFinished) { + return NotFinished; + } + + MOZ_ASSERT(marker().isDrained()); + + // We must not yield after this point before we start sweeping the group. + safeToYield = false; + + MaybeCheckWeakMapMarking(this); + + return Finished; +} + +// Causes the given WeakCache to be swept when run. +class ImmediateSweepWeakCacheTask : public GCParallelTask { + Zone* zone; + JS::detail::WeakCacheBase& cache; + + ImmediateSweepWeakCacheTask(const ImmediateSweepWeakCacheTask&) = delete; + + public: + ImmediateSweepWeakCacheTask(GCRuntime* gc, Zone* zone, + JS::detail::WeakCacheBase& wc) + : GCParallelTask(gc, gcstats::PhaseKind::SWEEP_WEAK_CACHES), + zone(zone), + cache(wc) {} + + ImmediateSweepWeakCacheTask(ImmediateSweepWeakCacheTask&& other) + : GCParallelTask(std::move(other)), + zone(other.zone), + cache(other.cache) {} + + void run(AutoLockHelperThreadState& lock) override { + AutoUnlockHelperThreadState unlock(lock); + AutoSetThreadIsSweeping threadIsSweeping(zone); + SweepingTracer trc(gc->rt); + cache.traceWeak(&trc, &gc->storeBuffer()); + } +}; + +void GCRuntime::updateAtomsBitmap() { + DenseBitmap marked; + if (atomMarking.computeBitmapFromChunkMarkBits(rt, marked)) { + for (GCZonesIter zone(this); !zone.done(); zone.next()) { + atomMarking.refineZoneBitmapForCollectedZone(zone, marked); + } + } else { + // Ignore OOM in computeBitmapFromChunkMarkBits. The + // refineZoneBitmapForCollectedZone call can only remove atoms from the + // zone bitmap, so it is conservative to just not call it. + } + + atomMarking.markAtomsUsedByUncollectedZones(rt); + + // For convenience sweep these tables non-incrementally as part of bitmap + // sweeping; they are likely to be much smaller than the main atoms table. + SweepingTracer trc(rt); + rt->symbolRegistry().traceWeak(&trc); +} + +void GCRuntime::sweepCCWrappers() { + SweepingTracer trc(rt); + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + zone->traceWeakCCWEdges(&trc); + } +} + +void GCRuntime::sweepRealmGlobals() { + SweepingTracer trc(rt); + for (SweepGroupRealmsIter r(this); !r.done(); r.next()) { + AutoSetThreadIsSweeping threadIsSweeping(r->zone()); + r->traceWeakGlobalEdge(&trc); + } +} + +void GCRuntime::sweepMisc() { + SweepingTracer trc(rt); + for (SweepGroupRealmsIter r(this); !r.done(); r.next()) { + AutoSetThreadIsSweeping threadIsSweeping(r->zone()); + r->traceWeakSavedStacks(&trc); + r->traceWeakRegExps(&trc); + } + for (SweepGroupCompartmentsIter c(this); !c.done(); c.next()) { + AutoSetThreadIsSweeping threadIsSweeping(c->zone()); + c->traceWeakNativeIterators(&trc); + } +} + +void GCRuntime::sweepCompressionTasks() { + JSRuntime* runtime = rt; + + // Attach finished compression tasks. + AutoLockHelperThreadState lock; + AttachFinishedCompressions(runtime, lock); + SweepPendingCompressions(lock); +} + +void GCRuntime::sweepWeakMaps() { + SweepingTracer trc(rt); + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + /* No need to look up any more weakmap keys from this sweep group. */ + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing weak keys in beginSweepingSweepGroup()"); + } + + // Lock the storebuffer since this may access it when rehashing or resizing + // the tables. + AutoLockStoreBuffer lock(&storeBuffer()); + zone->sweepWeakMaps(&trc); + } +} + +void GCRuntime::sweepUniqueIds() { + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + AutoSetThreadIsSweeping threadIsSweeping(zone); + zone->sweepUniqueIds(); + } +} + +void JS::Zone::sweepUniqueIds() { + SweepingTracer trc(runtimeFromAnyThread()); + uniqueIds().traceWeak(&trc); +} + +/* static */ +bool UniqueIdGCPolicy::traceWeak(JSTracer* trc, Cell** keyp, uint64_t* valuep) { + // Since this is only ever used for sweeping, we can optimize it for that + // case. (Compacting GC updates this table manually when it moves a cell.) + MOZ_ASSERT(trc->kind() == JS::TracerKind::Sweeping); + return (*keyp)->isMarkedAny(); +} + +void GCRuntime::sweepFinalizationObserversOnMainThread() { + // This calls back into the browser which expects to be called from the main + // thread. + gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS); + gcstats::AutoPhase ap2(stats(), + gcstats::PhaseKind::SWEEP_FINALIZATION_OBSERVERS); + SweepingTracer trc(rt); + AutoLockStoreBuffer lock(&storeBuffer()); + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + traceWeakFinalizationObserverEdges(&trc, zone); + } +} + +void GCRuntime::startTask(GCParallelTask& task, + AutoLockHelperThreadState& lock) { + if (!CanUseExtraThreads()) { + AutoUnlockHelperThreadState unlock(lock); + task.runFromMainThread(); + stats().recordParallelPhase(task.phaseKind, task.duration()); + return; + } + + task.startWithLockHeld(lock); +} + +void GCRuntime::joinTask(GCParallelTask& task, + AutoLockHelperThreadState& lock) { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::JOIN_PARALLEL_TASKS); + task.joinWithLockHeld(lock); +} + +void GCRuntime::sweepDebuggerOnMainThread(JS::GCContext* gcx) { + SweepingTracer trc(rt); + AutoLockStoreBuffer lock(&storeBuffer()); + + // Detach unreachable debuggers and global objects from each other. + // This can modify weakmaps and so must happen before weakmap sweeping. + DebugAPI::sweepAll(gcx); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS); + + // Sweep debug environment information. This performs lookups in the Zone's + // unique IDs table and so must not happen in parallel with sweeping that + // table. + { + gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::SWEEP_MISC); + for (SweepGroupRealmsIter r(rt); !r.done(); r.next()) { + r->traceWeakDebugEnvironmentEdges(&trc); + } + } +} + +void GCRuntime::sweepJitDataOnMainThread(JS::GCContext* gcx) { + SweepingTracer trc(rt); + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_JIT_DATA); + + if (initialState != State::NotActive) { + // Cancel any active or pending off thread compilations. We also did + // this before marking (in DiscardJITCodeForGC) so this is a no-op + // for non-incremental GCs. + js::CancelOffThreadIonCompile(rt, JS::Zone::Sweep); + } + + // Bug 1071218: the following method has not yet been refactored to + // work on a single zone-group at once. + + // Sweep entries containing about-to-be-finalized JitCode and + // update relocated TypeSet::Types inside the JitcodeGlobalTable. + jit::JitRuntime::TraceWeakJitcodeGlobalTable(rt, &trc); + } + + if (initialState != State::NotActive) { + gcstats::AutoPhase apdc(stats(), gcstats::PhaseKind::SWEEP_DISCARD_CODE); + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + zone->discardJitCode(gcx); + } + } + + // JitZone/JitRealm must be swept *after* discarding JIT code, because + // Zone::discardJitCode might access CacheIRStubInfos deleted here. + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_JIT_DATA); + + for (SweepGroupRealmsIter r(rt); !r.done(); r.next()) { + r->traceWeakEdgesInJitRealm(&trc); + } + + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + if (jit::JitZone* jitZone = zone->jitZone()) { + jitZone->traceWeak(&trc); + } + } + } +} + +using WeakCacheTaskVector = + mozilla::Vector; + +// Call a functor for all weak caches that need to be swept in the current +// sweep group. +template +static inline bool IterateWeakCaches(JSRuntime* rt, Functor f) { + for (SweepGroupZonesIter zone(rt); !zone.done(); zone.next()) { + for (JS::detail::WeakCacheBase* cache : zone->weakCaches()) { + if (!f(cache, zone.get())) { + return false; + } + } + } + + for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) { + if (!f(cache, nullptr)) { + return false; + } + } + + return true; +} + +static bool PrepareWeakCacheTasks(JSRuntime* rt, + WeakCacheTaskVector* immediateTasks) { + // Start incremental sweeping for caches that support it or add to a vector + // of sweep tasks to run on a helper thread. + + MOZ_ASSERT(immediateTasks->empty()); + + GCRuntime* gc = &rt->gc; + bool ok = + IterateWeakCaches(rt, [&](JS::detail::WeakCacheBase* cache, Zone* zone) { + if (cache->empty()) { + return true; + } + + // Caches that support incremental sweeping will be swept later. + if (zone && cache->setIncrementalBarrierTracer(&gc->sweepingTracer)) { + return true; + } + + return immediateTasks->emplaceBack(gc, zone, *cache); + }); + + if (!ok) { + immediateTasks->clearAndFree(); + } + + return ok; +} + +static void SweepAllWeakCachesOnMainThread(JSRuntime* rt) { + // If we ran out of memory, do all the work on the main thread. + gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::SWEEP_WEAK_CACHES); + SweepingTracer trc(rt); + IterateWeakCaches(rt, [&](JS::detail::WeakCacheBase* cache, Zone* zone) { + if (cache->needsIncrementalBarrier()) { + cache->setIncrementalBarrierTracer(nullptr); + } + cache->traceWeak(&trc, &rt->gc.storeBuffer()); + return true; + }); +} + +void GCRuntime::sweepEmbeddingWeakPointers(JS::GCContext* gcx) { + using namespace gcstats; + + AutoLockStoreBuffer lock(&storeBuffer()); + + AutoPhase ap(stats(), PhaseKind::FINALIZE_START); + callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_PREPARE); + { + AutoPhase ap2(stats(), PhaseKind::WEAK_ZONES_CALLBACK); + callWeakPointerZonesCallbacks(&sweepingTracer); + } + { + AutoPhase ap2(stats(), PhaseKind::WEAK_COMPARTMENT_CALLBACK); + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + callWeakPointerCompartmentCallbacks(&sweepingTracer, comp); + } + } + } + callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_START); +} + +IncrementalProgress GCRuntime::beginSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget) { + /* + * Begin sweeping the group of zones in currentSweepGroup, performing + * actions that must be done before yielding to caller. + */ + + using namespace gcstats; + + AutoSCC scc(stats(), sweepGroupIndex); + + bool sweepingAtoms = false; + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + /* Set the GC state to sweeping. */ + zone->changeGCState(Zone::MarkBlackAndGray, Zone::Sweep); + + /* Purge the ArenaLists before sweeping. */ + zone->arenas.checkSweepStateNotInUse(); + zone->arenas.unmarkPreMarkedFreeCells(); + zone->arenas.clearFreeLists(); + + if (zone->isAtomsZone()) { + sweepingAtoms = true; + } + } + +#ifdef DEBUG + for (auto cell : cellsToAssertNotGray.ref()) { + JS::AssertCellIsNotGray(cell); + } + cellsToAssertNotGray.ref().clearAndFree(); +#endif + + // Updating the atom marking bitmaps. This marks atoms referenced by + // uncollected zones so cannot be done in parallel with the other sweeping + // work below. + if (sweepingAtoms) { + AutoPhase ap(stats(), PhaseKind::UPDATE_ATOMS_BITMAP); + updateAtomsBitmap(); + } + +#ifdef JS_GC_ZEAL + validateIncrementalMarking(); +#endif + + AutoSetThreadIsSweeping threadIsSweeping; + + // This must happen before sweeping realm globals. + sweepDebuggerOnMainThread(gcx); + + // FinalizationRegistry sweeping touches weak maps and so must not run in + // parallel with that. This triggers a read barrier and can add marking work + // for zones that are still marking. Must happen before sweeping realm + // globals. + sweepFinalizationObserversOnMainThread(); + + // This must happen before updating embedding weak pointers. + sweepRealmGlobals(); + + sweepEmbeddingWeakPointers(gcx); + + { + AutoLockHelperThreadState lock; + + AutoPhase ap(stats(), PhaseKind::SWEEP_COMPARTMENTS); + + AutoRunParallelTask sweepCCWrappers(this, &GCRuntime::sweepCCWrappers, + PhaseKind::SWEEP_CC_WRAPPER, + GCUse::Sweeping, lock); + AutoRunParallelTask sweepMisc(this, &GCRuntime::sweepMisc, + PhaseKind::SWEEP_MISC, GCUse::Sweeping, lock); + AutoRunParallelTask sweepCompTasks(this, &GCRuntime::sweepCompressionTasks, + PhaseKind::SWEEP_COMPRESSION, + GCUse::Sweeping, lock); + AutoRunParallelTask sweepWeakMaps(this, &GCRuntime::sweepWeakMaps, + PhaseKind::SWEEP_WEAKMAPS, + GCUse::Sweeping, lock); + AutoRunParallelTask sweepUniqueIds(this, &GCRuntime::sweepUniqueIds, + PhaseKind::SWEEP_UNIQUEIDS, + GCUse::Sweeping, lock); + + WeakCacheTaskVector sweepCacheTasks; + bool canSweepWeakCachesOffThread = + PrepareWeakCacheTasks(rt, &sweepCacheTasks); + if (canSweepWeakCachesOffThread) { + weakCachesToSweep.ref().emplace(currentSweepGroup); + for (auto& task : sweepCacheTasks) { + startTask(task, lock); + } + } + + { + AutoUnlockHelperThreadState unlock(lock); + sweepJitDataOnMainThread(gcx); + + if (!canSweepWeakCachesOffThread) { + MOZ_ASSERT(sweepCacheTasks.empty()); + SweepAllWeakCachesOnMainThread(rt); + } + } + + for (auto& task : sweepCacheTasks) { + joinTask(task, lock); + } + } + + if (sweepingAtoms) { + startSweepingAtomsTable(); + } + + // Queue all GC things in all zones for sweeping, either on the foreground + // or on the background thread. + + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + for (const auto& phase : BackgroundFinalizePhases) { + initBackgroundSweep(zone, gcx, phase); + } + + zone->arenas.queueForegroundThingsForSweep(); + } + + MOZ_ASSERT(!sweepZone); + + safeToYield = true; + markOnBackgroundThreadDuringSweeping = CanUseExtraThreads(); + + return Finished; +} + +#ifdef JS_GC_ZEAL +bool GCRuntime::shouldYieldForZeal(ZealMode mode) { + bool yield = useZeal && hasZealMode(mode); + + // Only yield on the first sweep slice for this mode. + bool firstSweepSlice = initialState != State::Sweep; + if (mode == ZealMode::IncrementalMultipleSlices && !firstSweepSlice) { + yield = false; + } + + return yield; +} +#endif + +IncrementalProgress GCRuntime::endSweepingSweepGroup(JS::GCContext* gcx, + SliceBudget& budget) { + // This is to prevent a race between markTask checking the zone state and + // us changing it below. + if (joinBackgroundMarkTask() == NotFinished) { + return NotFinished; + } + + assertNoMarkingWork(); + + // Disable background marking during sweeping until we start sweeping the next + // zone group. + markOnBackgroundThreadDuringSweeping = false; + + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END); + AutoLockStoreBuffer lock(&storeBuffer()); + callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_END); + } + + /* Free LIFO blocks on a background thread if possible. */ + startBackgroundFree(); + + /* Update the GC state for zones we have swept. */ + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + if (jit::JitZone* jitZone = zone->jitZone()) { + // Clear out any small pools that we're hanging on to. + jitZone->execAlloc().purge(); + } + AutoLockGC lock(this); + zone->changeGCState(Zone::Sweep, Zone::Finished); + zone->arenas.unmarkPreMarkedFreeCells(); + zone->arenas.checkNoArenasToUpdate(); + zone->pretenuring.clearCellCountsInNewlyCreatedArenas(); + } + + /* + * Start background thread to sweep zones if required, sweeping any atoms + * zones last if present. + */ + ZoneList zones; + for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) { + if (zone->isAtomsZone()) { + zones.append(zone); + } else { + zones.prepend(zone); + } + } + + queueZonesAndStartBackgroundSweep(std::move(zones)); + + return Finished; +} + +IncrementalProgress GCRuntime::markDuringSweeping(JS::GCContext* gcx, + SliceBudget& budget) { + MOZ_ASSERT(markTask.isIdle()); + + if (markOnBackgroundThreadDuringSweeping) { + if (!marker().isDrained() || hasDelayedMarking()) { + AutoLockHelperThreadState lock; + MOZ_ASSERT(markTask.isIdle(lock)); + markTask.setBudget(budget); + markTask.startOrRunIfIdle(lock); + } + return Finished; // This means don't yield to the mutator here. + } + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK); + return markUntilBudgetExhausted(budget, AllowParallelMarking); +} + +void GCRuntime::beginSweepPhase(JS::GCReason reason, AutoGCSession& session) { + /* + * Sweep phase. + * + * Finalize as we sweep, outside of lock but with RuntimeHeapIsBusy() + * true so that any attempt to allocate a GC-thing from a finalizer will + * fail, rather than nest badly and leave the unmarked newborn to be swept. + */ + + MOZ_ASSERT(!abortSweepAfterCurrentGroup); + MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping); + +#ifdef DEBUG + releaseHeldRelocatedArenas(); + verifyAllChunks(); +#endif + +#ifdef JS_GC_ZEAL + computeNonIncrementalMarkingForValidation(session); +#endif + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP); + + AssertNoWrappersInGrayList(rt); + dropStringWrappers(); + + groupZonesForSweeping(reason); + + sweepActions->assertFinished(); +} + +bool GCRuntime::foregroundFinalize(JS::GCContext* gcx, Zone* zone, + AllocKind thingKind, + SliceBudget& sliceBudget, + SortedArenaList& sweepList) { + ArenaLists& lists = zone->arenas; + lists.checkNoArenasToUpdateForKind(thingKind); + + // Non-empty arenas are reused for use for new allocations as soon as the + // finalizers for that allocation kind have run. Empty arenas are only + // released when everything in the zone has been swept (see + // GCRuntime::sweepBackgroundThings for more details). + if (!FinalizeArenas(gcx, lists.collectingArenaList(thingKind), sweepList, + thingKind, sliceBudget)) { + // Copy the current contents of sweepList so that ArenaIter can find them. + lists.setIncrementalSweptArenas(thingKind, sweepList); + return false; + } + + sweepList.extractEmpty(&lists.savedEmptyArenas.ref()); + lists.mergeFinalizedArenas(thingKind, sweepList); + lists.clearIncrementalSweptArenas(); + + return true; +} + +BackgroundMarkTask::BackgroundMarkTask(GCRuntime* gc) + : GCParallelTask(gc, gcstats::PhaseKind::MARK, GCUse::Marking), + budget(SliceBudget::unlimited()) {} + +void js::gc::BackgroundMarkTask::run(AutoLockHelperThreadState& lock) { + AutoUnlockHelperThreadState unlock(lock); + + // Time reporting is handled separately for parallel tasks. + gc->sweepMarkResult = gc->markUntilBudgetExhausted( + this->budget, GCRuntime::SingleThreadedMarking, DontReportMarkTime); +} + +IncrementalProgress GCRuntime::joinBackgroundMarkTask() { + AutoLockHelperThreadState lock; + if (markTask.isIdle(lock)) { + return Finished; + } + + joinTask(markTask, lock); + + IncrementalProgress result = sweepMarkResult; + sweepMarkResult = Finished; + return result; +} + +template +static void SweepThing(JS::GCContext* gcx, T* thing) { + if (!TenuredThingIsMarkedAny(thing)) { + thing->sweep(gcx); + } +} + +template +static bool SweepArenaList(JS::GCContext* gcx, Arena** arenasToSweep, + SliceBudget& sliceBudget) { + while (Arena* arena = *arenasToSweep) { + MOZ_ASSERT(arena->zone->isGCSweeping()); + + for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) { + SweepThing(gcx, cell.as()); + } + + Arena* next = arena->next; + MOZ_ASSERT_IF(next, next->zone == arena->zone); + *arenasToSweep = next; + + AllocKind kind = MapTypeToAllocKind::kind; + sliceBudget.step(Arena::thingsPerArena(kind)); + if (sliceBudget.isOverBudget()) { + return false; + } + } + + return true; +} + +void GCRuntime::startSweepingAtomsTable() { + auto& maybeAtoms = maybeAtomsToSweep.ref(); + MOZ_ASSERT(maybeAtoms.isNothing()); + + AtomsTable* atomsTable = rt->atomsForSweeping(); + if (!atomsTable) { + return; + } + + // Create secondary tables to hold new atoms added while we're sweeping the + // main tables incrementally. + if (!atomsTable->startIncrementalSweep(maybeAtoms)) { + SweepingTracer trc(rt); + atomsTable->traceWeak(&trc); + } +} + +IncrementalProgress GCRuntime::sweepAtomsTable(JS::GCContext* gcx, + SliceBudget& budget) { + if (!atomsZone()->isGCSweeping()) { + return Finished; + } + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_ATOMS_TABLE); + + auto& maybeAtoms = maybeAtomsToSweep.ref(); + if (!maybeAtoms) { + return Finished; + } + + if (!rt->atomsForSweeping()->sweepIncrementally(maybeAtoms.ref(), budget)) { + return NotFinished; + } + + maybeAtoms.reset(); + + return Finished; +} + +static size_t IncrementalSweepWeakCache(GCRuntime* gc, + const WeakCacheToSweep& item) { + AutoSetThreadIsSweeping threadIsSweeping(item.zone); + + JS::detail::WeakCacheBase* cache = item.cache; + MOZ_ASSERT(cache->needsIncrementalBarrier()); + + SweepingTracer trc(gc->rt); + size_t steps = cache->traceWeak(&trc, &gc->storeBuffer()); + cache->setIncrementalBarrierTracer(nullptr); + + return steps; +} + +WeakCacheSweepIterator::WeakCacheSweepIterator(JS::Zone* sweepGroup) + : sweepZone(sweepGroup), sweepCache(sweepZone->weakCaches().getFirst()) { + settle(); +} + +bool WeakCacheSweepIterator::done() const { return !sweepZone; } + +WeakCacheToSweep WeakCacheSweepIterator::get() const { + MOZ_ASSERT(!done()); + + return {sweepCache, sweepZone}; +} + +void WeakCacheSweepIterator::next() { + MOZ_ASSERT(!done()); + + sweepCache = sweepCache->getNext(); + settle(); +} + +void WeakCacheSweepIterator::settle() { + while (sweepZone) { + while (sweepCache && !sweepCache->needsIncrementalBarrier()) { + sweepCache = sweepCache->getNext(); + } + + if (sweepCache) { + break; + } + + sweepZone = sweepZone->nextNodeInGroup(); + if (sweepZone) { + sweepCache = sweepZone->weakCaches().getFirst(); + } + } + + MOZ_ASSERT((!sweepZone && !sweepCache) || + (sweepCache && sweepCache->needsIncrementalBarrier())); +} + +IncrementalProgress GCRuntime::sweepWeakCaches(JS::GCContext* gcx, + SliceBudget& budget) { + if (weakCachesToSweep.ref().isNothing()) { + return Finished; + } + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS); + + WeakCacheSweepIterator& work = weakCachesToSweep.ref().ref(); + + AutoLockHelperThreadState lock; + + { + AutoRunParallelWork runWork(this, IncrementalSweepWeakCache, + gcstats::PhaseKind::SWEEP_WEAK_CACHES, + GCUse::Sweeping, work, budget, lock); + AutoUnlockHelperThreadState unlock(lock); + } + + if (work.done()) { + weakCachesToSweep.ref().reset(); + return Finished; + } + + return NotFinished; +} + +IncrementalProgress GCRuntime::finalizeAllocKind(JS::GCContext* gcx, + SliceBudget& budget) { + MOZ_ASSERT(sweepZone->isGCSweeping()); + + // Set the number of things per arena for this AllocKind. + size_t thingsPerArena = Arena::thingsPerArena(sweepAllocKind); + auto& sweepList = incrementalSweepList.ref(); + sweepList.setThingsPerArena(thingsPerArena); + + AutoSetThreadIsFinalizing threadIsFinalizing(gcx); + + if (!foregroundFinalize(gcx, sweepZone, sweepAllocKind, budget, sweepList)) { + return NotFinished; + } + + // Reset the slots of the sweep list that we used. + sweepList.reset(thingsPerArena); + + return Finished; +} + +IncrementalProgress GCRuntime::sweepPropMapTree(JS::GCContext* gcx, + SliceBudget& budget) { + // Remove dead SharedPropMaps from the tree. This happens incrementally on the + // main thread. PropMaps are finalized later on the a background thread. + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_PROP_MAP); + + ArenaLists& al = sweepZone->arenas; + + if (!SweepArenaList( + gcx, &al.gcCompactPropMapArenasToUpdate.ref(), budget)) { + return NotFinished; + } + if (!SweepArenaList( + gcx, &al.gcNormalPropMapArenasToUpdate.ref(), budget)) { + return NotFinished; + } + + return Finished; +} + +// An iterator for a standard container that provides an STL-like begin()/end() +// interface. This iterator provides a done()/get()/next() style interface. +template +class ContainerIter { + using Iter = decltype(std::declval().begin()); + using Elem = decltype(*std::declval()); + + Iter iter; + const Iter end; + + public: + explicit ContainerIter(const Container& container) + : iter(container.begin()), end(container.end()) {} + + bool done() const { return iter == end; } + + Elem get() const { return *iter; } + + void next() { + MOZ_ASSERT(!done()); + ++iter; + } +}; + +// IncrementalIter is a template class that makes a normal iterator into one +// that can be used to perform incremental work by using external state that +// persists between instantiations. The state is only initialised on the first +// use and subsequent uses carry on from the previous state. +template +struct IncrementalIter { + using State = mozilla::Maybe; + using Elem = decltype(std::declval().get()); + + private: + State& maybeIter; + + public: + template + explicit IncrementalIter(State& maybeIter, Args&&... args) + : maybeIter(maybeIter) { + if (maybeIter.isNothing()) { + maybeIter.emplace(std::forward(args)...); + } + } + + ~IncrementalIter() { + if (done()) { + maybeIter.reset(); + } + } + + bool done() const { return maybeIter.ref().done(); } + + Elem get() const { return maybeIter.ref().get(); } + + void next() { maybeIter.ref().next(); } +}; + +// Iterate through the sweep groups created by +// GCRuntime::groupZonesForSweeping(). +class js::gc::SweepGroupsIter { + GCRuntime* gc; + + public: + explicit SweepGroupsIter(JSRuntime* rt) : gc(&rt->gc) { + MOZ_ASSERT(gc->currentSweepGroup); + } + + bool done() const { return !gc->currentSweepGroup; } + + Zone* get() const { return gc->currentSweepGroup; } + + void next() { + MOZ_ASSERT(!done()); + gc->getNextSweepGroup(); + } +}; + +namespace sweepaction { + +// Implementation of the SweepAction interface that calls a method on GCRuntime. +class SweepActionCall final : public SweepAction { + using Method = IncrementalProgress (GCRuntime::*)(JS::GCContext* gcx, + SliceBudget& budget); + + Method method; + + public: + explicit SweepActionCall(Method m) : method(m) {} + IncrementalProgress run(Args& args) override { + return (args.gc->*method)(args.gcx, args.budget); + } + void assertFinished() const override {} +}; + +// Implementation of the SweepAction interface that yields in a specified zeal +// mode. +class SweepActionMaybeYield final : public SweepAction { +#ifdef JS_GC_ZEAL + ZealMode mode; + bool isYielding; +#endif + + public: + explicit SweepActionMaybeYield(ZealMode mode) +#ifdef JS_GC_ZEAL + : mode(mode), + isYielding(false) +#endif + { + } + + IncrementalProgress run(Args& args) override { +#ifdef JS_GC_ZEAL + if (!isYielding && args.gc->shouldYieldForZeal(mode)) { + isYielding = true; + return NotFinished; + } + + isYielding = false; +#endif + return Finished; + } + + void assertFinished() const override { MOZ_ASSERT(!isYielding); } + + // These actions should be skipped if GC zeal is not configured. +#ifndef JS_GC_ZEAL + bool shouldSkip() override { return true; } +#endif +}; + +// Implementation of the SweepAction interface that calls a list of actions in +// sequence. +class SweepActionSequence final : public SweepAction { + using ActionVector = Vector, 0, SystemAllocPolicy>; + using Iter = IncrementalIter>; + + ActionVector actions; + typename Iter::State iterState; + + public: + bool init(UniquePtr* acts, size_t count) { + for (size_t i = 0; i < count; i++) { + auto& action = acts[i]; + if (!action) { + return false; + } + if (action->shouldSkip()) { + continue; + } + if (!actions.emplaceBack(std::move(action))) { + return false; + } + } + return true; + } + + IncrementalProgress run(Args& args) override { + for (Iter iter(iterState, actions); !iter.done(); iter.next()) { + if (iter.get()->run(args) == NotFinished) { + return NotFinished; + } + } + return Finished; + } + + void assertFinished() const override { + MOZ_ASSERT(iterState.isNothing()); + for (const auto& action : actions) { + action->assertFinished(); + } + } +}; + +template +class SweepActionForEach final : public SweepAction { + using Elem = decltype(std::declval().get()); + using IncrIter = IncrementalIter; + + Init iterInit; + Elem* elemOut; + UniquePtr action; + typename IncrIter::State iterState; + + public: + SweepActionForEach(const Init& init, Elem* maybeElemOut, + UniquePtr action) + : iterInit(init), elemOut(maybeElemOut), action(std::move(action)) {} + + IncrementalProgress run(Args& args) override { + MOZ_ASSERT_IF(elemOut, *elemOut == Elem()); + auto clearElem = mozilla::MakeScopeExit([&] { setElem(Elem()); }); + for (IncrIter iter(iterState, iterInit); !iter.done(); iter.next()) { + setElem(iter.get()); + if (action->run(args) == NotFinished) { + return NotFinished; + } + } + return Finished; + } + + void assertFinished() const override { + MOZ_ASSERT(iterState.isNothing()); + MOZ_ASSERT_IF(elemOut, *elemOut == Elem()); + action->assertFinished(); + } + + private: + void setElem(const Elem& value) { + if (elemOut) { + *elemOut = value; + } + } +}; + +static UniquePtr Call(IncrementalProgress (GCRuntime::*method)( + JS::GCContext* gcx, SliceBudget& budget)) { + return MakeUnique(method); +} + +static UniquePtr MaybeYield(ZealMode zealMode) { + return MakeUnique(zealMode); +} + +template +static UniquePtr Sequence(UniquePtr first, + Rest... rest) { + UniquePtr actions[] = {std::move(first), std::move(rest)...}; + auto seq = MakeUnique(); + if (!seq || !seq->init(actions, std::size(actions))) { + return nullptr; + } + + return UniquePtr(std::move(seq)); +} + +static UniquePtr RepeatForSweepGroup( + JSRuntime* rt, UniquePtr action) { + if (!action) { + return nullptr; + } + + using Action = SweepActionForEach; + return js::MakeUnique(rt, nullptr, std::move(action)); +} + +static UniquePtr ForEachZoneInSweepGroup( + JSRuntime* rt, Zone** zoneOut, UniquePtr action) { + if (!action) { + return nullptr; + } + + using Action = SweepActionForEach; + return js::MakeUnique(rt, zoneOut, std::move(action)); +} + +static UniquePtr ForEachAllocKind(AllocKinds kinds, + AllocKind* kindOut, + UniquePtr action) { + if (!action) { + return nullptr; + } + + using Action = SweepActionForEach, AllocKinds>; + return js::MakeUnique(kinds, kindOut, std::move(action)); +} + +} // namespace sweepaction + +bool GCRuntime::initSweepActions() { + using namespace sweepaction; + using sweepaction::Call; + + sweepActions.ref() = RepeatForSweepGroup( + rt, + Sequence( + Call(&GCRuntime::beginMarkingSweepGroup), + Call(&GCRuntime::markGrayRootsInCurrentGroup), + MaybeYield(ZealMode::YieldWhileGrayMarking), + Call(&GCRuntime::markGray), Call(&GCRuntime::endMarkingSweepGroup), + Call(&GCRuntime::beginSweepingSweepGroup), + MaybeYield(ZealMode::IncrementalMultipleSlices), + MaybeYield(ZealMode::YieldBeforeSweepingAtoms), + Call(&GCRuntime::sweepAtomsTable), + MaybeYield(ZealMode::YieldBeforeSweepingCaches), + Call(&GCRuntime::sweepWeakCaches), + ForEachZoneInSweepGroup( + rt, &sweepZone.ref(), + Sequence(MaybeYield(ZealMode::YieldBeforeSweepingObjects), + ForEachAllocKind(ForegroundObjectFinalizePhase.kinds, + &sweepAllocKind.ref(), + Call(&GCRuntime::finalizeAllocKind)), + MaybeYield(ZealMode::YieldBeforeSweepingNonObjects), + ForEachAllocKind(ForegroundNonObjectFinalizePhase.kinds, + &sweepAllocKind.ref(), + Call(&GCRuntime::finalizeAllocKind)), + MaybeYield(ZealMode::YieldBeforeSweepingPropMapTrees), + Call(&GCRuntime::sweepPropMapTree))), + Call(&GCRuntime::endSweepingSweepGroup))); + + return sweepActions != nullptr; +} + +IncrementalProgress GCRuntime::performSweepActions(SliceBudget& budget) { + AutoMajorGCProfilerEntry s(this); + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP); + + JS::GCContext* gcx = rt->gcContext(); + AutoSetThreadIsSweeping threadIsSweeping(gcx); + AutoPoisonFreedJitCode pjc(gcx); + + // Don't trigger pre-barriers when finalizing. + AutoDisableBarriers disableBarriers(this); + + // Drain the mark stack, possibly in a parallel task if we're in a part of + // sweeping that allows it. + // + // In the first sweep slice where we must not yield to the mutator until we've + // starting sweeping a sweep group but in that case the stack must be empty + // already. + +#ifdef DEBUG + MOZ_ASSERT(initialState <= State::Sweep); + if (initialState != State::Sweep) { + assertNoMarkingWork(); + } +#endif + + if (initialState == State::Sweep && + markDuringSweeping(gcx, budget) == NotFinished) { + return NotFinished; + } + + // Then continue running sweep actions. + + SweepAction::Args args{this, gcx, budget}; + IncrementalProgress sweepProgress = sweepActions->run(args); + IncrementalProgress markProgress = joinBackgroundMarkTask(); + + if (sweepProgress == Finished && markProgress == Finished) { + return Finished; + } + + MOZ_ASSERT(isIncremental); + return NotFinished; +} + +bool GCRuntime::allCCVisibleZonesWereCollected() { + // Calculate whether the gray marking state is now valid. + // + // The gray bits change from invalid to valid if we finished a full GC from + // the point of view of the cycle collector. We ignore the following: + // + // - Helper thread zones, as these are not reachable from the main heap. + // - The atoms zone, since strings and symbols are never marked gray. + // - Empty zones. + // + // These exceptions ensure that when the CC requests a full GC the gray mark + // state ends up valid even it we don't collect all of the zones. + + for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) { + if (!zone->isCollecting() && !zone->arenas.arenaListsAreEmpty()) { + return false; + } + } + + return true; +} + +void GCRuntime::endSweepPhase(bool destroyingRuntime) { + MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping); + + sweepActions->assertFinished(); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP); + + MOZ_ASSERT_IF(destroyingRuntime, !useBackgroundThreads); + + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::DESTROY); + + // Sweep shared script bytecode now all zones have been swept and finalizers + // for BaseScripts have released their references. + SweepScriptData(rt); + } + + { + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END); + AutoLockStoreBuffer lock(&storeBuffer()); + callFinalizeCallbacks(rt->gcContext(), JSFINALIZE_COLLECTION_END); + + if (allCCVisibleZonesWereCollected()) { + grayBitsValid = true; + } + } + + if (isIncremental) { + findDeadCompartments(); + } + +#ifdef JS_GC_ZEAL + finishMarkingValidation(); +#endif + +#ifdef DEBUG + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + for (auto i : AllAllocKinds()) { + MOZ_ASSERT_IF(!IsBackgroundFinalized(i) || !useBackgroundThreads, + zone->arenas.collectingArenaList(i).isEmpty()); + } + } +#endif + + AssertNoWrappersInGrayList(rt); +} diff --git a/js/src/gc/Tenuring.cpp b/js/src/gc/Tenuring.cpp new file mode 100644 index 0000000000..457aa4876c --- /dev/null +++ b/js/src/gc/Tenuring.cpp @@ -0,0 +1,1016 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Implementation of nursery eviction (tenuring). + */ + +#include "gc/Tenuring.h" + +#include "mozilla/PodOperations.h" + +#include "gc/Cell.h" +#include "gc/GCInternals.h" +#include "gc/GCProbes.h" +#include "gc/Pretenuring.h" +#include "gc/Zone.h" +#include "jit/JitCode.h" +#include "proxy/Proxy.h" +#include "vm/BigIntType.h" +#include "vm/JSScript.h" +#include "vm/NativeObject.h" +#include "vm/Runtime.h" +#include "vm/TypedArrayObject.h" + +#include "gc/Heap-inl.h" +#include "gc/Marking-inl.h" +#include "gc/ObjectKind-inl.h" +#include "gc/StoreBuffer-inl.h" +#include "gc/TraceMethods-inl.h" +#include "vm/JSObject-inl.h" +#include "vm/PlainObject-inl.h" +#ifdef ENABLE_RECORD_TUPLE +# include "vm/TupleType.h" +#endif + +using namespace js; +using namespace js::gc; + +using mozilla::PodCopy; + +constexpr size_t MAX_DEDUPLICATABLE_STRING_LENGTH = 500; + +TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery) + : JSTracer(rt, JS::TracerKind::Tenuring, + JS::WeakMapTraceAction::TraceKeysAndValues), + nursery_(*nursery) { + stringDeDupSet.emplace(); +} + +size_t TenuringTracer::getTenuredSize() const { + return tenuredSize + tenuredCells * sizeof(NurseryCellHeader); +} + +size_t TenuringTracer::getTenuredCells() const { return tenuredCells; } + +static inline void UpdateAllocSiteOnTenure(Cell* cell) { + AllocSite* site = NurseryCellHeader::from(cell)->allocSite(); + site->incTenuredCount(); +} + +void TenuringTracer::onObjectEdge(JSObject** objp, const char* name) { + JSObject* obj = *objp; + if (!IsInsideNursery(obj)) { + return; + } + + if (obj->isForwarded()) { + const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(obj); + *objp = static_cast(overlay->forwardingAddress()); + return; + } + + UpdateAllocSiteOnTenure(obj); + + // Take a fast path for tenuring a plain object which is by far the most + // common case. + if (obj->is()) { + *objp = movePlainObjectToTenured(&obj->as()); + return; + } + + *objp = moveToTenuredSlow(obj); +} + +void TenuringTracer::onStringEdge(JSString** strp, const char* name) { + JSString* str = *strp; + if (!IsInsideNursery(str)) { + return; + } + + if (str->isForwarded()) { + const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(str); + *strp = static_cast(overlay->forwardingAddress()); + return; + } + + UpdateAllocSiteOnTenure(str); + + *strp = moveToTenured(str); +} + +void TenuringTracer::onBigIntEdge(JS::BigInt** bip, const char* name) { + JS::BigInt* bi = *bip; + if (!IsInsideNursery(bi)) { + return; + } + + if (bi->isForwarded()) { + const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(bi); + *bip = static_cast(overlay->forwardingAddress()); + return; + } + + UpdateAllocSiteOnTenure(bi); + + *bip = moveToTenured(bi); +} + +void TenuringTracer::onSymbolEdge(JS::Symbol** symp, const char* name) {} +void TenuringTracer::onScriptEdge(BaseScript** scriptp, const char* name) {} +void TenuringTracer::onShapeEdge(Shape** shapep, const char* name) {} +void TenuringTracer::onRegExpSharedEdge(RegExpShared** sharedp, + const char* name) {} +void TenuringTracer::onBaseShapeEdge(BaseShape** basep, const char* name) {} +void TenuringTracer::onGetterSetterEdge(GetterSetter** gsp, const char* name) {} +void TenuringTracer::onPropMapEdge(PropMap** mapp, const char* name) {} +void TenuringTracer::onJitCodeEdge(jit::JitCode** codep, const char* name) {} +void TenuringTracer::onScopeEdge(Scope** scopep, const char* name) {} + +void TenuringTracer::traverse(JS::Value* thingp) { + MOZ_ASSERT(!nursery().isInside(thingp)); + + Value value = *thingp; + CheckTracedThing(this, value); + + // We only care about a few kinds of GC thing here and this generates much + // tighter code than using MapGCThingTyped. + Value post; + if (value.isObject()) { + JSObject* obj = &value.toObject(); + onObjectEdge(&obj, "value"); + post = JS::ObjectValue(*obj); + } +#ifdef ENABLE_RECORD_TUPLE + else if (value.isExtendedPrimitive()) { + JSObject* obj = &value.toExtendedPrimitive(); + onObjectEdge(&obj, "value"); + post = JS::ExtendedPrimitiveValue(*obj); + } +#endif + else if (value.isString()) { + JSString* str = value.toString(); + onStringEdge(&str, "value"); + post = JS::StringValue(str); + } else if (value.isBigInt()) { + JS::BigInt* bi = value.toBigInt(); + onBigIntEdge(&bi, "value"); + post = JS::BigIntValue(bi); + } else { + MOZ_ASSERT_IF(value.isGCThing(), !IsInsideNursery(value.toGCThing())); + return; + } + + if (post != value) { + *thingp = post; + } +} + +template +void js::gc::StoreBuffer::MonoTypeBuffer::trace(TenuringTracer& mover) { + mozilla::ReentrancyGuard g(*owner_); + MOZ_ASSERT(owner_->isEnabled()); + if (last_) { + last_.trace(mover); + } + for (typename StoreSet::Range r = stores_.all(); !r.empty(); r.popFront()) { + r.front().trace(mover); + } +} + +namespace js { +namespace gc { +template void StoreBuffer::MonoTypeBuffer::trace( + TenuringTracer&); +template void StoreBuffer::MonoTypeBuffer::trace( + TenuringTracer&); +template struct StoreBuffer::MonoTypeBuffer; +template struct StoreBuffer::MonoTypeBuffer; +template struct StoreBuffer::MonoTypeBuffer; +} // namespace gc +} // namespace js + +void js::gc::StoreBuffer::SlotsEdge::trace(TenuringTracer& mover) const { + NativeObject* obj = object(); + MOZ_ASSERT(IsCellPointerValid(obj)); + + // Beware JSObject::swap exchanging a native object for a non-native one. + if (!obj->is()) { + return; + } + + MOZ_ASSERT(!IsInsideNursery(obj), "obj shouldn't live in nursery."); + + if (kind() == ElementKind) { + uint32_t initLen = obj->getDenseInitializedLength(); + uint32_t numShifted = obj->getElementsHeader()->numShiftedElements(); + uint32_t clampedStart = start_; + clampedStart = numShifted < clampedStart ? clampedStart - numShifted : 0; + clampedStart = std::min(clampedStart, initLen); + uint32_t clampedEnd = start_ + count_; + clampedEnd = numShifted < clampedEnd ? clampedEnd - numShifted : 0; + clampedEnd = std::min(clampedEnd, initLen); + MOZ_ASSERT(clampedStart <= clampedEnd); + mover.traceSlots( + static_cast(obj->getDenseElements() + clampedStart) + ->unbarrieredAddress(), + clampedEnd - clampedStart); + } else { + uint32_t start = std::min(start_, obj->slotSpan()); + uint32_t end = std::min(start_ + count_, obj->slotSpan()); + MOZ_ASSERT(start <= end); + mover.traceObjectSlots(obj, start, end); + } +} + +static inline void TraceWholeCell(TenuringTracer& mover, JSObject* object) { + MOZ_ASSERT_IF(object->storeBuffer(), + !object->storeBuffer()->markingNondeduplicatable); + mover.traceObject(object); +} + +// Non-deduplicatable marking is necessary because of the following 2 reasons: +// +// 1. Tenured string chars cannot be updated: +// +// If any of the tenured string's bases were deduplicated during tenuring, +// the tenured string's chars pointer would need to be adjusted. This would +// then require updating any other tenured strings that are dependent on the +// first tenured string, and we have no way to find them without scanning +// the entire tenured heap. +// +// 2. Tenured string cannot store its nursery base or base's chars: +// +// Tenured strings have no place to stash a pointer to their nursery base or +// its chars. You need to be able to traverse any dependent string's chain +// of bases up to a nursery "root base" that points to the malloced chars +// that the dependent strings started out pointing to, so that you can +// calculate the offset of any dependent string and update the ptr+offset if +// the root base gets deduplicated to a different allocation. Tenured +// strings in this base chain will stop you from reaching the nursery +// version of the root base; you can only get to the tenured version, and it +// has no place to store the original chars pointer. +static inline void PreventDeduplicationOfReachableStrings(JSString* str) { + MOZ_ASSERT(str->isTenured()); + MOZ_ASSERT(!str->isForwarded()); + + JSLinearString* baseOrRelocOverlay = str->nurseryBaseOrRelocOverlay(); + + // Walk along the chain of dependent strings' base string pointers + // to mark them all non-deduplicatable. + while (true) { + // baseOrRelocOverlay can be one of the three cases: + // 1. forwarded nursery string: + // The forwarded string still retains the flag that can tell whether + // this string is a dependent string with a base. Its + // StringRelocationOverlay holds a saved pointer to its base in the + // nursery. + // 2. not yet forwarded nursery string: + // Retrieve the base field directly from the string. + // 3. tenured string: + // The nursery base chain ends here, so stop traversing. + if (baseOrRelocOverlay->isForwarded()) { + JSLinearString* tenuredBase = Forwarded(baseOrRelocOverlay); + if (!tenuredBase->hasBase()) { + break; + } + baseOrRelocOverlay = StringRelocationOverlay::fromCell(baseOrRelocOverlay) + ->savedNurseryBaseOrRelocOverlay(); + } else { + JSLinearString* base = baseOrRelocOverlay; + if (base->isTenured()) { + break; + } + if (base->isDeduplicatable()) { + base->setNonDeduplicatable(); + } + if (!base->hasBase()) { + break; + } + baseOrRelocOverlay = base->nurseryBaseOrRelocOverlay(); + } + } +} + +static inline void TraceWholeCell(TenuringTracer& mover, JSString* str) { + MOZ_ASSERT_IF(str->storeBuffer(), + str->storeBuffer()->markingNondeduplicatable); + + // Mark all strings reachable from the tenured string `str` as + // non-deduplicatable. These strings are the bases of the tenured dependent + // string. + if (str->hasBase()) { + PreventDeduplicationOfReachableStrings(str); + } + + str->traceChildren(&mover); +} + +static inline void TraceWholeCell(TenuringTracer& mover, BaseScript* script) { + script->traceChildren(&mover); +} + +static inline void TraceWholeCell(TenuringTracer& mover, + jit::JitCode* jitcode) { + jitcode->traceChildren(&mover); +} + +template +static void TraceBufferedCells(TenuringTracer& mover, Arena* arena, + ArenaCellSet* cells) { + for (size_t i = 0; i < MaxArenaCellIndex; i += cells->BitsPerWord) { + ArenaCellSet::WordT bitset = cells->getWord(i / cells->BitsPerWord); + while (bitset) { + size_t bit = i + js::detail::CountTrailingZeroes(bitset); + auto cell = + reinterpret_cast(uintptr_t(arena) + ArenaCellIndexBytes * bit); + TraceWholeCell(mover, cell); + bitset &= bitset - 1; // Clear the low bit. + } + } +} + +void ArenaCellSet::trace(TenuringTracer& mover) { + for (ArenaCellSet* cells = this; cells; cells = cells->next) { + cells->check(); + + Arena* arena = cells->arena; + arena->bufferedCells() = &ArenaCellSet::Empty; + + JS::TraceKind kind = MapAllocToTraceKind(arena->getAllocKind()); + switch (kind) { + case JS::TraceKind::Object: + TraceBufferedCells(mover, arena, cells); + break; + case JS::TraceKind::String: + TraceBufferedCells(mover, arena, cells); + break; + case JS::TraceKind::Script: + TraceBufferedCells(mover, arena, cells); + break; + case JS::TraceKind::JitCode: + TraceBufferedCells(mover, arena, cells); + break; + default: + MOZ_CRASH("Unexpected trace kind"); + } + } +} + +void js::gc::StoreBuffer::WholeCellBuffer::trace(TenuringTracer& mover) { + MOZ_ASSERT(owner_->isEnabled()); + +#ifdef DEBUG + // Verify that all string whole cells are traced first before any other + // strings are visited for any reason. + MOZ_ASSERT(!owner_->markingNondeduplicatable); + owner_->markingNondeduplicatable = true; +#endif + // Trace all of the strings to mark the non-deduplicatable bits, then trace + // all other whole cells. + if (stringHead_) { + stringHead_->trace(mover); + } +#ifdef DEBUG + owner_->markingNondeduplicatable = false; +#endif + if (nonStringHead_) { + nonStringHead_->trace(mover); + } + + stringHead_ = nonStringHead_ = nullptr; +} + +template +void js::gc::StoreBuffer::CellPtrEdge::trace(TenuringTracer& mover) const { + static_assert(std::is_base_of_v, "T must be a Cell type"); + static_assert(!GCTypeIsTenured(), "T must not be a tenured Cell type"); + + T* thing = *edge; + if (!thing) { + return; + } + + MOZ_ASSERT(IsCellPointerValid(thing)); + MOZ_ASSERT(thing->getTraceKind() == JS::MapTypeToTraceKind::kind); + + if (std::is_same_v) { + // Nursery string deduplication requires all tenured string -> nursery + // string edges to be registered with the whole cell buffer in order to + // correctly set the non-deduplicatable bit. + MOZ_ASSERT(!mover.runtime()->gc.isPointerWithinTenuredCell( + edge, JS::TraceKind::String)); + } + + DispatchToOnEdge(&mover, edge, "CellPtrEdge"); +} + +void js::gc::StoreBuffer::ValueEdge::trace(TenuringTracer& mover) const { + if (deref()) { + mover.traverse(edge); + } +} + +// Visit all object children of the object and trace them. +void js::gc::TenuringTracer::traceObject(JSObject* obj) { + const JSClass* clasp = obj->getClass(); + MOZ_ASSERT(clasp); + + if (clasp->hasTrace()) { + clasp->doTrace(this, obj); + } + + if (!obj->is()) { + return; + } + + NativeObject* nobj = &obj->as(); + if (!nobj->hasEmptyElements()) { + HeapSlotArray elements = nobj->getDenseElements(); + Value* elems = elements.begin()->unbarrieredAddress(); + traceSlots(elems, elems + nobj->getDenseInitializedLength()); + } + + traceObjectSlots(nobj, 0, nobj->slotSpan()); +} + +void js::gc::TenuringTracer::traceObjectSlots(NativeObject* nobj, + uint32_t start, uint32_t end) { + auto traceRange = [this](HeapSlot* slotStart, HeapSlot* slotEnd) { + traceSlots(slotStart->unbarrieredAddress(), slotEnd->unbarrieredAddress()); + }; + nobj->forEachSlotRange(start, end, traceRange); +} + +void js::gc::TenuringTracer::traceSlots(Value* vp, Value* end) { + for (; vp != end; ++vp) { + traverse(vp); + } +} + +inline void js::gc::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots) { + traceSlots(vp, vp + nslots); +} + +void js::gc::TenuringTracer::traceString(JSString* str) { + str->traceChildren(this); +} + +void js::gc::TenuringTracer::traceBigInt(JS::BigInt* bi) { + bi->traceChildren(this); +} + +#ifdef DEBUG +static inline uintptr_t OffsetFromChunkStart(void* p) { + return uintptr_t(p) & gc::ChunkMask; +} +static inline ptrdiff_t OffsetToChunkEnd(void* p) { + return ChunkSize - (uintptr_t(p) & gc::ChunkMask); +} +#endif + +/* Insert the given relocation entry into the list of things to visit. */ +inline void js::gc::TenuringTracer::insertIntoObjectFixupList( + RelocationOverlay* entry) { + entry->setNext(objHead); + objHead = entry; +} + +template +inline T* js::gc::TenuringTracer::allocTenured(Zone* zone, AllocKind kind) { + return static_cast(static_cast(AllocateCellInGC(zone, kind))); +} + +JSString* js::gc::TenuringTracer::allocTenuredString(JSString* src, Zone* zone, + AllocKind dstKind) { + JSString* dst = allocTenured(zone, dstKind); + tenuredSize += moveStringToTenured(dst, src, dstKind); + tenuredCells++; + + return dst; +} + +JSObject* js::gc::TenuringTracer::moveToTenuredSlow(JSObject* src) { + MOZ_ASSERT(IsInsideNursery(src)); + MOZ_ASSERT(!src->is()); + + AllocKind dstKind = src->allocKindForTenure(nursery()); + auto dst = allocTenured(src->nurseryZone(), dstKind); + + size_t srcSize = Arena::thingSize(dstKind); + + // Arrays and Tuples do not necessarily have the same AllocKind between src + // and dst. We deal with this by copying elements manually, possibly + // re-inlining them if there is adequate room inline in dst. + // + // For Arrays and Tuples we're reducing tenuredSize to the smaller srcSize + // because moveElementsToTenured() accounts for all Array or Tuple elements, + // even if they are inlined. + if (src->is()) { + TypedArrayObject* tarray = &src->as(); + // Typed arrays with inline data do not necessarily have the same + // AllocKind between src and dst. The nursery does not allocate an + // inline data buffer that has the same size as the slow path will do. + // In the slow path, the Typed Array Object stores the inline data + // in the allocated space that fits the AllocKind. In the fast path, + // the nursery will allocate another buffer that is directly behind the + // minimal JSObject. That buffer size plus the JSObject size is not + // necessarily as large as the slow path's AllocKind size. + if (tarray->hasInlineElements()) { + AllocKind srcKind = GetGCObjectKind(TypedArrayObject::FIXED_DATA_START); + size_t headerSize = Arena::thingSize(srcKind); + srcSize = headerSize + tarray->byteLength(); + } + } else if (src->canHaveFixedElements()) { + srcSize = sizeof(NativeObject); + } + + tenuredSize += srcSize; + tenuredCells++; + + // Copy the Cell contents. + MOZ_ASSERT(OffsetFromChunkStart(src) >= sizeof(ChunkBase)); + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize)); + js_memcpy(dst, src, srcSize); + + // Move the slots and elements, if we need to. + if (src->is()) { + NativeObject* ndst = &dst->as(); + NativeObject* nsrc = &src->as(); + tenuredSize += moveSlotsToTenured(ndst, nsrc); + tenuredSize += moveElementsToTenured(ndst, nsrc, dstKind); + } + + JSObjectMovedOp op = dst->getClass()->extObjectMovedOp(); + MOZ_ASSERT_IF(src->is(), op == proxy_ObjectMoved); + if (op) { + // Tell the hazard analysis that the object moved hook can't GC. + JS::AutoSuppressGCAnalysis nogc; + tenuredSize += op(dst, src); + } else { + MOZ_ASSERT_IF(src->getClass()->hasFinalize(), + CanNurseryAllocateFinalizedClass(src->getClass())); + } + + RelocationOverlay* overlay = RelocationOverlay::forwardCell(src, dst); + insertIntoObjectFixupList(overlay); + + gcprobes::PromoteToTenured(src, dst); + return dst; +} + +inline JSObject* js::gc::TenuringTracer::movePlainObjectToTenured( + PlainObject* src) { + // Fast path version of moveToTenuredSlow() for specialized for PlainObject. + + MOZ_ASSERT(IsInsideNursery(src)); + + AllocKind dstKind = src->allocKindForTenure(); + auto dst = allocTenured(src->nurseryZone(), dstKind); + + size_t srcSize = Arena::thingSize(dstKind); + tenuredSize += srcSize; + tenuredCells++; + + // Copy the Cell contents. + MOZ_ASSERT(OffsetFromChunkStart(src) >= sizeof(ChunkBase)); + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize)); + js_memcpy(dst, src, srcSize); + + // Move the slots and elements. + tenuredSize += moveSlotsToTenured(dst, src); + tenuredSize += moveElementsToTenured(dst, src, dstKind); + + MOZ_ASSERT(!dst->getClass()->extObjectMovedOp()); + + RelocationOverlay* overlay = RelocationOverlay::forwardCell(src, dst); + insertIntoObjectFixupList(overlay); + + gcprobes::PromoteToTenured(src, dst); + return dst; +} + +size_t js::gc::TenuringTracer::moveSlotsToTenured(NativeObject* dst, + NativeObject* src) { + /* Fixed slots have already been copied over. */ + if (!src->hasDynamicSlots()) { + return 0; + } + + Zone* zone = src->nurseryZone(); + size_t count = src->numDynamicSlots(); + + uint64_t uid = src->maybeUniqueId(); + + size_t allocSize = ObjectSlots::allocSize(count); + + ObjectSlots* srcHeader = src->getSlotsHeader(); + if (!nursery().isInside(srcHeader)) { + AddCellMemory(dst, allocSize, MemoryUse::ObjectSlots); + nursery().removeMallocedBufferDuringMinorGC(srcHeader); + return 0; + } + + { + AutoEnterOOMUnsafeRegion oomUnsafe; + HeapSlot* allocation = + zone->pod_malloc(ObjectSlots::allocCount(count)); + if (!allocation) { + oomUnsafe.crash(allocSize, "Failed to allocate slots while tenuring."); + } + + ObjectSlots* slotsHeader = new (allocation) + ObjectSlots(count, srcHeader->dictionarySlotSpan(), uid); + dst->slots_ = slotsHeader->slots(); + } + + AddCellMemory(dst, allocSize, MemoryUse::ObjectSlots); + + PodCopy(dst->slots_, src->slots_, count); + if (count) { + nursery().setSlotsForwardingPointer(src->slots_, dst->slots_, count); + } + + return allocSize; +} + +size_t js::gc::TenuringTracer::moveElementsToTenured(NativeObject* dst, + NativeObject* src, + AllocKind dstKind) { + if (src->hasEmptyElements()) { + return 0; + } + + Zone* zone = src->nurseryZone(); + + ObjectElements* srcHeader = src->getElementsHeader(); + size_t nslots = srcHeader->numAllocatedElements(); + size_t allocSize = nslots * sizeof(HeapSlot); + + void* srcAllocatedHeader = src->getUnshiftedElementsHeader(); + + /* TODO Bug 874151: Prefer to put element data inline if we have space. */ + if (!nursery().isInside(srcAllocatedHeader)) { + MOZ_ASSERT(src->elements_ == dst->elements_); + nursery().removeMallocedBufferDuringMinorGC(srcAllocatedHeader); + + AddCellMemory(dst, allocSize, MemoryUse::ObjectElements); + + return 0; + } + + // Shifted elements are copied too. + uint32_t numShifted = srcHeader->numShiftedElements(); + + /* Unlike other objects, Arrays and Tuples can have fixed elements. */ + if (src->canHaveFixedElements() && nslots <= GetGCKindSlots(dstKind)) { + dst->as().setFixedElements(); + js_memcpy(dst->getElementsHeader(), srcAllocatedHeader, allocSize); + dst->elements_ += numShifted; + dst->getElementsHeader()->flags |= ObjectElements::FIXED; + nursery().setElementsForwardingPointer(srcHeader, dst->getElementsHeader(), + srcHeader->capacity); + return allocSize; + } + + MOZ_ASSERT(nslots >= 2); + + ObjectElements* dstHeader; + { + AutoEnterOOMUnsafeRegion oomUnsafe; + dstHeader = + reinterpret_cast(zone->pod_malloc(nslots)); + if (!dstHeader) { + oomUnsafe.crash(allocSize, "Failed to allocate elements while tenuring."); + } + } + + AddCellMemory(dst, allocSize, MemoryUse::ObjectElements); + + js_memcpy(dstHeader, srcAllocatedHeader, allocSize); + dst->elements_ = dstHeader->elements() + numShifted; + dst->getElementsHeader()->flags &= ~ObjectElements::FIXED; + nursery().setElementsForwardingPointer(srcHeader, dst->getElementsHeader(), + srcHeader->capacity); + return allocSize; +} + +inline void js::gc::TenuringTracer::insertIntoStringFixupList( + StringRelocationOverlay* entry) { + entry->setNext(stringHead); + stringHead = entry; +} + +JSString* js::gc::TenuringTracer::moveToTenured(JSString* src) { + MOZ_ASSERT(IsInsideNursery(src)); + MOZ_ASSERT(!src->isExternal()); + + AllocKind dstKind = src->getAllocKind(); + Zone* zone = src->nurseryZone(); + + // If this string is in the StringToAtomCache, try to deduplicate it by using + // the atom. Don't do this for dependent strings because they're more + // complicated. See StringRelocationOverlay and DeduplicationStringHasher + // comments. + if (src->isLinear() && src->inStringToAtomCache() && + src->isDeduplicatable() && !src->hasBase()) { + JSLinearString* linear = &src->asLinear(); + JSAtom* atom = runtime()->caches().stringToAtomCache.lookupInMap(linear); + MOZ_ASSERT(atom, "Why was the cache purged before minor GC?"); + + // Only deduplicate if both strings have the same encoding, to not confuse + // dependent strings. + if (src->hasTwoByteChars() == atom->hasTwoByteChars()) { + // The StringToAtomCache isn't used for inline strings (due to the minimum + // length) so canOwnDependentChars must be true for both src and atom. + // This means if there are dependent strings floating around using str's + // chars, they will be able to use the chars from the atom. + static_assert(StringToAtomCache::MinStringLength > + JSFatInlineString::MAX_LENGTH_LATIN1); + static_assert(StringToAtomCache::MinStringLength > + JSFatInlineString::MAX_LENGTH_TWO_BYTE); + MOZ_ASSERT(src->canOwnDependentChars()); + MOZ_ASSERT(atom->canOwnDependentChars()); + + StringRelocationOverlay::forwardCell(src, atom); + gcprobes::PromoteToTenured(src, atom); + return atom; + } + } + + JSString* dst; + + // A live nursery string can only get deduplicated when: + // 1. Its length is smaller than MAX_DEDUPLICATABLE_STRING_LENGTH: + // Hashing a long string can affect performance. + // 2. It is linear: + // Deduplicating every node in it would end up doing O(n^2) hashing work. + // 3. It is deduplicatable: + // The JSString NON_DEDUP_BIT flag is unset. + // 4. It matches an entry in stringDeDupSet. + + if (src->length() < MAX_DEDUPLICATABLE_STRING_LENGTH && src->isLinear() && + src->isDeduplicatable() && stringDeDupSet.isSome()) { + if (auto p = stringDeDupSet->lookup(src)) { + // Deduplicate to the looked-up string! + dst = *p; + zone->stringStats.ref().noteDeduplicated(src->length(), src->allocSize()); + StringRelocationOverlay::forwardCell(src, dst); + gcprobes::PromoteToTenured(src, dst); + return dst; + } + + dst = allocTenuredString(src, zone, dstKind); + + if (!stringDeDupSet->putNew(dst)) { + // When there is oom caused by the stringDeDupSet, stop deduplicating + // strings. + stringDeDupSet.reset(); + } + } else { + dst = allocTenuredString(src, zone, dstKind); + dst->clearNonDeduplicatable(); + } + + zone->stringStats.ref().noteTenured(src->allocSize()); + + auto* overlay = StringRelocationOverlay::forwardCell(src, dst); + MOZ_ASSERT(dst->isDeduplicatable()); + + if (dst->hasBase() || dst->isRope()) { + // dst or one of its leaves might have a base that will be deduplicated. + // Insert the overlay into the fixup list to relocate it later. + insertIntoStringFixupList(overlay); + } + + gcprobes::PromoteToTenured(src, dst); + return dst; +} + +template +void js::gc::TenuringTracer::relocateDependentStringChars( + JSDependentString* tenuredDependentStr, JSLinearString* baseOrRelocOverlay, + size_t* offset, bool* rootBaseNotYetForwarded, JSLinearString** rootBase) { + MOZ_ASSERT(*offset == 0); + MOZ_ASSERT(*rootBaseNotYetForwarded == false); + MOZ_ASSERT(*rootBase == nullptr); + + JS::AutoCheckCannotGC nogc; + + const CharT* dependentStrChars = + tenuredDependentStr->nonInlineChars(nogc); + + // Traverse the dependent string nursery base chain to find the base that + // it's using chars from. + while (true) { + if (baseOrRelocOverlay->isForwarded()) { + JSLinearString* tenuredBase = Forwarded(baseOrRelocOverlay); + StringRelocationOverlay* relocOverlay = + StringRelocationOverlay::fromCell(baseOrRelocOverlay); + + if (!tenuredBase->hasBase()) { + // The nursery root base is relocOverlay, it is tenured to tenuredBase. + // Relocate tenuredDependentStr chars and reassign the tenured root base + // as its base. + JSLinearString* tenuredRootBase = tenuredBase; + const CharT* rootBaseChars = relocOverlay->savedNurseryChars(); + *offset = dependentStrChars - rootBaseChars; + MOZ_ASSERT(*offset < tenuredRootBase->length()); + tenuredDependentStr->relocateNonInlineChars( + tenuredRootBase->nonInlineChars(nogc), *offset); + tenuredDependentStr->setBase(tenuredRootBase); + return; + } + + baseOrRelocOverlay = relocOverlay->savedNurseryBaseOrRelocOverlay(); + + } else { + JSLinearString* base = baseOrRelocOverlay; + + if (!base->hasBase()) { + // The root base is not forwarded yet, it is simply base. + *rootBase = base; + + // The root base can be in either the nursery or the tenured heap. + // dependentStr chars needs to be relocated after traceString if the + // root base is in the nursery. + if (!(*rootBase)->isTenured()) { + *rootBaseNotYetForwarded = true; + const CharT* rootBaseChars = (*rootBase)->nonInlineChars(nogc); + *offset = dependentStrChars - rootBaseChars; + MOZ_ASSERT(*offset < base->length(), "Tenured root base"); + } + + tenuredDependentStr->setBase(*rootBase); + + return; + } + + baseOrRelocOverlay = base->nurseryBaseOrRelocOverlay(); + } + } +} + +JS::BigInt* js::gc::TenuringTracer::moveToTenured(JS::BigInt* src) { + MOZ_ASSERT(IsInsideNursery(src)); + + AllocKind dstKind = src->getAllocKind(); + Zone* zone = src->nurseryZone(); + zone->tenuredBigInts++; + + JS::BigInt* dst = allocTenured(zone, dstKind); + tenuredSize += moveBigIntToTenured(dst, src, dstKind); + tenuredCells++; + + RelocationOverlay::forwardCell(src, dst); + + gcprobes::PromoteToTenured(src, dst); + return dst; +} + +void js::gc::TenuringTracer::collectToObjectFixedPoint() { + while (RelocationOverlay* p = objHead) { + objHead = objHead->next(); + auto* obj = static_cast(p->forwardingAddress()); + traceObject(obj); + } +} + +void js::gc::TenuringTracer::collectToStringFixedPoint() { + while (StringRelocationOverlay* p = stringHead) { + stringHead = stringHead->next(); + + auto* tenuredStr = static_cast(p->forwardingAddress()); + // To ensure the NON_DEDUP_BIT was reset properly. + MOZ_ASSERT(tenuredStr->isDeduplicatable()); + + // The nursery root base might not be forwarded before + // traceString(tenuredStr). traceString(tenuredStr) will forward the root + // base if that's the case. Dependent string chars needs to be relocated + // after traceString if root base was not forwarded. + size_t offset = 0; + bool rootBaseNotYetForwarded = false; + JSLinearString* rootBase = nullptr; + + if (tenuredStr->isDependent()) { + if (tenuredStr->hasTwoByteChars()) { + relocateDependentStringChars( + &tenuredStr->asDependent(), p->savedNurseryBaseOrRelocOverlay(), + &offset, &rootBaseNotYetForwarded, &rootBase); + } else { + relocateDependentStringChars( + &tenuredStr->asDependent(), p->savedNurseryBaseOrRelocOverlay(), + &offset, &rootBaseNotYetForwarded, &rootBase); + } + } + + traceString(tenuredStr); + + if (rootBaseNotYetForwarded) { + MOZ_ASSERT(rootBase->isForwarded(), + "traceString() should make it forwarded"); + JS::AutoCheckCannotGC nogc; + + JSLinearString* tenuredRootBase = Forwarded(rootBase); + MOZ_ASSERT(offset < tenuredRootBase->length()); + + if (tenuredStr->hasTwoByteChars()) { + tenuredStr->asDependent().relocateNonInlineChars( + tenuredRootBase->twoByteChars(nogc), offset); + } else { + tenuredStr->asDependent().relocateNonInlineChars( + tenuredRootBase->latin1Chars(nogc), offset); + } + tenuredStr->setBase(tenuredRootBase); + } + } +} + +size_t js::gc::TenuringTracer::moveStringToTenured(JSString* dst, JSString* src, + AllocKind dstKind) { + size_t size = Arena::thingSize(dstKind); + + // At the moment, strings always have the same AllocKind between src and + // dst. This may change in the future. + MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind()); + + // Copy the Cell contents. + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size)); + js_memcpy(dst, src, size); + + if (src->ownsMallocedChars()) { + void* chars = src->asLinear().nonInlineCharsRaw(); + nursery().removeMallocedBufferDuringMinorGC(chars); + AddCellMemory(dst, dst->asLinear().allocSize(), MemoryUse::StringContents); + } + + return size; +} + +size_t js::gc::TenuringTracer::moveBigIntToTenured(JS::BigInt* dst, + JS::BigInt* src, + AllocKind dstKind) { + size_t size = Arena::thingSize(dstKind); + + // At the moment, BigInts always have the same AllocKind between src and + // dst. This may change in the future. + MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind()); + + // Copy the Cell contents. + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size)); + js_memcpy(dst, src, size); + + MOZ_ASSERT(dst->zone() == src->nurseryZone()); + + if (src->hasHeapDigits()) { + size_t length = dst->digitLength(); + if (!nursery().isInside(src->heapDigits_)) { + nursery().removeMallocedBufferDuringMinorGC(src->heapDigits_); + } else { + Zone* zone = src->nurseryZone(); + { + AutoEnterOOMUnsafeRegion oomUnsafe; + dst->heapDigits_ = zone->pod_malloc(length); + if (!dst->heapDigits_) { + oomUnsafe.crash(sizeof(JS::BigInt::Digit) * length, + "Failed to allocate digits while tenuring."); + } + } + + PodCopy(dst->heapDigits_, src->heapDigits_, length); + nursery().setDirectForwardingPointer(src->heapDigits_, dst->heapDigits_); + + size += length * sizeof(JS::BigInt::Digit); + } + + AddCellMemory(dst, length * sizeof(JS::BigInt::Digit), + MemoryUse::BigIntDigits); + } + + return size; +} + +MinorSweepingTracer::MinorSweepingTracer(JSRuntime* rt) + : GenericTracerImpl(rt, JS::TracerKind::MinorSweeping, + JS::WeakMapTraceAction::TraceKeysAndValues) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime())); + MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting()); +} + +template +inline void MinorSweepingTracer::onEdge(T** thingp, const char* name) { + T* thing = *thingp; + if (thing->isTenured()) { + return; + } + + if (IsForwarded(thing)) { + *thingp = Forwarded(thing); + return; + } + + *thingp = nullptr; +} diff --git a/js/src/gc/Tenuring.h b/js/src/gc/Tenuring.h new file mode 100644 index 0000000000..34ef950e1d --- /dev/null +++ b/js/src/gc/Tenuring.h @@ -0,0 +1,169 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Tenuring_h +#define gc_Tenuring_h + +#include "mozilla/Maybe.h" + +#include "gc/AllocKind.h" +#include "js/GCAPI.h" +#include "js/TracingAPI.h" +#include "util/Text.h" + +namespace js { + +class NativeObject; +class Nursery; +class PlainObject; + +namespace gc { + +class RelocationOverlay; +class StringRelocationOverlay; + +template +struct DeduplicationStringHasher { + using Lookup = Key; + + static inline HashNumber hash(const Lookup& lookup) { + JS::AutoCheckCannotGC nogc; + HashNumber strHash; + + // Include flags in the hash. A string relocation overlay stores either + // the nursery root base chars or the dependent string nursery base, but + // does not indicate which one. If strings with different string types + // were deduplicated, for example, a dependent string gets deduplicated + // into an extensible string, the base chain would be broken and the root + // base would be unreachable. + + if (lookup->asLinear().hasLatin1Chars()) { + strHash = mozilla::HashString(lookup->asLinear().latin1Chars(nogc), + lookup->length()); + } else { + MOZ_ASSERT(lookup->asLinear().hasTwoByteChars()); + strHash = mozilla::HashString(lookup->asLinear().twoByteChars(nogc), + lookup->length()); + } + + return mozilla::HashGeneric(strHash, lookup->zone(), lookup->flags()); + } + + static MOZ_ALWAYS_INLINE bool match(const Key& key, const Lookup& lookup) { + if (!key->sameLengthAndFlags(*lookup) || + key->asTenured().zone() != lookup->zone() || + key->asTenured().getAllocKind() != lookup->getAllocKind()) { + return false; + } + + JS::AutoCheckCannotGC nogc; + + if (key->asLinear().hasLatin1Chars()) { + MOZ_ASSERT(lookup->asLinear().hasLatin1Chars()); + return EqualChars(key->asLinear().latin1Chars(nogc), + lookup->asLinear().latin1Chars(nogc), lookup->length()); + } else { + MOZ_ASSERT(key->asLinear().hasTwoByteChars()); + MOZ_ASSERT(lookup->asLinear().hasTwoByteChars()); + return EqualChars(key->asLinear().twoByteChars(nogc), + lookup->asLinear().twoByteChars(nogc), + lookup->length()); + } + } +}; + +class TenuringTracer final : public JSTracer { + Nursery& nursery_; + + // Amount of data moved to the tenured generation during collection. + size_t tenuredSize = 0; + // Number of cells moved to the tenured generation. + size_t tenuredCells = 0; + + // These lists are threaded through the Nursery using the space from + // already moved things. The lists are used to fix up the moved things and + // to find things held live by intra-Nursery pointers. + gc::RelocationOverlay* objHead = nullptr; + gc::StringRelocationOverlay* stringHead = nullptr; + + using StringDeDupSet = + HashSet, + SystemAllocPolicy>; + + // deDupSet is emplaced at the beginning of the nursery collection and reset + // at the end of the nursery collection. It can also be reset during nursery + // collection when out of memory to insert new entries. + mozilla::Maybe stringDeDupSet; + +#define DEFINE_ON_EDGE_METHOD(name, type, _1, _2) \ + void on##name##Edge(type** thingp, const char* name) override; + JS_FOR_EACH_TRACEKIND(DEFINE_ON_EDGE_METHOD) +#undef DEFINE_ON_EDGE_METHOD + + public: + TenuringTracer(JSRuntime* rt, Nursery* nursery); + + Nursery& nursery() { return nursery_; } + + // Move all objects and everything they can reach to the tenured heap. Called + // after all roots have been traced. + void collectToObjectFixedPoint(); + + // Move all strings and all strings they can reach to the tenured heap, and + // additionally do any fixups for when strings are pointing into memory that + // was deduplicated. Called after collectToObjectFixedPoint(). + void collectToStringFixedPoint(); + + size_t getTenuredSize() const; + size_t getTenuredCells() const; + + void traverse(JS::Value* thingp); + + // The store buffers need to be able to call these directly. + void traceObject(JSObject* src); + void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t end); + void traceSlots(JS::Value* vp, uint32_t nslots); + void traceString(JSString* src); + void traceBigInt(JS::BigInt* src); + + private: + // The dependent string chars needs to be relocated if the base which it's + // using chars from has been deduplicated. + template + void relocateDependentStringChars(JSDependentString* tenuredDependentStr, + JSLinearString* baseOrRelocOverlay, + size_t* offset, + bool* rootBaseNotYetForwarded, + JSLinearString** rootBase); + + inline void insertIntoObjectFixupList(gc::RelocationOverlay* entry); + inline void insertIntoStringFixupList(gc::StringRelocationOverlay* entry); + + template + inline T* allocTenured(JS::Zone* zone, gc::AllocKind kind); + JSString* allocTenuredString(JSString* src, JS::Zone* zone, + gc::AllocKind dstKind); + + inline JSObject* movePlainObjectToTenured(PlainObject* src); + JSObject* moveToTenuredSlow(JSObject* src); + JSString* moveToTenured(JSString* src); + JS::BigInt* moveToTenured(JS::BigInt* src); + + size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, + gc::AllocKind dstKind); + size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src); + size_t moveStringToTenured(JSString* dst, JSString* src, + gc::AllocKind dstKind); + size_t moveBigIntToTenured(JS::BigInt* dst, JS::BigInt* src, + gc::AllocKind dstKind); + + void traceSlots(JS::Value* vp, JS::Value* end); +}; + +} // namespace gc +} // namespace js + +#endif // gc_Tenuring_h diff --git a/js/src/gc/TraceKind.h b/js/src/gc/TraceKind.h new file mode 100644 index 0000000000..fdcb162239 --- /dev/null +++ b/js/src/gc/TraceKind.h @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_TraceKind_h +#define gc_TraceKind_h + +#include "js/TraceKind.h" + +namespace js { +namespace gc { + +// Map from all trace kinds to the base GC type. +template +struct MapTraceKindToType {}; + +#define DEFINE_TRACE_KIND_MAP(name, type, _, _1) \ + template <> \ + struct MapTraceKindToType { \ + using Type = type; \ + }; +JS_FOR_EACH_TRACEKIND(DEFINE_TRACE_KIND_MAP); +#undef DEFINE_TRACE_KIND_MAP + +// Map from a possibly-derived type to the base GC type. +template +struct BaseGCType { + using type = + typename MapTraceKindToType::kind>::Type; + static_assert(std::is_base_of_v, "Failed to find base type"); +}; + +template +struct TraceKindCanBeGray {}; +#define EXPAND_TRACEKIND_DEF(_, type, canBeGray, _1) \ + template <> \ + struct TraceKindCanBeGray { \ + static constexpr bool value = canBeGray; \ + }; +JS_FOR_EACH_TRACEKIND(EXPAND_TRACEKIND_DEF) +#undef EXPAND_TRACEKIND_DEF + +struct TraceKindCanBeGrayFunctor { + template + bool operator()() { + return TraceKindCanBeGray::value; + } +}; + +static inline bool TraceKindCanBeMarkedGray(JS::TraceKind kind) { + return DispatchTraceKindTyped(TraceKindCanBeGrayFunctor(), kind); +} + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_TraceKind_h */ diff --git a/js/src/gc/TraceMethods-inl.h b/js/src/gc/TraceMethods-inl.h new file mode 100644 index 0000000000..67d4142e0a --- /dev/null +++ b/js/src/gc/TraceMethods-inl.h @@ -0,0 +1,384 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Trace methods for all GC things, defined in a separate header to allow + * inlining. + * + * This also includes eager inline marking versions. Both paths must end up + * traversing equivalent subgraphs. + */ + +#ifndef gc_TraceMethods_inl_h +#define gc_TraceMethods_inl_h + +#include "gc/GCMarker.h" +#include "gc/Tracer.h" +#include "jit/JitCode.h" +#include "vm/BigIntType.h" +#include "vm/GetterSetter.h" +#include "vm/GlobalObject.h" +#include "vm/JSScript.h" +#include "vm/PropMap.h" +#include "vm/Realm.h" +#include "vm/Scope.h" +#include "vm/Shape.h" +#include "vm/StringType.h" +#include "vm/SymbolType.h" +#include "wasm/WasmJS.h" + +inline void js::BaseScript::traceChildren(JSTracer* trc) { + TraceNullableEdge(trc, &function_, "function"); + TraceEdge(trc, &sourceObject_, "sourceObject"); + + warmUpData_.trace(trc); + + if (data_) { + data_->trace(trc); + } +} + +inline void js::Shape::traceChildren(JSTracer* trc) { + TraceCellHeaderEdge(trc, this, "base"); + if (isNative()) { + asNative().traceChildren(trc); + } +} + +inline void js::NativeShape::traceChildren(JSTracer* trc) { + TraceNullableEdge(trc, &propMap_, "propertymap"); +} + +template +void js::GCMarker::eagerlyMarkChildren(Shape* shape) { + MOZ_ASSERT(shape->isMarked(markColor())); + + BaseShape* base = shape->base(); + checkTraversedEdge(shape, base); + if (mark(base)) { + base->traceChildren(tracer()); + } + + if (shape->isNative()) { + if (PropMap* map = shape->asNative().propMap()) { + markAndTraverseEdge(shape, map); + } + } +} + +inline void JSString::traceChildren(JSTracer* trc) { + if (hasBase()) { + traceBase(trc); + } else if (isRope()) { + asRope().traceChildren(trc); + } +} +template +void js::GCMarker::eagerlyMarkChildren(JSString* str) { + if (str->isLinear()) { + eagerlyMarkChildren(&str->asLinear()); + } else { + eagerlyMarkChildren(&str->asRope()); + } +} + +inline void JSString::traceBase(JSTracer* trc) { + MOZ_ASSERT(hasBase()); + js::TraceManuallyBarrieredEdge(trc, &d.s.u3.base, "base"); +} +template +void js::GCMarker::eagerlyMarkChildren(JSLinearString* linearStr) { + gc::AssertShouldMarkInZone(this, linearStr); + MOZ_ASSERT(linearStr->isMarkedAny()); + MOZ_ASSERT(linearStr->JSString::isLinear()); + + // Use iterative marking to avoid blowing out the stack. + while (linearStr->hasBase()) { + linearStr = linearStr->base(); + + // It's possible to observe a rope as the base of a linear string if we + // process barriers during rope flattening. See the assignment of base in + // JSRope::flattenInternal's finish_node section. + if (static_cast(linearStr)->isRope()) { + MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting()); + break; + } + + MOZ_ASSERT(linearStr->JSString::isLinear()); + MOZ_ASSERT(!linearStr->isPermanentAtom()); + gc::AssertShouldMarkInZone(this, linearStr); + if (!mark(static_cast(linearStr))) { + break; + } + } +} + +inline void JSRope::traceChildren(JSTracer* trc) { + js::TraceManuallyBarrieredEdge(trc, &d.s.u2.left, "left child"); + js::TraceManuallyBarrieredEdge(trc, &d.s.u3.right, "right child"); +} +template +void js::GCMarker::eagerlyMarkChildren(JSRope* rope) { + // This function tries to scan the whole rope tree using the marking stack + // as temporary storage. If that becomes full, the unscanned ropes are + // added to the delayed marking list. When the function returns, the + // marking stack is at the same depth as it was on entry. This way we avoid + // using tags when pushing ropes to the stack as ropes never leak to other + // users of the stack. This also assumes that a rope can only point to + // other ropes or linear strings, it cannot refer to GC things of other + // types. + size_t savedPos = stack.position(); + MOZ_DIAGNOSTIC_ASSERT(rope->getTraceKind() == JS::TraceKind::String); + while (true) { + MOZ_DIAGNOSTIC_ASSERT(rope->getTraceKind() == JS::TraceKind::String); + MOZ_DIAGNOSTIC_ASSERT(rope->JSString::isRope()); + gc::AssertShouldMarkInZone(this, rope); + MOZ_ASSERT(rope->isMarkedAny()); + JSRope* next = nullptr; + + JSString* right = rope->rightChild(); + if (mark(right)) { + MOZ_ASSERT(!right->isPermanentAtom()); + if (right->isLinear()) { + eagerlyMarkChildren(&right->asLinear()); + } else { + next = &right->asRope(); + } + } + + JSString* left = rope->leftChild(); + if (mark(left)) { + MOZ_ASSERT(!left->isPermanentAtom()); + if (left->isLinear()) { + eagerlyMarkChildren(&left->asLinear()); + } else { + // When both children are ropes, set aside the right one to + // scan it later. + if (next && !stack.pushTempRope(next)) { + delayMarkingChildrenOnOOM(next); + } + next = &left->asRope(); + } + } + if (next) { + rope = next; + } else if (savedPos != stack.position()) { + MOZ_ASSERT(savedPos < stack.position()); + rope = stack.popPtr().asTempRope(); + } else { + break; + } + } + MOZ_ASSERT(savedPos == stack.position()); +} + +inline void JS::Symbol::traceChildren(JSTracer* trc) { + js::TraceNullableCellHeaderEdge(trc, this, "symbol description"); +} + +template +void js::RuntimeScopeData::trace(JSTracer* trc) { + TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length); +} + +inline void js::FunctionScope::RuntimeData::trace(JSTracer* trc) { + TraceNullableEdge(trc, &canonicalFunction, "scope canonical function"); + TraceNullableBindingNames(trc, GetScopeDataTrailingNamesPointer(this), + length); +} +inline void js::ModuleScope::RuntimeData::trace(JSTracer* trc) { + TraceNullableEdge(trc, &module, "scope module"); + TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length); +} +inline void js::WasmInstanceScope::RuntimeData::trace(JSTracer* trc) { + TraceNullableEdge(trc, &instance, "wasm instance"); + TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length); +} + +inline void js::Scope::traceChildren(JSTracer* trc) { + TraceNullableEdge(trc, &environmentShape_, "scope env shape"); + TraceNullableEdge(trc, &enclosingScope_, "scope enclosing"); + applyScopeDataTyped([trc](auto data) { data->trace(trc); }); +} + +template +void js::GCMarker::eagerlyMarkChildren(Scope* scope) { + do { + if (Shape* shape = scope->environmentShape()) { + markAndTraverseEdge(scope, shape); + } + mozilla::Span> names; + switch (scope->kind()) { + case ScopeKind::Function: { + FunctionScope::RuntimeData& data = scope->as().data(); + if (data.canonicalFunction) { + markAndTraverseObjectEdge(scope, data.canonicalFunction); + } + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::FunctionBodyVar: { + VarScope::RuntimeData& data = scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::Lexical: + case ScopeKind::SimpleCatch: + case ScopeKind::Catch: + case ScopeKind::NamedLambda: + case ScopeKind::StrictNamedLambda: + case ScopeKind::FunctionLexical: { + LexicalScope::RuntimeData& data = scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::ClassBody: { + ClassBodyScope::RuntimeData& data = scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::Global: + case ScopeKind::NonSyntactic: { + GlobalScope::RuntimeData& data = scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::Eval: + case ScopeKind::StrictEval: { + EvalScope::RuntimeData& data = scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::Module: { + ModuleScope::RuntimeData& data = scope->as().data(); + if (data.module) { + markAndTraverseObjectEdge(scope, data.module); + } + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::With: + break; + + case ScopeKind::WasmInstance: { + WasmInstanceScope::RuntimeData& data = + scope->as().data(); + markAndTraverseObjectEdge(scope, data.instance); + names = GetScopeDataTrailingNames(&data); + break; + } + + case ScopeKind::WasmFunction: { + WasmFunctionScope::RuntimeData& data = + scope->as().data(); + names = GetScopeDataTrailingNames(&data); + break; + } + } + if (scope->kind_ == ScopeKind::Function) { + for (auto& binding : names) { + if (JSAtom* name = binding.name()) { + markAndTraverseStringEdge(scope, name); + } + } + } else { + for (auto& binding : names) { + markAndTraverseStringEdge(scope, binding.name()); + } + } + scope = scope->enclosing(); + } while (scope && mark(scope)); +} + +inline void js::BaseShape::traceChildren(JSTracer* trc) { + // Note: the realm's global can be nullptr if we GC while creating the global. + if (JSObject* global = realm()->unsafeUnbarrieredMaybeGlobal()) { + TraceManuallyBarrieredEdge(trc, &global, "baseshape_global"); + } + + if (proto_.isObject()) { + TraceEdge(trc, &proto_, "baseshape_proto"); + } +} + +inline void js::GetterSetter::traceChildren(JSTracer* trc) { + if (getter()) { + TraceCellHeaderEdge(trc, this, "gettersetter_getter"); + } + if (setter()) { + TraceEdge(trc, &setter_, "gettersetter_setter"); + } +} + +inline void js::PropMap::traceChildren(JSTracer* trc) { + if (hasPrevious()) { + TraceEdge(trc, &asLinked()->data_.previous, "propmap_previous"); + } + + if (isShared()) { + SharedPropMap::TreeData& treeData = asShared()->treeDataRef(); + if (SharedPropMap* parent = treeData.parent.maybeMap()) { + TraceManuallyBarrieredEdge(trc, &parent, "propmap_parent"); + if (parent != treeData.parent.map()) { + treeData.setParent(parent, treeData.parent.index()); + } + } + } + + for (uint32_t i = 0; i < PropMap::Capacity; i++) { + if (hasKey(i)) { + TraceEdge(trc, &keys_[i], "propmap_key"); + } + } + + if (canHaveTable() && asLinked()->hasTable()) { + asLinked()->data_.table->trace(trc); + } +} + +template +void js::GCMarker::eagerlyMarkChildren(PropMap* map) { + MOZ_ASSERT(map->isMarkedAny()); + do { + for (uint32_t i = 0; i < PropMap::Capacity; i++) { + if (map->hasKey(i)) { + markAndTraverseEdge(map, map->getKey(i)); + } + } + + if (map->canHaveTable()) { + // Special case: if a map has a table then all its pointers must point to + // this map or an ancestor. Since these pointers will be traced by this + // loop they do not need to be traced here as well. + MOZ_ASSERT(map->asLinked()->canSkipMarkingTable()); + } + + if (map->isDictionary()) { + map = map->asDictionary()->previous(); + } else { + // For shared maps follow the |parent| link and not the |previous| link. + // They're different when a map had a branch that wasn't at the end of the + // map, but in this case they must have the same |previous| map. This is + // asserted in SharedPropMap::addChild. In other words, marking all + // |parent| maps will also mark all |previous| maps. + map = map->asShared()->treeDataRef().parent.maybeMap(); + } + } while (map && mark(map)); +} + +inline void JS::BigInt::traceChildren(JSTracer* trc) {} + +// JitCode::traceChildren is not defined inline due to its dependence on +// MacroAssembler. + +#endif // gc_TraceMethods_inl_h diff --git a/js/src/gc/Tracer.cpp b/js/src/gc/Tracer.cpp new file mode 100644 index 0000000000..d150e4d2b7 --- /dev/null +++ b/js/src/gc/Tracer.cpp @@ -0,0 +1,297 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Tracer.h" + +#include "mozilla/DebugOnly.h" + +#include "NamespaceImports.h" + +#include "gc/PublicIterators.h" +#include "jit/JitCode.h" +#include "util/Memory.h" +#include "util/Text.h" +#include "vm/BigIntType.h" +#include "vm/JSContext.h" +#include "vm/JSFunction.h" +#include "vm/JSScript.h" +#include "vm/RegExpShared.h" +#include "vm/Scope.h" +#include "vm/Shape.h" +#include "vm/StringType.h" +#include "vm/SymbolType.h" + +#include "gc/TraceMethods-inl.h" +#include "vm/Shape-inl.h" + +using namespace js; +using namespace js::gc; +using mozilla::DebugOnly; + +template void RuntimeScopeData::trace(JSTracer* trc); +template void RuntimeScopeData::trace(JSTracer* trc); +template void RuntimeScopeData::trace(JSTracer* trc); +template void RuntimeScopeData::trace(JSTracer* trc); +template void RuntimeScopeData::trace(JSTracer* trc); +template void RuntimeScopeData::trace( + JSTracer* trc); + +void JS::TracingContext::getEdgeName(const char* name, char* buffer, + size_t bufferSize) { + MOZ_ASSERT(bufferSize > 0); + if (functor_) { + (*functor_)(this, buffer, bufferSize); + return; + } + if (index_ != InvalidIndex) { + snprintf(buffer, bufferSize, "%s[%zu]", name, index_); + return; + } + snprintf(buffer, bufferSize, "%s", name); +} + +/*** Public Tracing API *****************************************************/ + +JS_PUBLIC_API void JS::TraceChildren(JSTracer* trc, GCCellPtr thing) { + ApplyGCThingTyped(thing.asCell(), thing.kind(), [trc](auto t) { + MOZ_ASSERT_IF(t->runtimeFromAnyThread() != trc->runtime(), + t->isPermanentAndMayBeShared()); + t->traceChildren(trc); + }); +} + +void js::gc::TraceIncomingCCWs(JSTracer* trc, + const JS::CompartmentSet& compartments) { + for (CompartmentsIter source(trc->runtime()); !source.done(); source.next()) { + if (compartments.has(source)) { + continue; + } + // Iterate over all compartments that |source| has wrappers for. + for (Compartment::WrappedObjectCompartmentEnum dest(source); !dest.empty(); + dest.popFront()) { + if (!compartments.has(dest)) { + continue; + } + // Iterate over all wrappers from |source| to |dest| compartments. + for (Compartment::ObjectWrapperEnum e(source, dest); !e.empty(); + e.popFront()) { + JSObject* obj = e.front().key(); + MOZ_ASSERT(compartments.has(obj->compartment())); + mozilla::DebugOnly prior = obj; + TraceManuallyBarrieredEdge(trc, &obj, + "cross-compartment wrapper target"); + MOZ_ASSERT(obj == prior); + } + } + } +} + +/*** Cycle Collector Helpers ************************************************/ + +// This function is used by the Cycle Collector (CC) to trace through -- or in +// CC parlance, traverse -- a Shape. The CC does not care about Shapes, +// BaseShapes or PropMaps, only the JSObjects held live by them. Thus, we only +// report non-Shape things. +void gc::TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape) { + shape->base()->traceChildren(trc); + // Don't trace the PropMap because the CC doesn't care about PropertyKey. +} + +/*** Traced Edge Printer ****************************************************/ + +static size_t CountDecimalDigits(size_t num) { + size_t numDigits = 0; + do { + num /= 10; + numDigits++; + } while (num > 0); + + return numDigits; +} + +static const char* StringKindHeader(JSString* str) { + MOZ_ASSERT(str->isLinear()); + + if (str->isAtom()) { + if (str->isPermanentAtom()) { + return "permanent atom: "; + } + return "atom: "; + } + + if (str->isExtensible()) { + return "extensible: "; + } + + if (str->isInline()) { + if (str->isFatInline()) { + return "fat inline: "; + } + return "inline: "; + } + + if (str->isDependent()) { + return "dependent: "; + } + + if (str->isExternal()) { + return "external: "; + } + + return "linear: "; +} + +void js::gc::GetTraceThingInfo(char* buf, size_t bufsize, void* thing, + JS::TraceKind kind, bool details) { + const char* name = nullptr; /* silence uninitialized warning */ + size_t n; + + if (bufsize == 0) { + return; + } + + switch (kind) { + case JS::TraceKind::BaseShape: + name = "base_shape"; + break; + + case JS::TraceKind::GetterSetter: + name = "getter_setter"; + break; + + case JS::TraceKind::PropMap: + name = "prop_map"; + break; + + case JS::TraceKind::JitCode: + name = "jitcode"; + break; + + case JS::TraceKind::Null: + name = "null_pointer"; + break; + + case JS::TraceKind::Object: { + name = static_cast(thing)->getClass()->name; + break; + } + + case JS::TraceKind::RegExpShared: + name = "reg_exp_shared"; + break; + + case JS::TraceKind::Scope: + name = "scope"; + break; + + case JS::TraceKind::Script: + name = "script"; + break; + + case JS::TraceKind::Shape: + name = "shape"; + break; + + case JS::TraceKind::String: + name = ((JSString*)thing)->isDependent() ? "substring" : "string"; + break; + + case JS::TraceKind::Symbol: + name = "symbol"; + break; + + case JS::TraceKind::BigInt: + name = "BigInt"; + break; + + default: + name = "INVALID"; + break; + } + + n = strlen(name); + if (n > bufsize - 1) { + n = bufsize - 1; + } + js_memcpy(buf, name, n + 1); + buf += n; + bufsize -= n; + *buf = '\0'; + + if (details && bufsize > 2) { + switch (kind) { + case JS::TraceKind::Object: { + JSObject* obj = (JSObject*)thing; + if (obj->is()) { + JSFunction* fun = &obj->as(); + if (fun->displayAtom()) { + *buf++ = ' '; + bufsize--; + PutEscapedString(buf, bufsize, fun->displayAtom(), 0); + } + } else { + snprintf(buf, bufsize, " "); + } + break; + } + + case JS::TraceKind::Script: { + auto* script = static_cast(thing); + snprintf(buf, bufsize, " %s:%u", script->filename(), script->lineno()); + break; + } + + case JS::TraceKind::String: { + *buf++ = ' '; + bufsize--; + JSString* str = (JSString*)thing; + + if (str->isLinear()) { + const char* header = StringKindHeader(str); + bool willFit = str->length() + strlen(" ") + strlen(header) + + CountDecimalDigits(str->length()) < + bufsize; + + n = snprintf(buf, bufsize, "<%slength %zu%s> ", header, str->length(), + willFit ? "" : " (truncated)"); + buf += n; + bufsize -= n; + + PutEscapedString(buf, bufsize, &str->asLinear(), 0); + } else { + snprintf(buf, bufsize, "", str->length()); + } + break; + } + + case JS::TraceKind::Symbol: { + *buf++ = ' '; + bufsize--; + auto* sym = static_cast(thing); + if (JSAtom* desc = sym->description()) { + PutEscapedString(buf, bufsize, desc, 0); + } else { + snprintf(buf, bufsize, ""); + } + break; + } + + case JS::TraceKind::Scope: { + auto* scope = static_cast(thing); + snprintf(buf, bufsize, " %s", js::ScopeKindString(scope->kind())); + break; + } + + default: + break; + } + } + buf[bufsize - 1] = '\0'; +} + +JS::CallbackTracer::CallbackTracer(JSContext* cx, JS::TracerKind kind, + JS::TraceOptions options) + : CallbackTracer(cx->runtime(), kind, options) {} diff --git a/js/src/gc/Tracer.h b/js/src/gc/Tracer.h new file mode 100644 index 0000000000..e1f10980de --- /dev/null +++ b/js/src/gc/Tracer.h @@ -0,0 +1,401 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef js_Tracer_h +#define js_Tracer_h + +#include "gc/Barrier.h" +#include "gc/TraceKind.h" +#include "js/HashTable.h" +#include "js/TracingAPI.h" + +namespace JS { +using CompartmentSet = + js::HashSet, + js::SystemAllocPolicy>; +} // namespace JS + +namespace js { + +class TaggedProto; + +// Internal Tracing API +// +// Tracing is an abstract visitation of each edge in a JS heap graph.[1] The +// most common (and performance sensitive) use of this infrastructure is for GC +// "marking" as part of the mark-and-sweep collector; however, this +// infrastructure is much more general than that and is used for many other +// purposes as well. +// +// One commonly misunderstood subtlety of the tracing architecture is the role +// of graph vertices versus graph edges. Graph vertices are the heap +// allocations -- GC things -- that are returned by Allocate. Graph edges are +// pointers -- including tagged pointers like Value and jsid -- that link the +// allocations into a complex heap. The tracing API deals *only* with edges. +// Any action taken on the target of a graph edge is independent of the tracing +// itself. +// +// Another common misunderstanding relates to the role of the JSTracer. The +// JSTracer instance determines what tracing does when visiting an edge; it +// does not itself participate in the tracing process, other than to be passed +// through as opaque data. It works like a closure in that respect. +// +// Tracing implementations internal to SpiderMonkey should use these interfaces +// instead of the public interfaces in js/TracingAPI.h. Unlike the public +// tracing methods, these work on internal types and avoid an external call. +// +// Note that the implementations for these methods are, surprisingly, in +// js/src/gc/Marking.cpp. This is so that the compiler can inline as much as +// possible in the common, marking pathways. Conceptually, however, they remain +// as part of the generic "tracing" architecture, rather than the more specific +// marking implementation of tracing. +// +// 1 - In SpiderMonkey, we call this concept tracing rather than visiting +// because "visiting" is already used by the compiler. Also, it's been +// called "tracing" forever and changing it would be extremely difficult at +// this point. + +class GCMarker; + +// Debugging functions to check tracing invariants. +#ifdef DEBUG +template +void CheckTracedThing(JSTracer* trc, T* thing); +template +void CheckTracedThing(JSTracer* trc, const T& thing); +#else +template +inline void CheckTracedThing(JSTracer* trc, T* thing) {} +template +inline void CheckTracedThing(JSTracer* trc, const T& thing) {} +#endif + +namespace gc { + +// Our barrier templates are parameterized on the pointer types so that we can +// share the definitions with Value and jsid. Thus, we need to strip the +// pointer before sending the type to BaseGCType and re-add it on the other +// side. As such: +template +struct PtrBaseGCType { + using type = T; +}; +template +struct PtrBaseGCType { + using type = typename BaseGCType::type*; +}; + +// Cast a possibly-derived T** pointer to a base class pointer. +template +typename PtrBaseGCType::type* ConvertToBase(T* thingp) { + return reinterpret_cast::type*>(thingp); +} + +// Internal methods to trace edges. + +#define DEFINE_TRACE_FUNCTION(name, type, _1, _2) \ + MOZ_ALWAYS_INLINE bool TraceEdgeInternal(JSTracer* trc, type** thingp, \ + const char* name) { \ + CheckTracedThing(trc, *thingp); \ + trc->on##name##Edge(thingp, name); \ + return *thingp; \ + } +JS_FOR_EACH_TRACEKIND(DEFINE_TRACE_FUNCTION) +#undef DEFINE_TRACE_FUNCTION + +bool TraceEdgeInternal(JSTracer* trc, Value* thingp, const char* name); +bool TraceEdgeInternal(JSTracer* trc, jsid* thingp, const char* name); +bool TraceEdgeInternal(JSTracer* trc, TaggedProto* thingp, const char* name); + +template +void TraceRangeInternal(JSTracer* trc, size_t len, T* vec, const char* name); +template +bool TraceWeakMapKeyInternal(JSTracer* trc, Zone* zone, T* thingp, + const char* name); + +#ifdef DEBUG +void AssertRootMarkingPhase(JSTracer* trc); +void AssertShouldMarkInZone(GCMarker* marker, gc::Cell* thing); +#else +inline void AssertRootMarkingPhase(JSTracer* trc) {} +inline void AssertShouldMarkInZone(GCMarker* marker, gc::Cell* thing) {} +#endif + +} // namespace gc + +// Trace through a strong edge in the live object graph on behalf of +// tracing. The effect of tracing the edge depends on the JSTracer being +// used. For pointer types, |*thingp| must not be null. +// +// Note that weak edges are handled separately. GC things with weak edges must +// not trace those edges during marking tracing (which would keep the referent +// alive) but instead arrange for the edge to be swept by calling +// js::gc::IsAboutToBeFinalized or TraceWeakEdge during sweeping. +// +// GC things that are weakly held in containers can use WeakMap or a container +// wrapped in the WeakCache<> template to perform the appropriate sweeping. + +template +inline void TraceEdge(JSTracer* trc, const WriteBarriered* thingp, + const char* name) { + gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unbarrieredAddress()), + name); +} + +template +inline void TraceEdge(JSTracer* trc, WeakHeapPtr* thingp, const char* name) { + gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unbarrieredAddress()), + name); +} + +template +inline void TraceCellHeaderEdge(JSTracer* trc, + gc::CellWithTenuredGCPointer* thingp, + const char* name) { + T* thing = thingp->headerPtr(); + gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name); + if (thing != thingp->headerPtr()) { + thingp->unbarrieredSetHeaderPtr(thing); + } +} + +template +inline void TraceCellHeaderEdge(JSTracer* trc, + gc::TenuredCellWithGCPointer* thingp, + const char* name) { + T* thing = thingp->headerPtr(); + gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name); + if (thing != thingp->headerPtr()) { + thingp->unbarrieredSetHeaderPtr(thing); + } +} + +// Trace through a possibly-null edge in the live object graph on behalf of +// tracing. + +template +inline void TraceNullableEdge(JSTracer* trc, const WriteBarriered* thingp, + const char* name) { + if (InternalBarrierMethods::isMarkable(thingp->get())) { + TraceEdge(trc, thingp, name); + } +} + +template +inline void TraceNullableEdge(JSTracer* trc, WeakHeapPtr* thingp, + const char* name) { + if (InternalBarrierMethods::isMarkable(thingp->unbarrieredGet())) { + TraceEdge(trc, thingp, name); + } +} + +template +inline void TraceNullableCellHeaderEdge( + JSTracer* trc, gc::CellWithTenuredGCPointer* thingp, + const char* name) { + T* thing = thingp->headerPtr(); + if (thing) { + gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name); + if (thing != thingp->headerPtr()) { + thingp->unbarrieredSetHeaderPtr(thing); + } + } +} + +// Trace through a "root" edge. These edges are the initial edges in the object +// graph traversal. Root edges are asserted to only be traversed in the initial +// phase of a GC. + +template +inline void TraceRoot(JSTracer* trc, T* thingp, const char* name) { + gc::AssertRootMarkingPhase(trc); + gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name); +} + +template +inline void TraceRoot(JSTracer* trc, const HeapPtr* thingp, + const char* name) { + TraceRoot(trc, thingp->unbarrieredAddress(), name); +} + +// Idential to TraceRoot, except that this variant will not crash if |*thingp| +// is null. + +template +inline void TraceNullableRoot(JSTracer* trc, T* thingp, const char* name) { + gc::AssertRootMarkingPhase(trc); + if (InternalBarrierMethods::isMarkable(*thingp)) { + gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name); + } +} + +template +inline void TraceNullableRoot(JSTracer* trc, WeakHeapPtr* thingp, + const char* name) { + TraceNullableRoot(trc, thingp->unbarrieredAddress(), name); +} + +// Like TraceEdge, but for edges that do not use one of the automatic barrier +// classes and, thus, must be treated specially for moving GC. This method is +// separate from TraceEdge to make accidental use of such edges more obvious. + +template +inline void TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, + const char* name) { + gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name); +} + +// Trace through a weak edge. If *thingp is not marked at the end of marking, +// it is replaced by nullptr, and this method will return false to indicate that +// the edge no longer exists. +template +inline bool TraceManuallyBarrieredWeakEdge(JSTracer* trc, T* thingp, + const char* name) { + return gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name); +} + +// The result of tracing a weak edge, which can be either: +// +// - the target is dead (and the edge has been cleared), or +// - the target is alive (and the edge may have been updated) +// +// This includes the initial and final values of the edge to allow cleanup if +// the target is dead or access to the referent if it is alive. +template +struct TraceWeakResult { + const bool live_; + const T initial_; + const T final_; + + bool isLive() const { return live_; } + bool isDead() const { return !live_; } + + MOZ_IMPLICIT operator bool() const { return isLive(); } + + T initialTarget() const { + MOZ_ASSERT(isDead()); + return initial_; + } + + T finalTarget() const { + MOZ_ASSERT(isLive()); + return final_; + } +}; + +template +inline TraceWeakResult TraceWeakEdge(JSTracer* trc, BarrieredBase* thingp, + const char* name) { + T* addr = thingp->unbarrieredAddress(); + T initial = *addr; + bool live = !InternalBarrierMethods::isMarkable(initial) || + gc::TraceEdgeInternal(trc, gc::ConvertToBase(addr), name); + return TraceWeakResult{live, initial, *addr}; +} + +// Trace all edges contained in the given array. + +template +void TraceRange(JSTracer* trc, size_t len, BarrieredBase* vec, + const char* name) { + gc::TraceRangeInternal(trc, len, + gc::ConvertToBase(vec[0].unbarrieredAddress()), name); +} + +// Trace all root edges in the given array. + +template +void TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name) { + gc::AssertRootMarkingPhase(trc); + gc::TraceRangeInternal(trc, len, gc::ConvertToBase(vec), name); +} + +// As below but with manual barriers. +template +void TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src, + T* dst, const char* name); + +// Trace an edge that crosses compartment boundaries. If the compartment of the +// destination thing is not being GC'd, then the edge will not be traced. +template +void TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, + const WriteBarriered* dst, const char* name) { + TraceManuallyBarrieredCrossCompartmentEdge( + trc, src, gc::ConvertToBase(dst->unbarrieredAddress()), name); +} + +// Trace an edge that's guaranteed to be same-zone but may cross a compartment +// boundary. This should NOT be used for object => object edges, as those have +// to be in the cross-compartment wrapper map. +// +// WARNING: because this turns off certain compartment checks, you most likely +// don't want to use this! If you still think you need this function, talk to a +// GC peer first. +template +void TraceSameZoneCrossCompartmentEdge(JSTracer* trc, + const WriteBarriered* dst, + const char* name); + +// Trace a weak map key. For debugger weak maps these may be cross compartment, +// but the compartment must always be within the current sweep group. +template +void TraceWeakMapKeyEdgeInternal(JSTracer* trc, Zone* weakMapZone, T** thingp, + const char* name); + +template +inline void TraceWeakMapKeyEdge(JSTracer* trc, Zone* weakMapZone, + const WriteBarriered* thingp, + const char* name) { + TraceWeakMapKeyEdgeInternal( + trc, weakMapZone, gc::ConvertToBase(thingp->unbarrieredAddress()), name); +} + +// Trace a root edge that uses the base GC thing type, instead of a more +// specific type. +void TraceGenericPointerRoot(JSTracer* trc, gc::Cell** thingp, + const char* name); + +// Trace a non-root edge that uses the base GC thing type, instead of a more +// specific type. +void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, + const char* name); + +void TraceGCCellPtrRoot(JSTracer* trc, JS::GCCellPtr* thingp, const char* name); + +void TraceManuallyBarrieredGCCellPtr(JSTracer* trc, JS::GCCellPtr* thingp, + const char* name); + +namespace gc { + +// Trace through a shape or group iteratively during cycle collection to avoid +// deep or infinite recursion. +void TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape); + +/** + * Trace every value within |compartments| that is wrapped by a + * cross-compartment wrapper from a compartment that is not an element of + * |compartments|. + */ +void TraceIncomingCCWs(JSTracer* trc, const JS::CompartmentSet& compartments); + +/* Get information about a GC thing. Used when dumping the heap. */ +void GetTraceThingInfo(char* buf, size_t bufsize, void* thing, + JS::TraceKind kind, bool includeDetails); + +// Overloaded function to call the correct GenericTracer method based on the +// argument type. +#define DEFINE_DISPATCH_FUNCTION(name, type, _1, _2) \ + inline void DispatchToOnEdge(JSTracer* trc, type** thingp, \ + const char* name) { \ + trc->on##name##Edge(thingp, name); \ + } +JS_FOR_EACH_TRACEKIND(DEFINE_DISPATCH_FUNCTION) +#undef DEFINE_DISPATCH_FUNCTION + +} // namespace gc +} // namespace js + +#endif /* js_Tracer_h */ diff --git a/js/src/gc/Verifier.cpp b/js/src/gc/Verifier.cpp new file mode 100644 index 0000000000..299d73c51f --- /dev/null +++ b/js/src/gc/Verifier.cpp @@ -0,0 +1,1135 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Maybe.h" +#include "mozilla/Sprintf.h" + +#include +#include + +#ifdef MOZ_VALGRIND +# include +#endif + +#include "gc/GCInternals.h" +#include "gc/GCLock.h" +#include "gc/PublicIterators.h" +#include "gc/WeakMap.h" +#include "gc/Zone.h" +#include "js/friend/DumpFunctions.h" // js::DumpObject +#include "js/HashTable.h" +#include "vm/JSContext.h" + +#include "gc/ArenaList-inl.h" +#include "gc/GC-inl.h" +#include "gc/Marking-inl.h" +#include "gc/PrivateIterators-inl.h" + +using namespace js; +using namespace js::gc; + +using mozilla::DebugOnly; + +#ifdef JS_GC_ZEAL + +/* + * Write barrier verification + * + * The next few functions are for write barrier verification. + * + * The VerifyBarriers function is a shorthand. It checks if a verification phase + * is currently running. If not, it starts one. Otherwise, it ends the current + * phase and starts a new one. + * + * The user can adjust the frequency of verifications, which causes + * VerifyBarriers to be a no-op all but one out of N calls. However, if the + * |always| parameter is true, it starts a new phase no matter what. + * + * Pre-Barrier Verifier: + * When StartVerifyBarriers is called, a snapshot is taken of all objects in + * the GC heap and saved in an explicit graph data structure. Later, + * EndVerifyBarriers traverses the heap again. Any pointer values that were in + * the snapshot and are no longer found must be marked; otherwise an assertion + * triggers. Note that we must not GC in between starting and finishing a + * verification phase. + */ + +struct EdgeValue { + JS::GCCellPtr thing; + const char* label; +}; + +struct VerifyNode { + JS::GCCellPtr thing; + uint32_t count; + EdgeValue edges[1]; +}; + +typedef HashMap, SystemAllocPolicy> + NodeMap; + +/* + * The verifier data structures are simple. The entire graph is stored in a + * single block of memory. At the beginning is a VerifyNode for the root + * node. It is followed by a sequence of EdgeValues--the exact number is given + * in the node. After the edges come more nodes and their edges. + * + * The edgeptr and term fields are used to allocate out of the block of memory + * for the graph. If we run out of memory (i.e., if edgeptr goes beyond term), + * we just abandon the verification. + * + * The nodemap field is a hashtable that maps from the address of the GC thing + * to the VerifyNode that represents it. + */ +class js::VerifyPreTracer final : public JS::CallbackTracer { + JS::AutoDisableGenerationalGC noggc; + + void onChild(JS::GCCellPtr thing, const char* name) override; + + public: + /* The gcNumber when the verification began. */ + uint64_t number; + + /* This counts up to gcZealFrequency to decide whether to verify. */ + int count; + + /* This graph represents the initial GC "snapshot". */ + VerifyNode* curnode; + VerifyNode* root; + char* edgeptr; + char* term; + NodeMap nodemap; + + explicit VerifyPreTracer(JSRuntime* rt) + : JS::CallbackTracer(rt, JS::TracerKind::Callback, + JS::WeakEdgeTraceAction::Skip), + noggc(rt->mainContextFromOwnThread()), + number(rt->gc.gcNumber()), + count(0), + curnode(nullptr), + root(nullptr), + edgeptr(nullptr), + term(nullptr) { + // We don't care about weak edges here. Since they are not marked they + // cannot cause the problem that the pre-write barrier protects against. + } + + ~VerifyPreTracer() { js_free(root); } +}; + +/* + * This function builds up the heap snapshot by adding edges to the current + * node. + */ +void VerifyPreTracer::onChild(JS::GCCellPtr thing, const char* name) { + MOZ_ASSERT(!IsInsideNursery(thing.asCell())); + + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) { + return; + } + + edgeptr += sizeof(EdgeValue); + if (edgeptr >= term) { + edgeptr = term; + return; + } + + VerifyNode* node = curnode; + uint32_t i = node->count; + + node->edges[i].thing = thing; + node->edges[i].label = name; + node->count++; +} + +static VerifyNode* MakeNode(VerifyPreTracer* trc, JS::GCCellPtr thing) { + NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing.asCell()); + if (!p) { + VerifyNode* node = (VerifyNode*)trc->edgeptr; + trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue); + if (trc->edgeptr >= trc->term) { + trc->edgeptr = trc->term; + return nullptr; + } + + node->thing = thing; + node->count = 0; + if (!trc->nodemap.add(p, thing.asCell(), node)) { + trc->edgeptr = trc->term; + return nullptr; + } + + return node; + } + return nullptr; +} + +static VerifyNode* NextNode(VerifyNode* node) { + if (node->count == 0) { + return (VerifyNode*)((char*)node + sizeof(VerifyNode) - sizeof(EdgeValue)); + } else { + return (VerifyNode*)((char*)node + sizeof(VerifyNode) + + sizeof(EdgeValue) * (node->count - 1)); + } +} + +template +static void ClearMarkBits(GCRuntime* gc) { + // This does not clear the mark bits for permanent atoms, whose arenas are + // removed from the arena lists by GCRuntime::freezePermanentAtoms. + + for (ZonesIterT zone(gc); !zone.done(); zone.next()) { + for (auto kind : AllAllocKinds()) { + for (ArenaIter arena(zone, kind); !arena.done(); arena.next()) { + arena->unmarkAll(); + } + } + } +} + +void gc::GCRuntime::startVerifyPreBarriers() { + if (verifyPreData || isIncrementalGCInProgress()) { + return; + } + + JSContext* cx = rt->mainContextFromOwnThread(); + + if (IsIncrementalGCUnsafe(rt) != GCAbortReason::None) { + return; + } + + number++; + + VerifyPreTracer* trc = js_new(rt); + if (!trc) { + return; + } + + AutoPrepareForTracing prep(cx); + + ClearMarkBits(this); + + gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::TRACE_HEAP); + + const size_t size = 64 * 1024 * 1024; + trc->root = (VerifyNode*)js_malloc(size); + if (!trc->root) { + goto oom; + } + trc->edgeptr = (char*)trc->root; + trc->term = trc->edgeptr + size; + + /* Create the root node. */ + trc->curnode = MakeNode(trc, JS::GCCellPtr()); + + MOZ_ASSERT(incrementalState == State::NotActive); + incrementalState = State::MarkRoots; + + /* Make all the roots be edges emanating from the root node. */ + traceRuntime(trc, prep); + + VerifyNode* node; + node = trc->curnode; + if (trc->edgeptr == trc->term) { + goto oom; + } + + /* For each edge, make a node for it if one doesn't already exist. */ + while ((char*)node < trc->edgeptr) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& e = node->edges[i]; + VerifyNode* child = MakeNode(trc, e.thing); + if (child) { + trc->curnode = child; + JS::TraceChildren(trc, e.thing); + } + if (trc->edgeptr == trc->term) { + goto oom; + } + } + + node = NextNode(node); + } + + verifyPreData = trc; + incrementalState = State::Mark; + marker().start(); + + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + zone->changeGCState(Zone::NoGC, Zone::VerifyPreBarriers); + zone->setNeedsIncrementalBarrier(true); + zone->arenas.clearFreeLists(); + } + + return; + +oom: + incrementalState = State::NotActive; + js_delete(trc); + verifyPreData = nullptr; +} + +static bool IsMarkedOrAllocated(TenuredCell* cell) { + return cell->isMarkedAny(); +} + +struct CheckEdgeTracer final : public JS::CallbackTracer { + VerifyNode* node; + explicit CheckEdgeTracer(JSRuntime* rt) + : JS::CallbackTracer(rt), node(nullptr) {} + void onChild(JS::GCCellPtr thing, const char* name) override; +}; + +static const uint32_t MAX_VERIFIER_EDGES = 1000; + +/* + * This function is called by EndVerifyBarriers for every heap edge. If the edge + * already existed in the original snapshot, we "cancel it out" by overwriting + * it with nullptr. EndVerifyBarriers later asserts that the remaining + * non-nullptr edges (i.e., the ones from the original snapshot that must have + * been modified) must point to marked objects. + */ +void CheckEdgeTracer::onChild(JS::GCCellPtr thing, const char* name) { + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) { + return; + } + + /* Avoid n^2 behavior. */ + if (node->count > MAX_VERIFIER_EDGES) { + return; + } + + for (uint32_t i = 0; i < node->count; i++) { + if (node->edges[i].thing == thing) { + node->edges[i].thing = JS::GCCellPtr(); + return; + } + } +} + +static bool IsMarkedOrAllocated(const EdgeValue& edge) { + if (!edge.thing || IsMarkedOrAllocated(&edge.thing.asCell()->asTenured())) { + return true; + } + + // Permanent atoms and well-known symbols aren't marked during graph + // traversal. + if (edge.thing.is() && + edge.thing.as().isPermanentAtom()) { + return true; + } + if (edge.thing.is() && + edge.thing.as().isWellKnownSymbol()) { + return true; + } + + return false; +} + +void gc::GCRuntime::endVerifyPreBarriers() { + VerifyPreTracer* trc = verifyPreData; + + if (!trc) { + return; + } + + MOZ_ASSERT(!JS::IsGenerationalGCEnabled(rt)); + + // Now that barrier marking has finished, prepare the heap to allow this + // method to trace cells and discover their outgoing edges. + AutoPrepareForTracing prep(rt->mainContextFromOwnThread()); + + bool compartmentCreated = false; + + /* We need to disable barriers before tracing, which may invoke barriers. */ + for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) { + if (zone->isVerifyingPreBarriers()) { + zone->changeGCState(Zone::VerifyPreBarriers, Zone::NoGC); + } else { + compartmentCreated = true; + } + + MOZ_ASSERT(!zone->wasGCStarted()); + MOZ_ASSERT(!zone->needsIncrementalBarrier()); + } + + verifyPreData = nullptr; + MOZ_ASSERT(incrementalState == State::Mark); + incrementalState = State::NotActive; + + if (!compartmentCreated && IsIncrementalGCUnsafe(rt) == GCAbortReason::None) { + CheckEdgeTracer cetrc(rt); + + /* Start after the roots. */ + VerifyNode* node = NextNode(trc->root); + while ((char*)node < trc->edgeptr) { + cetrc.node = node; + JS::TraceChildren(&cetrc, node->thing); + + if (node->count <= MAX_VERIFIER_EDGES) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& edge = node->edges[i]; + if (!IsMarkedOrAllocated(edge)) { + char msgbuf[1024]; + SprintfLiteral( + msgbuf, + "[barrier verifier] Unmarked edge: %s %p '%s' edge to %s %p", + JS::GCTraceKindToAscii(node->thing.kind()), + node->thing.asCell(), edge.label, + JS::GCTraceKindToAscii(edge.thing.kind()), edge.thing.asCell()); + MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__); + MOZ_CRASH(); + } + } + } + + node = NextNode(node); + } + } + + marker().reset(); + marker().stop(); + resetDelayedMarking(); + + js_delete(trc); +} + +/*** Barrier Verifier Scheduling ***/ + +void gc::GCRuntime::verifyPreBarriers() { + if (verifyPreData) { + endVerifyPreBarriers(); + } else { + startVerifyPreBarriers(); + } +} + +void gc::VerifyBarriers(JSRuntime* rt, VerifierType type) { + if (type == PreBarrierVerifier) { + rt->gc.verifyPreBarriers(); + } +} + +void gc::GCRuntime::maybeVerifyPreBarriers(bool always) { + if (!hasZealMode(ZealMode::VerifierPre)) { + return; + } + + if (rt->mainContextFromOwnThread()->suppressGC) { + return; + } + + if (verifyPreData) { + if (++verifyPreData->count < zealFrequency && !always) { + return; + } + + endVerifyPreBarriers(); + } + + startVerifyPreBarriers(); +} + +void js::gc::MaybeVerifyBarriers(JSContext* cx, bool always) { + GCRuntime* gc = &cx->runtime()->gc; + gc->maybeVerifyPreBarriers(always); +} + +void js::gc::GCRuntime::finishVerifier() { + if (verifyPreData) { + js_delete(verifyPreData.ref()); + verifyPreData = nullptr; + } +} + +struct GCChunkHasher { + typedef gc::TenuredChunk* Lookup; + + /* + * Strip zeros for better distribution after multiplying by the golden + * ratio. + */ + static HashNumber hash(gc::TenuredChunk* chunk) { + MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask)); + return HashNumber(uintptr_t(chunk) >> gc::ChunkShift); + } + + static bool match(gc::TenuredChunk* k, gc::TenuredChunk* l) { + MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask)); + MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask)); + return k == l; + } +}; + +class js::gc::MarkingValidator { + public: + explicit MarkingValidator(GCRuntime* gc); + void nonIncrementalMark(AutoGCSession& session); + void validate(); + + private: + GCRuntime* gc; + bool initialized; + + using BitmapMap = HashMap, GCChunkHasher, + SystemAllocPolicy>; + BitmapMap map; +}; + +js::gc::MarkingValidator::MarkingValidator(GCRuntime* gc) + : gc(gc), initialized(false) {} + +void js::gc::MarkingValidator::nonIncrementalMark(AutoGCSession& session) { + /* + * Perform a non-incremental mark for all collecting zones and record + * the results for later comparison. + * + * Currently this does not validate gray marking. + */ + + JSRuntime* runtime = gc->rt; + GCMarker* gcmarker = &gc->marker(); + + MOZ_ASSERT(!gcmarker->isWeakMarking()); + + /* Wait for off-thread parsing which can allocate. */ + WaitForAllHelperThreads(); + + gc->waitBackgroundAllocEnd(); + gc->waitBackgroundSweepEnd(); + + /* Save existing mark bits. */ + { + AutoLockGC lock(gc); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); + chunk.next()) { + MarkBitmap* bitmap = &chunk->markBits; + auto entry = MakeUnique(); + if (!entry) { + return; + } + + memcpy((void*)entry->bitmap, (void*)bitmap->bitmap, + sizeof(bitmap->bitmap)); + + if (!map.putNew(chunk, std::move(entry))) { + return; + } + } + } + + /* + * Temporarily clear the weakmaps' mark flags for the compartments we are + * collecting. + */ + + WeakMapColors markedWeakMaps; + + /* + * For saving, smush all of the keys into one big table and split them back + * up into per-zone tables when restoring. + */ + gc::EphemeronEdgeTable savedEphemeronEdges( + SystemAllocPolicy(), runtime->randomHashCodeScrambler()); + if (!savedEphemeronEdges.init()) { + return; + } + + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + if (!WeakMapBase::saveZoneMarkedWeakMaps(zone, markedWeakMaps)) { + return; + } + + AutoEnterOOMUnsafeRegion oomUnsafe; + for (auto r = zone->gcEphemeronEdges().mutableAll(); !r.empty(); + r.popFront()) { + MOZ_ASSERT(r.front().key->asTenured().zone() == zone); + if (!savedEphemeronEdges.put(r.front().key, std::move(r.front().value))) { + oomUnsafe.crash("saving weak keys table for validator"); + } + } + + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing weak keys table for validator"); + } + } + + /* Save and restore test mark queue state. */ +# ifdef DEBUG + size_t savedQueuePos = gc->queuePos; + mozilla::Maybe savedQueueColor = gc->queueMarkColor; +# endif + + /* + * After this point, the function should run to completion, so we shouldn't + * do anything fallible. + */ + initialized = true; + + /* Re-do all the marking, but non-incrementally. */ + js::gc::State state = gc->incrementalState; + gc->incrementalState = State::MarkRoots; + + { + gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::PREPARE); + + { + gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::UNMARK); + + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + WeakMapBase::unmarkZone(zone); + } + + MOZ_ASSERT(gcmarker->isDrained()); + gcmarker->reset(); + + ClearMarkBits(gc); + } + } + + { + gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::MARK); + + gc->traceRuntimeForMajorGC(gcmarker->tracer(), session); + + gc->incrementalState = State::Mark; + gc->drainMarkStack(); + } + + gc->incrementalState = State::Sweep; + { + gcstats::AutoPhase ap1(gc->stats(), gcstats::PhaseKind::SWEEP); + gcstats::AutoPhase ap2(gc->stats(), gcstats::PhaseKind::MARK); + + gc->markAllWeakReferences(); + + /* Update zone state for gray marking. */ + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + zone->changeGCState(zone->initialMarkingState(), Zone::MarkBlackAndGray); + } + + AutoSetMarkColor setColorGray(*gcmarker, MarkColor::Gray); + + gc->markAllGrayReferences(gcstats::PhaseKind::MARK_GRAY); + gc->markAllWeakReferences(); + + /* Restore zone state. */ + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + zone->changeGCState(Zone::MarkBlackAndGray, zone->initialMarkingState()); + } + MOZ_ASSERT(gc->marker().isDrained()); + } + + /* Take a copy of the non-incremental mark state and restore the original. */ + { + AutoLockGC lock(gc); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); + chunk.next()) { + MarkBitmap* bitmap = &chunk->markBits; + auto ptr = map.lookup(chunk); + MOZ_RELEASE_ASSERT(ptr, "Chunk not found in map"); + MarkBitmap* entry = ptr->value().get(); + for (size_t i = 0; i < MarkBitmap::WordCount; i++) { + uintptr_t v = entry->bitmap[i]; + entry->bitmap[i] = uintptr_t(bitmap->bitmap[i]); + bitmap->bitmap[i] = v; + } + } + } + + for (GCZonesIter zone(gc); !zone.done(); zone.next()) { + WeakMapBase::unmarkZone(zone); + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing weak keys table for validator"); + } + } + + WeakMapBase::restoreMarkedWeakMaps(markedWeakMaps); + + for (auto r = savedEphemeronEdges.mutableAll(); !r.empty(); r.popFront()) { + AutoEnterOOMUnsafeRegion oomUnsafe; + Zone* zone = r.front().key->asTenured().zone(); + if (!zone->gcEphemeronEdges().put(r.front().key, + std::move(r.front().value))) { + oomUnsafe.crash("restoring weak keys table for validator"); + } + } + +# ifdef DEBUG + gc->queuePos = savedQueuePos; + gc->queueMarkColor = savedQueueColor; +# endif + + gc->incrementalState = state; +} + +void js::gc::MarkingValidator::validate() { + /* + * Validates the incremental marking for a single compartment by comparing + * the mark bits to those previously recorded for a non-incremental mark. + */ + + if (!initialized) { + return; + } + + MOZ_ASSERT(!gc->marker().isWeakMarking()); + + gc->waitBackgroundSweepEnd(); + + bool ok = true; + AutoLockGC lock(gc->rt); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) { + BitmapMap::Ptr ptr = map.lookup(chunk); + if (!ptr) { + continue; /* Allocated after we did the non-incremental mark. */ + } + + MarkBitmap* bitmap = ptr->value().get(); + MarkBitmap* incBitmap = &chunk->markBits; + + for (size_t i = 0; i < ArenasPerChunk; i++) { + if (chunk->decommittedPages[chunk->pageIndex(i)]) { + continue; + } + Arena* arena = &chunk->arenas[i]; + if (!arena->allocated()) { + continue; + } + if (!arena->zone->isGCSweeping()) { + continue; + } + + AllocKind kind = arena->getAllocKind(); + uintptr_t thing = arena->thingsStart(); + uintptr_t end = arena->thingsEnd(); + while (thing < end) { + auto* cell = reinterpret_cast(thing); + + /* + * If a non-incremental GC wouldn't have collected a cell, then + * an incremental GC won't collect it. + */ + if (bitmap->isMarkedAny(cell)) { + if (!incBitmap->isMarkedAny(cell)) { + ok = false; + const char* color = TenuredCell::getColor(bitmap, cell).name(); + fprintf(stderr, + "%p: cell not marked, but would be marked %s by " + "non-incremental marking\n", + cell, color); +# ifdef DEBUG + cell->dump(); + fprintf(stderr, "\n"); +# endif + } + } + + /* + * If the cycle collector isn't allowed to collect an object + * after a non-incremental GC has run, then it isn't allowed to + * collected it after an incremental GC. + */ + if (!bitmap->isMarkedGray(cell)) { + if (incBitmap->isMarkedGray(cell)) { + ok = false; + const char* color = TenuredCell::getColor(bitmap, cell).name(); + fprintf(stderr, + "%p: cell marked gray, but would be marked %s by " + "non-incremental marking\n", + cell, color); +# ifdef DEBUG + cell->dump(); + fprintf(stderr, "\n"); +# endif + } + } + + thing += Arena::thingSize(kind); + } + } + } + + MOZ_RELEASE_ASSERT(ok, "Incremental marking verification failed"); +} + +void GCRuntime::computeNonIncrementalMarkingForValidation( + AutoGCSession& session) { + MOZ_ASSERT(!markingValidator); + if (isIncremental && hasZealMode(ZealMode::IncrementalMarkingValidator)) { + markingValidator = js_new(this); + } + if (markingValidator) { + markingValidator->nonIncrementalMark(session); + } +} + +void GCRuntime::validateIncrementalMarking() { + if (markingValidator) { + markingValidator->validate(); + } +} + +void GCRuntime::finishMarkingValidation() { + js_delete(markingValidator.ref()); + markingValidator = nullptr; +} + +#endif /* JS_GC_ZEAL */ + +#if defined(JS_GC_ZEAL) || defined(DEBUG) + +class HeapCheckTracerBase : public JS::CallbackTracer { + public: + explicit HeapCheckTracerBase(JSRuntime* rt, JS::TraceOptions options); + bool traceHeap(AutoTraceSession& session); + virtual void checkCell(Cell* cell, const char* name) = 0; + + protected: + void dumpCellInfo(Cell* cell); + void dumpCellPath(const char* name); + + Cell* parentCell() { + return parentIndex == -1 ? nullptr : stack[parentIndex].thing.asCell(); + } + + size_t failures; + + private: + void onChild(JS::GCCellPtr thing, const char* name) override; + + struct WorkItem { + WorkItem(JS::GCCellPtr thing, const char* name, int parentIndex) + : thing(thing), + name(name), + parentIndex(parentIndex), + processed(false) {} + + JS::GCCellPtr thing; + const char* name; + int parentIndex; + bool processed; + }; + + JSRuntime* rt; + bool oom; + HashSet, SystemAllocPolicy> visited; + Vector stack; + int parentIndex; +}; + +HeapCheckTracerBase::HeapCheckTracerBase(JSRuntime* rt, + JS::TraceOptions options) + : CallbackTracer(rt, JS::TracerKind::Callback, options), + failures(0), + rt(rt), + oom(false), + parentIndex(-1) {} + +void HeapCheckTracerBase::onChild(JS::GCCellPtr thing, const char* name) { + Cell* cell = thing.asCell(); + checkCell(cell, name); + + if (visited.lookup(cell)) { + return; + } + + if (!visited.put(cell)) { + oom = true; + return; + } + + // Don't trace into GC things owned by another runtime. + if (cell->runtimeFromAnyThread() != rt) { + return; + } + + WorkItem item(thing, name, parentIndex); + if (!stack.append(item)) { + oom = true; + } +} + +bool HeapCheckTracerBase::traceHeap(AutoTraceSession& session) { + // The analysis thinks that traceRuntime might GC by calling a GC callback. + JS::AutoSuppressGCAnalysis nogc; + if (!rt->isBeingDestroyed()) { + rt->gc.traceRuntime(this, session); + } + + while (!stack.empty() && !oom) { + WorkItem item = stack.back(); + if (item.processed) { + stack.popBack(); + } else { + parentIndex = stack.length() - 1; + stack.back().processed = true; + TraceChildren(this, item.thing); + } + } + + return !oom; +} + +void HeapCheckTracerBase::dumpCellInfo(Cell* cell) { + auto kind = cell->getTraceKind(); + JSObject* obj = + kind == JS::TraceKind::Object ? static_cast(cell) : nullptr; + + fprintf(stderr, "%s %s", cell->color().name(), GCTraceKindToAscii(kind)); + if (obj) { + fprintf(stderr, " %s", obj->getClass()->name); + } + fprintf(stderr, " %p", cell); + if (obj) { + fprintf(stderr, " (compartment %p)", obj->compartment()); + } +} + +void HeapCheckTracerBase::dumpCellPath(const char* name) { + for (int index = parentIndex; index != -1; index = stack[index].parentIndex) { + const WorkItem& parent = stack[index]; + Cell* cell = parent.thing.asCell(); + fprintf(stderr, " from "); + dumpCellInfo(cell); + fprintf(stderr, " %s edge\n", name); + name = parent.name; + } + fprintf(stderr, " from root %s\n", name); +} + +class CheckHeapTracer final : public HeapCheckTracerBase { + public: + enum GCType { Moving, NonMoving }; + + explicit CheckHeapTracer(JSRuntime* rt, GCType type); + void check(AutoTraceSession& session); + + private: + void checkCell(Cell* cell, const char* name) override; + GCType gcType; +}; + +CheckHeapTracer::CheckHeapTracer(JSRuntime* rt, GCType type) + : HeapCheckTracerBase(rt, JS::WeakMapTraceAction::TraceKeysAndValues), + gcType(type) {} + +inline static bool IsValidGCThingPointer(Cell* cell) { + return (uintptr_t(cell) & CellAlignMask) == 0; +} + +void CheckHeapTracer::checkCell(Cell* cell, const char* name) { + // Moving + if (!IsValidGCThingPointer(cell) || + ((gcType == GCType::Moving) && !IsGCThingValidAfterMovingGC(cell)) || + ((gcType == GCType::NonMoving) && cell->isForwarded())) { + failures++; + fprintf(stderr, "Bad pointer %p\n", cell); + dumpCellPath(name); + } +} + +void CheckHeapTracer::check(AutoTraceSession& session) { + if (!traceHeap(session)) { + return; + } + + if (failures) { + fprintf(stderr, "Heap check: %zu failure(s)\n", failures); + } + MOZ_RELEASE_ASSERT(failures == 0); +} + +void js::gc::CheckHeapAfterGC(JSRuntime* rt) { + AutoTraceSession session(rt); + CheckHeapTracer::GCType gcType; + + if (rt->gc.nursery().isEmpty()) { + gcType = CheckHeapTracer::GCType::Moving; + } else { + gcType = CheckHeapTracer::GCType::NonMoving; + } + + CheckHeapTracer tracer(rt, gcType); + tracer.check(session); +} + +class CheckGrayMarkingTracer final : public HeapCheckTracerBase { + public: + explicit CheckGrayMarkingTracer(JSRuntime* rt); + bool check(AutoTraceSession& session); + + private: + void checkCell(Cell* cell, const char* name) override; +}; + +CheckGrayMarkingTracer::CheckGrayMarkingTracer(JSRuntime* rt) + : HeapCheckTracerBase(rt, JS::TraceOptions(JS::WeakMapTraceAction::Skip, + JS::WeakEdgeTraceAction::Skip)) { + // Weak gray->black edges are allowed. +} + +void CheckGrayMarkingTracer::checkCell(Cell* cell, const char* name) { + Cell* parent = parentCell(); + if (!parent) { + return; + } + + if (parent->isMarkedBlack() && cell->isMarkedGray()) { + failures++; + + fprintf(stderr, "Found black to gray edge to "); + dumpCellInfo(cell); + fprintf(stderr, "\n"); + dumpCellPath(name); + +# ifdef DEBUG + if (parent->is()) { + fprintf(stderr, "\nSource: "); + DumpObject(parent->as(), stderr); + } + if (cell->is()) { + fprintf(stderr, "\nTarget: "); + DumpObject(cell->as(), stderr); + } +# endif + } +} + +bool CheckGrayMarkingTracer::check(AutoTraceSession& session) { + if (!traceHeap(session)) { + return true; // Ignore failure. + } + + return failures == 0; +} + +JS_PUBLIC_API bool js::CheckGrayMarkingState(JSRuntime* rt) { + MOZ_ASSERT(!JS::RuntimeHeapIsCollecting()); + MOZ_ASSERT(!rt->gc.isIncrementalGCInProgress()); + if (!rt->gc.areGrayBitsValid()) { + return true; + } + + gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP); + AutoTraceSession session(rt); + CheckGrayMarkingTracer tracer(rt); + + return tracer.check(session); +} + +static JSObject* MaybeGetDelegate(Cell* cell) { + if (!cell->is()) { + return nullptr; + } + + JSObject* object = cell->as(); + return js::UncheckedUnwrapWithoutExpose(object); +} + +bool js::gc::CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key, + Cell* value) { + bool ok = true; + + Zone* zone = map->zone(); + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + MOZ_ASSERT(zone->isGCMarking()); + + JSObject* object = map->memberOf; + MOZ_ASSERT_IF(object, object->zone() == zone); + + // Debugger weak maps can have keys in different zones. + Zone* keyZone = key->zoneFromAnyThread(); + MOZ_ASSERT_IF(!map->allowKeysInOtherZones(), + keyZone == zone || keyZone->isAtomsZone()); + + Zone* valueZone = value->zoneFromAnyThread(); + MOZ_ASSERT(valueZone == zone || valueZone->isAtomsZone()); + + if (object && object->color() != map->mapColor) { + fprintf(stderr, "WeakMap object is marked differently to the map\n"); + fprintf(stderr, "(map %p is %s, object %p is %s)\n", map, + map->mapColor.name(), object, object->color().name()); + ok = false; + } + + // Values belonging to other runtimes or in uncollected zones are treated as + // black. + JSRuntime* mapRuntime = zone->runtimeFromAnyThread(); + auto effectiveColor = [=](Cell* cell, Zone* cellZone) -> CellColor { + if (cell->runtimeFromAnyThread() != mapRuntime) { + return CellColor::Black; + } + if (cellZone->isGCMarkingOrSweeping()) { + return cell->color(); + } + return CellColor::Black; + }; + + CellColor valueColor = effectiveColor(value, valueZone); + CellColor keyColor = effectiveColor(key, keyZone); + + if (valueColor < std::min(map->mapColor, keyColor)) { + fprintf(stderr, "WeakMap value is less marked than map and key\n"); + fprintf(stderr, "(map %p is %s, key %p is %s, value %p is %s)\n", map, + map->mapColor.name(), key, keyColor.name(), value, + valueColor.name()); +# ifdef DEBUG + fprintf(stderr, "Key:\n"); + key->dump(); + if (auto delegate = MaybeGetDelegate(key); delegate) { + fprintf(stderr, "Delegate:\n"); + delegate->dump(); + } + fprintf(stderr, "Value:\n"); + value->dump(); +# endif + + ok = false; + } + + JSObject* delegate = MaybeGetDelegate(key); + if (!delegate) { + return ok; + } + + CellColor delegateColor = effectiveColor(delegate, delegate->zone()); + if (keyColor < std::min(map->mapColor, delegateColor)) { + fprintf(stderr, "WeakMap key is less marked than map or delegate\n"); + fprintf(stderr, "(map %p is %s, delegate %p is %s, key %p is %s)\n", map, + map->mapColor.name(), delegate, delegateColor.name(), key, + keyColor.name()); + ok = false; + } + + return ok; +} + +#endif // defined(JS_GC_ZEAL) || defined(DEBUG) + +#ifdef DEBUG +// Return whether an arbitrary pointer is within a cell with the given +// traceKind. Only for assertions. +bool GCRuntime::isPointerWithinTenuredCell(void* ptr, JS::TraceKind traceKind) { + AutoLockGC lock(this); + for (auto chunk = allNonEmptyChunks(lock); !chunk.done(); chunk.next()) { + MOZ_ASSERT(!chunk->isNurseryChunk()); + if (ptr >= &chunk->arenas[0] && ptr < &chunk->arenas[ArenasPerChunk]) { + auto* arena = reinterpret_cast(uintptr_t(ptr) & ~ArenaMask); + if (!arena->allocated()) { + return false; + } + + return MapAllocToTraceKind(arena->getAllocKind()) == traceKind; + } + } + + return false; +} +#endif // DEBUG diff --git a/js/src/gc/WeakMap-inl.h b/js/src/gc/WeakMap-inl.h new file mode 100644 index 0000000000..1b5626211f --- /dev/null +++ b/js/src/gc/WeakMap-inl.h @@ -0,0 +1,413 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_WeakMap_inl_h +#define gc_WeakMap_inl_h + +#include "gc/WeakMap.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include +#include + +#include "gc/GCLock.h" +#include "gc/Marking.h" +#include "gc/Zone.h" +#include "js/TraceKind.h" +#include "vm/JSContext.h" + +#include "gc/StableCellHasher-inl.h" + +namespace js { +namespace gc { + +namespace detail { + +// Return the effective cell color given the current marking state. +// This must be kept in sync with ShouldMark in Marking.cpp. +template +static CellColor GetEffectiveColor(GCMarker* marker, const T& item) { + Cell* cell = ToMarkable(item); + if (!cell->isTenured()) { + return CellColor::Black; + } + const TenuredCell& t = cell->asTenured(); + if (!t.zoneFromAnyThread()->shouldMarkInZone(marker->markColor())) { + return CellColor::Black; + } + MOZ_ASSERT(t.runtimeFromAnyThread() == marker->runtime()); + return t.color(); +} + +// Only objects have delegates, so default to returning nullptr. Note that some +// compilation units will only ever use the object version. +static MOZ_MAYBE_UNUSED JSObject* GetDelegateInternal(gc::Cell* key) { + return nullptr; +} + +static MOZ_MAYBE_UNUSED JSObject* GetDelegateInternal(JSObject* key) { + JSObject* delegate = UncheckedUnwrapWithoutExpose(key); + return (key == delegate) ? nullptr : delegate; +} + +// Use a helper function to do overload resolution to handle cases like +// Heap: find everything that is convertible to JSObject* (and +// avoid calling barriers). +template +static inline JSObject* GetDelegate(const T& key) { + return GetDelegateInternal(key); +} + +template <> +inline JSObject* GetDelegate(gc::Cell* const&) = delete; + +} /* namespace detail */ +} /* namespace gc */ + +// Weakmap entry -> value edges are only visible if the map is traced, which +// only happens if the map zone is being collected. If the map and the value +// were in different zones, then we could have a case where the map zone is not +// collecting but the value zone is, and incorrectly free a value that is +// reachable solely through weakmaps. +template +void WeakMap::assertMapIsSameZoneWithValue(const V& v) { +#ifdef DEBUG + gc::Cell* cell = gc::ToMarkable(v); + if (cell) { + Zone* cellZone = cell->zoneFromAnyThread(); + MOZ_ASSERT(zone() == cellZone || cellZone->isAtomsZone()); + } +#endif +} + +template +WeakMap::WeakMap(JSContext* cx, JSObject* memOf) + : WeakMap(cx->zone(), memOf) {} + +template +WeakMap::WeakMap(JS::Zone* zone, JSObject* memOf) + : Base(zone), WeakMapBase(memOf, zone) { + using ElemType = typename K::ElementType; + using NonPtrType = std::remove_pointer_t; + + // The object's TraceKind needs to be added to CC graph if this object is + // used as a WeakMap key, otherwise the key is considered to be pointed from + // somewhere unknown, and results in leaking the subgraph which contains the + // key. See the comments in NoteWeakMapsTracer::trace for more details. + static_assert(JS::IsCCTraceKind(NonPtrType::TraceKind), + "Object's TraceKind should be added to CC graph."); + + zone->gcWeakMapList().insertFront(this); + if (zone->gcState() > Zone::Prepare) { + mapColor = CellColor::Black; + } +} + +// If the entry is live, ensure its key and value are marked. Also make sure the +// key is at least as marked as min(map, delegate), so it cannot get discarded +// and then recreated by rewrapping the delegate. +// +// Optionally adds edges to the ephemeron edges table for any keys (or +// delegates) where future changes to their mark color would require marking the +// value (or the key). +template +bool WeakMap::markEntry(GCMarker* marker, K& key, V& value, + bool populateWeakKeysTable) { +#ifdef DEBUG + MOZ_ASSERT(mapColor); + if (marker->isParallelMarking()) { + marker->runtime()->gc.assertCurrentThreadHasLockedGC(); + } +#endif + + bool marked = false; + CellColor markColor = marker->markColor(); + CellColor keyColor = gc::detail::GetEffectiveColor(marker, key); + JSObject* delegate = gc::detail::GetDelegate(key); + JSTracer* trc = marker->tracer(); + + if (delegate) { + CellColor delegateColor = gc::detail::GetEffectiveColor(marker, delegate); + // The key needs to stay alive while both the delegate and map are live. + CellColor proxyPreserveColor = std::min(delegateColor, mapColor); + if (keyColor < proxyPreserveColor) { + MOZ_ASSERT(markColor >= proxyPreserveColor); + if (markColor == proxyPreserveColor) { + TraceWeakMapKeyEdge(trc, zone(), &key, + "proxy-preserved WeakMap entry key"); + MOZ_ASSERT(key->color() >= proxyPreserveColor); + marked = true; + keyColor = proxyPreserveColor; + } + } + } + + gc::Cell* cellValue = gc::ToMarkable(value); + if (keyColor) { + if (cellValue) { + CellColor targetColor = std::min(mapColor, keyColor); + CellColor valueColor = gc::detail::GetEffectiveColor(marker, cellValue); + if (valueColor < targetColor) { + MOZ_ASSERT(markColor >= targetColor); + if (markColor == targetColor) { + TraceEdge(trc, &value, "WeakMap entry value"); + MOZ_ASSERT(cellValue->color() >= targetColor); + marked = true; + } + } + } + } + + if (populateWeakKeysTable) { + // Note that delegateColor >= keyColor because marking a key marks its + // delegate, so we only need to check whether keyColor < mapColor to tell + // this. + + if (keyColor < mapColor) { + MOZ_ASSERT(trc->weakMapAction() == JS::WeakMapTraceAction::Expand); + // The final color of the key is not yet known. Record this weakmap and + // the lookup key in the list of weak keys. If the key has a delegate, + // then the lookup key is the delegate (because marking the key will end + // up marking the delegate and thereby mark the entry.) + gc::TenuredCell* tenuredValue = nullptr; + if (cellValue && cellValue->isTenured()) { + tenuredValue = &cellValue->asTenured(); + } + + if (!this->addImplicitEdges(key, delegate, tenuredValue)) { + marker->abortLinearWeakMarking(); + } + } + } + + return marked; +} + +template +void WeakMap::trace(JSTracer* trc) { + MOZ_ASSERT(isInList()); + + TraceNullableEdge(trc, &memberOf, "WeakMap owner"); + + if (trc->isMarkingTracer()) { + MOZ_ASSERT(trc->weakMapAction() == JS::WeakMapTraceAction::Expand); + auto marker = GCMarker::fromTracer(trc); + + // Lock if we are marking in parallel to synchronize updates to: + // - the weak map's color + // - the ephemeron edges table + mozilla::Maybe lock; + if (marker->isParallelMarking()) { + lock.emplace(marker->runtime()); + } + + // Don't downgrade the map color from black to gray. This can happen when a + // barrier pushes the map object onto the black mark stack when it's + // already present on the gray mark stack, which is marked later. + if (mapColor < marker->markColor()) { + mapColor = marker->markColor(); + (void)markEntries(marker); + } + return; + } + + if (trc->weakMapAction() == JS::WeakMapTraceAction::Skip) { + return; + } + + // Trace keys only if weakMapAction() says to. + if (trc->weakMapAction() == JS::WeakMapTraceAction::TraceKeysAndValues) { + for (Enum e(*this); !e.empty(); e.popFront()) { + TraceWeakMapKeyEdge(trc, zone(), &e.front().mutableKey(), + "WeakMap entry key"); + } + } + + // Always trace all values (unless weakMapAction() is Skip). + for (Range r = Base::all(); !r.empty(); r.popFront()) { + TraceEdge(trc, &r.front().value(), "WeakMap entry value"); + } +} + +bool WeakMapBase::addImplicitEdges(gc::Cell* key, gc::Cell* delegate, + gc::TenuredCell* value) { + if (delegate) { + auto& edgeTable = delegate->zone()->gcEphemeronEdges(delegate); + auto* p = edgeTable.get(delegate); + + gc::EphemeronEdgeVector newVector; + gc::EphemeronEdgeVector& edges = p ? p->value : newVector; + + // Add a -> key edge: the key must be preserved for + // future lookups until either the weakmap or the delegate dies. + gc::EphemeronEdge keyEdge{mapColor, key}; + if (!edges.append(keyEdge)) { + return false; + } + + if (value) { + gc::EphemeronEdge valueEdge{mapColor, value}; + if (!edges.append(valueEdge)) { + return false; + } + } + + if (!p) { + return edgeTable.put(delegate, std::move(newVector)); + } + + return true; + } + + // No delegate. Insert just the key -> value edge. + + if (!value) { + return true; + } + + auto& edgeTable = key->zone()->gcEphemeronEdges(key); + auto* p = edgeTable.get(key); + gc::EphemeronEdge valueEdge{mapColor, value}; + if (p) { + return p->value.append(valueEdge); + } else { + gc::EphemeronEdgeVector edges; + MOZ_ALWAYS_TRUE(edges.append(valueEdge)); + return edgeTable.put(key, std::move(edges)); + } +} + +template +bool WeakMap::markEntries(GCMarker* marker) { + // This method is called whenever the map's mark color changes. Mark values + // (and keys with delegates) as required for the new color and populate the + // ephemeron edges if we're in incremental marking mode. + +#ifdef DEBUG + if (marker->isParallelMarking()) { + marker->runtime()->gc.assertCurrentThreadHasLockedGC(); + } +#endif + + MOZ_ASSERT(mapColor); + bool markedAny = false; + + // If we don't populate the weak keys table now then we do it when we enter + // weak marking mode. + bool populateWeakKeysTable = + marker->incrementalWeakMapMarkingEnabled || marker->isWeakMarking(); + + for (Enum e(*this); !e.empty(); e.popFront()) { + if (markEntry(marker, e.front().mutableKey(), e.front().value(), + populateWeakKeysTable)) { + markedAny = true; + } + } + + return markedAny; +} + +template +void WeakMap::traceWeakEdges(JSTracer* trc) { + // Remove all entries whose keys remain unmarked. + for (Enum e(*this); !e.empty(); e.popFront()) { + if (!TraceWeakEdge(trc, &e.front().mutableKey(), "WeakMap key")) { + e.removeFront(); + } + } + +#if DEBUG + // Once we've swept, all remaining edges should stay within the known-live + // part of the graph. + assertEntriesNotAboutToBeFinalized(); +#endif +} + +// memberOf can be nullptr, which means that the map is not part of a JSObject. +template +void WeakMap::traceMappings(WeakMapTracer* tracer) { + for (Range r = Base::all(); !r.empty(); r.popFront()) { + gc::Cell* key = gc::ToMarkable(r.front().key()); + gc::Cell* value = gc::ToMarkable(r.front().value()); + if (key && value) { + tracer->trace(memberOf, JS::GCCellPtr(r.front().key().get()), + JS::GCCellPtr(r.front().value().get())); + } + } +} + +template +bool WeakMap::findSweepGroupEdges() { + // For weakmap keys with delegates in a different zone, add a zone edge to + // ensure that the delegate zone finishes marking before the key zone. + JS::AutoSuppressGCAnalysis nogc; + for (Range r = all(); !r.empty(); r.popFront()) { + const K& key = r.front().key(); + + // If the key type doesn't have delegates, then this will always return + // nullptr and the optimizer can remove the entire body of this function. + JSObject* delegate = gc::detail::GetDelegate(key); + if (!delegate) { + continue; + } + + // Marking a WeakMap key's delegate will mark the key, so process the + // delegate zone no later than the key zone. + Zone* delegateZone = delegate->zone(); + Zone* keyZone = key->zone(); + if (delegateZone != keyZone && delegateZone->isGCMarking() && + keyZone->isGCMarking()) { + if (!delegateZone->addSweepGroupEdgeTo(keyZone)) { + return false; + } + } + } + return true; +} + +template +size_t WeakMap::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return mallocSizeOf(this) + shallowSizeOfExcludingThis(mallocSizeOf); +} + +#if DEBUG +template +void WeakMap::assertEntriesNotAboutToBeFinalized() { + for (Range r = Base::all(); !r.empty(); r.popFront()) { + UnbarrieredKey k = r.front().key(); + MOZ_ASSERT(!gc::IsAboutToBeFinalizedUnbarriered(k)); + JSObject* delegate = gc::detail::GetDelegate(k); + if (delegate) { + MOZ_ASSERT(!gc::IsAboutToBeFinalizedUnbarriered(delegate), + "weakmap marking depends on a key tracing its delegate"); + } + MOZ_ASSERT(!gc::IsAboutToBeFinalized(r.front().value())); + } +} +#endif + +#ifdef JS_GC_ZEAL +template +bool WeakMap::checkMarking() const { + bool ok = true; + for (Range r = Base::all(); !r.empty(); r.popFront()) { + gc::Cell* key = gc::ToMarkable(r.front().key()); + gc::Cell* value = gc::ToMarkable(r.front().value()); + if (key && value) { + if (!gc::CheckWeakMapEntryMarking(this, key, value)) { + ok = false; + } + } + } + return ok; +} +#endif + +} /* namespace js */ + +#endif /* gc_WeakMap_inl_h */ diff --git a/js/src/gc/WeakMap.cpp b/js/src/gc/WeakMap.cpp new file mode 100644 index 0000000000..82684c7c9c --- /dev/null +++ b/js/src/gc/WeakMap.cpp @@ -0,0 +1,175 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/WeakMap-inl.h" + +#include + +#include "gc/PublicIterators.h" +#include "vm/JSObject.h" + +#include "gc/Marking-inl.h" + +using namespace js; +using namespace js::gc; + +WeakMapBase::WeakMapBase(JSObject* memOf, Zone* zone) + : memberOf(memOf), zone_(zone), mapColor(CellColor::White) { + MOZ_ASSERT_IF(memberOf, memberOf->compartment()->zone() == zone); +} + +WeakMapBase::~WeakMapBase() { + MOZ_ASSERT(CurrentThreadIsGCFinalizing() || + CurrentThreadCanAccessZone(zone_)); +} + +void WeakMapBase::unmarkZone(JS::Zone* zone) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcEphemeronEdges().clear()) { + oomUnsafe.crash("clearing ephemeron edges table"); + } + MOZ_ASSERT(zone->gcNurseryEphemeronEdges().count() == 0); + + for (WeakMapBase* m : zone->gcWeakMapList()) { + m->mapColor = CellColor::White; + } +} + +void Zone::traceWeakMaps(JSTracer* trc) { + MOZ_ASSERT(trc->weakMapAction() != JS::WeakMapTraceAction::Skip); + for (WeakMapBase* m : gcWeakMapList()) { + m->trace(trc); + TraceNullableEdge(trc, &m->memberOf, "memberOf"); + } +} + +#if defined(JS_GC_ZEAL) || defined(DEBUG) +bool WeakMapBase::checkMarkingForZone(JS::Zone* zone) { + // This is called at the end of marking. + MOZ_ASSERT(zone->isGCMarking()); + + bool ok = true; + for (WeakMapBase* m : zone->gcWeakMapList()) { + if (m->mapColor && !m->checkMarking()) { + ok = false; + } + } + + return ok; +} +#endif + +bool WeakMapBase::markZoneIteratively(JS::Zone* zone, GCMarker* marker) { + bool markedAny = false; + for (WeakMapBase* m : zone->gcWeakMapList()) { + if (m->mapColor && m->markEntries(marker)) { + markedAny = true; + } + } + return markedAny; +} + +bool WeakMapBase::findSweepGroupEdgesForZone(JS::Zone* zone) { + for (WeakMapBase* m : zone->gcWeakMapList()) { + if (!m->findSweepGroupEdges()) { + return false; + } + } + return true; +} + +void Zone::sweepWeakMaps(JSTracer* trc) { + for (WeakMapBase* m = gcWeakMapList().getFirst(); m;) { + WeakMapBase* next = m->getNext(); + if (m->mapColor) { + m->traceWeakEdges(trc); + } else { + m->clearAndCompact(); + m->removeFrom(gcWeakMapList()); + } + m = next; + } + +#ifdef DEBUG + for (WeakMapBase* m : gcWeakMapList()) { + MOZ_ASSERT(m->isInList() && m->mapColor); + } +#endif +} + +void WeakMapBase::traceAllMappings(WeakMapTracer* tracer) { + JSRuntime* rt = tracer->runtime; + for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) { + for (WeakMapBase* m : zone->gcWeakMapList()) { + // The WeakMapTracer callback is not allowed to GC. + JS::AutoSuppressGCAnalysis nogc; + m->traceMappings(tracer); + } + } +} + +bool WeakMapBase::saveZoneMarkedWeakMaps(JS::Zone* zone, + WeakMapColors& markedWeakMaps) { + for (WeakMapBase* m : zone->gcWeakMapList()) { + if (m->mapColor && !markedWeakMaps.put(m, m->mapColor)) { + return false; + } + } + return true; +} + +void WeakMapBase::restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps) { + for (WeakMapColors::Range r = markedWeakMaps.all(); !r.empty(); + r.popFront()) { + WeakMapBase* map = r.front().key(); + MOZ_ASSERT(map->zone()->isGCMarking()); + MOZ_ASSERT(map->mapColor == CellColor::White); + map->mapColor = r.front().value(); + } +} + +ObjectWeakMap::ObjectWeakMap(JSContext* cx) : map(cx, nullptr) {} + +JSObject* ObjectWeakMap::lookup(const JSObject* obj) { + if (ObjectValueWeakMap::Ptr p = map.lookup(const_cast(obj))) { + return &p->value().toObject(); + } + return nullptr; +} + +bool ObjectWeakMap::add(JSContext* cx, JSObject* obj, JSObject* target) { + MOZ_ASSERT(obj && target); + + Value targetVal(ObjectValue(*target)); + if (!map.putNew(obj, targetVal)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +void ObjectWeakMap::remove(JSObject* key) { + MOZ_ASSERT(key); + map.remove(key); +} + +void ObjectWeakMap::clear() { map.clear(); } + +void ObjectWeakMap::trace(JSTracer* trc) { map.trace(trc); } + +size_t ObjectWeakMap::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return map.shallowSizeOfExcludingThis(mallocSizeOf); +} + +#ifdef JSGC_HASH_TABLE_CHECKS +void ObjectWeakMap::checkAfterMovingGC() { + for (ObjectValueWeakMap::Range r = map.all(); !r.empty(); r.popFront()) { + CheckGCThingAfterMovingGC(r.front().key().get()); + CheckGCThingAfterMovingGC(&r.front().value().toObject()); + } +} +#endif // JSGC_HASH_TABLE_CHECKS diff --git a/js/src/gc/WeakMap.h b/js/src/gc/WeakMap.h new file mode 100644 index 0000000000..4693bd838c --- /dev/null +++ b/js/src/gc/WeakMap.h @@ -0,0 +1,354 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_WeakMap_h +#define gc_WeakMap_h + +#include "mozilla/LinkedList.h" + +#include "gc/Barrier.h" +#include "gc/Tracer.h" +#include "gc/ZoneAllocator.h" +#include "js/HashTable.h" +#include "js/HeapAPI.h" + +namespace JS { +class Zone; +} + +namespace js { + +class GCMarker; +class WeakMapBase; +struct WeakMapTracer; + +extern void DumpWeakMapLog(JSRuntime* rt); + +namespace gc { + +struct WeakMarkable; + +#if defined(JS_GC_ZEAL) || defined(DEBUG) +// Check whether a weak map entry is marked correctly. +bool CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key, Cell* value); +#endif + +} // namespace gc + +// A subclass template of js::HashMap whose keys and values may be +// garbage-collected. When a key is collected, the table entry disappears, +// dropping its reference to the value. +// +// More precisely: +// +// A WeakMap entry is live if and only if both the WeakMap and the entry's +// key are live. An entry holds a strong reference to its value. +// +// You must call this table's 'trace' method when its owning object is reached +// by the garbage collection tracer. Once a table is known to be live, the +// implementation takes care of the special weak marking (ie, marking through +// the implicit edges stored in the map) and of removing (sweeping) table +// entries when collection is complete. + +// WeakMaps are marked with an incremental linear-time algorithm that handles +// all orderings of map and key marking. The basic algorithm is: +// +// At first while marking, do nothing special when marking WeakMap keys (there +// is no straightforward way to know whether a particular object is being used +// as a key in some weakmap.) When a WeakMap is marked, scan through it to mark +// all entries with live keys, and collect all unmarked keys into a "weak keys" +// table. +// +// At some point, everything reachable has been marked. At this point, enter +// "weak marking mode". In this mode, whenever any object is marked, look it up +// in the weak keys table to see if it is the key for any WeakMap entry and if +// so, mark the value. When entering weak marking mode, scan the weak key table +// to find all keys that have been marked since we added them to the table, and +// mark those entries. +// +// In addition, we want weakmap marking to work incrementally. So WeakMap +// mutations are barriered to keep the weak keys table up to date: entries are +// removed if their key is removed from the table, etc. +// +// You can break down various ways that WeakMap values get marked based on the +// order that the map and key are marked. All of these assume the map and key +// get marked at some point: +// +// key marked, then map marked: +// - value was marked with map in `markEntries()` +// map marked, key already in map, key marked before weak marking mode: +// - key added to gcEphemeronEdges when map marked in `markEntries()` +// - value marked during `enterWeakMarkingMode` +// map marked, key already in map, key marked after weak marking mode: +// - when key is marked, gcEphemeronEdges[key] triggers marking of value in +// `markImplicitEdges()` +// map marked, key inserted into map, key marked: +// - value was live when inserted and must get marked at some point +// + +using WeakMapColors = HashMap, SystemAllocPolicy>; + +// Common base class for all WeakMap specializations, used for calling +// subclasses' GC-related methods. +class WeakMapBase : public mozilla::LinkedListElement { + friend class js::GCMarker; + + public: + using CellColor = js::gc::CellColor; + + WeakMapBase(JSObject* memOf, JS::Zone* zone); + virtual ~WeakMapBase(); + + JS::Zone* zone() const { return zone_; } + + // Garbage collector entry points. + + // Unmark all weak maps in a zone. + static void unmarkZone(JS::Zone* zone); + + // Check all weak maps in a zone that have been marked as live in this garbage + // collection, and mark the values of all entries that have become strong + // references to them. Return true if we marked any new values, indicating + // that we need to make another pass. In other words, mark my marked maps' + // marked members' mid-collection. + static bool markZoneIteratively(JS::Zone* zone, GCMarker* marker); + + // Add zone edges for weakmaps with key delegates in a different zone. + [[nodiscard]] static bool findSweepGroupEdgesForZone(JS::Zone* zone); + + // Sweep the marked weak maps in a zone, updating moved keys. + static void sweepZoneAfterMinorGC(JS::Zone* zone); + + // Trace all weak map bindings. Used by the cycle collector. + static void traceAllMappings(WeakMapTracer* tracer); + + // Save information about which weak maps are marked for a zone. + static bool saveZoneMarkedWeakMaps(JS::Zone* zone, + WeakMapColors& markedWeakMaps); + + // Restore information about which weak maps are marked for many zones. + static void restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps); + +#if defined(JS_GC_ZEAL) || defined(DEBUG) + static bool checkMarkingForZone(JS::Zone* zone); +#endif + + protected: + // Instance member functions called by the above. Instantiations of WeakMap + // override these with definitions appropriate for their Key and Value types. + virtual void trace(JSTracer* tracer) = 0; + virtual bool findSweepGroupEdges() = 0; + virtual void traceWeakEdges(JSTracer* trc) = 0; + virtual void traceMappings(WeakMapTracer* tracer) = 0; + virtual void clearAndCompact() = 0; + + // We have a key that, if it or its delegate is marked, may lead to a WeakMap + // value getting marked. Insert it or its delegate (if any) into the + // appropriate zone's gcEphemeronEdges or gcNurseryEphemeronEdges. + inline bool addImplicitEdges(gc::Cell* key, gc::Cell* delegate, + gc::TenuredCell* value); + + virtual bool markEntries(GCMarker* marker) = 0; + +#ifdef JS_GC_ZEAL + virtual bool checkMarking() const = 0; + virtual bool allowKeysInOtherZones() const { return false; } + friend bool gc::CheckWeakMapEntryMarking(const WeakMapBase*, gc::Cell*, + gc::Cell*); +#endif + + // Object that this weak map is part of, if any. + HeapPtr memberOf; + + // Zone containing this weak map. + JS::Zone* zone_; + + // Whether this object has been marked during garbage collection and which + // color it was marked. + gc::CellColor mapColor; + + friend class JS::Zone; +}; + +template +class WeakMap + : private HashMap, ZoneAllocPolicy>, + public WeakMapBase { + public: + using Base = HashMap, ZoneAllocPolicy>; + + using Lookup = typename Base::Lookup; + using Entry = typename Base::Entry; + using Range = typename Base::Range; + using Ptr = typename Base::Ptr; + using AddPtr = typename Base::AddPtr; + + struct Enum : public Base::Enum { + explicit Enum(WeakMap& map) : Base::Enum(static_cast(map)) {} + }; + + using Base::all; + using Base::clear; + using Base::count; + using Base::empty; + using Base::has; + using Base::shallowSizeOfExcludingThis; + + // Resolve ambiguity with LinkedListElement<>::remove. + using Base::remove; + + using UnbarrieredKey = typename RemoveBarrier::Type; + + explicit WeakMap(JSContext* cx, JSObject* memOf = nullptr); + explicit WeakMap(JS::Zone* zone, JSObject* memOf = nullptr); + + // Add a read barrier to prevent an incorrectly gray value from escaping the + // weak map. See the UnmarkGrayTracer::onChild comment in gc/Marking.cpp. + Ptr lookup(const Lookup& l) const { + Ptr p = Base::lookup(l); + if (p) { + exposeGCThingToActiveJS(p->value()); + } + return p; + } + + Ptr lookupUnbarriered(const Lookup& l) const { return Base::lookup(l); } + + AddPtr lookupForAdd(const Lookup& l) { + AddPtr p = Base::lookupForAdd(l); + if (p) { + exposeGCThingToActiveJS(p->value()); + } + return p; + } + + template + [[nodiscard]] bool add(AddPtr& p, KeyInput&& k, ValueInput&& v) { + MOZ_ASSERT(k); + return Base::add(p, std::forward(k), std::forward(v)); + } + + template + [[nodiscard]] bool relookupOrAdd(AddPtr& p, KeyInput&& k, ValueInput&& v) { + MOZ_ASSERT(k); + return Base::relookupOrAdd(p, std::forward(k), + std::forward(v)); + } + + template + [[nodiscard]] bool put(KeyInput&& k, ValueInput&& v) { + MOZ_ASSERT(k); + return Base::put(std::forward(k), std::forward(v)); + } + + template + [[nodiscard]] bool putNew(KeyInput&& k, ValueInput&& v) { + MOZ_ASSERT(k); + return Base::putNew(std::forward(k), std::forward(v)); + } + + template + void putNewInfallible(KeyInput&& k, ValueInput&& v) { + MOZ_ASSERT(k); + Base::putNewInfallible(std::forward(k), std::forward(k)); + } + +#ifdef DEBUG + template + bool hasEntry(KeyInput&& key, ValueInput&& value) { + Ptr p = Base::lookup(std::forward(key)); + return p && p->value() == value; + } +#endif + + bool markEntry(GCMarker* marker, Key& key, Value& value, + bool populateWeakKeysTable); + + void trace(JSTracer* trc) override; + + size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); + + protected: + inline void assertMapIsSameZoneWithValue(const Value& v); + + bool markEntries(GCMarker* marker) override; + + protected: + // Find sweep group edges for delegates, if the key type has delegates. (If + // not, the optimizer should make this a nop.) + bool findSweepGroupEdges() override; + + /** + * If a wrapper is used as a key in a weakmap, the garbage collector should + * keep that object around longer than it otherwise would. We want to avoid + * collecting the wrapper (and removing the weakmap entry) as long as the + * wrapped object is alive (because the object can be rewrapped and looked up + * again). As long as the wrapper is used as a weakmap key, it will not be + * collected (and remain in the weakmap) until the wrapped object is + * collected. + */ + private: + void exposeGCThingToActiveJS(const JS::Value& v) const { + JS::ExposeValueToActiveJS(v); + } + void exposeGCThingToActiveJS(JSObject* obj) const { + JS::ExposeObjectToActiveJS(obj); + } + + void traceWeakEdges(JSTracer* trc) override; + + void clearAndCompact() override { + Base::clear(); + Base::compact(); + } + + // memberOf can be nullptr, which means that the map is not part of a + // JSObject. + void traceMappings(WeakMapTracer* tracer) override; + + protected: +#if DEBUG + void assertEntriesNotAboutToBeFinalized(); +#endif + +#ifdef JS_GC_ZEAL + bool checkMarking() const override; +#endif +}; + +using ObjectValueWeakMap = WeakMap, HeapPtr>; + +// Generic weak map for mapping objects to other objects. +class ObjectWeakMap { + ObjectValueWeakMap map; + + public: + explicit ObjectWeakMap(JSContext* cx); + + JS::Zone* zone() const { return map.zone(); } + + JSObject* lookup(const JSObject* obj); + bool add(JSContext* cx, JSObject* obj, JSObject* target); + void remove(JSObject* key); + void clear(); + + void trace(JSTracer* trc); + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); + size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return mallocSizeOf(this) + sizeOfExcludingThis(mallocSizeOf); + } + + ObjectValueWeakMap& valueMap() { return map; } + +#ifdef JSGC_HASH_TABLE_CHECKS + void checkAfterMovingGC(); +#endif +}; + +} /* namespace js */ + +#endif /* gc_WeakMap_h */ diff --git a/js/src/gc/WeakMapPtr.cpp b/js/src/gc/WeakMapPtr.cpp new file mode 100644 index 0000000000..f58fc52372 --- /dev/null +++ b/js/src/gc/WeakMapPtr.cpp @@ -0,0 +1,114 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "js/WeakMapPtr.h" + +#include "gc/WeakMap-inl.h" + +// +// Machinery for the externally-linkable JS::WeakMapPtr, which wraps js::WeakMap +// for a few public data types. +// + +using namespace js; + +namespace WeakMapDetails { + +template +struct DataType {}; + +template <> +struct DataType { + using BarrieredType = HeapPtr; + using HasherType = StableCellHasher; + static JSObject* NullValue() { return nullptr; } +}; + +template <> +struct DataType { + using BarrieredType = HeapPtr; + static JS::Value NullValue() { return JS::UndefinedValue(); } +}; + +template +struct Utils { + using KeyType = typename DataType::BarrieredType; + using ValueType = typename DataType::BarrieredType; + typedef WeakMap Type; + using PtrType = Type*; + static PtrType cast(void* ptr) { return static_cast(ptr); } +}; + +} // namespace WeakMapDetails + +template +void JS::WeakMapPtr::destroy() { + MOZ_ASSERT(initialized()); + js_delete(WeakMapDetails::Utils::cast(ptr)); + ptr = nullptr; +} + +template +bool JS::WeakMapPtr::init(JSContext* cx) { + MOZ_ASSERT(!initialized()); + typename WeakMapDetails::Utils::PtrType map = + cx->new_::Type>(cx); + if (!map) { + return false; + } + ptr = map; + return true; +} + +template +void JS::WeakMapPtr::trace(JSTracer* trc) { + MOZ_ASSERT(initialized()); + return WeakMapDetails::Utils::cast(ptr)->trace(trc); +} + +template +V JS::WeakMapPtr::lookup(const K& key) { + MOZ_ASSERT(initialized()); + typename WeakMapDetails::Utils::Type::Ptr result = + WeakMapDetails::Utils::cast(ptr)->lookup(key); + if (!result) { + return WeakMapDetails::DataType::NullValue(); + } + return result->value(); +} + +template +bool JS::WeakMapPtr::put(JSContext* cx, const K& key, const V& value) { + MOZ_ASSERT(initialized()); + return WeakMapDetails::Utils::cast(ptr)->put(key, value); +} + +template +V JS::WeakMapPtr::removeValue(const K& key) { + typedef typename WeakMapDetails::Utils::Type Map; + using Ptr = typename Map::Ptr; + + MOZ_ASSERT(initialized()); + + Map* map = WeakMapDetails::Utils::cast(ptr); + if (Ptr result = map->lookup(key)) { + V value = result->value(); + map->remove(result); + return value; + } + return WeakMapDetails::DataType::NullValue(); +} + +// +// Supported specializations of JS::WeakMap: +// + +template class JS_PUBLIC_API JS::WeakMapPtr; + +#ifdef DEBUG +// Nobody's using this at the moment, but we want to make sure it compiles. +template class JS_PUBLIC_API JS::WeakMapPtr; +#endif diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp new file mode 100644 index 0000000000..982b12f17f --- /dev/null +++ b/js/src/gc/Zone.cpp @@ -0,0 +1,979 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Zone.h" +#include "js/shadow/Zone.h" // JS::shadow::Zone + +#include "mozilla/Sprintf.h" +#include "mozilla/TimeStamp.h" + +#include + +#include "gc/FinalizationObservers.h" +#include "gc/GCContext.h" +#include "gc/PublicIterators.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/Invalidation.h" +#include "jit/JitZone.h" +#include "vm/Runtime.h" +#include "vm/Time.h" + +#include "debugger/DebugAPI-inl.h" +#include "gc/GC-inl.h" +#include "gc/Marking-inl.h" +#include "gc/Nursery-inl.h" +#include "gc/WeakMap-inl.h" +#include "vm/JSScript-inl.h" +#include "vm/Realm-inl.h" + +using namespace js; +using namespace js::gc; + +Zone* const Zone::NotOnList = reinterpret_cast(1); + +ZoneAllocator::ZoneAllocator(JSRuntime* rt, Kind kind) + : JS::shadow::Zone(rt, rt->gc.marker().tracer(), kind), + jitHeapThreshold(jit::MaxCodeBytesPerProcess * 0.8) {} + +ZoneAllocator::~ZoneAllocator() { +#ifdef DEBUG + mallocTracker.checkEmptyOnDestroy(); + MOZ_ASSERT(gcHeapSize.bytes() == 0); + MOZ_ASSERT(mallocHeapSize.bytes() == 0); + MOZ_ASSERT(jitHeapSize.bytes() == 0); +#endif +} + +void ZoneAllocator::fixupAfterMovingGC() { +#ifdef DEBUG + mallocTracker.fixupAfterMovingGC(); +#endif +} + +void js::ZoneAllocator::updateSchedulingStateOnGCStart() { + gcHeapSize.updateOnGCStart(); + mallocHeapSize.updateOnGCStart(); + jitHeapSize.updateOnGCStart(); + perZoneGCTime = mozilla::TimeDuration(); +} + +void js::ZoneAllocator::updateGCStartThresholds(GCRuntime& gc) { + bool isAtomsZone = JS::Zone::from(this)->isAtomsZone(); + gcHeapThreshold.updateStartThreshold( + gcHeapSize.retainedBytes(), smoothedAllocationRate.ref(), + smoothedCollectionRate.ref(), gc.tunables, gc.schedulingState, + isAtomsZone); + + mallocHeapThreshold.updateStartThreshold(mallocHeapSize.retainedBytes(), + gc.tunables, gc.schedulingState); +} + +void js::ZoneAllocator::setGCSliceThresholds(GCRuntime& gc, + bool waitingOnBGTask) { + gcHeapThreshold.setSliceThreshold(this, gcHeapSize, gc.tunables, + waitingOnBGTask); + mallocHeapThreshold.setSliceThreshold(this, mallocHeapSize, gc.tunables, + waitingOnBGTask); + jitHeapThreshold.setSliceThreshold(this, jitHeapSize, gc.tunables, + waitingOnBGTask); +} + +void js::ZoneAllocator::clearGCSliceThresholds() { + gcHeapThreshold.clearSliceThreshold(); + mallocHeapThreshold.clearSliceThreshold(); + jitHeapThreshold.clearSliceThreshold(); +} + +bool ZoneAllocator::addSharedMemory(void* mem, size_t nbytes, MemoryUse use) { + // nbytes can be zero here for SharedArrayBuffers. + + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_)); + + auto ptr = sharedMemoryUseCounts.lookupForAdd(mem); + MOZ_ASSERT_IF(ptr, ptr->value().use == use); + + if (!ptr && !sharedMemoryUseCounts.add(ptr, mem, gc::SharedMemoryUse(use))) { + return false; + } + + ptr->value().count++; + + // Allocations can grow, so add any increase over the previous size and record + // the new size. + if (nbytes > ptr->value().nbytes) { + mallocHeapSize.addBytes(nbytes - ptr->value().nbytes); + ptr->value().nbytes = nbytes; + } + + maybeTriggerGCOnMalloc(); + + return true; +} + +void ZoneAllocator::removeSharedMemory(void* mem, size_t nbytes, + MemoryUse use) { + // nbytes can be zero here for SharedArrayBuffers. + + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_)); + MOZ_ASSERT(CurrentThreadIsGCFinalizing()); + + auto ptr = sharedMemoryUseCounts.lookup(mem); + + MOZ_ASSERT(ptr); + MOZ_ASSERT(ptr->value().count != 0); + MOZ_ASSERT(ptr->value().use == use); + MOZ_ASSERT(ptr->value().nbytes >= nbytes); + + ptr->value().count--; + if (ptr->value().count == 0) { + mallocHeapSize.removeBytes(ptr->value().nbytes, true); + sharedMemoryUseCounts.remove(ptr); + } +} + +template +void js::TrackedAllocPolicy::decMemory(size_t nbytes) { + bool updateRetainedSize = false; + if constexpr (kind == TrackingKind::Cell) { + // Only subtract freed cell memory from retained size for cell associations + // during sweeping. + JS::GCContext* gcx = TlsGCContext.get(); + updateRetainedSize = gcx->isFinalizing(); + } + + zone_->decNonGCMemory(this, nbytes, MemoryUse::TrackedAllocPolicy, + updateRetainedSize); +} + +namespace js { +template class TrackedAllocPolicy; +template class TrackedAllocPolicy; +} // namespace js + +JS::Zone::Zone(JSRuntime* rt, Kind kind) + : ZoneAllocator(rt, kind), + arenas(this), + data(nullptr), + tenuredBigInts(0), + markedStrings(0), + finalizedStrings(0), + suppressAllocationMetadataBuilder(false), + allocNurseryObjects_(true), + allocNurseryStrings_(true), + allocNurseryBigInts_(true), + pretenuring(this), + compartments_(), + crossZoneStringWrappers_(this), + gcEphemeronEdges_(SystemAllocPolicy(), rt->randomHashCodeScrambler()), + gcNurseryEphemeronEdges_(SystemAllocPolicy(), + rt->randomHashCodeScrambler()), + shapeZone_(this), + gcScheduled_(false), + gcScheduledSaved_(false), + gcPreserveCode_(false), + keepPropMapTables_(false), + wasCollected_(false), + listNext_(NotOnList), + keptObjects(this) { + /* Ensure that there are no vtables to mess us up here. */ + MOZ_ASSERT(reinterpret_cast(this) == + static_cast(this)); + MOZ_ASSERT_IF(isAtomsZone(), rt->gc.zones().empty()); + + updateGCStartThresholds(rt->gc); + rt->gc.nursery().setAllocFlagsForZone(this); +} + +Zone::~Zone() { + MOZ_ASSERT_IF(regExps_.ref(), regExps().empty()); + + DebugAPI::deleteDebugScriptMap(debugScriptMap); + js_delete(finalizationObservers_.ref().release()); + + MOZ_ASSERT(gcWeakMapList().isEmpty()); + + JSRuntime* rt = runtimeFromAnyThread(); + if (this == rt->gc.systemZone) { + MOZ_ASSERT(isSystemZone()); + rt->gc.systemZone = nullptr; + } + + js_delete(jitZone_.ref()); +} + +bool Zone::init() { + regExps_.ref() = make_unique(this); + return regExps_.ref() && gcEphemeronEdges().init() && + gcNurseryEphemeronEdges().init(); +} + +void Zone::setNeedsIncrementalBarrier(bool needs) { + needsIncrementalBarrier_ = needs; +} + +void Zone::changeGCState(GCState prev, GCState next) { + MOZ_ASSERT(RuntimeHeapIsBusy()); + MOZ_ASSERT(gcState() == prev); + + // This can be called when barriers have been temporarily disabled by + // AutoDisableBarriers. In that case, don't update needsIncrementalBarrier_ + // and barriers will be re-enabled by ~AutoDisableBarriers() if necessary. + bool barriersDisabled = isGCMarking() && !needsIncrementalBarrier(); + + gcState_ = next; + + // Update the barriers state when we transition between marking and + // non-marking states, unless barriers have been disabled. + if (!barriersDisabled) { + needsIncrementalBarrier_ = isGCMarking(); + } +} + +template +static void EraseIf(js::gc::EphemeronEdgeVector& entries, Pred pred) { + auto* begin = entries.begin(); + auto* const end = entries.end(); + + auto* newEnd = begin; + for (auto* p = begin; p != end; p++) { + if (!pred(*p)) { + *newEnd++ = *p; + } + } + + size_t removed = end - newEnd; + entries.shrinkBy(removed); +} + +static void SweepEphemeronEdgesWhileMinorSweeping( + js::gc::EphemeronEdgeVector& entries) { + EraseIf(entries, [](js::gc::EphemeronEdge& edge) -> bool { + return IsAboutToBeFinalizedDuringMinorSweep(&edge.target); + }); +} + +void Zone::sweepAfterMinorGC(JSTracer* trc) { + sweepEphemeronTablesAfterMinorGC(); + crossZoneStringWrappers().sweepAfterMinorGC(trc); + + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + comp->sweepAfterMinorGC(trc); + } +} + +void Zone::sweepEphemeronTablesAfterMinorGC() { + for (auto r = gcNurseryEphemeronEdges().mutableAll(); !r.empty(); + r.popFront()) { + // Sweep gcNurseryEphemeronEdges to move live (forwarded) keys to + // gcEphemeronEdges, scanning through all the entries for such keys to + // update them. + // + // Forwarded and dead keys may also appear in their delegates' entries, + // so sweep those too (see below.) + + // The tricky case is when the key has a delegate that was already + // tenured. Then it will be in its compartment's gcEphemeronEdges, but we + // still need to update the key (which will be in the entries + // associated with it.) + gc::Cell* key = r.front().key; + MOZ_ASSERT(!key->isTenured()); + if (!Nursery::getForwardedPointer(&key)) { + // Dead nursery cell => discard. + continue; + } + + // Key been moved. The value is an array of pairs; update all + // cells in that array. + EphemeronEdgeVector& entries = r.front().value; + SweepEphemeronEdgesWhileMinorSweeping(entries); + + // Live (moved) nursery cell. Append entries to gcEphemeronEdges. + EphemeronEdgeTable& tenuredEdges = gcEphemeronEdges(); + auto* entry = tenuredEdges.get(key); + if (!entry) { + if (!tenuredEdges.put(key, gc::EphemeronEdgeVector())) { + AutoEnterOOMUnsafeRegion oomUnsafe; + oomUnsafe.crash("Failed to tenure weak keys entry"); + } + entry = tenuredEdges.get(key); + } + + if (!entry->value.appendAll(entries)) { + AutoEnterOOMUnsafeRegion oomUnsafe; + oomUnsafe.crash("Failed to tenure weak keys entry"); + } + + // If the key has a delegate, then it will map to a WeakKeyEntryVector + // containing the key that needs to be updated. + + JSObject* delegate = gc::detail::GetDelegate(key->as()); + if (!delegate) { + continue; + } + MOZ_ASSERT(delegate->isTenured()); + + // If delegate was formerly nursery-allocated, we will sweep its entries + // when we visit its gcNurseryEphemeronEdges (if we haven't already). Note + // that we don't know the nursery address of the delegate, since the + // location it was stored in has already been updated. + // + // Otherwise, it will be in gcEphemeronEdges and we sweep it here. + auto* p = delegate->zone()->gcEphemeronEdges().get(delegate); + if (p) { + SweepEphemeronEdgesWhileMinorSweeping(p->value); + } + } + + if (!gcNurseryEphemeronEdges().clear()) { + AutoEnterOOMUnsafeRegion oomUnsafe; + oomUnsafe.crash("OOM while clearing gcNurseryEphemeronEdges."); + } +} + +void Zone::traceWeakCCWEdges(JSTracer* trc) { + crossZoneStringWrappers().traceWeak(trc); + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + comp->traceCrossCompartmentObjectWrapperEdges(trc); + } +} + +/* static */ +void Zone::fixupAllCrossCompartmentWrappersAfterMovingGC(JSTracer* trc) { + MOZ_ASSERT(trc->runtime()->gc.isHeapCompacting()); + + for (ZonesIter zone(trc->runtime(), WithAtoms); !zone.done(); zone.next()) { + // Trace the wrapper map to update keys (wrapped values) in other + // compartments that may have been moved. + zone->crossZoneStringWrappers().traceWeak(trc); + + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + comp->fixupCrossCompartmentObjectWrappersAfterMovingGC(trc); + } + } +} + +void Zone::dropStringWrappersOnGC() { + MOZ_ASSERT(JS::RuntimeHeapIsCollecting()); + crossZoneStringWrappers().clear(); +} + +#ifdef JSGC_HASH_TABLE_CHECKS + +void Zone::checkAllCrossCompartmentWrappersAfterMovingGC() { + checkStringWrappersAfterMovingGC(); + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + comp->checkObjectWrappersAfterMovingGC(); + } +} + +void Zone::checkStringWrappersAfterMovingGC() { + for (StringWrapperMap::Enum e(crossZoneStringWrappers()); !e.empty(); + e.popFront()) { + // Assert that the postbarriers have worked and that nothing is left in the + // wrapper map that points into the nursery, and that the hash table entries + // are discoverable. + auto key = e.front().key(); + CheckGCThingAfterMovingGC(key.get()); + + auto ptr = crossZoneStringWrappers().lookup(key); + MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &e.front()); + } +} +#endif + +void Zone::discardJitCode(JS::GCContext* gcx, const DiscardOptions& options) { + if (!isPreservingCode()) { + forceDiscardJitCode(gcx, options); + } +} + +void Zone::forceDiscardJitCode(JS::GCContext* gcx, + const DiscardOptions& options) { + if (!jitZone()) { + return; + } + + if (options.discardJitScripts && options.discardBaselineCode) { + lastDiscardedCodeTime_ = mozilla::TimeStamp::Now(); + } + + if (options.discardBaselineCode || options.discardJitScripts) { +#ifdef DEBUG + // Assert no JitScripts are marked as active. + for (auto iter = cellIter(); !iter.done(); iter.next()) { + BaseScript* base = iter.unbarrieredGet(); + if (jit::JitScript* jitScript = base->maybeJitScript()) { + MOZ_ASSERT(!jitScript->active()); + } + } +#endif + + // Mark JitScripts on the stack as active. + jit::MarkActiveJitScripts(this); + } + + // Invalidate all Ion code in this zone. + jit::InvalidateAll(gcx, this); + + for (auto base = cellIterUnsafe(); !base.done(); base.next()) { + jit::JitScript* jitScript = base->maybeJitScript(); + if (!jitScript) { + continue; + } + + JSScript* script = base->asJSScript(); + jit::FinishInvalidation(gcx, script); + + // Discard baseline script if it's not marked as active. + if (options.discardBaselineCode) { + if (jitScript->hasBaselineScript() && !jitScript->active()) { + jit::FinishDiscardBaselineScript(gcx, script); + } + } + +#ifdef JS_CACHEIR_SPEW + maybeUpdateWarmUpCount(script); +#endif + + // Warm-up counter for scripts are reset on GC. After discarding code we + // need to let it warm back up to get information such as which + // opcodes are setting array holes or accessing getter properties. + script->resetWarmUpCounterForGC(); + + // Try to release the script's JitScript. This should happen after + // releasing JIT code because we can't do this when the script still has + // JIT code. + if (options.discardJitScripts) { + script->maybeReleaseJitScript(gcx); + jitScript = script->maybeJitScript(); + if (!jitScript) { + // Try to discard the ScriptCounts too. + if (!script->realm()->collectCoverageForDebug() && + !gcx->runtime()->profilingScripts) { + script->destroyScriptCounts(); + } + continue; + } + } + + // If we did not release the JitScript, we need to purge optimized IC + // stubs because the optimizedStubSpace will be purged below. + if (options.discardBaselineCode) { + jitScript->purgeOptimizedStubs(script); + } + + if (options.resetNurseryAllocSites || options.resetPretenuredAllocSites) { + jitScript->resetAllocSites(options.resetNurseryAllocSites, + options.resetPretenuredAllocSites); + } + + // Finally, reset the active flag. + jitScript->resetActive(); + } + + // Also clear references to jit code from RegExpShared cells at this point. + // This avoid holding onto ExecutablePools. + for (auto regExp = cellIterUnsafe(); !regExp.done(); + regExp.next()) { + regExp->discardJitCode(); + } + + /* + * When scripts contains pointers to nursery things, the store buffer + * can contain entries that point into the optimized stub space. Since + * this method can be called outside the context of a GC, this situation + * could result in us trying to mark invalid store buffer entries. + * + * Defer freeing any allocated blocks until after the next minor GC. + */ + if (options.discardBaselineCode) { + jitZone()->optimizedStubSpace()->freeAllAfterMinorGC(this); + jitZone()->purgeIonCacheIRStubInfo(); + } + + // Generate a profile marker + if (gcx->runtime()->geckoProfiler().enabled()) { + char discardingJitScript = options.discardJitScripts ? 'Y' : 'N'; + char discardingBaseline = options.discardBaselineCode ? 'Y' : 'N'; + char discardingIon = 'Y'; + + char discardingRegExp = 'Y'; + char discardingNurserySites = options.resetNurseryAllocSites ? 'Y' : 'N'; + char discardingPretenuredSites = + options.resetPretenuredAllocSites ? 'Y' : 'N'; + + char buf[100]; + SprintfLiteral(buf, + "JitScript:%c Baseline:%c Ion:%c " + "RegExp:%c NurserySites:%c PretenuredSites:%c", + discardingJitScript, discardingBaseline, discardingIon, + discardingRegExp, discardingNurserySites, + discardingPretenuredSites); + gcx->runtime()->geckoProfiler().markEvent("DiscardJit", buf); + } +} + +void JS::Zone::resetAllocSitesAndInvalidate(bool resetNurserySites, + bool resetPretenuredSites) { + MOZ_ASSERT(resetNurserySites || resetPretenuredSites); + + if (!jitZone()) { + return; + } + + JSContext* cx = runtime_->mainContextFromOwnThread(); + for (auto base = cellIterUnsafe(); !base.done(); base.next()) { + jit::JitScript* jitScript = base->maybeJitScript(); + if (!jitScript) { + continue; + } + + if (!jitScript->resetAllocSites(resetNurserySites, resetPretenuredSites)) { + continue; + } + + JSScript* script = base->asJSScript(); + CancelOffThreadIonCompile(script); + + if (!script->hasIonScript()) { + continue; + } + + jit::Invalidate(cx, script, + /* resetUses = */ true, + /* cancelOffThread = */ true); + } +} + +void JS::Zone::beforeClearDelegateInternal(JSObject* wrapper, + JSObject* delegate) { + MOZ_ASSERT(js::gc::detail::GetDelegate(wrapper) == delegate); + MOZ_ASSERT(needsIncrementalBarrier()); + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(this)); + runtimeFromMainThread()->gc.marker().severWeakDelegate(wrapper, delegate); +} + +void JS::Zone::afterAddDelegateInternal(JSObject* wrapper) { + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(this)); + JSObject* delegate = js::gc::detail::GetDelegate(wrapper); + if (delegate) { + runtimeFromMainThread()->gc.marker().restoreWeakDelegate(wrapper, delegate); + } +} + +#ifdef JSGC_HASH_TABLE_CHECKS +void JS::Zone::checkUniqueIdTableAfterMovingGC() { + for (auto r = uniqueIds().all(); !r.empty(); r.popFront()) { + js::gc::CheckGCThingAfterMovingGC(r.front().key()); + } +} +#endif + +js::jit::JitZone* Zone::createJitZone(JSContext* cx) { + MOZ_ASSERT(!jitZone_); + MOZ_ASSERT(cx->runtime()->hasJitRuntime()); + + UniquePtr jitZone(cx->new_()); + if (!jitZone) { + return nullptr; + } + + jitZone_ = jitZone.release(); + return jitZone_; +} + +bool Zone::hasMarkedRealms() { + for (RealmsInZoneIter realm(this); !realm.done(); realm.next()) { + if (realm->marked()) { + return true; + } + } + return false; +} + +void Zone::notifyObservingDebuggers() { + AutoAssertNoGC nogc; + MOZ_ASSERT(JS::RuntimeHeapIsCollecting(), + "This method should be called during GC."); + + JSRuntime* rt = runtimeFromMainThread(); + + for (RealmsInZoneIter realms(this); !realms.done(); realms.next()) { + GlobalObject* global = realms->unsafeUnbarrieredMaybeGlobal(); + if (!global) { + continue; + } + + DebugAPI::notifyParticipatesInGC(global, rt->gc.majorGCCount()); + } +} + +bool Zone::isOnList() const { return listNext_ != NotOnList; } + +Zone* Zone::nextZone() const { + MOZ_ASSERT(isOnList()); + return listNext_; +} + +void Zone::fixupAfterMovingGC() { + ZoneAllocator::fixupAfterMovingGC(); + shapeZone().fixupPropMapShapeTableAfterMovingGC(); +} + +void Zone::purgeAtomCache() { + atomCache().clearAndCompact(); + + // Also purge the dtoa caches so that subsequent lookups populate atom + // cache too. + for (RealmsInZoneIter r(this); !r.done(); r.next()) { + r->dtoaCache.purge(); + } +} + +void Zone::addSizeOfIncludingThis( + mozilla::MallocSizeOf mallocSizeOf, JS::CodeSizes* code, size_t* regexpZone, + size_t* jitZone, size_t* baselineStubsOptimized, size_t* uniqueIdMap, + size_t* initialPropMapTable, size_t* shapeTables, size_t* atomsMarkBitmaps, + size_t* compartmentObjects, size_t* crossCompartmentWrappersTables, + size_t* compartmentsPrivateData, size_t* scriptCountsMapArg) { + *regexpZone += regExps().sizeOfIncludingThis(mallocSizeOf); + if (jitZone_) { + jitZone_->addSizeOfIncludingThis(mallocSizeOf, code, jitZone, + baselineStubsOptimized); + } + *uniqueIdMap += uniqueIds().shallowSizeOfExcludingThis(mallocSizeOf); + shapeZone().addSizeOfExcludingThis(mallocSizeOf, initialPropMapTable, + shapeTables); + *atomsMarkBitmaps += markedAtoms().sizeOfExcludingThis(mallocSizeOf); + *crossCompartmentWrappersTables += + crossZoneStringWrappers().sizeOfExcludingThis(mallocSizeOf); + + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + comp->addSizeOfIncludingThis(mallocSizeOf, compartmentObjects, + crossCompartmentWrappersTables, + compartmentsPrivateData); + } + + if (scriptCountsMap) { + *scriptCountsMapArg += + scriptCountsMap->shallowSizeOfIncludingThis(mallocSizeOf); + for (auto r = scriptCountsMap->all(); !r.empty(); r.popFront()) { + *scriptCountsMapArg += + r.front().value()->sizeOfIncludingThis(mallocSizeOf); + } + } +} + +void* ZoneAllocator::onOutOfMemory(js::AllocFunction allocFunc, + arena_id_t arena, size_t nbytes, + void* reallocPtr) { + if (!js::CurrentThreadCanAccessRuntime(runtime_)) { + return nullptr; + } + // The analysis sees that JSRuntime::onOutOfMemory could report an error, + // which with a JSErrorInterceptor could GC. But we're passing a null cx (to + // a default parameter) so the error will not be reported. + JS::AutoSuppressGCAnalysis suppress; + return runtimeFromMainThread()->onOutOfMemory(allocFunc, arena, nbytes, + reallocPtr); +} + +void ZoneAllocator::reportAllocationOverflow() const { + js::ReportAllocationOverflow(static_cast(nullptr)); +} + +ZoneList::ZoneList() : head(nullptr), tail(nullptr) {} + +ZoneList::ZoneList(Zone* zone) : head(zone), tail(zone) { + MOZ_RELEASE_ASSERT(!zone->isOnList()); + zone->listNext_ = nullptr; +} + +ZoneList::~ZoneList() { MOZ_ASSERT(isEmpty()); } + +void ZoneList::check() const { +#ifdef DEBUG + MOZ_ASSERT((head == nullptr) == (tail == nullptr)); + if (!head) { + return; + } + + Zone* zone = head; + for (;;) { + MOZ_ASSERT(zone && zone->isOnList()); + if (zone == tail) break; + zone = zone->listNext_; + } + MOZ_ASSERT(!zone->listNext_); +#endif +} + +bool ZoneList::isEmpty() const { return head == nullptr; } + +Zone* ZoneList::front() const { + MOZ_ASSERT(!isEmpty()); + MOZ_ASSERT(head->isOnList()); + return head; +} + +void ZoneList::prepend(Zone* zone) { prependList(ZoneList(zone)); } + +void ZoneList::append(Zone* zone) { appendList(ZoneList(zone)); } + +void ZoneList::prependList(ZoneList&& other) { + check(); + other.check(); + + if (other.isEmpty()) { + return; + } + + MOZ_ASSERT(tail != other.tail); + + if (!isEmpty()) { + other.tail->listNext_ = head; + } else { + tail = other.tail; + } + head = other.head; + + other.head = nullptr; + other.tail = nullptr; +} + +void ZoneList::appendList(ZoneList&& other) { + check(); + other.check(); + + if (other.isEmpty()) { + return; + } + + MOZ_ASSERT(tail != other.tail); + + if (!isEmpty()) { + tail->listNext_ = other.head; + } else { + head = other.head; + } + tail = other.tail; + + other.head = nullptr; + other.tail = nullptr; +} + +Zone* ZoneList::removeFront() { + MOZ_ASSERT(!isEmpty()); + check(); + + Zone* front = head; + head = head->listNext_; + if (!head) { + tail = nullptr; + } + + front->listNext_ = Zone::NotOnList; + + return front; +} + +void ZoneList::clear() { + while (!isEmpty()) { + removeFront(); + } +} + +JS_PUBLIC_API void JS::shadow::RegisterWeakCache( + JS::Zone* zone, detail::WeakCacheBase* cachep) { + zone->registerWeakCache(cachep); +} + +void Zone::traceRootsInMajorGC(JSTracer* trc) { + if (trc->isMarkingTracer() && !isGCMarking()) { + return; + } + + // Trace zone script-table roots. See comment below for justification re: + // calling this only during major (non-nursery) collections. + traceScriptTableRoots(trc); + + if (FinalizationObservers* observers = finalizationObservers()) { + observers->traceRoots(trc); + } +} + +void Zone::traceScriptTableRoots(JSTracer* trc) { + static_assert(std::is_convertible_v, + "BaseScript must not be nursery-allocated for script-table " + "tracing to work"); + + // Performance optimization: the script-table keys are JSScripts, which + // cannot be in the nursery, so we can skip this tracing if we are only in a + // minor collection. We static-assert this fact above. + MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting()); + + // N.B.: the script-table keys are weak *except* in an exceptional case: when + // then --dump-bytecode command line option or the PCCount JSFriend API is + // used, then the scripts for all counts must remain alive. We only trace + // when the `trc->runtime()->profilingScripts` flag is set. This flag is + // cleared in JSRuntime::destroyRuntime() during shutdown to ensure that + // scripts are collected before the runtime goes away completely. + if (scriptCountsMap && trc->runtime()->profilingScripts) { + for (ScriptCountsMap::Range r = scriptCountsMap->all(); !r.empty(); + r.popFront()) { + BaseScript* script = r.front().key(); + MOZ_ASSERT(script->hasScriptCounts()); + TraceRoot(trc, &script, "profilingScripts"); + } + } + + // Trace the debugger's DebugScript weak map. + if (debugScriptMap) { + DebugAPI::traceDebugScriptMap(trc, debugScriptMap); + } +} + +void Zone::fixupScriptMapsAfterMovingGC(JSTracer* trc) { + // Map entries are removed by BaseScript::finalize, but we need to update the + // script pointers here in case they are moved by the GC. + + if (scriptCountsMap) { + scriptCountsMap->traceWeak(trc); + } + + if (scriptLCovMap) { + scriptLCovMap->traceWeak(trc); + } + +#ifdef MOZ_VTUNE + if (scriptVTuneIdMap) { + scriptVTuneIdMap->traceWeak(trc); + } +#endif + +#ifdef JS_CACHEIR_SPEW + if (scriptFinalWarmUpCountMap) { + scriptFinalWarmUpCountMap->traceWeak(trc); + } +#endif +} + +#ifdef JSGC_HASH_TABLE_CHECKS +void Zone::checkScriptMapsAfterMovingGC() { + if (scriptCountsMap) { + for (auto r = scriptCountsMap->all(); !r.empty(); r.popFront()) { + BaseScript* script = r.front().key(); + MOZ_ASSERT(script->zone() == this); + CheckGCThingAfterMovingGC(script); + auto ptr = scriptCountsMap->lookup(script); + MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front()); + } + } + + if (scriptLCovMap) { + for (auto r = scriptLCovMap->all(); !r.empty(); r.popFront()) { + BaseScript* script = r.front().key(); + MOZ_ASSERT(script->zone() == this); + CheckGCThingAfterMovingGC(script); + auto ptr = scriptLCovMap->lookup(script); + MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front()); + } + } + +# ifdef MOZ_VTUNE + if (scriptVTuneIdMap) { + for (auto r = scriptVTuneIdMap->all(); !r.empty(); r.popFront()) { + BaseScript* script = r.front().key(); + MOZ_ASSERT(script->zone() == this); + CheckGCThingAfterMovingGC(script); + auto ptr = scriptVTuneIdMap->lookup(script); + MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front()); + } + } +# endif // MOZ_VTUNE + +# ifdef JS_CACHEIR_SPEW + if (scriptFinalWarmUpCountMap) { + for (auto r = scriptFinalWarmUpCountMap->all(); !r.empty(); r.popFront()) { + BaseScript* script = r.front().key(); + MOZ_ASSERT(script->zone() == this); + CheckGCThingAfterMovingGC(script); + auto ptr = scriptFinalWarmUpCountMap->lookup(script); + MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front()); + } + } +# endif // JS_CACHEIR_SPEW +} +#endif + +void Zone::clearScriptCounts(Realm* realm) { + if (!scriptCountsMap) { + return; + } + + // Clear all hasScriptCounts_ flags of BaseScript, in order to release all + // ScriptCounts entries of the given realm. + for (auto i = scriptCountsMap->modIter(); !i.done(); i.next()) { + BaseScript* script = i.get().key(); + if (script->realm() != realm) { + continue; + } + // We can't destroy the ScriptCounts yet if the script has Baseline code, + // because Baseline code bakes in pointers to the counters. The ScriptCounts + // will be destroyed in Zone::discardJitCode when discarding the JitScript. + if (script->hasBaselineScript()) { + continue; + } + script->clearHasScriptCounts(); + i.remove(); + } +} + +void Zone::clearScriptLCov(Realm* realm) { + if (!scriptLCovMap) { + return; + } + + for (auto i = scriptLCovMap->modIter(); !i.done(); i.next()) { + BaseScript* script = i.get().key(); + if (script->realm() == realm) { + i.remove(); + } + } +} + +void Zone::clearRootsForShutdownGC() { + // Finalization callbacks are not called if we're shutting down. + if (finalizationObservers()) { + finalizationObservers()->clearRecords(); + } + + clearKeptObjects(); +} + +void Zone::finishRoots() { + for (RealmsInZoneIter r(this); !r.done(); r.next()) { + r->finishRoots(); + } +} + +void Zone::traceKeptObjects(JSTracer* trc) { keptObjects.ref().trace(trc); } + +bool Zone::keepDuringJob(HandleObject target) { + return keptObjects.ref().put(target); +} + +void Zone::clearKeptObjects() { keptObjects.ref().clear(); } + +bool Zone::ensureFinalizationObservers() { + if (finalizationObservers_.ref()) { + return true; + } + + finalizationObservers_ = js::MakeUnique(this); + return bool(finalizationObservers_.ref()); +} diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h new file mode 100644 index 0000000000..ba3de7ec3e --- /dev/null +++ b/js/src/gc/Zone.h @@ -0,0 +1,653 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Zone_h +#define gc_Zone_h + +#include "mozilla/Array.h" +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#include "mozilla/LinkedList.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/PodOperations.h" +#include "mozilla/TimeStamp.h" + +#include "jstypes.h" + +#include "ds/Bitmap.h" +#include "gc/ArenaList.h" +#include "gc/Barrier.h" +#include "gc/FindSCCs.h" +#include "gc/GCMarker.h" +#include "gc/NurseryAwareHashMap.h" +#include "gc/Pretenuring.h" +#include "gc/Statistics.h" +#include "gc/ZoneAllocator.h" +#include "js/GCHashTable.h" +#include "js/Vector.h" +#include "vm/AtomsTable.h" +#include "vm/JSObject.h" +#include "vm/JSScript.h" +#include "vm/ShapeZone.h" + +namespace js { + +class DebugScriptMap; +class RegExpZone; +class WeakRefObject; + +namespace jit { +class JitZone; +} // namespace jit + +namespace gc { + +class FinalizationObservers; +class ZoneList; + +using ZoneComponentFinder = ComponentFinder; + +struct UniqueIdGCPolicy { + static bool traceWeak(JSTracer* trc, Cell** keyp, uint64_t* valuep); +}; + +// Maps a Cell* to a unique, 64bit id. +using UniqueIdMap = GCHashMap, + SystemAllocPolicy, UniqueIdGCPolicy>; + +template +class ZoneAllCellIter; + +template +class ZoneCellIter; + +} // namespace gc + +// If two different nursery strings are wrapped into the same zone, and have +// the same contents, then deduplication may make them duplicates. +// `DuplicatesPossible` will allow this and map both wrappers to the same (now +// tenured) source string. +using StringWrapperMap = + NurseryAwareHashMap; + +class MOZ_NON_TEMPORARY_CLASS ExternalStringCache { + static const size_t NumEntries = 4; + mozilla::Array entries_; + + ExternalStringCache(const ExternalStringCache&) = delete; + void operator=(const ExternalStringCache&) = delete; + + public: + ExternalStringCache() { purge(); } + void purge() { mozilla::PodArrayZero(entries_); } + + MOZ_ALWAYS_INLINE JSString* lookup(const char16_t* chars, size_t len) const; + MOZ_ALWAYS_INLINE void put(JSString* s); +}; + +class MOZ_NON_TEMPORARY_CLASS FunctionToStringCache { + struct Entry { + BaseScript* script; + JSString* string; + + void set(BaseScript* scriptArg, JSString* stringArg) { + script = scriptArg; + string = stringArg; + } + }; + static const size_t NumEntries = 2; + mozilla::Array entries_; + + FunctionToStringCache(const FunctionToStringCache&) = delete; + void operator=(const FunctionToStringCache&) = delete; + + public: + FunctionToStringCache() { purge(); } + void purge() { mozilla::PodArrayZero(entries_); } + + MOZ_ALWAYS_INLINE JSString* lookup(BaseScript* script) const; + MOZ_ALWAYS_INLINE void put(BaseScript* script, JSString* string); +}; + +} // namespace js + +namespace JS { + +// [SMDOC] GC Zones +// +// A zone is a collection of compartments. Every compartment belongs to exactly +// one zone. In Firefox, there is roughly one zone per tab along with a system +// zone for everything else. Zones mainly serve as boundaries for garbage +// collection. Unlike compartments, they have no special security properties. +// +// Every GC thing belongs to exactly one zone. GC things from the same zone but +// different compartments can share an arena (4k page). GC things from different +// zones cannot be stored in the same arena. The garbage collector is capable of +// collecting one zone at a time; it cannot collect at the granularity of +// compartments. +// +// GC things are tied to zones and compartments as follows: +// +// - JSObjects belong to a compartment and cannot be shared between +// compartments. If an object needs to point to a JSObject in a different +// compartment, regardless of zone, it must go through a cross-compartment +// wrapper. Each compartment keeps track of its outgoing wrappers in a table. +// JSObjects find their compartment via their ObjectGroup. +// +// - JSStrings do not belong to any particular compartment, but they do belong +// to a zone. Thus, two different compartments in the same zone can point to a +// JSString. When a string needs to be wrapped, we copy it if it's in a +// different zone and do nothing if it's in the same zone. Thus, transferring +// strings within a zone is very efficient. +// +// - Shapes and base shapes belong to a zone and are shared between compartments +// in that zone where possible. Accessor shapes store getter and setter +// JSObjects which belong to a single compartment, so these shapes and all +// their descendants can't be shared with other compartments. +// +// - Scripts are also compartment-local and cannot be shared. A script points to +// its compartment. +// +// - ObjectGroup and JitCode objects belong to a compartment and cannot be +// shared. There is no mechanism to obtain the compartment from a JitCode +// object. +// +// A zone remains alive as long as any GC things in the zone are alive. A +// compartment remains alive as long as any JSObjects, scripts, shapes, or base +// shapes within it are alive. +// +// We always guarantee that a zone has at least one live compartment by refusing +// to delete the last compartment in a live zone. +class Zone : public js::ZoneAllocator, public js::gc::GraphNodeBase { + public: + js::gc::ArenaLists arenas; + + // Per-zone data for use by an embedder. + js::MainThreadData data; + + js::MainThreadData tenuredBigInts; + + // Number of marked/finalized JSStrings/JSFatInlineStrings during major GC. + js::MainThreadOrGCTaskData markedStrings; + js::MainThreadOrGCTaskData finalizedStrings; + + // When true, skip calling the metadata callback. We use this: + // - to avoid invoking the callback recursively; + // - to avoid observing lazy prototype setup (which confuses callbacks that + // want to use the types being set up!); + // - to avoid attaching allocation stacks to allocation stack nodes, which + // is silly + // And so on. + js::MainThreadData suppressAllocationMetadataBuilder; + + // Flags permanently set when nursery allocation is disabled for this zone. + js::MainThreadData nurseryStringsDisabled; + js::MainThreadData nurseryBigIntsDisabled; + + private: + // Flags dynamically updated based on more than one condition, including the + // flags above. + js::MainThreadOrIonCompileData allocNurseryObjects_; + js::MainThreadOrIonCompileData allocNurseryStrings_; + js::MainThreadOrIonCompileData allocNurseryBigInts_; + + // Minimum Heap value which results in tenured allocation. + js::MainThreadData minObjectHeapToTenure_; + js::MainThreadData minStringHeapToTenure_; + js::MainThreadData minBigintHeapToTenure_; + + public: + // Script side-tables. These used to be held by Realm, but are now placed + // here in order to allow JSScript to access them during finalize (see bug + // 1568245; this change in 1575350). The tables are initialized lazily by + // JSScript. + js::UniquePtr scriptCountsMap; + js::UniquePtr scriptLCovMap; + js::MainThreadData debugScriptMap; +#ifdef MOZ_VTUNE + js::UniquePtr scriptVTuneIdMap; +#endif +#ifdef JS_CACHEIR_SPEW + js::UniquePtr scriptFinalWarmUpCountMap; +#endif + + js::MainThreadData previousGCStringStats; + js::MainThreadData stringStats; + +#ifdef DEBUG + js::MainThreadData gcSweepGroupIndex; +#endif + + js::gc::PretenuringZone pretenuring; + + private: + // Side map for storing unique ids for cells, independent of address. + js::MainThreadOrGCTaskData uniqueIds_; + + // Number of allocations since the most recent minor GC for this thread. + uint32_t tenuredAllocsSinceMinorGC_ = 0; + + // Live weakmaps in this zone. + js::MainThreadOrGCTaskData> + gcWeakMapList_; + + // The set of compartments in this zone. + using CompartmentVector = + js::Vector; + js::MainThreadOrGCTaskData compartments_; + + // All cross-zone string wrappers in the zone. + js::MainThreadOrGCTaskData crossZoneStringWrappers_; + + // List of non-ephemeron weak containers to sweep during + // beginSweepingSweepGroup. + js::MainThreadOrGCTaskData> + weakCaches_; + + // Mapping from not yet marked keys to a vector of all values that the key + // maps to in any live weak map. Separate tables for nursery and tenured + // keys. + js::MainThreadOrGCTaskData gcEphemeronEdges_; + js::MainThreadOrGCTaskData + gcNurseryEphemeronEdges_; + + js::MainThreadData> regExps_; + + // Bitmap of atoms marked by this zone. + js::MainThreadOrGCTaskData markedAtoms_; + + // Set of atoms recently used by this Zone. Purged on GC. + js::MainThreadOrGCTaskData atomCache_; + + // Cache storing allocated external strings. Purged on GC. + js::MainThreadOrGCTaskData externalStringCache_; + + // Cache for Function.prototype.toString. Purged on GC. + js::MainThreadOrGCTaskData functionToStringCache_; + + // Cache for Function.prototype.bind mapping an atom `name` to atom + // `"bound " + name`. Purged on GC. + using BoundPrefixCache = + js::HashMap, + js::SystemAllocPolicy>; + js::MainThreadData boundPrefixCache_; + + // Information about Shapes and BaseShapes. + js::MainThreadData shapeZone_; + + // Information about finalization registries, created on demand. + js::MainThreadOrGCTaskData> + finalizationObservers_; + + js::MainThreadOrGCTaskData jitZone_; + + // Last time at which JIT code was discarded for this zone. This is only set + // when JitScripts and Baseline code are discarded as well. + js::MainThreadData lastDiscardedCodeTime_; + + js::MainThreadData gcScheduled_; + js::MainThreadData gcScheduledSaved_; + js::MainThreadData gcPreserveCode_; + js::MainThreadData keepPropMapTables_; + js::MainThreadData wasCollected_; + + // Allow zones to be linked into a list + js::MainThreadOrGCTaskData listNext_; + static Zone* const NotOnList; + friend class js::gc::ZoneList; + + using KeptAliveSet = + JS::GCHashSet, + js::StableCellHasher>, + js::ZoneAllocPolicy>; + friend class js::WeakRefObject; + js::MainThreadOrGCTaskData keptObjects; + + public: + static JS::Zone* from(ZoneAllocator* zoneAlloc) { + return static_cast(zoneAlloc); + } + + explicit Zone(JSRuntime* rt, Kind kind = NormalZone); + ~Zone(); + + [[nodiscard]] bool init(); + + void destroy(JS::GCContext* gcx); + + [[nodiscard]] bool findSweepGroupEdges(Zone* atomsZone); + + struct DiscardOptions { + DiscardOptions() {} + bool discardBaselineCode = true; + bool discardJitScripts = false; + bool resetNurseryAllocSites = false; + bool resetPretenuredAllocSites = false; + }; + + void discardJitCode(JS::GCContext* gcx, + const DiscardOptions& options = DiscardOptions()); + + // Discard JIT code regardless of isPreservingCode(). + void forceDiscardJitCode(JS::GCContext* gcx, + const DiscardOptions& options = DiscardOptions()); + + void resetAllocSitesAndInvalidate(bool resetNurserySites, + bool resetPretenuredSites); + + void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, + JS::CodeSizes* code, size_t* regexpZone, + size_t* jitZone, size_t* baselineStubsOptimized, + size_t* uniqueIdMap, size_t* initialPropMapTable, + size_t* shapeTables, size_t* atomsMarkBitmaps, + size_t* compartmentObjects, + size_t* crossCompartmentWrappersTables, + size_t* compartmentsPrivateData, + size_t* scriptCountsMapArg); + + // Iterate over all cells in the zone. See the definition of ZoneCellIter + // in gc/GC-inl.h for the possible arguments and documentation. + template + js::gc::ZoneCellIter cellIter(Args&&... args) { + return js::gc::ZoneCellIter(const_cast(this), + std::forward(args)...); + } + + // As above, but can return about-to-be-finalised things. + template + js::gc::ZoneAllCellIter cellIterUnsafe(Args&&... args) { + return js::gc::ZoneAllCellIter(const_cast(this), + std::forward(args)...); + } + + bool hasMarkedRealms(); + + void scheduleGC() { + MOZ_ASSERT(!RuntimeHeapIsBusy()); + gcScheduled_ = true; + } + void unscheduleGC() { gcScheduled_ = false; } + bool isGCScheduled() { return gcScheduled_; } + + void setPreservingCode(bool preserving) { gcPreserveCode_ = preserving; } + bool isPreservingCode() const { return gcPreserveCode_; } + + mozilla::TimeStamp lastDiscardedCodeTime() const { + return lastDiscardedCodeTime_; + } + + void changeGCState(GCState prev, GCState next); + + bool isCollecting() const { + MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(runtimeFromMainThread())); + return isCollectingFromAnyThread(); + } + + inline bool isCollectingFromAnyThread() const { + return needsIncrementalBarrier() || wasGCStarted(); + } + + GCState initialMarkingState() const; + + bool shouldMarkInZone(js::gc::MarkColor color) const { + // Check whether the zone is in one or both of the MarkBlackOnly and + // MarkBlackAndGray states, depending on the mark color. Also check for + // VerifyPreBarriers when the mark color is black (we don't do any gray + // marking when verifying pre-barriers). + if (color == js::gc::MarkColor::Black) { + return isGCMarkingOrVerifyingPreBarriers(); + } + + return isGCMarkingBlackAndGray(); + } + + // Was this zone collected in the last GC. + bool wasCollected() const { return wasCollected_; } + void setWasCollected(bool v) { wasCollected_ = v; } + + void setNeedsIncrementalBarrier(bool needs); + const BarrierState* addressOfNeedsIncrementalBarrier() const { + return &needsIncrementalBarrier_; + } + + static constexpr size_t offsetOfNeedsIncrementalBarrier() { + return offsetof(Zone, needsIncrementalBarrier_); + } + + js::jit::JitZone* getJitZone(JSContext* cx) { + return jitZone_ ? jitZone_ : createJitZone(cx); + } + js::jit::JitZone* jitZone() { return jitZone_; } + + void prepareForCompacting(); + + void traceRootsInMajorGC(JSTracer* trc); + + void sweepAfterMinorGC(JSTracer* trc); + void sweepUniqueIds(); + void sweepCompartments(JS::GCContext* gcx, bool keepAtleastOne, bool lastGC); + + // Remove dead weak maps from gcWeakMapList_ and remove entries from the + // remaining weak maps whose keys are dead. + void sweepWeakMaps(JSTracer* trc); + + // Trace all weak maps in this zone. Used to update edges after a moving GC. + void traceWeakMaps(JSTracer* trc); + + js::gc::UniqueIdMap& uniqueIds() { return uniqueIds_.ref(); } + + void notifyObservingDebuggers(); + + void noteTenuredAlloc() { tenuredAllocsSinceMinorGC_++; } + + uint32_t* addressOfTenuredAllocCount() { return &tenuredAllocsSinceMinorGC_; } + + uint32_t getAndResetTenuredAllocsSinceMinorGC() { + uint32_t res = tenuredAllocsSinceMinorGC_; + tenuredAllocsSinceMinorGC_ = 0; + return res; + } + + mozilla::LinkedList& gcWeakMapList() { + return gcWeakMapList_.ref(); + } + + CompartmentVector& compartments() { return compartments_.ref(); } + + js::StringWrapperMap& crossZoneStringWrappers() { + return crossZoneStringWrappers_.ref(); + } + const js::StringWrapperMap& crossZoneStringWrappers() const { + return crossZoneStringWrappers_.ref(); + } + + void dropStringWrappersOnGC(); + + void traceWeakCCWEdges(JSTracer* trc); + static void fixupAllCrossCompartmentWrappersAfterMovingGC(JSTracer* trc); + + void fixupAfterMovingGC(); + void fixupScriptMapsAfterMovingGC(JSTracer* trc); + + void setNurseryAllocFlags(bool allocObjects, bool allocStrings, + bool allocBigInts); + + bool allocKindInNursery(JS::TraceKind kind) const { + switch (kind) { + case JS::TraceKind::Object: + return allocNurseryObjects_; + case JS::TraceKind::String: + return allocNurseryStrings_; + case JS::TraceKind::BigInt: + return allocNurseryBigInts_; + default: + MOZ_CRASH("Unsupported kind for nursery allocation"); + } + } + bool allocNurseryObjects() const { return allocNurseryObjects_; } + bool allocNurseryStrings() const { return allocNurseryStrings_; } + bool allocNurseryBigInts() const { return allocNurseryBigInts_; } + + js::gc::Heap minHeapToTenure(JS::TraceKind kind) const { + switch (kind) { + case JS::TraceKind::Object: + return minObjectHeapToTenure_; + case JS::TraceKind::String: + return minStringHeapToTenure_; + case JS::TraceKind::BigInt: + return minBigintHeapToTenure_; + default: + MOZ_CRASH("Unsupported kind for nursery allocation"); + } + } + + mozilla::LinkedList& weakCaches() { + return weakCaches_.ref(); + } + void registerWeakCache(detail::WeakCacheBase* cachep) { + weakCaches().insertBack(cachep); + } + + void beforeClearDelegate(JSObject* wrapper, JSObject* delegate) { + if (needsIncrementalBarrier()) { + beforeClearDelegateInternal(wrapper, delegate); + } + } + + void afterAddDelegate(JSObject* wrapper) { + if (needsIncrementalBarrier()) { + afterAddDelegateInternal(wrapper); + } + } + + void beforeClearDelegateInternal(JSObject* wrapper, JSObject* delegate); + void afterAddDelegateInternal(JSObject* wrapper); + js::gc::EphemeronEdgeTable& gcEphemeronEdges() { + return gcEphemeronEdges_.ref(); + } + js::gc::EphemeronEdgeTable& gcNurseryEphemeronEdges() { + return gcNurseryEphemeronEdges_.ref(); + } + + js::gc::EphemeronEdgeTable& gcEphemeronEdges(const js::gc::Cell* cell) { + return cell->isTenured() ? gcEphemeronEdges() : gcNurseryEphemeronEdges(); + } + + // Perform all pending weakmap entry marking for this zone after + // transitioning to weak marking mode. + js::gc::IncrementalProgress enterWeakMarkingMode(js::GCMarker* marker, + js::SliceBudget& budget); + + // A set of edges from this zone to other zones used during GC to calculate + // sweep groups. + NodeSet& gcSweepGroupEdges() { + return gcGraphEdges; // Defined in GraphNodeBase base class. + } + bool hasSweepGroupEdgeTo(Zone* otherZone) const { + return gcGraphEdges.has(otherZone); + } + [[nodiscard]] bool addSweepGroupEdgeTo(Zone* otherZone) { + MOZ_ASSERT(otherZone->isGCMarking()); + return gcSweepGroupEdges().put(otherZone); + } + void clearSweepGroupEdges() { gcSweepGroupEdges().clear(); } + + js::RegExpZone& regExps() { return *regExps_.ref(); } + + js::SparseBitmap& markedAtoms() { return markedAtoms_.ref(); } + + js::AtomSet& atomCache() { return atomCache_.ref(); } + + void purgeAtomCache(); + + js::ExternalStringCache& externalStringCache() { + return externalStringCache_.ref(); + }; + + js::FunctionToStringCache& functionToStringCache() { + return functionToStringCache_.ref(); + } + + BoundPrefixCache& boundPrefixCache() { return boundPrefixCache_.ref(); } + + js::ShapeZone& shapeZone() { return shapeZone_.ref(); } + + bool keepPropMapTables() const { return keepPropMapTables_; } + void setKeepPropMapTables(bool b) { keepPropMapTables_ = b; } + + void clearRootsForShutdownGC(); + void finishRoots(); + + void traceScriptTableRoots(JSTracer* trc); + + void clearScriptCounts(Realm* realm); + void clearScriptLCov(Realm* realm); + + // Add the target of JS WeakRef to a kept-alive set maintained by GC. + // See: https://tc39.es/proposal-weakrefs/#sec-keepduringjob + bool keepDuringJob(HandleObject target); + + void traceKeptObjects(JSTracer* trc); + + // Clear the kept-alive set. + // See: https://tc39.es/proposal-weakrefs/#sec-clear-kept-objects + void clearKeptObjects(); + + js::gc::AllocSite* unknownAllocSite(JS::TraceKind kind) { + return &pretenuring.unknownAllocSite(kind); + } + js::gc::AllocSite* optimizedAllocSite() { + return &pretenuring.optimizedAllocSite; + } + uint32_t nurseryAllocCount(JS::TraceKind kind) const { + return pretenuring.nurseryAllocCount(kind); + } + +#ifdef JSGC_HASH_TABLE_CHECKS + void checkAllCrossCompartmentWrappersAfterMovingGC(); + void checkStringWrappersAfterMovingGC(); + + // Assert that the UniqueId table has been redirected successfully. + void checkUniqueIdTableAfterMovingGC(); + + void checkScriptMapsAfterMovingGC(); +#endif + +#ifdef DEBUG + // For testing purposes, return the index of the sweep group which this zone + // was swept in in the last GC. + unsigned lastSweepGroupIndex() { return gcSweepGroupIndex; } +#endif + + private: + js::jit::JitZone* createJitZone(JSContext* cx); + + bool isQueuedForBackgroundSweep() { return isOnList(); } + + void sweepEphemeronTablesAfterMinorGC(); + + js::gc::FinalizationObservers* finalizationObservers() { + return finalizationObservers_.ref().get(); + } + bool ensureFinalizationObservers(); + + bool isOnList() const; + Zone* nextZone() const; + + friend bool js::CurrentThreadCanAccessZone(Zone* zone); + friend class js::gc::GCRuntime; +}; + +} // namespace JS + +namespace js { +namespace gc { +const char* StateName(JS::Zone::GCState state); +} // namespace gc +} // namespace js + +#endif // gc_Zone_h diff --git a/js/src/gc/ZoneAllocator.h b/js/src/gc/ZoneAllocator.h new file mode 100644 index 0000000000..de2dd7da28 --- /dev/null +++ b/js/src/gc/ZoneAllocator.h @@ -0,0 +1,354 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Public header for allocating memory associated with GC things. + */ + +#ifndef gc_ZoneAllocator_h +#define gc_ZoneAllocator_h + +#include "mozilla/Maybe.h" + +#include "jsfriendapi.h" +#include "jstypes.h" +#include "gc/Cell.h" +#include "gc/Scheduling.h" +#include "js/GCAPI.h" +#include "js/shadow/Zone.h" // JS::shadow::Zone +#include "vm/MallocProvider.h" + +namespace JS { +class JS_PUBLIC_API Zone; +} // namespace JS + +namespace js { + +class ZoneAllocator; + +#ifdef DEBUG +bool CurrentThreadIsGCFinalizing(); +#endif + +namespace gc { +void MaybeMallocTriggerZoneGC(JSRuntime* rt, ZoneAllocator* zoneAlloc, + const HeapSize& heap, + const HeapThreshold& threshold, + JS::GCReason reason); +} + +// Base class of JS::Zone that provides malloc memory allocation and accounting. +class ZoneAllocator : public JS::shadow::Zone, + public js::MallocProvider { + protected: + explicit ZoneAllocator(JSRuntime* rt, Kind kind); + ~ZoneAllocator(); + void fixupAfterMovingGC(); + + public: + static ZoneAllocator* from(JS::Zone* zone) { + // This is a safe upcast, but the compiler hasn't seen the definition yet. + return reinterpret_cast(zone); + } + + [[nodiscard]] void* onOutOfMemory(js::AllocFunction allocFunc, + arena_id_t arena, size_t nbytes, + void* reallocPtr = nullptr); + void reportAllocationOverflow() const; + + void updateSchedulingStateOnGCStart(); + void updateGCStartThresholds(gc::GCRuntime& gc); + void setGCSliceThresholds(gc::GCRuntime& gc, bool waitingOnBGTask); + void clearGCSliceThresholds(); + + // Memory accounting APIs for malloc memory owned by GC cells. + + void addCellMemory(js::gc::Cell* cell, size_t nbytes, js::MemoryUse use) { + MOZ_ASSERT(cell); + MOZ_ASSERT(nbytes); + + mallocHeapSize.addBytes(nbytes); + +#ifdef DEBUG + mallocTracker.trackGCMemory(cell, nbytes, use); +#endif + + maybeTriggerGCOnMalloc(); + } + + void removeCellMemory(js::gc::Cell* cell, size_t nbytes, js::MemoryUse use, + bool updateRetainedSize = false) { + MOZ_ASSERT(cell); + MOZ_ASSERT(nbytes); + MOZ_ASSERT_IF(CurrentThreadIsGCFinalizing(), updateRetainedSize); + + mallocHeapSize.removeBytes(nbytes, updateRetainedSize); + +#ifdef DEBUG + mallocTracker.untrackGCMemory(cell, nbytes, use); +#endif + } + + void swapCellMemory(js::gc::Cell* a, js::gc::Cell* b, js::MemoryUse use) { +#ifdef DEBUG + mallocTracker.swapGCMemory(a, b, use); +#endif + } + + void registerNonGCMemory(void* mem, MemoryUse use) { +#ifdef DEBUG + return mallocTracker.registerNonGCMemory(mem, use); +#endif + } + void unregisterNonGCMemory(void* mem, MemoryUse use) { +#ifdef DEBUG + return mallocTracker.unregisterNonGCMemory(mem, use); +#endif + } + void moveOtherMemory(void* dst, void* src, MemoryUse use) { +#ifdef DEBUG + return mallocTracker.moveNonGCMemory(dst, src, use); +#endif + } + + void incNonGCMemory(void* mem, size_t nbytes, MemoryUse use) { + MOZ_ASSERT(nbytes); + mallocHeapSize.addBytes(nbytes); + +#ifdef DEBUG + mallocTracker.incNonGCMemory(mem, nbytes, use); +#endif + + maybeTriggerGCOnMalloc(); + } + void decNonGCMemory(void* mem, size_t nbytes, MemoryUse use, + bool updateRetainedSize) { + MOZ_ASSERT(nbytes); + + mallocHeapSize.removeBytes(nbytes, updateRetainedSize); + +#ifdef DEBUG + mallocTracker.decNonGCMemory(mem, nbytes, use); +#endif + } + + // Account for allocations that may be referenced by more than one GC thing. + bool addSharedMemory(void* mem, size_t nbytes, MemoryUse use); + void removeSharedMemory(void* mem, size_t nbytes, MemoryUse use); + + void incJitMemory(size_t nbytes) { + MOZ_ASSERT(nbytes); + jitHeapSize.addBytes(nbytes); + maybeTriggerZoneGC(jitHeapSize, jitHeapThreshold, + JS::GCReason::TOO_MUCH_JIT_CODE); + } + void decJitMemory(size_t nbytes) { + MOZ_ASSERT(nbytes); + jitHeapSize.removeBytes(nbytes, true); + } + + // Check malloc allocation threshold and trigger a zone GC if necessary. + void maybeTriggerGCOnMalloc() { + maybeTriggerZoneGC(mallocHeapSize, mallocHeapThreshold, + JS::GCReason::TOO_MUCH_MALLOC); + } + + private: + void maybeTriggerZoneGC(const js::gc::HeapSize& heap, + const js::gc::HeapThreshold& threshold, + JS::GCReason reason) { + if (heap.bytes() >= threshold.startBytes()) { + gc::MaybeMallocTriggerZoneGC(runtimeFromAnyThread(), this, heap, + threshold, reason); + } + } + + void updateCollectionRate(mozilla::TimeDuration mainThreadGCTime, + size_t initialBytesForAllZones); + + void updateAllocationRate(mozilla::TimeDuration mutatorTime); + + public: + // The size of allocated GC arenas in this zone. + gc::PerZoneGCHeapSize gcHeapSize; + + // Threshold used to trigger GC based on GC heap size. + gc::GCHeapThreshold gcHeapThreshold; + + // Amount of malloc data owned by tenured GC things in this zone, including + // external allocations supplied by JS::AddAssociatedMemory. + gc::HeapSize mallocHeapSize; + + // Threshold used to trigger GC based on malloc allocations. + gc::MallocHeapThreshold mallocHeapThreshold; + + // Amount of exectuable JIT code owned by GC things in this zone. + gc::HeapSize jitHeapSize; + + // Threshold used to trigger GC based on JIT allocations. + gc::JitHeapThreshold jitHeapThreshold; + + // Use counts for memory that can be referenced by more than one GC thing. + // Memory recorded here is also recorded in mallocHeapSize. This structure + // is used to avoid over-counting in mallocHeapSize. + gc::SharedMemoryMap sharedMemoryUseCounts; + + // Collection rate estimate for this zone in MB/s, and state used to calculate + // it. Updated every time this zone is collected. + MainThreadData> smoothedCollectionRate; + MainThreadOrGCTaskData perZoneGCTime; + + // Allocation rate estimate in MB/s of mutator time, and state used to + // calculate it. + MainThreadData> smoothedAllocationRate; + MainThreadData prevGCHeapSize; + + private: +#ifdef DEBUG + // In debug builds, malloc allocations can be tracked to make debugging easier + // (possible?) if allocation and free sizes don't balance. + gc::MemoryTracker mallocTracker; +#endif + + friend class gc::GCRuntime; +}; + +// Whether memory is associated with a single cell or whether it is associated +// with the zone as a whole (for memory used by the system). +enum class TrackingKind { Cell, Zone }; + +/* + * Allocation policy that performs memory tracking for malloced memory. This + * should be used for all containers associated with a GC thing or a zone. + * + * Since it doesn't hold a JSContext (those may not live long enough), it can't + * report out-of-memory conditions itself; the caller must check for OOM and + * take the appropriate action. + * + * FIXME bug 647103 - replace these *AllocPolicy names. + */ +template +class TrackedAllocPolicy : public MallocProvider> { + ZoneAllocator* zone_; + +#ifdef DEBUG + friend class js::gc::MemoryTracker; // Can clear |zone_| on merge. +#endif + + public: + MOZ_IMPLICIT TrackedAllocPolicy(ZoneAllocator* z) : zone_(z) { + zone()->registerNonGCMemory(this, MemoryUse::TrackedAllocPolicy); + } + MOZ_IMPLICIT TrackedAllocPolicy(JS::Zone* z) + : TrackedAllocPolicy(ZoneAllocator::from(z)) {} + TrackedAllocPolicy(TrackedAllocPolicy& other) + : TrackedAllocPolicy(other.zone_) {} + TrackedAllocPolicy(TrackedAllocPolicy&& other) : zone_(other.zone_) { + zone()->moveOtherMemory(this, &other, MemoryUse::TrackedAllocPolicy); + other.zone_ = nullptr; + } + ~TrackedAllocPolicy() { + if (zone_) { + zone_->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy); + } + } + + TrackedAllocPolicy& operator=(const TrackedAllocPolicy& other) { + zone()->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy); + zone_ = other.zone(); + zone()->registerNonGCMemory(this, MemoryUse::TrackedAllocPolicy); + return *this; + } + TrackedAllocPolicy& operator=(TrackedAllocPolicy&& other) { + MOZ_ASSERT(this != &other); + zone()->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy); + zone_ = other.zone(); + zone()->moveOtherMemory(this, &other, MemoryUse::TrackedAllocPolicy); + other.zone_ = nullptr; + return *this; + } + + // Public methods required to fulfill the AllocPolicy interface. + + template + void free_(T* p, size_t numElems) { + if (p) { + decMemory(numElems * sizeof(T)); + js_free(p); + } + } + + [[nodiscard]] bool checkSimulatedOOM() const { + return !js::oom::ShouldFailWithOOM(); + } + + void reportAllocOverflow() const { reportAllocationOverflow(); } + + // Internal methods called by the MallocProvider implementation. + + [[nodiscard]] void* onOutOfMemory(js::AllocFunction allocFunc, + arena_id_t arena, size_t nbytes, + void* reallocPtr = nullptr) { + return zone()->onOutOfMemory(allocFunc, arena, nbytes, reallocPtr); + } + void reportAllocationOverflow() const { zone()->reportAllocationOverflow(); } + void updateMallocCounter(size_t nbytes) { + zone()->incNonGCMemory(this, nbytes, MemoryUse::TrackedAllocPolicy); + } + + private: + ZoneAllocator* zone() const { + MOZ_ASSERT(zone_); + return zone_; + } + void decMemory(size_t nbytes); +}; + +using ZoneAllocPolicy = TrackedAllocPolicy; +using CellAllocPolicy = TrackedAllocPolicy; + +// Functions for memory accounting on the zone. + +// Associate malloc memory with a GC thing. This call should be matched by a +// following call to RemoveCellMemory with the same size and use. The total +// amount of malloc memory associated with a zone is used to trigger GC. +// +// You should use InitReservedSlot / InitObjectPrivate in preference to this +// where possible. + +inline void AddCellMemory(gc::TenuredCell* cell, size_t nbytes, MemoryUse use) { + if (nbytes) { + ZoneAllocator::from(cell->zone())->addCellMemory(cell, nbytes, use); + } +} +inline void AddCellMemory(gc::Cell* cell, size_t nbytes, MemoryUse use) { + if (cell->isTenured()) { + AddCellMemory(&cell->asTenured(), nbytes, use); + } +} + +// Remove association between malloc memory and a GC thing. This call should +// follow a call to AddCellMemory with the same size and use. + +inline void RemoveCellMemory(gc::TenuredCell* cell, size_t nbytes, + MemoryUse use) { + MOZ_ASSERT(!CurrentThreadIsGCFinalizing(), + "Use GCContext methods to remove associated memory in finalizers"); + + if (nbytes) { + auto zoneBase = ZoneAllocator::from(cell->zoneFromAnyThread()); + zoneBase->removeCellMemory(cell, nbytes, use, false); + } +} +inline void RemoveCellMemory(gc::Cell* cell, size_t nbytes, MemoryUse use) { + if (cell->isTenured()) { + RemoveCellMemory(&cell->asTenured(), nbytes, use); + } +} + +} // namespace js + +#endif // gc_ZoneAllocator_h diff --git a/js/src/gc/moz.build b/js/src/gc/moz.build new file mode 100644 index 0000000000..489b5a96ae --- /dev/null +++ b/js/src/gc/moz.build @@ -0,0 +1,61 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FINAL_LIBRARY = "js" + +# Includes should be relative to parent path +LOCAL_INCLUDES += ["!..", ".."] + +include("../js-config.mozbuild") +include("../js-cxxflags.mozbuild") + +# Generate GC statistics phase data. +GeneratedFile( + "StatsPhasesGenerated.h", + script="GenerateStatsPhases.py", + entry_point="generateHeader", +) +GeneratedFile( + "StatsPhasesGenerated.inc", + script="GenerateStatsPhases.py", + entry_point="generateCpp", +) + +UNIFIED_SOURCES += [ + "Allocator.cpp", + "AtomMarking.cpp", + "Barrier.cpp", + "Compacting.cpp", + "FinalizationObservers.cpp", + "GC.cpp", + "GCAPI.cpp", + "GCParallelTask.cpp", + "Heap.cpp", + "MallocedBlockCache.cpp", + "Marking.cpp", + "Memory.cpp", + "Nursery.cpp", + "ParallelMarking.cpp", + "Pretenuring.cpp", + "PublicIterators.cpp", + "RootMarking.cpp", + "Scheduling.cpp", + "Statistics.cpp", + "Sweeping.cpp", + "Tenuring.cpp", + "Tracer.cpp", + "Verifier.cpp", + "WeakMap.cpp", + "WeakMapPtr.cpp", + "Zone.cpp", +] + +# StoreBuffer.cpp cannot be built in unified mode because its template +# instantiations may or may not be needed depending on what it gets bundled +# with. +SOURCES += [ + "StoreBuffer.cpp", +] -- cgit v1.2.3