summaryrefslogtreecommitdiffstats
path: root/js/src/gc
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/gc')
-rw-r--r--js/src/gc/AllocKind.h326
-rw-r--r--js/src/gc/Allocator.cpp672
-rw-r--r--js/src/gc/Allocator.h173
-rw-r--r--js/src/gc/ArenaList-inl.h327
-rw-r--r--js/src/gc/ArenaList.h386
-rw-r--r--js/src/gc/AtomMarking-inl.h96
-rw-r--r--js/src/gc/AtomMarking.cpp294
-rw-r--r--js/src/gc/AtomMarking.h86
-rw-r--r--js/src/gc/Barrier.cpp144
-rw-r--r--js/src/gc/Barrier.h1257
-rw-r--r--js/src/gc/Cell.h930
-rw-r--r--js/src/gc/ClearEdgesTracer.h27
-rw-r--r--js/src/gc/Compacting.cpp967
-rw-r--r--js/src/gc/FinalizationObservers.cpp509
-rw-r--r--js/src/gc/FinalizationObservers.h129
-rw-r--r--js/src/gc/FindSCCs.h207
-rw-r--r--js/src/gc/GC-inl.h344
-rw-r--r--js/src/gc/GC.cpp5101
-rw-r--r--js/src/gc/GC.h243
-rw-r--r--js/src/gc/GCAPI.cpp798
-rw-r--r--js/src/gc/GCContext-inl.h40
-rw-r--r--js/src/gc/GCContext.h257
-rw-r--r--js/src/gc/GCEnum.h160
-rw-r--r--js/src/gc/GCInternals.h344
-rw-r--r--js/src/gc/GCLock.h110
-rw-r--r--js/src/gc/GCMarker.h598
-rw-r--r--js/src/gc/GCParallelTask.cpp231
-rw-r--r--js/src/gc/GCParallelTask.h246
-rw-r--r--js/src/gc/GCProbes.h49
-rw-r--r--js/src/gc/GCRuntime.h1444
-rw-r--r--js/src/gc/GenerateStatsPhases.py404
-rw-r--r--js/src/gc/HashUtil.h84
-rw-r--r--js/src/gc/Heap-inl.h70
-rw-r--r--js/src/gc/Heap.cpp635
-rw-r--r--js/src/gc/Heap.h846
-rw-r--r--js/src/gc/IteratorUtils.h121
-rw-r--r--js/src/gc/MallocedBlockCache.cpp144
-rw-r--r--js/src/gc/MallocedBlockCache.h91
-rw-r--r--js/src/gc/Marking-inl.h196
-rw-r--r--js/src/gc/Marking.cpp2774
-rw-r--r--js/src/gc/Marking.h150
-rw-r--r--js/src/gc/MaybeRooted.h141
-rw-r--r--js/src/gc/Memory.cpp1050
-rw-r--r--js/src/gc/Memory.h84
-rw-r--r--js/src/gc/Nursery-inl.h172
-rw-r--r--js/src/gc/Nursery.cpp2083
-rw-r--r--js/src/gc/Nursery.h645
-rw-r--r--js/src/gc/NurseryAwareHashMap.h209
-rw-r--r--js/src/gc/ObjectKind-inl.h162
-rw-r--r--js/src/gc/ParallelMarking.cpp359
-rw-r--r--js/src/gc/ParallelMarking.h123
-rw-r--r--js/src/gc/ParallelWork.h136
-rw-r--r--js/src/gc/Policy.h101
-rw-r--r--js/src/gc/Pretenuring.cpp459
-rw-r--r--js/src/gc/Pretenuring.h348
-rw-r--r--js/src/gc/PrivateIterators-inl.h164
-rw-r--r--js/src/gc/PublicIterators.cpp272
-rw-r--r--js/src/gc/PublicIterators.h158
-rw-r--r--js/src/gc/RelocationOverlay.h66
-rw-r--r--js/src/gc/RootMarking.cpp467
-rw-r--r--js/src/gc/Scheduling.cpp873
-rw-r--r--js/src/gc/Scheduling.h917
-rw-r--r--js/src/gc/StableCellHasher-inl.h245
-rw-r--r--js/src/gc/StableCellHasher.h46
-rw-r--r--js/src/gc/Statistics.cpp1811
-rw-r--r--js/src/gc/Statistics.h606
-rw-r--r--js/src/gc/StoreBuffer-inl.h97
-rw-r--r--js/src/gc/StoreBuffer.cpp255
-rw-r--r--js/src/gc/StoreBuffer.h662
-rw-r--r--js/src/gc/Sweeping.cpp2383
-rw-r--r--js/src/gc/Tenuring.cpp1016
-rw-r--r--js/src/gc/Tenuring.h169
-rw-r--r--js/src/gc/TraceKind.h59
-rw-r--r--js/src/gc/TraceMethods-inl.h384
-rw-r--r--js/src/gc/Tracer.cpp297
-rw-r--r--js/src/gc/Tracer.h401
-rw-r--r--js/src/gc/Verifier.cpp1135
-rw-r--r--js/src/gc/WeakMap-inl.h413
-rw-r--r--js/src/gc/WeakMap.cpp175
-rw-r--r--js/src/gc/WeakMap.h354
-rw-r--r--js/src/gc/WeakMapPtr.cpp114
-rw-r--r--js/src/gc/Zone.cpp979
-rw-r--r--js/src/gc/Zone.h653
-rw-r--r--js/src/gc/ZoneAllocator.h354
-rw-r--r--js/src/gc/moz.build61
85 files changed, 43668 insertions, 0 deletions
diff --git a/js/src/gc/AllocKind.h b/js/src/gc/AllocKind.h
new file mode 100644
index 0000000000..3427d0aba6
--- /dev/null
+++ b/js/src/gc/AllocKind.h
@@ -0,0 +1,326 @@
+
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definition of GC cell kinds.
+ */
+
+#ifndef gc_AllocKind_h
+#define gc_AllocKind_h
+
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/EnumeratedRange.h"
+
+#include <iterator>
+#include <stdint.h>
+
+#include "js/TraceKind.h"
+
+class JSDependentString;
+class JSExternalString;
+class JSFatInlineString;
+class JSLinearString;
+class JSRope;
+class JSThinInlineString;
+
+namespace js {
+
+class CompactPropMap;
+class FatInlineAtom;
+class NormalAtom;
+class NormalPropMap;
+class DictionaryPropMap;
+class DictionaryShape;
+class SharedShape;
+class ProxyShape;
+class WasmGCShape;
+
+namespace gc {
+
+// The GC allocation kinds.
+//
+// These are defined by macros which enumerate the different allocation kinds
+// and supply the following information:
+//
+// - the corresponding AllocKind
+// - their JS::TraceKind
+// - their C++ base type
+// - a C++ type of the correct size
+// - whether they can be finalized on the background thread
+// - whether they can be allocated in the nursery (this is true for foreground
+// finalized objects but these will can only actually be allocated in the
+// nursery if JSCLASS_SKIP_NURSERY_FINALIZE is set)
+// - whether they can be compacted
+
+// clang-format off
+#define FOR_EACH_OBJECT_ALLOCKIND(D) \
+ /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \
+ D(FUNCTION, Object, JSObject, JSObject_Slots4, true, true, true) \
+ D(FUNCTION_EXTENDED, Object, JSObject, JSObject_Slots6, true, true, true) \
+ D(OBJECT0, Object, JSObject, JSObject_Slots0, false, true, true) \
+ D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0, true, true, true) \
+ D(OBJECT2, Object, JSObject, JSObject_Slots2, false, true, true) \
+ D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2, true, true, true) \
+ D(ARRAYBUFFER4, Object, JSObject, JSObject_Slots4, true, true, true) \
+ D(OBJECT4, Object, JSObject, JSObject_Slots4, false, true, true) \
+ D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4, true, true, true) \
+ D(ARRAYBUFFER8, Object, JSObject, JSObject_Slots8, true, true, true) \
+ D(OBJECT8, Object, JSObject, JSObject_Slots8, false, true, true) \
+ D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8, true, true, true) \
+ D(ARRAYBUFFER12, Object, JSObject, JSObject_Slots12, true, true, true) \
+ D(OBJECT12, Object, JSObject, JSObject_Slots12, false, true, true) \
+ D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12, true, true, true) \
+ D(ARRAYBUFFER16, Object, JSObject, JSObject_Slots16, true, true, true) \
+ D(OBJECT16, Object, JSObject, JSObject_Slots16, false, true, true) \
+ D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16, true, true, true)
+
+#define FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(D) \
+ /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \
+ D(SCRIPT, Script, js::BaseScript, js::BaseScript, false, false, true) \
+ D(SHAPE, Shape, js::Shape, js::SizedShape, true, false, true) \
+ D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape, true, false, true) \
+ D(GETTER_SETTER, GetterSetter, js::GetterSetter, js::GetterSetter, true, false, true) \
+ D(COMPACT_PROP_MAP, PropMap, js::CompactPropMap, js::CompactPropMap, true, false, true) \
+ D(NORMAL_PROP_MAP, PropMap, js::NormalPropMap, js::NormalPropMap, true, false, true) \
+ D(DICT_PROP_MAP, PropMap, js::DictionaryPropMap, js::DictionaryPropMap, true, false, true) \
+ D(EXTERNAL_STRING, String, JSExternalString, JSExternalString, true, false, true) \
+ D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom, true, false, false) \
+ D(ATOM, String, js::NormalAtom, js::NormalAtom, true, false, false) \
+ D(SYMBOL, Symbol, JS::Symbol, JS::Symbol, true, false, false) \
+ D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode, false, false, false) \
+ D(SCOPE, Scope, js::Scope, js::Scope, true, false, true) \
+ D(REGEXP_SHARED, RegExpShared, js::RegExpShared, js::RegExpShared, true, false, true)
+
+#define FOR_EACH_NONOBJECT_NURSERY_ALLOCKIND(D) \
+ /* AllocKind TraceKind TypeName SizedType BGFinal Nursery Compact */ \
+ D(BIGINT, BigInt, JS::BigInt, JS::BigInt, true, true, true)
+
+#define FOR_EACH_NURSERY_STRING_ALLOCKIND(D) \
+ D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString, true, true, true) \
+ D(STRING, String, JSString, JSString, true, true, true)
+// clang-format on
+
+#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \
+ FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(D) \
+ FOR_EACH_NONOBJECT_NURSERY_ALLOCKIND(D) \
+ FOR_EACH_NURSERY_STRING_ALLOCKIND(D)
+
+#define FOR_EACH_ALLOCKIND(D) \
+ FOR_EACH_OBJECT_ALLOCKIND(D) \
+ FOR_EACH_NONOBJECT_ALLOCKIND(D)
+
+#define DEFINE_ALLOC_KIND(allocKind, _1, _2, _3, _4, _5, _6) allocKind,
+enum class AllocKind : uint8_t {
+ // clang-format off
+ FOR_EACH_OBJECT_ALLOCKIND(DEFINE_ALLOC_KIND)
+
+ OBJECT_LIMIT,
+ OBJECT_LAST = OBJECT_LIMIT - 1,
+
+ FOR_EACH_NONOBJECT_ALLOCKIND(DEFINE_ALLOC_KIND)
+
+ LIMIT,
+ LAST = LIMIT - 1,
+
+ FIRST = 0,
+ OBJECT_FIRST = FUNCTION // Hardcoded to first object kind.
+ // clang-format on
+};
+#undef DEFINE_ALLOC_KIND
+
+static_assert(int(AllocKind::FIRST) == 0,
+ "Various places depend on AllocKind starting at 0");
+static_assert(int(AllocKind::OBJECT_FIRST) == 0,
+ "OBJECT_FIRST must be defined as the first object kind");
+
+constexpr size_t AllocKindCount = size_t(AllocKind::LIMIT);
+
+/*
+ * A flag specifying either the tenured heap or a default heap (which may be
+ * either the nursery or the tenured heap).
+ *
+ * This allows an allocation site to request a heap based upon the estimated
+ * lifetime or lifetime requirements of objects allocated from that site.
+ *
+ * Order is important as these are numerically compared.
+ */
+enum class Heap : uint8_t { Default = 0, Tenured = 1 };
+
+inline bool IsAllocKind(AllocKind kind) {
+ return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT;
+}
+
+inline bool IsValidAllocKind(AllocKind kind) {
+ return kind >= AllocKind::FIRST && kind <= AllocKind::LAST;
+}
+
+const char* AllocKindName(AllocKind kind);
+
+inline bool IsObjectAllocKind(AllocKind kind) {
+ return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST;
+}
+
+inline bool IsShapeAllocKind(AllocKind kind) {
+ return kind == AllocKind::SHAPE;
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over all alloc kinds.
+inline auto AllAllocKinds() {
+ return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT);
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over all object alloc kinds.
+inline auto ObjectAllocKinds() {
+ return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST,
+ AllocKind::OBJECT_LIMIT);
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over alloc kinds from |first| to |limit|, exclusive.
+inline auto SomeAllocKinds(AllocKind first = AllocKind::FIRST,
+ AllocKind limit = AllocKind::LIMIT) {
+ MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!");
+ MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!");
+ return mozilla::MakeEnumeratedRange(first, limit);
+}
+
+// AllAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
+// with each index corresponding to a particular alloc kind.
+template <typename ValueType>
+using AllAllocKindArray =
+ mozilla::EnumeratedArray<AllocKind, AllocKind::LIMIT, ValueType>;
+
+// ObjectAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
+// with each index corresponding to a particular object alloc kind.
+template <typename ValueType>
+using ObjectAllocKindArray =
+ mozilla::EnumeratedArray<AllocKind, AllocKind::OBJECT_LIMIT, ValueType>;
+
+/*
+ * Map from C++ type to alloc kind for non-object types. JSObject does not have
+ * a 1:1 mapping, so must use Arena::thingSize.
+ *
+ * The AllocKind is available as MapTypeToAllocKind<SomeType>::kind.
+ *
+ * There are specializations for strings and shapes since more than one derived
+ * type shares the same alloc kind.
+ */
+template <typename T>
+struct MapTypeToAllocKind {};
+#define EXPAND_MAPTYPETOALLOCKIND(allocKind, traceKind, type, sizedType, \
+ bgFinal, nursery, compact) \
+ template <> \
+ struct MapTypeToAllocKind<type> { \
+ static const AllocKind kind = AllocKind::allocKind; \
+ };
+FOR_EACH_NONOBJECT_ALLOCKIND(EXPAND_MAPTYPETOALLOCKIND)
+#undef EXPAND_MAPTYPETOALLOCKIND
+
+template <>
+struct MapTypeToAllocKind<JSDependentString> {
+ static const AllocKind kind = AllocKind::STRING;
+};
+template <>
+struct MapTypeToAllocKind<JSRope> {
+ static const AllocKind kind = AllocKind::STRING;
+};
+template <>
+struct MapTypeToAllocKind<JSLinearString> {
+ static const AllocKind kind = AllocKind::STRING;
+};
+template <>
+struct MapTypeToAllocKind<JSThinInlineString> {
+ static const AllocKind kind = AllocKind::STRING;
+};
+
+template <>
+struct MapTypeToAllocKind<js::SharedShape> {
+ static const AllocKind kind = AllocKind::SHAPE;
+};
+template <>
+struct MapTypeToAllocKind<js::DictionaryShape> {
+ static const AllocKind kind = AllocKind::SHAPE;
+};
+template <>
+struct MapTypeToAllocKind<js::ProxyShape> {
+ static const AllocKind kind = AllocKind::SHAPE;
+};
+template <>
+struct MapTypeToAllocKind<js::WasmGCShape> {
+ static const AllocKind kind = AllocKind::SHAPE;
+};
+
+static inline JS::TraceKind MapAllocToTraceKind(AllocKind kind) {
+ static const JS::TraceKind map[] = {
+#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType, bgFinal, \
+ nursery, compact) \
+ JS::TraceKind::traceKind,
+ FOR_EACH_ALLOCKIND(EXPAND_ELEMENT)
+#undef EXPAND_ELEMENT
+ };
+
+ static_assert(std::size(map) == AllocKindCount,
+ "AllocKind-to-TraceKind mapping must be in sync");
+ return map[size_t(kind)];
+}
+
+static inline bool IsNurseryAllocable(AllocKind kind) {
+ MOZ_ASSERT(IsValidAllocKind(kind));
+
+ static const bool map[] = {
+#define DEFINE_NURSERY_ALLOCABLE(_1, _2, _3, _4, _5, nursery, _6) nursery,
+ FOR_EACH_ALLOCKIND(DEFINE_NURSERY_ALLOCABLE)
+#undef DEFINE_NURSERY_ALLOCABLE
+ };
+
+ static_assert(std::size(map) == AllocKindCount,
+ "IsNurseryAllocable sanity check");
+ return map[size_t(kind)];
+}
+
+static inline bool IsBackgroundFinalized(AllocKind kind) {
+ MOZ_ASSERT(IsValidAllocKind(kind));
+
+ static const bool map[] = {
+#define DEFINE_BACKGROUND_FINALIZED(_1, _2, _3, _4, bgFinal, _5, _6) bgFinal,
+ FOR_EACH_ALLOCKIND(DEFINE_BACKGROUND_FINALIZED)
+#undef DEFINE_BACKGROUND_FINALIZED
+ };
+
+ static_assert(std::size(map) == AllocKindCount,
+ "IsBackgroundFinalized sanity check");
+ return map[size_t(kind)];
+}
+
+static inline bool IsForegroundFinalized(AllocKind kind) {
+ return !IsBackgroundFinalized(kind);
+}
+
+static inline bool IsCompactingKind(AllocKind kind) {
+ MOZ_ASSERT(IsValidAllocKind(kind));
+
+ static const bool map[] = {
+#define DEFINE_COMPACTING_KIND(_1, _2, _3, _4, _5, _6, compact) compact,
+ FOR_EACH_ALLOCKIND(DEFINE_COMPACTING_KIND)
+#undef DEFINE_COMPACTING_KIND
+ };
+
+ static_assert(std::size(map) == AllocKindCount,
+ "IsCompactingKind sanity check");
+ return map[size_t(kind)];
+}
+
+static inline bool IsMovableKind(AllocKind kind) {
+ return IsNurseryAllocable(kind) || IsCompactingKind(kind);
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_AllocKind_h */
diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp
new file mode 100644
index 0000000000..92d4653961
--- /dev/null
+++ b/js/src/gc/Allocator.cpp
@@ -0,0 +1,672 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Allocator.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/OperatorNewExtensions.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/GCProbes.h"
+#include "gc/Nursery.h"
+#include "threading/CpuCount.h"
+#include "util/Poison.h"
+#include "vm/BigIntType.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+#include "vm/StringType.h"
+
+#include "gc/ArenaList-inl.h"
+#include "gc/Heap-inl.h"
+#include "gc/PrivateIterators-inl.h"
+#include "vm/JSContext-inl.h"
+
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+using namespace js;
+using namespace js::gc;
+
+// Return a Heap value that can be compared numerically with an
+// allocation's requested heap to determine whether to allocate in the nursery
+// or the tenured heap.
+//
+// If nursery allocation is allowed this returns Heap::Tenured, meaning only
+// Heap::Tenured allocations will be tenured. If nursery allocation is not
+// allowed this returns Heap::Default, meaning all allocations are tenured.
+static Heap MinHeapToTenure(bool allowNurseryAlloc) {
+ static_assert(Heap::Tenured > Heap::Default);
+ return allowNurseryAlloc ? Heap::Tenured : Heap::Default;
+}
+
+void Zone::setNurseryAllocFlags(bool allocObjects, bool allocStrings,
+ bool allocBigInts) {
+ allocNurseryObjects_ = allocObjects;
+ allocNurseryStrings_ = allocStrings;
+ allocNurseryBigInts_ = allocBigInts;
+
+ minObjectHeapToTenure_ = MinHeapToTenure(allocNurseryObjects());
+ minStringHeapToTenure_ = MinHeapToTenure(allocNurseryStrings());
+ minBigintHeapToTenure_ = MinHeapToTenure(allocNurseryBigInts());
+}
+
+template <JS::TraceKind traceKind, AllowGC allowGC /* = CanGC */>
+void* gc::CellAllocator::AllocNurseryOrTenuredCell(JSContext* cx,
+ AllocKind allocKind,
+ gc::Heap heap,
+ AllocSite* site) {
+ MOZ_ASSERT(!cx->isHelperThreadContext());
+ MOZ_ASSERT_IF(heap != gc::Heap::Tenured, IsNurseryAllocable(allocKind));
+ MOZ_ASSERT(MapAllocToTraceKind(allocKind) == traceKind);
+ MOZ_ASSERT_IF(site && site->initialHeap() == Heap::Tenured,
+ heap == Heap::Tenured);
+
+ size_t thingSize = Arena::thingSize(allocKind);
+
+ JSRuntime* rt = cx->runtime();
+ if (!rt->gc.checkAllocatorState<allowGC>(cx, allocKind)) {
+ return nullptr;
+ }
+
+ if (heap < cx->zone()->minHeapToTenure(traceKind)) {
+ if (!site) {
+ site = cx->zone()->unknownAllocSite(traceKind);
+ }
+
+ void* obj =
+ rt->gc.tryNewNurseryCell<traceKind, allowGC>(cx, thingSize, site);
+ if (obj) {
+ return obj;
+ }
+
+ // Our most common non-jit allocation path is NoGC; thus, if we fail the
+ // alloc and cannot GC, we *must* return nullptr here so that the caller
+ // will do a CanGC allocation to clear the nursery. Failing to do so will
+ // cause all allocations on this path to land in Tenured, and we will not
+ // get the benefit of the nursery.
+ if (!allowGC) {
+ return nullptr;
+ }
+ }
+
+ return GCRuntime::tryNewTenuredThing<allowGC>(cx, allocKind, thingSize);
+}
+
+#define INSTANTIATE_ALLOC_NURSERY_CELL(traceKind, allowGc) \
+ template void* \
+ gc::CellAllocator::AllocNurseryOrTenuredCell<traceKind, allowGc>( \
+ JSContext*, AllocKind, gc::Heap, AllocSite*);
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::Object, NoGC)
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::Object, CanGC)
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::String, NoGC)
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::String, CanGC)
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::BigInt, NoGC)
+INSTANTIATE_ALLOC_NURSERY_CELL(JS::TraceKind::BigInt, CanGC)
+#undef INSTANTIATE_ALLOC_NURSERY_CELL
+
+// Attempt to allocate a new cell in the nursery. If there is not enough room in
+// the nursery or there is an OOM, this method will return nullptr.
+template <JS::TraceKind kind, AllowGC allowGC>
+void* GCRuntime::tryNewNurseryCell(JSContext* cx, size_t thingSize,
+ AllocSite* site) {
+ MOZ_ASSERT(cx->isNurseryAllocAllowed());
+ MOZ_ASSERT(cx->zone() == site->zone());
+ MOZ_ASSERT(!cx->zone()->isAtomsZone());
+ MOZ_ASSERT(cx->zone()->allocKindInNursery(kind));
+
+ void* ptr = cx->nursery().allocateCell(site, thingSize, kind);
+ if (ptr) {
+ return ptr;
+ }
+
+ if constexpr (allowGC) {
+ if (!cx->suppressGC) {
+ cx->runtime()->gc.minorGC(JS::GCReason::OUT_OF_NURSERY);
+
+ // Exceeding gcMaxBytes while tenuring can disable the Nursery.
+ if (cx->zone()->allocKindInNursery(kind)) {
+ return cx->nursery().allocateCell(site, thingSize, kind);
+ }
+ }
+ }
+
+ return nullptr;
+}
+
+template <AllowGC allowGC /* = CanGC */>
+void* gc::CellAllocator::AllocTenuredCell(JSContext* cx, gc::AllocKind kind,
+ size_t size) {
+ MOZ_ASSERT(!cx->isHelperThreadContext());
+ MOZ_ASSERT(!IsNurseryAllocable(kind));
+ MOZ_ASSERT(size == Arena::thingSize(kind));
+ MOZ_ASSERT(
+ size >= gc::MinCellSize,
+ "All allocations must be at least the allocator-imposed minimum size.");
+
+ if (!cx->runtime()->gc.checkAllocatorState<allowGC>(cx, kind)) {
+ return nullptr;
+ }
+
+ return GCRuntime::tryNewTenuredThing<allowGC>(cx, kind, size);
+}
+template void* gc::CellAllocator::AllocTenuredCell<NoGC>(JSContext*, AllocKind,
+ size_t);
+template void* gc::CellAllocator::AllocTenuredCell<CanGC>(JSContext*, AllocKind,
+ size_t);
+
+template <AllowGC allowGC>
+/* static */
+void* GCRuntime::tryNewTenuredThing(JSContext* cx, AllocKind kind,
+ size_t thingSize) {
+ // Bump allocate in the arena's current free-list span.
+ Zone* zone = cx->zone();
+ void* ptr = zone->arenas.freeLists().allocate(kind);
+ if (MOZ_UNLIKELY(!ptr)) {
+ // Get the next available free list and allocate out of it. This may
+ // acquire a new arena, which will lock the chunk list. If there are no
+ // chunks available it may also allocate new memory directly.
+ ptr = refillFreeList(cx, kind);
+
+ if (MOZ_UNLIKELY(!ptr)) {
+ if constexpr (allowGC) {
+ cx->runtime()->gc.attemptLastDitchGC(cx);
+ ptr = tryNewTenuredThing<NoGC>(cx, kind, thingSize);
+ if (ptr) {
+ return ptr;
+ }
+ ReportOutOfMemory(cx);
+ }
+
+ return nullptr;
+ }
+ }
+
+#ifdef DEBUG
+ checkIncrementalZoneState(cx, ptr);
+#endif
+
+ gcprobes::TenuredAlloc(ptr, kind);
+
+ // We count this regardless of the profiler's state, assuming that it costs
+ // just as much to count it, as to check the profiler's state and decide not
+ // to count it.
+ zone->noteTenuredAlloc();
+
+ return ptr;
+}
+
+void GCRuntime::attemptLastDitchGC(JSContext* cx) {
+ // Either there was no memory available for a new chunk or the heap hit its
+ // size limit. Try to perform an all-compartments, non-incremental, shrinking
+ // GC and wait for it to finish.
+
+ if (!lastLastDitchTime.IsNull() &&
+ TimeStamp::Now() - lastLastDitchTime <= tunables.minLastDitchGCPeriod()) {
+ return;
+ }
+
+ JS::PrepareForFullGC(cx);
+ gc(JS::GCOptions::Shrink, JS::GCReason::LAST_DITCH);
+ waitBackgroundAllocEnd();
+ waitBackgroundFreeEnd();
+
+ lastLastDitchTime = mozilla::TimeStamp::Now();
+}
+
+template <AllowGC allowGC>
+bool GCRuntime::checkAllocatorState(JSContext* cx, AllocKind kind) {
+ MOZ_ASSERT_IF(cx->zone()->isAtomsZone(),
+ kind == AllocKind::ATOM || kind == AllocKind::FAT_INLINE_ATOM ||
+ kind == AllocKind::SYMBOL || kind == AllocKind::JITCODE ||
+ kind == AllocKind::SCOPE);
+ MOZ_ASSERT_IF(!cx->zone()->isAtomsZone(),
+ kind != AllocKind::ATOM && kind != AllocKind::FAT_INLINE_ATOM);
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+
+ if constexpr (allowGC) {
+ // Crash if we could perform a GC action when it is not safe.
+ if (!cx->suppressGC) {
+ cx->verifyIsSafeToGC();
+ }
+
+ gcIfNeededAtAllocation(cx);
+ }
+
+ // For testing out of memory conditions.
+ if (js::oom::ShouldFailWithOOM()) {
+ // If we are doing a fallible allocation, percolate up the OOM instead of
+ // reporting it.
+ if constexpr (allowGC) {
+ ReportOutOfMemory(cx);
+ }
+ return false;
+ }
+
+ return true;
+}
+
+inline void GCRuntime::gcIfNeededAtAllocation(JSContext* cx) {
+#ifdef JS_GC_ZEAL
+ if (needZealousGC()) {
+ runDebugGC();
+ }
+#endif
+
+ // Invoking the interrupt callback can fail and we can't usefully
+ // handle that here. Just check in case we need to collect instead.
+ if (cx->hasAnyPendingInterrupt()) {
+ gcIfRequested();
+ }
+}
+
+#ifdef DEBUG
+void GCRuntime::checkIncrementalZoneState(JSContext* cx, void* ptr) {
+ MOZ_ASSERT(ptr);
+ TenuredCell* cell = reinterpret_cast<TenuredCell*>(ptr);
+ TenuredChunkBase* chunk = detail::GetCellChunkBase(cell);
+ if (cx->zone()->isGCMarkingOrSweeping()) {
+ MOZ_ASSERT(chunk->markBits.isMarkedBlack(cell));
+ } else {
+ MOZ_ASSERT(!chunk->markBits.isMarkedAny(cell));
+ }
+}
+#endif
+
+void* js::gc::AllocateCellInGC(Zone* zone, AllocKind thingKind) {
+ void* ptr = zone->arenas.allocateFromFreeList(thingKind);
+ if (!ptr) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ ptr = GCRuntime::refillFreeListInGC(zone, thingKind);
+ if (!ptr) {
+ oomUnsafe.crash(ChunkSize, "Failed to allocate new chunk during GC");
+ }
+ }
+ return ptr;
+}
+
+// /////////// Arena -> Thing Allocator //////////////////////////////////////
+
+void GCRuntime::startBackgroundAllocTaskIfIdle() {
+ AutoLockHelperThreadState lock;
+ if (!allocTask.wasStarted(lock)) {
+ // Join the previous invocation of the task. This will return immediately
+ // if the thread has never been started.
+ allocTask.joinWithLockHeld(lock);
+ allocTask.startWithLockHeld(lock);
+ }
+}
+
+/* static */
+void* GCRuntime::refillFreeList(JSContext* cx, AllocKind thingKind) {
+ MOZ_ASSERT(cx->zone()->arenas.freeLists().isEmpty(thingKind));
+ MOZ_ASSERT(!cx->isHelperThreadContext());
+
+ // It should not be possible to allocate on the main thread while we are
+ // inside a GC.
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy(), "allocating while under GC");
+
+ return cx->zone()->arenas.refillFreeListAndAllocate(
+ thingKind, ShouldCheckThresholds::CheckThresholds);
+}
+
+/* static */
+void* GCRuntime::refillFreeListInGC(Zone* zone, AllocKind thingKind) {
+ // Called by compacting GC to refill a free list while we are in a GC.
+ MOZ_ASSERT(JS::RuntimeHeapIsCollecting());
+ MOZ_ASSERT_IF(!JS::RuntimeHeapIsMinorCollecting(),
+ !zone->runtimeFromMainThread()->gc.isBackgroundSweeping());
+
+ return zone->arenas.refillFreeListAndAllocate(
+ thingKind, ShouldCheckThresholds::DontCheckThresholds);
+}
+
+void* ArenaLists::refillFreeListAndAllocate(
+ AllocKind thingKind, ShouldCheckThresholds checkThresholds) {
+ MOZ_ASSERT(freeLists().isEmpty(thingKind));
+
+ JSRuntime* rt = runtimeFromAnyThread();
+
+ mozilla::Maybe<AutoLockGCBgAlloc> maybeLock;
+
+ // See if we can proceed without taking the GC lock.
+ if (concurrentUse(thingKind) != ConcurrentUse::None) {
+ maybeLock.emplace(rt);
+ }
+
+ Arena* arena = arenaList(thingKind).takeNextArena();
+ if (arena) {
+ // Empty arenas should be immediately freed.
+ MOZ_ASSERT(!arena->isEmpty());
+
+ return freeLists().setArenaAndAllocate(arena, thingKind);
+ }
+
+ // Parallel threads have their own ArenaLists, but chunks are shared;
+ // if we haven't already, take the GC lock now to avoid racing.
+ if (maybeLock.isNothing()) {
+ maybeLock.emplace(rt);
+ }
+
+ TenuredChunk* chunk = rt->gc.pickChunk(maybeLock.ref());
+ if (!chunk) {
+ return nullptr;
+ }
+
+ // Although our chunk should definitely have enough space for another arena,
+ // there are other valid reasons why TenuredChunk::allocateArena() may fail.
+ arena = rt->gc.allocateArena(chunk, zone_, thingKind, checkThresholds,
+ maybeLock.ref());
+ if (!arena) {
+ return nullptr;
+ }
+
+ ArenaList& al = arenaList(thingKind);
+ MOZ_ASSERT(al.isCursorAtEnd());
+ al.insertBeforeCursor(arena);
+
+ return freeLists().setArenaAndAllocate(arena, thingKind);
+}
+
+inline void* FreeLists::setArenaAndAllocate(Arena* arena, AllocKind kind) {
+#ifdef DEBUG
+ auto old = freeLists_[kind];
+ if (!old->isEmpty()) {
+ old->getArena()->checkNoMarkedFreeCells();
+ }
+#endif
+
+ FreeSpan* span = arena->getFirstFreeSpan();
+ freeLists_[kind] = span;
+
+ Zone* zone = arena->zone;
+ if (MOZ_UNLIKELY(zone->isGCMarkingOrSweeping())) {
+ arena->arenaAllocatedDuringGC();
+ }
+
+ TenuredCell* thing = span->allocate(Arena::thingSize(kind));
+ MOZ_ASSERT(thing); // This allocation is infallible.
+
+ return thing;
+}
+
+void Arena::arenaAllocatedDuringGC() {
+ // Ensure that anything allocated during the mark or sweep phases of an
+ // incremental GC will be marked black by pre-marking all free cells in the
+ // arena we are about to allocate from.
+
+ MOZ_ASSERT(zone->isGCMarkingOrSweeping());
+ for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) {
+ MOZ_ASSERT(!cell->isMarkedAny());
+ cell->markBlack();
+ }
+}
+
+// /////////// TenuredChunk -> Arena Allocator ///////////////////////////////
+
+bool GCRuntime::wantBackgroundAllocation(const AutoLockGC& lock) const {
+ // To minimize memory waste, we do not want to run the background chunk
+ // allocation if we already have some empty chunks or when the runtime has
+ // a small heap size (and therefore likely has a small growth rate).
+ return allocTask.enabled() &&
+ emptyChunks(lock).count() < minEmptyChunkCount(lock) &&
+ (fullChunks(lock).count() + availableChunks(lock).count()) >= 4;
+}
+
+Arena* GCRuntime::allocateArena(TenuredChunk* chunk, Zone* zone,
+ AllocKind thingKind,
+ ShouldCheckThresholds checkThresholds,
+ const AutoLockGC& lock) {
+ MOZ_ASSERT(chunk->hasAvailableArenas());
+
+ // Fail the allocation if we are over our heap size limits.
+ if ((checkThresholds != ShouldCheckThresholds::DontCheckThresholds) &&
+ (heapSize.bytes() >= tunables.gcMaxBytes())) {
+ return nullptr;
+ }
+
+ Arena* arena = chunk->allocateArena(this, zone, thingKind, lock);
+ zone->gcHeapSize.addGCArena(heapSize);
+
+ // Trigger an incremental slice if needed.
+ if (checkThresholds != ShouldCheckThresholds::DontCheckThresholds) {
+ maybeTriggerGCAfterAlloc(zone);
+ }
+
+ return arena;
+}
+
+Arena* TenuredChunk::allocateArena(GCRuntime* gc, Zone* zone,
+ AllocKind thingKind,
+ const AutoLockGC& lock) {
+ if (info.numArenasFreeCommitted == 0) {
+ commitOnePage(gc);
+ MOZ_ASSERT(info.numArenasFreeCommitted == ArenasPerPage);
+ }
+
+ MOZ_ASSERT(info.numArenasFreeCommitted > 0);
+ Arena* arena = fetchNextFreeArena(gc);
+
+ arena->init(zone, thingKind, lock);
+ updateChunkListAfterAlloc(gc, lock);
+
+ verify();
+
+ return arena;
+}
+
+template <size_t N>
+static inline size_t FindFirstBitSet(
+ const mozilla::BitSet<N, uint32_t>& bitset) {
+ MOZ_ASSERT(!bitset.IsEmpty());
+
+ const auto& words = bitset.Storage();
+ for (size_t i = 0; i < words.Length(); i++) {
+ uint32_t word = words[i];
+ if (word) {
+ return i * 32 + mozilla::CountTrailingZeroes32(word);
+ }
+ }
+
+ MOZ_CRASH("No bits found");
+}
+
+void TenuredChunk::commitOnePage(GCRuntime* gc) {
+ MOZ_ASSERT(info.numArenasFreeCommitted == 0);
+ MOZ_ASSERT(info.numArenasFree >= ArenasPerPage);
+
+ uint32_t pageIndex = FindFirstBitSet(decommittedPages);
+ MOZ_ASSERT(decommittedPages[pageIndex]);
+
+ if (DecommitEnabled()) {
+ MarkPagesInUseSoft(pageAddress(pageIndex), PageSize);
+ }
+
+ decommittedPages[pageIndex] = false;
+
+ for (size_t i = 0; i < ArenasPerPage; i++) {
+ size_t arenaIndex = pageIndex * ArenasPerPage + i;
+ MOZ_ASSERT(!freeCommittedArenas[arenaIndex]);
+ freeCommittedArenas[arenaIndex] = true;
+ arenas[arenaIndex].setAsNotAllocated();
+ ++info.numArenasFreeCommitted;
+ gc->updateOnArenaFree();
+ }
+
+ verify();
+}
+
+inline void GCRuntime::updateOnFreeArenaAlloc(const TenuredChunkInfo& info) {
+ MOZ_ASSERT(info.numArenasFreeCommitted <= numArenasFreeCommitted);
+ --numArenasFreeCommitted;
+}
+
+Arena* TenuredChunk::fetchNextFreeArena(GCRuntime* gc) {
+ MOZ_ASSERT(info.numArenasFreeCommitted > 0);
+ MOZ_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree);
+
+ size_t index = FindFirstBitSet(freeCommittedArenas);
+ MOZ_ASSERT(freeCommittedArenas[index]);
+
+ freeCommittedArenas[index] = false;
+ --info.numArenasFreeCommitted;
+ --info.numArenasFree;
+ gc->updateOnFreeArenaAlloc(info);
+
+ return &arenas[index];
+}
+
+// /////////// System -> TenuredChunk Allocator //////////////////////////////
+
+TenuredChunk* GCRuntime::getOrAllocChunk(AutoLockGCBgAlloc& lock) {
+ TenuredChunk* chunk = emptyChunks(lock).pop();
+ if (chunk) {
+ // Reinitialize ChunkBase; arenas are all free and may or may not be
+ // committed.
+ SetMemCheckKind(chunk, sizeof(ChunkBase), MemCheckKind::MakeUndefined);
+ chunk->initBase(rt, nullptr);
+ MOZ_ASSERT(chunk->unused());
+ } else {
+ void* ptr = TenuredChunk::allocate(this);
+ if (!ptr) {
+ return nullptr;
+ }
+
+ chunk = TenuredChunk::emplace(ptr, this, /* allMemoryCommitted = */ true);
+ MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0);
+ }
+
+ if (wantBackgroundAllocation(lock)) {
+ lock.tryToStartBackgroundAllocation();
+ }
+
+ return chunk;
+}
+
+void GCRuntime::recycleChunk(TenuredChunk* chunk, const AutoLockGC& lock) {
+#ifdef DEBUG
+ MOZ_ASSERT(chunk->unused());
+ chunk->verify();
+#endif
+
+ // Poison ChunkBase to catch use after free.
+ AlwaysPoison(chunk, JS_FREED_CHUNK_PATTERN, sizeof(ChunkBase),
+ MemCheckKind::MakeNoAccess);
+
+ emptyChunks(lock).push(chunk);
+}
+
+TenuredChunk* GCRuntime::pickChunk(AutoLockGCBgAlloc& lock) {
+ if (availableChunks(lock).count()) {
+ return availableChunks(lock).head();
+ }
+
+ TenuredChunk* chunk = getOrAllocChunk(lock);
+ if (!chunk) {
+ return nullptr;
+ }
+
+#ifdef DEBUG
+ chunk->verify();
+ MOZ_ASSERT(chunk->unused());
+ MOZ_ASSERT(!fullChunks(lock).contains(chunk));
+ MOZ_ASSERT(!availableChunks(lock).contains(chunk));
+#endif
+
+ availableChunks(lock).push(chunk);
+
+ return chunk;
+}
+
+BackgroundAllocTask::BackgroundAllocTask(GCRuntime* gc, ChunkPool& pool)
+ : GCParallelTask(gc, gcstats::PhaseKind::NONE),
+ chunkPool_(pool),
+ enabled_(CanUseExtraThreads() && GetCPUCount() >= 2) {
+ // This can occur outside GCs so doesn't have a stats phase.
+}
+
+void BackgroundAllocTask::run(AutoLockHelperThreadState& lock) {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ AutoLockGC gcLock(gc);
+ while (!isCancelled() && gc->wantBackgroundAllocation(gcLock)) {
+ TenuredChunk* chunk;
+ {
+ AutoUnlockGC unlock(gcLock);
+ void* ptr = TenuredChunk::allocate(gc);
+ if (!ptr) {
+ break;
+ }
+ chunk = TenuredChunk::emplace(ptr, gc, /* allMemoryCommitted = */ true);
+ }
+ chunkPool_.ref().push(chunk);
+ }
+}
+
+/* static */
+void* TenuredChunk::allocate(GCRuntime* gc) {
+ void* chunk = MapAlignedPages(ChunkSize, ChunkSize);
+ if (!chunk) {
+ return nullptr;
+ }
+
+ gc->stats().count(gcstats::COUNT_NEW_CHUNK);
+ return chunk;
+}
+
+static inline bool ShouldDecommitNewChunk(bool allMemoryCommitted,
+ const GCSchedulingState& state) {
+ if (!DecommitEnabled()) {
+ return false;
+ }
+
+ return !allMemoryCommitted || !state.inHighFrequencyGCMode();
+}
+
+TenuredChunk* TenuredChunk::emplace(void* ptr, GCRuntime* gc,
+ bool allMemoryCommitted) {
+ /* The chunk may still have some regions marked as no-access. */
+ MOZ_MAKE_MEM_UNDEFINED(ptr, ChunkSize);
+
+ /*
+ * Poison the chunk. Note that decommitAllArenas() below will mark the
+ * arenas as inaccessible (for memory sanitizers).
+ */
+ Poison(ptr, JS_FRESH_TENURED_PATTERN, ChunkSize, MemCheckKind::MakeUndefined);
+
+ TenuredChunk* chunk = new (mozilla::KnownNotNull, ptr) TenuredChunk(gc->rt);
+
+ if (ShouldDecommitNewChunk(allMemoryCommitted, gc->schedulingState)) {
+ // Decommit the arenas. We do this after poisoning so that if the OS does
+ // not have to recycle the pages, we still get the benefit of poisoning.
+ chunk->decommitAllArenas();
+ } else {
+ // The chunk metadata is initialized as decommitted regardless, to avoid
+ // having to initialize the arenas at this time.
+ chunk->initAsDecommitted();
+ }
+
+ chunk->verify();
+
+ return chunk;
+}
+
+void TenuredChunk::decommitAllArenas() {
+ MOZ_ASSERT(unused());
+ MarkPagesUnusedSoft(&arenas[0], ArenasPerChunk * ArenaSize);
+ initAsDecommitted();
+}
+
+void TenuredChunkBase::initAsDecommitted() {
+ // Set the state of all arenas to free and decommitted. They might not
+ // actually be decommitted, but in that case the re-commit operation is a
+ // no-op so it doesn't matter.
+ decommittedPages.SetAll();
+ freeCommittedArenas.ResetAll();
+ info.numArenasFree = ArenasPerChunk;
+ info.numArenasFreeCommitted = 0;
+}
diff --git a/js/src/gc/Allocator.h b/js/src/gc/Allocator.h
new file mode 100644
index 0000000000..1cd3ff6aad
--- /dev/null
+++ b/js/src/gc/Allocator.h
@@ -0,0 +1,173 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Allocator_h
+#define gc_Allocator_h
+
+#include "mozilla/OperatorNewExtensions.h"
+
+#include <stdint.h>
+
+#include "gc/AllocKind.h"
+#include "gc/Cell.h"
+#include "js/Class.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+
+// [SMDOC] AllowGC template parameter
+//
+// AllowGC is a template parameter for functions that support both with and
+// without GC operation.
+//
+// The CanGC variant of the function can trigger a garbage collection, and
+// should set a pending exception on failure.
+//
+// The NoGC variant of the function cannot trigger a garbage collection, and
+// should not set any pending exception on failure. This variant can be called
+// in fast paths where the caller has unrooted pointers. The failure means we
+// need to perform GC to allocate an object. The caller can fall back to a slow
+// path that roots pointers before calling a CanGC variant of the function,
+// without having to clear a pending exception.
+enum AllowGC { NoGC = 0, CanGC = 1 };
+
+namespace gc {
+
+class AllocSite;
+struct Cell;
+class TenuredCell;
+
+// Allocator implementation functions. SpiderMonkey code outside this file
+// should use:
+//
+// cx->newCell<T>(...)
+//
+// or optionally:
+//
+// cx->newCell<T, AllowGC::NoGC>(...)
+//
+// `friend` js::gc::CellAllocator in a subtype T of Cell in order to allow it to
+// be allocated with cx->newCell<T>(...). The friend declaration will allow
+// calling T's constructor.
+//
+// The parameters will be passed to a type-specific function or constructor. For
+// nursery-allocatable types, see e.g. the NewString, NewObject, and NewBigInt
+// methods. For all other types, the parameters will be forwarded to the
+// constructor.
+class CellAllocator {
+ public:
+ template <typename T, js::AllowGC allowGC = CanGC, typename... Args>
+ static T* NewCell(JSContext* cx, Args&&... args);
+
+ private:
+ // Allocate a cell in the nursery, unless |heap| is Heap::Tenured or nursery
+ // allocation is disabled for |traceKind| in the current zone.
+ template <JS::TraceKind traceKind, AllowGC allowGC = CanGC>
+ static void* AllocNurseryOrTenuredCell(JSContext* cx, gc::AllocKind allocKind,
+ gc::Heap heap, AllocSite* site);
+
+ // Allocate a cell in the tenured heap.
+ template <AllowGC allowGC = CanGC>
+ static void* AllocTenuredCell(JSContext* cx, gc::AllocKind kind, size_t size);
+
+ // Allocate a string. Use cx->newCell<T>([heap]).
+ //
+ // Use for nursery-allocatable strings. Returns a value cast to the correct
+ // type. Non-nursery-allocatable strings will go through the fallback
+ // tenured-only allocation path.
+ template <typename T, AllowGC allowGC = CanGC, typename... Args>
+ static T* NewString(JSContext* cx, gc::Heap heap, Args&&... args) {
+ static_assert(std::is_base_of_v<JSString, T>);
+ gc::AllocKind kind = gc::MapTypeToAllocKind<T>::kind;
+ void* ptr = AllocNurseryOrTenuredCell<JS::TraceKind::String, allowGC>(
+ cx, kind, heap, nullptr);
+ if (!ptr) {
+ return nullptr;
+ }
+ return new (mozilla::KnownNotNull, ptr) T(std::forward<Args>(args)...);
+ }
+
+ template <typename T, AllowGC allowGC /* = CanGC */>
+ static T* NewBigInt(JSContext* cx, Heap heap) {
+ void* ptr = AllocNurseryOrTenuredCell<JS::TraceKind::BigInt, allowGC>(
+ cx, gc::AllocKind::BIGINT, heap, nullptr);
+ if (ptr) {
+ return new (mozilla::KnownNotNull, ptr) T();
+ }
+ return nullptr;
+ }
+
+ template <typename T, AllowGC allowGC = CanGC>
+ static T* NewObject(JSContext* cx, gc::AllocKind kind, gc::Heap heap,
+ const JSClass* clasp, gc::AllocSite* site = nullptr) {
+ MOZ_ASSERT(IsObjectAllocKind(kind));
+ MOZ_ASSERT_IF(heap != gc::Heap::Tenured && clasp->hasFinalize() &&
+ !clasp->isProxyObject(),
+ CanNurseryAllocateFinalizedClass(clasp));
+ void* cell = AllocNurseryOrTenuredCell<JS::TraceKind::Object, allowGC>(
+ cx, kind, heap, site);
+ if (!cell) {
+ return nullptr;
+ }
+ return new (mozilla::KnownNotNull, cell) T();
+ }
+
+ // Allocate all other kinds of GC thing.
+ template <typename T, AllowGC allowGC = CanGC, typename... Args>
+ static T* NewTenuredCell(JSContext* cx, Args&&... args) {
+ gc::AllocKind kind = gc::MapTypeToAllocKind<T>::kind;
+ void* cell = AllocTenuredCell<allowGC>(cx, kind, sizeof(T));
+ if (!cell) {
+ return nullptr;
+ }
+ return new (mozilla::KnownNotNull, cell) T(std::forward<Args>(args)...);
+ }
+};
+
+} // namespace gc
+
+// This is the entry point for all allocation, though callers should still not
+// use this directly. Use cx->newCell<T>(...) instead.
+//
+// After a successful allocation the caller must fully initialize the thing
+// before calling any function that can potentially trigger GC. This will
+// ensure that GC tracing never sees junk values stored in the partially
+// initialized thing.
+template <typename T, AllowGC allowGC, typename... Args>
+T* gc::CellAllocator::NewCell(JSContext* cx, Args&&... args) {
+ static_assert(std::is_base_of_v<gc::Cell, T>);
+
+ // Objects. See the valid parameter list in NewObject, above.
+ if constexpr (std::is_base_of_v<JSObject, T>) {
+ return NewObject<T, allowGC>(cx, std::forward<Args>(args)...);
+ }
+
+ // BigInt
+ else if constexpr (std::is_base_of_v<JS::BigInt, T>) {
+ return NewBigInt<T, allowGC>(cx, std::forward<Args>(args)...);
+ }
+
+ // "Normal" strings (all of which can be nursery allocated). Atoms and
+ // external strings will fall through to the generic code below. All other
+ // strings go through NewString, which will forward the arguments to the
+ // appropriate string class's constructor.
+ else if constexpr (std::is_base_of_v<JSString, T> &&
+ !std::is_base_of_v<JSAtom, T> &&
+ !std::is_base_of_v<JSExternalString, T>) {
+ return NewString<T, allowGC>(cx, std::forward<Args>(args)...);
+ }
+
+ else {
+ // Allocate a new tenured GC thing that's not nursery-allocatable. Use
+ // cx->newCell<T>(...), where the parameters are forwarded to the type's
+ // constructor.
+ return NewTenuredCell<T, allowGC>(cx, std::forward<Args>(args)...);
+ }
+}
+
+} // namespace js
+
+#endif // gc_Allocator_h
diff --git a/js/src/gc/ArenaList-inl.h b/js/src/gc/ArenaList-inl.h
new file mode 100644
index 0000000000..92925a227c
--- /dev/null
+++ b/js/src/gc/ArenaList-inl.h
@@ -0,0 +1,327 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_ArenaList_inl_h
+#define gc_ArenaList_inl_h
+
+#include "gc/ArenaList.h"
+
+#include "gc/Heap.h"
+#include "gc/Zone.h"
+
+void js::gc::SortedArenaListSegment::append(Arena* arena) {
+ MOZ_ASSERT(arena);
+ MOZ_ASSERT_IF(head, head->getAllocKind() == arena->getAllocKind());
+ *tailp = arena;
+ tailp = &arena->next;
+}
+
+inline js::gc::ArenaList::ArenaList() { clear(); }
+
+inline js::gc::ArenaList::ArenaList(ArenaList&& other) { moveFrom(other); }
+
+inline js::gc::ArenaList::~ArenaList() { MOZ_ASSERT(isEmpty()); }
+
+void js::gc::ArenaList::moveFrom(ArenaList& other) {
+ other.check();
+
+ head_ = other.head_;
+ cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_;
+ other.clear();
+
+ check();
+}
+
+js::gc::ArenaList& js::gc::ArenaList::operator=(ArenaList&& other) {
+ MOZ_ASSERT(isEmpty());
+ moveFrom(other);
+ return *this;
+}
+
+inline js::gc::ArenaList::ArenaList(const SortedArenaListSegment& segment) {
+ head_ = segment.head;
+ cursorp_ = segment.isEmpty() ? &head_ : segment.tailp;
+ check();
+}
+
+// This does checking just of |head_| and |cursorp_|.
+void js::gc::ArenaList::check() const {
+#ifdef DEBUG
+ // If the list is empty, it must have this form.
+ MOZ_ASSERT_IF(!head_, cursorp_ == &head_);
+
+ // If there's an arena following the cursor, it must not be full.
+ Arena* cursor = *cursorp_;
+ MOZ_ASSERT_IF(cursor, cursor->hasFreeThings());
+#endif
+}
+
+void js::gc::ArenaList::clear() {
+ head_ = nullptr;
+ cursorp_ = &head_;
+ check();
+}
+
+bool js::gc::ArenaList::isEmpty() const {
+ check();
+ return !head_;
+}
+
+js::gc::Arena* js::gc::ArenaList::head() const {
+ check();
+ return head_;
+}
+
+bool js::gc::ArenaList::isCursorAtHead() const {
+ check();
+ return cursorp_ == &head_;
+}
+
+bool js::gc::ArenaList::isCursorAtEnd() const {
+ check();
+ return !*cursorp_;
+}
+
+js::gc::Arena* js::gc::ArenaList::arenaAfterCursor() const {
+ check();
+ return *cursorp_;
+}
+
+js::gc::Arena* js::gc::ArenaList::takeNextArena() {
+ check();
+ Arena* arena = *cursorp_;
+ if (!arena) {
+ return nullptr;
+ }
+ cursorp_ = &arena->next;
+ check();
+ return arena;
+}
+
+void js::gc::ArenaList::insertAtCursor(Arena* a) {
+ check();
+ a->next = *cursorp_;
+ *cursorp_ = a;
+ // At this point, the cursor is sitting before |a|. Move it after |a|
+ // if necessary.
+ if (!a->hasFreeThings()) {
+ cursorp_ = &a->next;
+ }
+ check();
+}
+
+void js::gc::ArenaList::insertBeforeCursor(Arena* a) {
+ check();
+ a->next = *cursorp_;
+ *cursorp_ = a;
+ cursorp_ = &a->next;
+ check();
+}
+
+js::gc::ArenaList& js::gc::ArenaList::insertListWithCursorAtEnd(
+ ArenaList& other) {
+ check();
+ other.check();
+ MOZ_ASSERT(other.isCursorAtEnd());
+
+ if (other.isEmpty()) {
+ return *this;
+ }
+
+ // Insert the full arenas of |other| after those of |this|.
+ *other.cursorp_ = *cursorp_;
+ *cursorp_ = other.head_;
+ cursorp_ = other.cursorp_;
+ check();
+
+ other.clear();
+ return *this;
+}
+
+js::gc::Arena* js::gc::ArenaList::takeFirstArena() {
+ check();
+ Arena* arena = head_;
+ if (!arena) {
+ return nullptr;
+ }
+
+ head_ = arena->next;
+ if (cursorp_ == &arena->next) {
+ cursorp_ = &head_;
+ }
+
+ check();
+ return arena;
+}
+
+js::gc::SortedArenaList::SortedArenaList(size_t thingsPerArena) {
+ reset(thingsPerArena);
+}
+
+void js::gc::SortedArenaList::setThingsPerArena(size_t thingsPerArena) {
+ MOZ_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena);
+ thingsPerArena_ = thingsPerArena;
+}
+
+void js::gc::SortedArenaList::reset(size_t thingsPerArena) {
+ setThingsPerArena(thingsPerArena);
+ // Initialize the segments.
+ for (size_t i = 0; i <= thingsPerArena; ++i) {
+ segments[i].clear();
+ }
+}
+
+void js::gc::SortedArenaList::insertAt(Arena* arena, size_t nfree) {
+ MOZ_ASSERT(nfree <= thingsPerArena_);
+ segments[nfree].append(arena);
+}
+
+void js::gc::SortedArenaList::extractEmpty(Arena** empty) {
+ SortedArenaListSegment& segment = segments[thingsPerArena_];
+ if (segment.head) {
+ *segment.tailp = *empty;
+ *empty = segment.head;
+ segment.clear();
+ }
+}
+
+js::gc::ArenaList js::gc::SortedArenaList::toArenaList() {
+ // Link the non-empty segment tails up to the non-empty segment heads.
+ size_t tailIndex = 0;
+ for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) {
+ if (headAt(headIndex)) {
+ segments[tailIndex].linkTo(headAt(headIndex));
+ tailIndex = headIndex;
+ }
+ }
+ // Point the tail of the final non-empty segment at null. Note that if
+ // the list is empty, this will just set segments[0].head to null.
+ segments[tailIndex].linkTo(nullptr);
+ // Create an ArenaList with head and cursor set to the head and tail of
+ // the first segment (if that segment is empty, only the head is used).
+ return ArenaList(segments[0]);
+}
+
+#ifdef DEBUG
+
+bool js::gc::FreeLists::allEmpty() const {
+ for (auto i : AllAllocKinds()) {
+ if (!isEmpty(i)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool js::gc::FreeLists::isEmpty(AllocKind kind) const {
+ return freeLists_[kind]->isEmpty();
+}
+
+#endif
+
+void js::gc::FreeLists::clear() {
+ for (auto i : AllAllocKinds()) {
+#ifdef DEBUG
+ auto old = freeLists_[i];
+ if (!old->isEmpty()) {
+ old->getArena()->checkNoMarkedFreeCells();
+ }
+#endif
+ freeLists_[i] = &emptySentinel;
+ }
+}
+
+js::gc::TenuredCell* js::gc::FreeLists::allocate(AllocKind kind) {
+ return freeLists_[kind]->allocate(Arena::thingSize(kind));
+}
+
+void js::gc::FreeLists::unmarkPreMarkedFreeCells(AllocKind kind) {
+ FreeSpan* freeSpan = freeLists_[kind];
+ if (!freeSpan->isEmpty()) {
+ freeSpan->getArena()->unmarkPreMarkedFreeCells();
+ }
+}
+
+JSRuntime* js::gc::ArenaLists::runtime() {
+ return zone_->runtimeFromMainThread();
+}
+
+JSRuntime* js::gc::ArenaLists::runtimeFromAnyThread() {
+ return zone_->runtimeFromAnyThread();
+}
+
+js::gc::Arena* js::gc::ArenaLists::getFirstArena(AllocKind thingKind) const {
+ return arenaList(thingKind).head();
+}
+
+js::gc::Arena* js::gc::ArenaLists::getFirstCollectingArena(
+ AllocKind thingKind) const {
+ return collectingArenaList(thingKind).head();
+}
+
+js::gc::Arena* js::gc::ArenaLists::getFirstSweptArena(
+ AllocKind thingKind) const {
+ if (thingKind != incrementalSweptArenaKind.ref()) {
+ return nullptr;
+ }
+ return incrementalSweptArenas.ref().head();
+}
+
+js::gc::Arena* js::gc::ArenaLists::getArenaAfterCursor(
+ AllocKind thingKind) const {
+ return arenaList(thingKind).arenaAfterCursor();
+}
+
+bool js::gc::ArenaLists::arenaListsAreEmpty() const {
+ for (auto i : AllAllocKinds()) {
+ /*
+ * The arena cannot be empty if the background finalization is not yet
+ * done.
+ */
+ if (concurrentUse(i) == ConcurrentUse::BackgroundFinalize) {
+ return false;
+ }
+ if (!arenaList(i).isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool js::gc::ArenaLists::doneBackgroundFinalize(AllocKind kind) const {
+ return concurrentUse(kind) != ConcurrentUse::BackgroundFinalize;
+}
+
+bool js::gc::ArenaLists::needBackgroundFinalizeWait(AllocKind kind) const {
+ return concurrentUse(kind) == ConcurrentUse::BackgroundFinalize;
+}
+
+void js::gc::ArenaLists::clearFreeLists() { freeLists().clear(); }
+
+MOZ_ALWAYS_INLINE js::gc::TenuredCell* js::gc::ArenaLists::allocateFromFreeList(
+ AllocKind thingKind) {
+ return freeLists().allocate(thingKind);
+}
+
+void js::gc::ArenaLists::unmarkPreMarkedFreeCells() {
+ for (auto i : AllAllocKinds()) {
+ freeLists().unmarkPreMarkedFreeCells(i);
+ }
+}
+
+void js::gc::ArenaLists::checkEmptyFreeLists() {
+ MOZ_ASSERT(freeLists().allEmpty());
+}
+
+void js::gc::ArenaLists::checkEmptyArenaLists() {
+#ifdef DEBUG
+ for (auto i : AllAllocKinds()) {
+ checkEmptyArenaList(i);
+ }
+#endif
+}
+
+#endif // gc_ArenaList_inl_h
diff --git a/js/src/gc/ArenaList.h b/js/src/gc/ArenaList.h
new file mode 100644
index 0000000000..4ae4afde71
--- /dev/null
+++ b/js/src/gc/ArenaList.h
@@ -0,0 +1,386 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definitions of ArenaList and associated heap data structures.
+ */
+
+#ifndef gc_ArenaList_h
+#define gc_ArenaList_h
+
+#include "gc/AllocKind.h"
+#include "js/GCAPI.h"
+#include "js/HeapAPI.h"
+#include "js/TypeDecls.h"
+#include "threading/ProtectedData.h"
+
+namespace js {
+
+class Nursery;
+class SliceBudget;
+
+namespace gcstats {
+struct Statistics;
+}
+
+namespace gc {
+
+class Arena;
+class BackgroundUnmarkTask;
+struct FinalizePhase;
+class FreeSpan;
+class TenuredCell;
+class TenuringTracer;
+
+/*
+ * A single segment of a SortedArenaList. Each segment has a head and a tail,
+ * which track the start and end of a segment for O(1) append and concatenation.
+ */
+struct SortedArenaListSegment {
+ Arena* head;
+ Arena** tailp;
+
+ void clear() {
+ head = nullptr;
+ tailp = &head;
+ }
+
+ bool isEmpty() const { return tailp == &head; }
+
+ // Appends |arena| to this segment.
+ inline void append(Arena* arena);
+
+ // Points the tail of this segment at |arena|, which may be null. Note
+ // that this does not change the tail itself, but merely which arena
+ // follows it. This essentially turns the tail into a cursor (see also the
+ // description of ArenaList), but from the perspective of a SortedArenaList
+ // this makes no difference.
+ void linkTo(Arena* arena) { *tailp = arena; }
+};
+
+/*
+ * Arena lists contain a singly linked lists of arenas starting from a head
+ * pointer.
+ *
+ * They also have a cursor, which conceptually lies on arena boundaries,
+ * i.e. before the first arena, between two arenas, or after the last arena.
+ *
+ * Arenas are usually sorted in order of increasing free space, with the cursor
+ * following the Arena currently being allocated from. This ordering should not
+ * be treated as an invariant, however, as the free lists may be cleared,
+ * leaving arenas previously used for allocation partially full. Sorting order
+ * is restored during sweeping.
+ *
+ * Arenas following the cursor should not be full.
+ */
+class ArenaList {
+ // The cursor is implemented via an indirect pointer, |cursorp_|, to allow
+ // for efficient list insertion at the cursor point and other list
+ // manipulations.
+ //
+ // - If the list is empty: |head| is null, |cursorp_| points to |head|, and
+ // therefore |*cursorp_| is null.
+ //
+ // - If the list is not empty: |head| is non-null, and...
+ //
+ // - If the cursor is at the start of the list: |cursorp_| points to
+ // |head|, and therefore |*cursorp_| points to the first arena.
+ //
+ // - If cursor is at the end of the list: |cursorp_| points to the |next|
+ // field of the last arena, and therefore |*cursorp_| is null.
+ //
+ // - If the cursor is at neither the start nor the end of the list:
+ // |cursorp_| points to the |next| field of the arena preceding the
+ // cursor, and therefore |*cursorp_| points to the arena following the
+ // cursor.
+ //
+ // |cursorp_| is never null.
+ //
+ Arena* head_;
+ Arena** cursorp_;
+
+ // Transfers the contents of |other| to this list and clears |other|.
+ inline void moveFrom(ArenaList& other);
+
+ public:
+ inline ArenaList();
+ inline ArenaList(ArenaList&& other);
+ inline ~ArenaList();
+
+ inline ArenaList& operator=(ArenaList&& other);
+
+ // It doesn't make sense for arenas to be present in more than one list, so
+ // list copy operations are not provided.
+ ArenaList(const ArenaList& other) = delete;
+ ArenaList& operator=(const ArenaList& other) = delete;
+
+ inline explicit ArenaList(const SortedArenaListSegment& segment);
+
+ inline void check() const;
+
+ inline void clear();
+ inline bool isEmpty() const;
+
+ // This returns nullptr if the list is empty.
+ inline Arena* head() const;
+
+ inline bool isCursorAtHead() const;
+ inline bool isCursorAtEnd() const;
+
+ // This can return nullptr.
+ inline Arena* arenaAfterCursor() const;
+
+ // This returns the arena after the cursor and moves the cursor past it.
+ inline Arena* takeNextArena();
+
+ // This does two things.
+ // - Inserts |a| at the cursor.
+ // - Leaves the cursor sitting just before |a|, if |a| is not full, or just
+ // after |a|, if |a| is full.
+ inline void insertAtCursor(Arena* a);
+
+ // Inserts |a| at the cursor, then moves the cursor past it.
+ inline void insertBeforeCursor(Arena* a);
+
+ // This inserts the contents of |other|, which must be full, at the cursor of
+ // |this| and clears |other|.
+ inline ArenaList& insertListWithCursorAtEnd(ArenaList& other);
+
+ inline Arena* takeFirstArena();
+
+ Arena* removeRemainingArenas(Arena** arenap);
+ Arena** pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut);
+ Arena* relocateArenas(Arena* toRelocate, Arena* relocated,
+ js::SliceBudget& sliceBudget,
+ gcstats::Statistics& stats);
+
+#ifdef DEBUG
+ void dump();
+#endif
+};
+
+/*
+ * A class that holds arenas in sorted order by appending arenas to specific
+ * segments. Each segment has a head and a tail, which can be linked up to
+ * other segments to create a contiguous ArenaList.
+ */
+class SortedArenaList {
+ public:
+ // The minimum size, in bytes, of a GC thing.
+ static const size_t MinThingSize = 16;
+
+ static_assert(ArenaSize <= 4096,
+ "When increasing the Arena size, please consider how"
+ " this will affect the size of a SortedArenaList.");
+
+ static_assert(MinThingSize >= 16,
+ "When decreasing the minimum thing size, please consider"
+ " how this will affect the size of a SortedArenaList.");
+
+ private:
+ // The maximum number of GC things that an arena can hold.
+ static const size_t MaxThingsPerArena =
+ (ArenaSize - ArenaHeaderSize) / MinThingSize;
+
+ size_t thingsPerArena_;
+ SortedArenaListSegment segments[MaxThingsPerArena + 1];
+
+ // Convenience functions to get the nth head and tail.
+ Arena* headAt(size_t n) { return segments[n].head; }
+ Arena** tailAt(size_t n) { return segments[n].tailp; }
+
+ public:
+ inline explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena);
+
+ inline void setThingsPerArena(size_t thingsPerArena);
+
+ // Resets the first |thingsPerArena| segments of this list for further use.
+ inline void reset(size_t thingsPerArena = MaxThingsPerArena);
+
+ // Inserts an arena, which has room for |nfree| more things, in its segment.
+ inline void insertAt(Arena* arena, size_t nfree);
+
+ // Remove all empty arenas, inserting them as a linked list.
+ inline void extractEmpty(Arena** empty);
+
+ // Links up the tail of each non-empty segment to the head of the next
+ // non-empty segment, creating a contiguous list that is returned as an
+ // ArenaList. This is not a destructive operation: neither the head nor tail
+ // of any segment is modified. However, note that the Arenas in the
+ // resulting ArenaList should be treated as read-only unless the
+ // SortedArenaList is no longer needed: inserting or removing arenas would
+ // invalidate the SortedArenaList.
+ inline ArenaList toArenaList();
+};
+
+enum class ShouldCheckThresholds {
+ DontCheckThresholds = 0,
+ CheckThresholds = 1
+};
+
+// For each arena kind its free list is represented as the first span with free
+// things. Initially all the spans are initialized as empty. After we find a new
+// arena with available things we move its first free span into the list and set
+// the arena as fully allocated. That way we do not need to update the arena
+// after the initial allocation. When starting the GC we only move the head of
+// the of the list of spans back to the arena only for the arena that was not
+// fully allocated.
+class FreeLists {
+ AllAllocKindArray<FreeSpan*> freeLists_;
+
+ public:
+ // Because the JITs can allocate from the free lists, they cannot be null.
+ // We use a placeholder FreeSpan that is empty (and wihout an associated
+ // Arena) so the JITs can fall back gracefully.
+ static FreeSpan emptySentinel;
+
+ FreeLists();
+
+#ifdef DEBUG
+ inline bool allEmpty() const;
+ inline bool isEmpty(AllocKind kind) const;
+#endif
+
+ inline void clear();
+
+ MOZ_ALWAYS_INLINE TenuredCell* allocate(AllocKind kind);
+
+ inline void* setArenaAndAllocate(Arena* arena, AllocKind kind);
+
+ inline void unmarkPreMarkedFreeCells(AllocKind kind);
+
+ FreeSpan** addressOfFreeList(AllocKind thingKind) {
+ return &freeLists_[thingKind];
+ }
+};
+
+class ArenaLists {
+ enum class ConcurrentUse : uint32_t { None, BackgroundFinalize };
+
+ using ConcurrentUseState =
+ mozilla::Atomic<ConcurrentUse, mozilla::SequentiallyConsistent>;
+
+ JS::Zone* zone_;
+
+ // Whether this structure can be accessed by other threads.
+ UnprotectedData<AllAllocKindArray<ConcurrentUseState>> concurrentUseState_;
+
+ MainThreadData<FreeLists> freeLists_;
+
+ /* The main list of arenas for each alloc kind. */
+ MainThreadOrGCTaskData<AllAllocKindArray<ArenaList>> arenaLists_;
+
+ /*
+ * Arenas which are currently being collected. The collector can move arenas
+ * from arenaLists_ here and back again at various points in collection.
+ */
+ MainThreadOrGCTaskData<AllAllocKindArray<ArenaList>> collectingArenaLists_;
+
+ /* During incremental sweeping, a list of the arenas already swept. */
+ MainThreadOrGCTaskData<AllocKind> incrementalSweptArenaKind;
+ MainThreadOrGCTaskData<ArenaList> incrementalSweptArenas;
+
+ // Arena lists which have yet to be swept, but need additional foreground
+ // processing before they are swept.
+ MainThreadData<Arena*> gcCompactPropMapArenasToUpdate;
+ MainThreadData<Arena*> gcNormalPropMapArenasToUpdate;
+
+ // The list of empty arenas which are collected during the sweep phase and
+ // released at the end of sweeping every sweep group.
+ MainThreadOrGCTaskData<Arena*> savedEmptyArenas;
+
+ public:
+ explicit ArenaLists(JS::Zone* zone);
+ ~ArenaLists();
+
+ FreeLists& freeLists() { return freeLists_.ref(); }
+ const FreeLists& freeLists() const { return freeLists_.ref(); }
+
+ FreeSpan** addressOfFreeList(AllocKind thingKind) {
+ return freeLists_.refNoCheck().addressOfFreeList(thingKind);
+ }
+
+ inline Arena* getFirstArena(AllocKind thingKind) const;
+ inline Arena* getFirstCollectingArena(AllocKind thingKind) const;
+ inline Arena* getFirstSweptArena(AllocKind thingKind) const;
+ inline Arena* getArenaAfterCursor(AllocKind thingKind) const;
+
+ inline bool arenaListsAreEmpty() const;
+
+ inline bool doneBackgroundFinalize(AllocKind kind) const;
+ inline bool needBackgroundFinalizeWait(AllocKind kind) const;
+
+ /* Clear the free lists so we won't try to allocate from swept arenas. */
+ inline void clearFreeLists();
+
+ inline void unmarkPreMarkedFreeCells();
+
+ MOZ_ALWAYS_INLINE TenuredCell* allocateFromFreeList(AllocKind thingKind);
+
+ inline void checkEmptyFreeLists();
+ inline void checkEmptyArenaLists();
+ inline void checkEmptyFreeList(AllocKind kind);
+
+ void checkEmptyArenaList(AllocKind kind);
+
+ bool relocateArenas(Arena*& relocatedListOut, JS::GCReason reason,
+ js::SliceBudget& sliceBudget, gcstats::Statistics& stats);
+
+ void queueForegroundObjectsForSweep(JS::GCContext* gcx);
+ void queueForegroundThingsForSweep();
+
+ Arena* takeSweptEmptyArenas();
+
+ void setIncrementalSweptArenas(AllocKind kind, SortedArenaList& arenas);
+ void clearIncrementalSweptArenas();
+
+ void mergeFinalizedArenas(AllocKind thingKind,
+ SortedArenaList& finalizedArenas);
+
+ void moveArenasToCollectingLists();
+ void mergeArenasFromCollectingLists();
+
+ void checkGCStateNotInUse();
+ void checkSweepStateNotInUse();
+ void checkNoArenasToUpdate();
+ void checkNoArenasToUpdateForKind(AllocKind kind);
+
+ private:
+ ArenaList& arenaList(AllocKind i) { return arenaLists_.ref()[i]; }
+ const ArenaList& arenaList(AllocKind i) const { return arenaLists_.ref()[i]; }
+
+ ArenaList& collectingArenaList(AllocKind i) {
+ return collectingArenaLists_.ref()[i];
+ }
+ const ArenaList& collectingArenaList(AllocKind i) const {
+ return collectingArenaLists_.ref()[i];
+ }
+
+ ConcurrentUseState& concurrentUse(AllocKind i) {
+ return concurrentUseState_.ref()[i];
+ }
+ ConcurrentUse concurrentUse(AllocKind i) const {
+ return concurrentUseState_.ref()[i];
+ }
+
+ inline JSRuntime* runtime();
+ inline JSRuntime* runtimeFromAnyThread();
+
+ void initBackgroundSweep(AllocKind thingKind);
+
+ void* refillFreeListAndAllocate(AllocKind thingKind,
+ ShouldCheckThresholds checkThresholds);
+
+ friend class BackgroundUnmarkTask;
+ friend class GCRuntime;
+ friend class js::Nursery;
+ friend class TenuringTracer;
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_ArenaList_h */
diff --git a/js/src/gc/AtomMarking-inl.h b/js/src/gc/AtomMarking-inl.h
new file mode 100644
index 0000000000..9b6e869e72
--- /dev/null
+++ b/js/src/gc/AtomMarking-inl.h
@@ -0,0 +1,96 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/AtomMarking.h"
+
+#include "mozilla/Assertions.h"
+
+#include <type_traits>
+
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+#include "vm/SymbolType.h"
+
+#include "gc/Heap-inl.h"
+
+namespace js {
+namespace gc {
+
+inline size_t GetAtomBit(TenuredCell* thing) {
+ MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone());
+ Arena* arena = thing->arena();
+ size_t arenaBit = (reinterpret_cast<uintptr_t>(thing) - arena->address()) /
+ CellBytesPerMarkBit;
+ return arena->atomBitmapStart() * JS_BITS_PER_WORD + arenaBit;
+}
+
+template <typename T, bool Fallible>
+MOZ_ALWAYS_INLINE bool AtomMarkingRuntime::inlinedMarkAtomInternal(
+ JSContext* cx, T* thing) {
+ static_assert(std::is_same_v<T, JSAtom> || std::is_same_v<T, JS::Symbol>,
+ "Should only be called with JSAtom* or JS::Symbol* argument");
+
+ MOZ_ASSERT(cx->isMainThreadContext());
+ MOZ_ASSERT(cx->zone());
+ MOZ_ASSERT(!cx->zone()->isAtomsZone());
+
+ MOZ_ASSERT(thing);
+ js::gc::TenuredCell* cell = &thing->asTenured();
+ MOZ_ASSERT(cell->zoneFromAnyThread()->isAtomsZone());
+
+ // This doesn't check for pinned atoms since that might require taking a
+ // lock. This is not required for correctness.
+ if (thing->isPermanentAndMayBeShared()) {
+ return true;
+ }
+
+ size_t bit = GetAtomBit(cell);
+ MOZ_ASSERT(bit / JS_BITS_PER_WORD < allocatedWords);
+
+ if (Fallible) {
+ if (!cx->zone()->markedAtoms().setBitFallible(bit)) {
+ return false;
+ }
+ } else {
+ cx->zone()->markedAtoms().setBit(bit);
+ }
+
+ // Trigger a read barrier on the atom, in case there is an incremental
+ // GC in progress. This is necessary if the atom is being marked
+ // because a reference to it was obtained from another zone which is
+ // not being collected by the incremental GC.
+ ReadBarrier(thing);
+
+ // Children of the thing also need to be marked in the context's zone.
+ // We don't have a JSTracer for this so manually handle the cases in which
+ // an atom can reference other atoms.
+ markChildren(cx, thing);
+
+ return true;
+}
+
+void AtomMarkingRuntime::markChildren(JSContext* cx, JSAtom*) {}
+
+void AtomMarkingRuntime::markChildren(JSContext* cx, JS::Symbol* symbol) {
+ if (JSAtom* description = symbol->description()) {
+ markAtom(cx, description);
+ }
+}
+
+template <typename T>
+MOZ_ALWAYS_INLINE void AtomMarkingRuntime::inlinedMarkAtom(JSContext* cx,
+ T* thing) {
+ MOZ_ALWAYS_TRUE((inlinedMarkAtomInternal<T, false>(cx, thing)));
+}
+
+template <typename T>
+MOZ_ALWAYS_INLINE bool AtomMarkingRuntime::inlinedMarkAtomFallible(
+ JSContext* cx, T* thing) {
+ return inlinedMarkAtomInternal<T, true>(cx, thing);
+}
+
+} // namespace gc
+} // namespace js
diff --git a/js/src/gc/AtomMarking.cpp b/js/src/gc/AtomMarking.cpp
new file mode 100644
index 0000000000..78beb742d0
--- /dev/null
+++ b/js/src/gc/AtomMarking.cpp
@@ -0,0 +1,294 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/AtomMarking-inl.h"
+
+#include <type_traits>
+
+#include "gc/PublicIterators.h"
+
+#include "gc/GC-inl.h"
+#include "gc/Heap-inl.h"
+
+namespace js {
+namespace gc {
+
+// [SMDOC] GC Atom Marking
+//
+// Things in the atoms zone (which includes atomized strings and other things,
+// all of which we will refer to as 'atoms' here) may be pointed to freely by
+// things in other zones. To avoid the need to perform garbage collections of
+// the entire runtime to collect atoms, we compute a separate atom mark bitmap
+// for each zone that is always an overapproximation of the atoms that zone is
+// using. When an atom is not in the mark bitmap for any zone, it can be
+// destroyed.
+//
+// To minimize interference with the rest of the GC, atom marking and sweeping
+// is done by manipulating the mark bitmaps in the chunks used for the atoms.
+// When the atoms zone is being collected, the mark bitmaps for the chunk(s)
+// used by the atoms are updated normally during marking. After marking
+// finishes, the chunk mark bitmaps are translated to a more efficient atom mark
+// bitmap (see below) that is stored on the zones which the GC collected
+// (computeBitmapFromChunkMarkBits). Before sweeping begins, the chunk mark
+// bitmaps are updated with any atoms that might be referenced by zones which
+// weren't collected (markAtomsUsedByUncollectedZones). The GC sweeping will
+// then release all atoms which are not marked by any zone.
+//
+// The representation of atom mark bitmaps is as follows:
+//
+// Each arena in the atoms zone has an atomBitmapStart() value indicating the
+// word index into the bitmap of the first thing in the arena. Each arena uses
+// ArenaBitmapWords of data to store its bitmap, which uses the same
+// representation as chunk mark bitmaps: one bit is allocated per Cell, with
+// bits for space between things being unused when things are larger than a
+// single Cell.
+
+void AtomMarkingRuntime::registerArena(Arena* arena, const AutoLockGC& lock) {
+ MOZ_ASSERT(arena->getThingSize() != 0);
+ MOZ_ASSERT(arena->getThingSize() % CellAlignBytes == 0);
+ MOZ_ASSERT(arena->zone->isAtomsZone());
+
+ // We need to find a range of bits from the atoms bitmap for this arena.
+
+ // Look for a free range of bits compatible with this arena.
+ if (freeArenaIndexes.ref().length()) {
+ arena->atomBitmapStart() = freeArenaIndexes.ref().popCopy();
+ return;
+ }
+
+ // Allocate a range of bits from the end for this arena.
+ arena->atomBitmapStart() = allocatedWords;
+ allocatedWords += ArenaBitmapWords;
+}
+
+void AtomMarkingRuntime::unregisterArena(Arena* arena, const AutoLockGC& lock) {
+ MOZ_ASSERT(arena->zone->isAtomsZone());
+
+ // Leak these atom bits if we run out of memory.
+ (void)freeArenaIndexes.ref().emplaceBack(arena->atomBitmapStart());
+}
+
+bool AtomMarkingRuntime::computeBitmapFromChunkMarkBits(JSRuntime* runtime,
+ DenseBitmap& bitmap) {
+ MOZ_ASSERT(CurrentThreadIsPerformingGC());
+
+ if (!bitmap.ensureSpace(allocatedWords)) {
+ return false;
+ }
+
+ Zone* atomsZone = runtime->unsafeAtomsZone();
+ for (auto thingKind : AllAllocKinds()) {
+ for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) {
+ Arena* arena = aiter.get();
+ MarkBitmapWord* chunkWords = arena->chunk()->markBits.arenaBits(arena);
+ bitmap.copyBitsFrom(arena->atomBitmapStart(), ArenaBitmapWords,
+ chunkWords);
+ }
+ }
+
+ return true;
+}
+
+void AtomMarkingRuntime::refineZoneBitmapForCollectedZone(
+ Zone* zone, const DenseBitmap& bitmap) {
+ MOZ_ASSERT(zone->isCollectingFromAnyThread());
+
+ if (zone->isAtomsZone()) {
+ return;
+ }
+
+ // Take the bitwise and between the two mark bitmaps to get the best new
+ // overapproximation we can. |bitmap| might include bits that are not in
+ // the zone's mark bitmap, if additional zones were collected by the GC.
+ zone->markedAtoms().bitwiseAndWith(bitmap);
+}
+
+// Set any bits in the chunk mark bitmaps for atoms which are marked in bitmap.
+template <typename Bitmap>
+static void BitwiseOrIntoChunkMarkBits(JSRuntime* runtime, Bitmap& bitmap) {
+ // Make sure that by copying the mark bits for one arena in word sizes we
+ // do not affect the mark bits for other arenas.
+ static_assert(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD,
+ "ArenaBitmapWords must evenly divide ArenaBitmapBits");
+
+ Zone* atomsZone = runtime->unsafeAtomsZone();
+ for (auto thingKind : AllAllocKinds()) {
+ for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) {
+ Arena* arena = aiter.get();
+ MarkBitmapWord* chunkWords = arena->chunk()->markBits.arenaBits(arena);
+ bitmap.bitwiseOrRangeInto(arena->atomBitmapStart(), ArenaBitmapWords,
+ chunkWords);
+ }
+ }
+}
+
+void AtomMarkingRuntime::markAtomsUsedByUncollectedZones(JSRuntime* runtime) {
+ MOZ_ASSERT(CurrentThreadIsPerformingGC());
+
+ // Try to compute a simple union of the zone atom bitmaps before updating
+ // the chunk mark bitmaps. If this allocation fails then fall back to
+ // updating the chunk mark bitmaps separately for each zone.
+ DenseBitmap markedUnion;
+ if (markedUnion.ensureSpace(allocatedWords)) {
+ for (ZonesIter zone(runtime, SkipAtoms); !zone.done(); zone.next()) {
+ // We only need to update the chunk mark bits for zones which were
+ // not collected in the current GC. Atoms which are referenced by
+ // collected zones have already been marked.
+ if (!zone->isCollectingFromAnyThread()) {
+ zone->markedAtoms().bitwiseOrInto(markedUnion);
+ }
+ }
+ BitwiseOrIntoChunkMarkBits(runtime, markedUnion);
+ } else {
+ for (ZonesIter zone(runtime, SkipAtoms); !zone.done(); zone.next()) {
+ if (!zone->isCollectingFromAnyThread()) {
+ BitwiseOrIntoChunkMarkBits(runtime, zone->markedAtoms());
+ }
+ }
+ }
+}
+
+template <typename T>
+void AtomMarkingRuntime::markAtom(JSContext* cx, T* thing) {
+ return inlinedMarkAtom(cx, thing);
+}
+
+template void AtomMarkingRuntime::markAtom(JSContext* cx, JSAtom* thing);
+template void AtomMarkingRuntime::markAtom(JSContext* cx, JS::Symbol* thing);
+
+void AtomMarkingRuntime::markId(JSContext* cx, jsid id) {
+ if (id.isAtom()) {
+ markAtom(cx, id.toAtom());
+ return;
+ }
+ if (id.isSymbol()) {
+ markAtom(cx, id.toSymbol());
+ return;
+ }
+ MOZ_ASSERT(!id.isGCThing());
+}
+
+void AtomMarkingRuntime::markAtomValue(JSContext* cx, const Value& value) {
+ if (value.isString()) {
+ if (value.toString()->isAtom()) {
+ markAtom(cx, &value.toString()->asAtom());
+ }
+ return;
+ }
+ if (value.isSymbol()) {
+ markAtom(cx, value.toSymbol());
+ return;
+ }
+ MOZ_ASSERT_IF(value.isGCThing(), value.isObject() ||
+ value.isPrivateGCThing() ||
+ value.isBigInt());
+}
+
+#ifdef DEBUG
+template <typename T>
+bool AtomMarkingRuntime::atomIsMarked(Zone* zone, T* thing) {
+ static_assert(std::is_same_v<T, JSAtom> || std::is_same_v<T, JS::Symbol>,
+ "Should only be called with JSAtom* or JS::Symbol* argument");
+
+ MOZ_ASSERT(thing);
+ MOZ_ASSERT(!IsInsideNursery(thing));
+ MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone());
+
+ if (!zone->runtimeFromAnyThread()->permanentAtomsPopulated()) {
+ return true;
+ }
+
+ if (thing->isPermanentAndMayBeShared()) {
+ return true;
+ }
+
+ if constexpr (std::is_same_v<T, JSAtom>) {
+ if (thing->isPinned()) {
+ return true;
+ }
+ }
+
+ size_t bit = GetAtomBit(&thing->asTenured());
+ return zone->markedAtoms().readonlyThreadsafeGetBit(bit);
+}
+
+template bool AtomMarkingRuntime::atomIsMarked(Zone* zone, JSAtom* thing);
+template bool AtomMarkingRuntime::atomIsMarked(Zone* zone, JS::Symbol* thing);
+
+template <>
+bool AtomMarkingRuntime::atomIsMarked(Zone* zone, TenuredCell* thing) {
+ if (!thing) {
+ return true;
+ }
+
+ if (thing->is<JSString>()) {
+ JSString* str = thing->as<JSString>();
+ if (!str->isAtom()) {
+ return true;
+ }
+ return atomIsMarked(zone, &str->asAtom());
+ }
+
+ if (thing->is<JS::Symbol>()) {
+ return atomIsMarked(zone, thing->as<JS::Symbol>());
+ }
+
+ return true;
+}
+
+bool AtomMarkingRuntime::idIsMarked(Zone* zone, jsid id) {
+ if (id.isAtom()) {
+ return atomIsMarked(zone, id.toAtom());
+ }
+
+ if (id.isSymbol()) {
+ return atomIsMarked(zone, id.toSymbol());
+ }
+
+ MOZ_ASSERT(!id.isGCThing());
+ return true;
+}
+
+bool AtomMarkingRuntime::valueIsMarked(Zone* zone, const Value& value) {
+ if (value.isString()) {
+ if (value.toString()->isAtom()) {
+ return atomIsMarked(zone, &value.toString()->asAtom());
+ }
+ return true;
+ }
+
+ if (value.isSymbol()) {
+ return atomIsMarked(zone, value.toSymbol());
+ }
+
+ MOZ_ASSERT_IF(value.isGCThing(), value.hasObjectPayload() ||
+ value.isPrivateGCThing() ||
+ value.isBigInt());
+ return true;
+}
+
+#endif // DEBUG
+
+} // namespace gc
+
+#ifdef DEBUG
+
+bool AtomIsMarked(Zone* zone, JSAtom* atom) {
+ return zone->runtimeFromAnyThread()->gc.atomMarking.atomIsMarked(zone, atom);
+}
+
+bool AtomIsMarked(Zone* zone, jsid id) {
+ return zone->runtimeFromAnyThread()->gc.atomMarking.idIsMarked(zone, id);
+}
+
+bool AtomIsMarked(Zone* zone, const Value& value) {
+ return zone->runtimeFromAnyThread()->gc.atomMarking.valueIsMarked(zone,
+ value);
+}
+
+#endif // DEBUG
+
+} // namespace js
diff --git a/js/src/gc/AtomMarking.h b/js/src/gc/AtomMarking.h
new file mode 100644
index 0000000000..e7e97fb389
--- /dev/null
+++ b/js/src/gc/AtomMarking.h
@@ -0,0 +1,86 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_AtomMarking_h
+#define gc_AtomMarking_h
+
+#include "NamespaceImports.h"
+#include "js/Vector.h"
+#include "threading/ProtectedData.h"
+
+namespace js {
+
+class AutoLockGC;
+class DenseBitmap;
+
+namespace gc {
+
+class Arena;
+
+// This class manages state used for marking atoms during GCs.
+// See AtomMarking.cpp for details.
+class AtomMarkingRuntime {
+ // Unused arena atom bitmap indexes. Protected by the GC lock.
+ js::GCLockData<Vector<size_t, 0, SystemAllocPolicy>> freeArenaIndexes;
+
+ inline void markChildren(JSContext* cx, JSAtom*);
+ inline void markChildren(JSContext* cx, JS::Symbol* symbol);
+
+ public:
+ // The extent of all allocated and free words in atom mark bitmaps.
+ // This monotonically increases and may be read from without locking.
+ mozilla::Atomic<size_t, mozilla::SequentiallyConsistent> allocatedWords;
+
+ AtomMarkingRuntime() : allocatedWords(0) {}
+
+ // Mark an arena as holding things in the atoms zone.
+ void registerArena(Arena* arena, const AutoLockGC& lock);
+
+ // Mark an arena as no longer holding things in the atoms zone.
+ void unregisterArena(Arena* arena, const AutoLockGC& lock);
+
+ // Fill |bitmap| with an atom marking bitmap based on the things that are
+ // currently marked in the chunks used by atoms zone arenas. This returns
+ // false on an allocation failure (but does not report an exception).
+ bool computeBitmapFromChunkMarkBits(JSRuntime* runtime, DenseBitmap& bitmap);
+
+ // Update the atom marking bitmap in |zone| according to another
+ // overapproximation of the reachable atoms in |bitmap|.
+ void refineZoneBitmapForCollectedZone(Zone* zone, const DenseBitmap& bitmap);
+
+ // Set any bits in the chunk mark bitmaps for atoms which are marked in any
+ // uncollected zone in the runtime.
+ void markAtomsUsedByUncollectedZones(JSRuntime* runtime);
+
+ // Mark an atom or id as being newly reachable by the context's zone.
+ template <typename T>
+ void markAtom(JSContext* cx, T* thing);
+
+ // Version of markAtom that's always inlined, for performance-sensitive
+ // callers.
+ template <typename T, bool Fallible>
+ MOZ_ALWAYS_INLINE bool inlinedMarkAtomInternal(JSContext* cx, T* thing);
+ template <typename T>
+ MOZ_ALWAYS_INLINE void inlinedMarkAtom(JSContext* cx, T* thing);
+ template <typename T>
+ MOZ_ALWAYS_INLINE bool inlinedMarkAtomFallible(JSContext* cx, T* thing);
+
+ void markId(JSContext* cx, jsid id);
+ void markAtomValue(JSContext* cx, const Value& value);
+
+#ifdef DEBUG
+ // Return whether |thing/id| is in the atom marking bitmap for |zone|.
+ template <typename T>
+ bool atomIsMarked(Zone* zone, T* thing);
+ bool idIsMarked(Zone* zone, jsid id);
+ bool valueIsMarked(Zone* zone, const Value& value);
+#endif
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_AtomMarking_h
diff --git a/js/src/gc/Barrier.cpp b/js/src/gc/Barrier.cpp
new file mode 100644
index 0000000000..c0b9d28798
--- /dev/null
+++ b/js/src/gc/Barrier.cpp
@@ -0,0 +1,144 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Barrier.h"
+
+#include "gc/Marking.h"
+#include "jit/JitContext.h"
+#include "js/HashTable.h"
+#include "js/shadow/Zone.h" // JS::shadow::Zone
+#include "js/Value.h"
+#include "vm/BigIntType.h" // JS::BigInt
+#include "vm/EnvironmentObject.h"
+#include "vm/GeneratorObject.h"
+#include "vm/JSObject.h"
+#include "vm/PropMap.h"
+#include "wasm/WasmJS.h"
+
+#include "gc/StableCellHasher-inl.h"
+
+namespace js {
+
+bool RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone) {
+ MOZ_ASSERT(
+ CurrentThreadCanAccessRuntime(shadowZone->runtimeFromMainThread()));
+ return JS::RuntimeHeapIsMajorCollecting();
+}
+
+#ifdef DEBUG
+
+bool IsMarkedBlack(JSObject* obj) { return obj->isMarkedBlack(); }
+
+bool HeapSlot::preconditionForSet(NativeObject* owner, Kind kind,
+ uint32_t slot) const {
+ if (kind == Slot) {
+ return &owner->getSlotRef(slot) == this;
+ }
+
+ uint32_t numShifted = owner->getElementsHeader()->numShiftedElements();
+ MOZ_ASSERT(slot >= numShifted);
+ return &owner->getDenseElement(slot - numShifted) == (const Value*)this;
+}
+
+void HeapSlot::assertPreconditionForPostWriteBarrier(
+ NativeObject* obj, Kind kind, uint32_t slot, const Value& target) const {
+ if (kind == Slot) {
+ MOZ_ASSERT(obj->getSlotAddressUnchecked(slot)->get() == target);
+ } else {
+ uint32_t numShifted = obj->getElementsHeader()->numShiftedElements();
+ MOZ_ASSERT(slot >= numShifted);
+ MOZ_ASSERT(
+ static_cast<HeapSlot*>(obj->getDenseElements() + (slot - numShifted))
+ ->get() == target);
+ }
+
+ if (!obj->zone()->isGCPreparing()) {
+ AssertTargetIsNotGray(obj);
+ }
+}
+
+bool CurrentThreadIsIonCompiling() {
+ jit::JitContext* jcx = jit::MaybeGetJitContext();
+ return jcx && jcx->inIonBackend();
+}
+
+#endif // DEBUG
+
+#if !MOZ_IS_GCC
+template struct JS_PUBLIC_API StableCellHasher<JSObject*>;
+#endif
+
+} // namespace js
+
+// Post-write barrier, used by the C++ Heap<T> implementation.
+
+JS_PUBLIC_API void JS::HeapObjectPostWriteBarrier(JSObject** objp,
+ JSObject* prev,
+ JSObject* next) {
+ MOZ_ASSERT(objp);
+ js::InternalBarrierMethods<JSObject*>::postBarrier(objp, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapStringPostWriteBarrier(JSString** strp,
+ JSString* prev,
+ JSString* next) {
+ MOZ_ASSERT(strp);
+ js::InternalBarrierMethods<JSString*>::postBarrier(strp, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapBigIntPostWriteBarrier(JS::BigInt** bip,
+ JS::BigInt* prev,
+ JS::BigInt* next) {
+ MOZ_ASSERT(bip);
+ js::InternalBarrierMethods<JS::BigInt*>::postBarrier(bip, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapValuePostWriteBarrier(JS::Value* valuep,
+ const Value& prev,
+ const Value& next) {
+ MOZ_ASSERT(valuep);
+ js::InternalBarrierMethods<JS::Value>::postBarrier(valuep, prev, next);
+}
+
+// Combined pre- and post-write barriers, used by the rust Heap<T>
+// implementation.
+
+JS_PUBLIC_API void JS::HeapObjectWriteBarriers(JSObject** objp, JSObject* prev,
+ JSObject* next) {
+ MOZ_ASSERT(objp);
+ js::InternalBarrierMethods<JSObject*>::preBarrier(prev);
+ js::InternalBarrierMethods<JSObject*>::postBarrier(objp, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapStringWriteBarriers(JSString** strp, JSString* prev,
+ JSString* next) {
+ MOZ_ASSERT(strp);
+ js::InternalBarrierMethods<JSString*>::preBarrier(prev);
+ js::InternalBarrierMethods<JSString*>::postBarrier(strp, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapBigIntWriteBarriers(JS::BigInt** bip,
+ JS::BigInt* prev,
+ JS::BigInt* next) {
+ MOZ_ASSERT(bip);
+ js::InternalBarrierMethods<JS::BigInt*>::preBarrier(prev);
+ js::InternalBarrierMethods<JS::BigInt*>::postBarrier(bip, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapScriptWriteBarriers(JSScript** scriptp,
+ JSScript* prev, JSScript* next) {
+ MOZ_ASSERT(scriptp);
+ js::InternalBarrierMethods<JSScript*>::preBarrier(prev);
+ js::InternalBarrierMethods<JSScript*>::postBarrier(scriptp, prev, next);
+}
+
+JS_PUBLIC_API void JS::HeapValueWriteBarriers(JS::Value* valuep,
+ const Value& prev,
+ const Value& next) {
+ MOZ_ASSERT(valuep);
+ js::InternalBarrierMethods<JS::Value>::preBarrier(prev);
+ js::InternalBarrierMethods<JS::Value>::postBarrier(valuep, prev, next);
+}
diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h
new file mode 100644
index 0000000000..d4451a60e5
--- /dev/null
+++ b/js/src/gc/Barrier.h
@@ -0,0 +1,1257 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Barrier_h
+#define gc_Barrier_h
+
+#include <type_traits> // std::true_type
+
+#include "NamespaceImports.h"
+
+#include "gc/Cell.h"
+#include "gc/GCContext.h"
+#include "gc/StoreBuffer.h"
+#include "js/ComparisonOperators.h" // JS::detail::DefineComparisonOps
+#include "js/experimental/TypedData.h" // js::EnableIfABOVType
+#include "js/HeapAPI.h"
+#include "js/Id.h"
+#include "js/RootingAPI.h"
+#include "js/Value.h"
+#include "util/Poison.h"
+
+/*
+ * [SMDOC] GC Barriers
+ *
+ * Several kinds of barrier are necessary to allow the GC to function correctly.
+ * These are triggered by reading or writing to GC pointers in the heap and
+ * serve to tell the collector about changes to the graph of reachable GC
+ * things.
+ *
+ * Since it would be awkward to change every write to memory into a function
+ * call, this file contains a bunch of C++ classes and templates that use
+ * operator overloading to take care of barriers automatically. In most cases,
+ * all that's necessary is to replace:
+ *
+ * Type* field;
+ *
+ * with:
+ *
+ * HeapPtr<Type> field;
+ *
+ * All heap-based GC pointers and tagged pointers must use one of these classes,
+ * except in a couple of exceptional cases.
+ *
+ * These classes are designed to be used by the internals of the JS engine.
+ * Barriers designed to be used externally are provided in js/RootingAPI.h.
+ *
+ * Overview
+ * ========
+ *
+ * This file implements the following concrete classes:
+ *
+ * HeapPtr General wrapper for heap-based pointers that provides pre- and
+ * post-write barriers. Most clients should use this.
+ *
+ * GCPtr An optimisation of HeapPtr for objects which are only destroyed
+ * by GC finalization (this rules out use in Vector, for example).
+ *
+ * PreBarriered Provides a pre-barrier but not a post-barrier. Necessary when
+ * generational GC updates are handled manually, e.g. for hash
+ * table keys that don't use StableCellHasher.
+ *
+ * HeapSlot Provides pre and post-barriers, optimised for use in JSObject
+ * slots and elements.
+ *
+ * WeakHeapPtr Provides read and post-write barriers, for use with weak
+ * pointers.
+ *
+ * UnsafeBarePtr Provides no barriers. Don't add new uses of this, or only if
+ * you really know what you are doing.
+ *
+ * The following classes are implemented in js/RootingAPI.h (in the JS
+ * namespace):
+ *
+ * Heap General wrapper for external clients. Like HeapPtr but also
+ * handles cycle collector concerns. Most external clients should
+ * use this.
+ *
+ * Heap::Tenured Like Heap but doesn't allow nursery pointers. Allows storing
+ * flags in unused lower bits of the pointer.
+ *
+ * Which class to use?
+ * -------------------
+ *
+ * Answer the following questions to decide which barrier class is right for
+ * your use case:
+ *
+ * Is your code part of the JS engine?
+ * Yes, it's internal =>
+ * Is your pointer weak or strong?
+ * Strong =>
+ * Do you want automatic handling of nursery pointers?
+ * Yes, of course =>
+ * Can your object be destroyed outside of a GC?
+ * Yes => Use HeapPtr<T>
+ * No => Use GCPtr<T> (optimization)
+ * No, I'll do this myself =>
+ * Do you want pre-barriers so incremental marking works?
+ * Yes, of course => Use PreBarriered<T>
+ * No, and I'll fix all the bugs myself => Use UnsafeBarePtr<T>
+ * Weak => Use WeakHeapPtr<T>
+ * No, it's external =>
+ * Can your pointer refer to nursery objects?
+ * Yes => Use JS::Heap<T>
+ * Never => Use JS::Heap::Tenured<T> (optimization)
+ *
+ * If in doubt, use HeapPtr<T>.
+ *
+ * Write barriers
+ * ==============
+ *
+ * A write barrier is a mechanism used by incremental or generational GCs to
+ * ensure that every value that needs to be marked is marked. In general, the
+ * write barrier should be invoked whenever a write can cause the set of things
+ * traced through by the GC to change. This includes:
+ *
+ * - writes to object properties
+ * - writes to array slots
+ * - writes to fields like JSObject::shape_ that we trace through
+ * - writes to fields in private data
+ * - writes to non-markable fields like JSObject::private that point to
+ * markable data
+ *
+ * The last category is the trickiest. Even though the private pointer does not
+ * point to a GC thing, changing the private pointer may change the set of
+ * objects that are traced by the GC. Therefore it needs a write barrier.
+ *
+ * Every barriered write should have the following form:
+ *
+ * <pre-barrier>
+ * obj->field = value; // do the actual write
+ * <post-barrier>
+ *
+ * The pre-barrier is used for incremental GC and the post-barrier is for
+ * generational GC.
+ *
+ * Pre-write barrier
+ * -----------------
+ *
+ * To understand the pre-barrier, let's consider how incremental GC works. The
+ * GC itself is divided into "slices". Between each slice, JS code is allowed to
+ * run. Each slice should be short so that the user doesn't notice the
+ * interruptions. In our GC, the structure of the slices is as follows:
+ *
+ * 1. ... JS work, which leads to a request to do GC ...
+ * 2. [first GC slice, which performs all root marking and (maybe) more marking]
+ * 3. ... more JS work is allowed to run ...
+ * 4. [GC mark slice, which runs entirely in
+ * GCRuntime::markUntilBudgetExhausted]
+ * 5. ... more JS work ...
+ * 6. [GC mark slice, which runs entirely in
+ * GCRuntime::markUntilBudgetExhausted]
+ * 7. ... more JS work ...
+ * 8. [GC marking finishes; sweeping done non-incrementally; GC is done]
+ * 9. ... JS continues uninterrupted now that GC is finishes ...
+ *
+ * Of course, there may be a different number of slices depending on how much
+ * marking is to be done.
+ *
+ * The danger inherent in this scheme is that the JS code in steps 3, 5, and 7
+ * might change the heap in a way that causes the GC to collect an object that
+ * is actually reachable. The write barrier prevents this from happening. We use
+ * a variant of incremental GC called "snapshot at the beginning." This approach
+ * guarantees the invariant that if an object is reachable in step 2, then we
+ * will mark it eventually. The name comes from the idea that we take a
+ * theoretical "snapshot" of all reachable objects in step 2; all objects in
+ * that snapshot should eventually be marked. (Note that the write barrier
+ * verifier code takes an actual snapshot.)
+ *
+ * The basic correctness invariant of a snapshot-at-the-beginning collector is
+ * that any object reachable at the end of the GC (step 9) must either:
+ * (1) have been reachable at the beginning (step 2) and thus in the snapshot
+ * (2) or must have been newly allocated, in steps 3, 5, or 7.
+ * To deal with case (2), any objects allocated during an incremental GC are
+ * automatically marked black.
+ *
+ * This strategy is actually somewhat conservative: if an object becomes
+ * unreachable between steps 2 and 8, it would be safe to collect it. We won't,
+ * mainly for simplicity. (Also, note that the snapshot is entirely
+ * theoretical. We don't actually do anything special in step 2 that we wouldn't
+ * do in a non-incremental GC.
+ *
+ * It's the pre-barrier's job to maintain the snapshot invariant. Consider the
+ * write "obj->field = value". Let the prior value of obj->field be
+ * value0. Since it's possible that value0 may have been what obj->field
+ * contained in step 2, when the snapshot was taken, the barrier marks
+ * value0. Note that it only does this if we're in the middle of an incremental
+ * GC. Since this is rare, the cost of the write barrier is usually just an
+ * extra branch.
+ *
+ * In practice, we implement the pre-barrier differently based on the type of
+ * value0. E.g., see JSObject::preWriteBarrier, which is used if obj->field is
+ * a JSObject*. It takes value0 as a parameter.
+ *
+ * Post-write barrier
+ * ------------------
+ *
+ * For generational GC, we want to be able to quickly collect the nursery in a
+ * minor collection. Part of the way this is achieved is to only mark the
+ * nursery itself; tenured things, which may form the majority of the heap, are
+ * not traced through or marked. This leads to the problem of what to do about
+ * tenured objects that have pointers into the nursery: if such things are not
+ * marked, they may be discarded while there are still live objects which
+ * reference them. The solution is to maintain information about these pointers,
+ * and mark their targets when we start a minor collection.
+ *
+ * The pointers can be thought of as edges in an object graph, and the set of
+ * edges from the tenured generation into the nursery is known as the remembered
+ * set. Post barriers are used to track this remembered set.
+ *
+ * Whenever a slot which could contain such a pointer is written, we check
+ * whether the pointed-to thing is in the nursery (if storeBuffer() returns a
+ * buffer). If so we add the cell into the store buffer, which is the
+ * collector's representation of the remembered set. This means that when we
+ * come to do a minor collection we can examine the contents of the store buffer
+ * and mark any edge targets that are in the nursery.
+ *
+ * Read barriers
+ * =============
+ *
+ * Weak pointer read barrier
+ * -------------------------
+ *
+ * Weak pointers must have a read barrier to prevent the referent from being
+ * collected if it is read after the start of an incremental GC.
+ *
+ * The problem happens when, during an incremental GC, some code reads a weak
+ * pointer and writes it somewhere on the heap that has been marked black in a
+ * previous slice. Since the weak pointer will not otherwise be marked and will
+ * be swept and finalized in the last slice, this will leave the pointer just
+ * written dangling after the GC. To solve this, we immediately mark black all
+ * weak pointers that get read between slices so that it is safe to store them
+ * in an already marked part of the heap, e.g. in Rooted.
+ *
+ * Cycle collector read barrier
+ * ----------------------------
+ *
+ * Heap pointers external to the engine may be marked gray. The JS API has an
+ * invariant that no gray pointers may be passed, and this maintained by a read
+ * barrier that calls ExposeGCThingToActiveJS on such pointers. This is
+ * implemented by JS::Heap<T> in js/RootingAPI.h.
+ *
+ * Implementation Details
+ * ======================
+ *
+ * One additional note: not all object writes need to be pre-barriered. Writes
+ * to newly allocated objects do not need a pre-barrier. In these cases, we use
+ * the "obj->field.init(value)" method instead of "obj->field = value". We use
+ * the init naming idiom in many places to signify that a field is being
+ * assigned for the first time.
+ *
+ * This file implements the following hierarchy of classes:
+ *
+ * BarrieredBase base class of all barriers
+ * | |
+ * | WriteBarriered base class which provides common write operations
+ * | | | | |
+ * | | | | PreBarriered provides pre-barriers only
+ * | | | |
+ * | | | GCPtr provides pre- and post-barriers
+ * | | |
+ * | | HeapPtr provides pre- and post-barriers; is relocatable
+ * | | and deletable for use inside C++ managed memory
+ * | |
+ * | HeapSlot similar to GCPtr, but tailored to slots storage
+ * |
+ * ReadBarriered base class which provides common read operations
+ * |
+ * WeakHeapPtr provides read barriers only
+ *
+ *
+ * The implementation of the barrier logic is implemented in the
+ * Cell/TenuredCell base classes, which are called via:
+ *
+ * WriteBarriered<T>::pre
+ * -> InternalBarrierMethods<T*>::preBarrier
+ * -> Cell::preWriteBarrier
+ * -> InternalBarrierMethods<Value>::preBarrier
+ * -> InternalBarrierMethods<jsid>::preBarrier
+ * -> InternalBarrierMethods<T*>::preBarrier
+ * -> Cell::preWriteBarrier
+ *
+ * GCPtr<T>::post and HeapPtr<T>::post
+ * -> InternalBarrierMethods<T*>::postBarrier
+ * -> gc::PostWriteBarrierImpl
+ * -> InternalBarrierMethods<Value>::postBarrier
+ * -> StoreBuffer::put
+ *
+ * Barriers for use outside of the JS engine call into the same barrier
+ * implementations at InternalBarrierMethods<T>::post via an indirect call to
+ * Heap(.+)PostWriteBarrier.
+ *
+ * These clases are designed to be used to wrap GC thing pointers or values that
+ * act like them (i.e. JS::Value and jsid). It is possible to use them for
+ * other types by supplying the necessary barrier implementations but this
+ * is not usually necessary and should be done with caution.
+ */
+
+namespace js {
+
+class NativeObject;
+
+namespace gc {
+
+inline void ValueReadBarrier(const Value& v) {
+ MOZ_ASSERT(v.isGCThing());
+ ReadBarrierImpl(v.toGCThing());
+}
+
+inline void ValuePreWriteBarrier(const Value& v) {
+ MOZ_ASSERT(v.isGCThing());
+ PreWriteBarrierImpl(v.toGCThing());
+}
+
+inline void IdPreWriteBarrier(jsid id) {
+ MOZ_ASSERT(id.isGCThing());
+ PreWriteBarrierImpl(&id.toGCThing()->asTenured());
+}
+
+inline void CellPtrPreWriteBarrier(JS::GCCellPtr thing) {
+ MOZ_ASSERT(thing);
+ PreWriteBarrierImpl(thing.asCell());
+}
+
+} // namespace gc
+
+#ifdef DEBUG
+
+bool CurrentThreadIsTouchingGrayThings();
+
+bool IsMarkedBlack(JSObject* obj);
+
+#endif
+
+template <typename T, typename Enable = void>
+struct InternalBarrierMethods {};
+
+template <typename T>
+struct InternalBarrierMethods<T*> {
+ static_assert(std::is_base_of_v<gc::Cell, T>, "Expected a GC thing type");
+
+ static bool isMarkable(const T* v) { return v != nullptr; }
+
+ static void preBarrier(T* v) { gc::PreWriteBarrier(v); }
+
+ static void postBarrier(T** vp, T* prev, T* next) {
+ gc::PostWriteBarrier(vp, prev, next);
+ }
+
+ static void readBarrier(T* v) { gc::ReadBarrier(v); }
+
+#ifdef DEBUG
+ static void assertThingIsNotGray(T* v) { return T::assertThingIsNotGray(v); }
+#endif
+};
+
+template <>
+struct InternalBarrierMethods<Value> {
+ static bool isMarkable(const Value& v) { return v.isGCThing(); }
+
+ static void preBarrier(const Value& v) {
+ if (v.isGCThing()) {
+ gc::ValuePreWriteBarrier(v);
+ }
+ }
+
+ static MOZ_ALWAYS_INLINE void postBarrier(Value* vp, const Value& prev,
+ const Value& next) {
+ MOZ_ASSERT(!CurrentThreadIsIonCompiling());
+ MOZ_ASSERT(vp);
+
+ // If the target needs an entry, add it.
+ js::gc::StoreBuffer* sb;
+ if (next.isGCThing() && (sb = next.toGCThing()->storeBuffer())) {
+ // If we know that the prev has already inserted an entry, we can
+ // skip doing the lookup to add the new entry. Note that we cannot
+ // safely assert the presence of the entry because it may have been
+ // added via a different store buffer.
+ if (prev.isGCThing() && prev.toGCThing()->storeBuffer()) {
+ return;
+ }
+ sb->putValue(vp);
+ return;
+ }
+ // Remove the prev entry if the new value does not need it.
+ if (prev.isGCThing() && (sb = prev.toGCThing()->storeBuffer())) {
+ sb->unputValue(vp);
+ }
+ }
+
+ static void readBarrier(const Value& v) {
+ if (v.isGCThing()) {
+ gc::ValueReadBarrier(v);
+ }
+ }
+
+#ifdef DEBUG
+ static void assertThingIsNotGray(const Value& v) {
+ JS::AssertValueIsNotGray(v);
+ }
+#endif
+};
+
+template <>
+struct InternalBarrierMethods<jsid> {
+ static bool isMarkable(jsid id) { return id.isGCThing(); }
+ static void preBarrier(jsid id) {
+ if (id.isGCThing()) {
+ gc::IdPreWriteBarrier(id);
+ }
+ }
+ static void postBarrier(jsid* idp, jsid prev, jsid next) {}
+#ifdef DEBUG
+ static void assertThingIsNotGray(jsid id) { JS::AssertIdIsNotGray(id); }
+#endif
+};
+
+// Specialization for JS::ArrayBufferOrView subclasses.
+template <typename T>
+struct InternalBarrierMethods<T, EnableIfABOVType<T>> {
+ using BM = BarrierMethods<T>;
+
+ static bool isMarkable(const T& thing) { return bool(thing); }
+ static void preBarrier(const T& thing) {
+ gc::PreWriteBarrier(thing.asObjectUnbarriered());
+ }
+ static void postBarrier(T* tp, const T& prev, const T& next) {
+ BM::postWriteBarrier(tp, prev, next);
+ }
+ static void readBarrier(const T& thing) { BM::readBarrier(thing); }
+#ifdef DEBUG
+ static void assertThingIsNotGray(const T& thing) {
+ JSObject* obj = thing.asObjectUnbarriered();
+ if (obj) {
+ JS::AssertValueIsNotGray(JS::ObjectValue(*obj));
+ }
+ }
+#endif
+};
+
+template <typename T>
+static inline void AssertTargetIsNotGray(const T& v) {
+#ifdef DEBUG
+ if (!CurrentThreadIsTouchingGrayThings()) {
+ InternalBarrierMethods<T>::assertThingIsNotGray(v);
+ }
+#endif
+}
+
+// Base class of all barrier types.
+//
+// This is marked non-memmovable since post barriers added by derived classes
+// can add pointers to class instances to the store buffer.
+template <typename T>
+class MOZ_NON_MEMMOVABLE BarrieredBase {
+ protected:
+ // BarrieredBase is not directly instantiable.
+ explicit BarrieredBase(const T& v) : value(v) {}
+
+ // BarrieredBase subclasses cannot be copy constructed by default.
+ BarrieredBase(const BarrieredBase<T>& other) = default;
+
+ // Storage for all barrier classes. |value| must be a GC thing reference
+ // type: either a direct pointer to a GC thing or a supported tagged
+ // pointer that can reference GC things, such as JS::Value or jsid. Nested
+ // barrier types are NOT supported. See assertTypeConstraints.
+ T value;
+
+ public:
+ using ElementType = T;
+
+ // Note: this is public because C++ cannot friend to a specific template
+ // instantiation. Friending to the generic template leads to a number of
+ // unintended consequences, including template resolution ambiguity and a
+ // circular dependency with Tracing.h.
+ T* unbarrieredAddress() const { return const_cast<T*>(&value); }
+};
+
+// Base class for barriered pointer types that intercept only writes.
+template <class T>
+class WriteBarriered : public BarrieredBase<T>,
+ public WrappedPtrOperations<T, WriteBarriered<T>> {
+ protected:
+ using BarrieredBase<T>::value;
+
+ // WriteBarriered is not directly instantiable.
+ explicit WriteBarriered(const T& v) : BarrieredBase<T>(v) {}
+
+ public:
+ DECLARE_POINTER_CONSTREF_OPS(T);
+
+ // Use this if the automatic coercion to T isn't working.
+ const T& get() const { return this->value; }
+
+ // Use this if you want to change the value without invoking barriers.
+ // Obviously this is dangerous unless you know the barrier is not needed.
+ void unbarrieredSet(const T& v) { this->value = v; }
+
+ // For users who need to manually barrier the raw types.
+ static void preWriteBarrier(const T& v) {
+ InternalBarrierMethods<T>::preBarrier(v);
+ }
+
+ protected:
+ void pre() { InternalBarrierMethods<T>::preBarrier(this->value); }
+ MOZ_ALWAYS_INLINE void post(const T& prev, const T& next) {
+ InternalBarrierMethods<T>::postBarrier(&this->value, prev, next);
+ }
+};
+
+#define DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(Wrapper, T) \
+ DECLARE_POINTER_ASSIGN_OPS(Wrapper, T) \
+ Wrapper<T>& operator=(Wrapper<T>&& other) { \
+ setUnchecked(other.release()); \
+ return *this; \
+ }
+
+/*
+ * PreBarriered only automatically handles pre-barriers. Post-barriers must be
+ * manually implemented when using this class. GCPtr and HeapPtr should be used
+ * in all cases that do not require explicit low-level control of moving
+ * behavior.
+ *
+ * This class is useful for example for HashMap keys where automatically
+ * updating a moved nursery pointer would break the hash table.
+ */
+template <class T>
+class PreBarriered : public WriteBarriered<T> {
+ public:
+ PreBarriered() : WriteBarriered<T>(JS::SafelyInitialized<T>::create()) {}
+ /*
+ * Allow implicit construction for use in generic contexts.
+ */
+ MOZ_IMPLICIT PreBarriered(const T& v) : WriteBarriered<T>(v) {}
+
+ explicit PreBarriered(const PreBarriered<T>& other)
+ : WriteBarriered<T>(other.value) {}
+
+ PreBarriered(PreBarriered<T>&& other) : WriteBarriered<T>(other.release()) {}
+
+ ~PreBarriered() { this->pre(); }
+
+ void init(const T& v) { this->value = v; }
+
+ /* Use to set the pointer to nullptr. */
+ void clear() { set(JS::SafelyInitialized<T>::create()); }
+
+ DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(PreBarriered, T);
+
+ void set(const T& v) {
+ AssertTargetIsNotGray(v);
+ setUnchecked(v);
+ }
+
+ private:
+ void setUnchecked(const T& v) {
+ this->pre();
+ this->value = v;
+ }
+
+ T release() {
+ T tmp = this->value;
+ this->value = JS::SafelyInitialized<T>::create();
+ return tmp;
+ }
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::PreBarriered<T>> : std::true_type {
+ static const T& get(const js::PreBarriered<T>& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+/*
+ * A pre- and post-barriered heap pointer, for use inside the JS engine.
+ *
+ * It must only be stored in memory that has GC lifetime. GCPtr must not be
+ * used in contexts where it may be implicitly moved or deleted, e.g. most
+ * containers.
+ *
+ * The post-barriers implemented by this class are faster than those
+ * implemented by js::HeapPtr<T> or JS::Heap<T> at the cost of not
+ * automatically handling deletion or movement.
+ */
+template <class T>
+class GCPtr : public WriteBarriered<T> {
+ public:
+ GCPtr() : WriteBarriered<T>(JS::SafelyInitialized<T>::create()) {}
+
+ explicit GCPtr(const T& v) : WriteBarriered<T>(v) {
+ this->post(JS::SafelyInitialized<T>::create(), v);
+ }
+
+ explicit GCPtr(const GCPtr<T>& v) : WriteBarriered<T>(v) {
+ this->post(JS::SafelyInitialized<T>::create(), v);
+ }
+
+#ifdef DEBUG
+ ~GCPtr() {
+ // No barriers are necessary as this only happens when the GC is sweeping.
+ //
+ // If this assertion fails you may need to make the containing object use a
+ // HeapPtr instead, as this can be deleted from outside of GC.
+ MOZ_ASSERT(CurrentThreadIsGCSweeping() || CurrentThreadIsGCFinalizing());
+
+ Poison(this, JS_FREED_HEAP_PTR_PATTERN, sizeof(*this),
+ MemCheckKind::MakeNoAccess);
+ }
+#endif
+
+ void init(const T& v) {
+ AssertTargetIsNotGray(v);
+ this->value = v;
+ this->post(JS::SafelyInitialized<T>::create(), v);
+ }
+
+ DECLARE_POINTER_ASSIGN_OPS(GCPtr, T);
+
+ void set(const T& v) {
+ AssertTargetIsNotGray(v);
+ setUnchecked(v);
+ }
+
+ private:
+ void setUnchecked(const T& v) {
+ this->pre();
+ T tmp = this->value;
+ this->value = v;
+ this->post(tmp, this->value);
+ }
+
+ /*
+ * Unlike HeapPtr<T>, GCPtr<T> must be managed with GC lifetimes.
+ * Specifically, the memory used by the pointer itself must be live until
+ * at least the next minor GC. For that reason, move semantics are invalid
+ * and are deleted here. Please note that not all containers support move
+ * semantics, so this does not completely prevent invalid uses.
+ */
+ GCPtr(GCPtr<T>&&) = delete;
+ GCPtr<T>& operator=(GCPtr<T>&&) = delete;
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::GCPtr<T>> : std::true_type {
+ static const T& get(const js::GCPtr<T>& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+/*
+ * A pre- and post-barriered heap pointer, for use inside the JS engine. These
+ * heap pointers can be stored in C++ containers like GCVector and GCHashMap.
+ *
+ * The GC sometimes keeps pointers to pointers to GC things --- for example, to
+ * track references into the nursery. However, C++ containers like GCVector and
+ * GCHashMap usually reserve the right to relocate their elements any time
+ * they're modified, invalidating all pointers to the elements. HeapPtr
+ * has a move constructor which knows how to keep the GC up to date if it is
+ * moved to a new location.
+ *
+ * However, because of this additional communication with the GC, HeapPtr
+ * is somewhat slower, so it should only be used in contexts where this ability
+ * is necessary.
+ *
+ * Obviously, JSObjects, JSStrings, and the like get tenured and compacted, so
+ * whatever pointers they contain get relocated, in the sense used here.
+ * However, since the GC itself is moving those values, it takes care of its
+ * internal pointers to those pointers itself. HeapPtr is only necessary
+ * when the relocation would otherwise occur without the GC's knowledge.
+ */
+template <class T>
+class HeapPtr : public WriteBarriered<T> {
+ public:
+ HeapPtr() : WriteBarriered<T>(JS::SafelyInitialized<T>::create()) {}
+
+ // Implicitly adding barriers is a reasonable default.
+ MOZ_IMPLICIT HeapPtr(const T& v) : WriteBarriered<T>(v) {
+ this->post(JS::SafelyInitialized<T>::create(), this->value);
+ }
+
+ MOZ_IMPLICIT HeapPtr(const HeapPtr<T>& other) : WriteBarriered<T>(other) {
+ this->post(JS::SafelyInitialized<T>::create(), this->value);
+ }
+
+ HeapPtr(HeapPtr<T>&& other) : WriteBarriered<T>(other.release()) {
+ this->post(JS::SafelyInitialized<T>::create(), this->value);
+ }
+
+ ~HeapPtr() {
+ this->pre();
+ this->post(this->value, JS::SafelyInitialized<T>::create());
+ }
+
+ void init(const T& v) {
+ MOZ_ASSERT(this->value == JS::SafelyInitialized<T>::create());
+ AssertTargetIsNotGray(v);
+ this->value = v;
+ this->post(JS::SafelyInitialized<T>::create(), this->value);
+ }
+
+ DECLARE_POINTER_ASSIGN_AND_MOVE_OPS(HeapPtr, T);
+
+ void set(const T& v) {
+ AssertTargetIsNotGray(v);
+ setUnchecked(v);
+ }
+
+ /* Make this friend so it can access pre() and post(). */
+ template <class T1, class T2>
+ friend inline void BarrieredSetPair(Zone* zone, HeapPtr<T1*>& v1, T1* val1,
+ HeapPtr<T2*>& v2, T2* val2);
+
+ protected:
+ void setUnchecked(const T& v) {
+ this->pre();
+ postBarrieredSet(v);
+ }
+
+ void postBarrieredSet(const T& v) {
+ T tmp = this->value;
+ this->value = v;
+ this->post(tmp, this->value);
+ }
+
+ T release() {
+ T tmp = this->value;
+ postBarrieredSet(JS::SafelyInitialized<T>::create());
+ return tmp;
+ }
+};
+
+/*
+ * A pre-barriered heap pointer, for use inside the JS engine.
+ *
+ * Similar to GCPtr, but used for a pointer to a malloc-allocated structure
+ * containing GC thing pointers.
+ *
+ * It must only be stored in memory that has GC lifetime. It must not be used in
+ * contexts where it may be implicitly moved or deleted, e.g. most containers.
+ *
+ * A post-barrier is unnecessary since malloc-allocated structures cannot be in
+ * the nursery.
+ */
+template <class T>
+class GCStructPtr : public BarrieredBase<T> {
+ public:
+ // This is sometimes used to hold tagged pointers.
+ static constexpr uintptr_t MaxTaggedPointer = 0x2;
+
+ GCStructPtr() : BarrieredBase<T>(JS::SafelyInitialized<T>::create()) {}
+
+ // Implicitly adding barriers is a reasonable default.
+ MOZ_IMPLICIT GCStructPtr(const T& v) : BarrieredBase<T>(v) {}
+
+ GCStructPtr(const GCStructPtr<T>& other) : BarrieredBase<T>(other) {}
+
+ GCStructPtr(GCStructPtr<T>&& other) : BarrieredBase<T>(other.release()) {}
+
+ ~GCStructPtr() {
+ // No barriers are necessary as this only happens when the GC is sweeping.
+ MOZ_ASSERT_IF(isTraceable(),
+ CurrentThreadIsGCSweeping() || CurrentThreadIsGCFinalizing());
+ }
+
+ void init(const T& v) {
+ MOZ_ASSERT(this->get() == JS::SafelyInitialized<T>());
+ AssertTargetIsNotGray(v);
+ this->value = v;
+ }
+
+ void set(JS::Zone* zone, const T& v) {
+ pre(zone);
+ this->value = v;
+ }
+
+ T get() const { return this->value; }
+ operator T() const { return get(); }
+ T operator->() const { return get(); }
+
+ protected:
+ bool isTraceable() const { return uintptr_t(get()) > MaxTaggedPointer; }
+
+ void pre(JS::Zone* zone) {
+ if (isTraceable()) {
+ PreWriteBarrier(zone, get());
+ }
+ }
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::HeapPtr<T>> : std::true_type {
+ static const T& get(const js::HeapPtr<T>& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+// Base class for barriered pointer types that intercept reads and writes.
+template <typename T>
+class ReadBarriered : public BarrieredBase<T> {
+ protected:
+ // ReadBarriered is not directly instantiable.
+ explicit ReadBarriered(const T& v) : BarrieredBase<T>(v) {}
+
+ void read() const { InternalBarrierMethods<T>::readBarrier(this->value); }
+ void post(const T& prev, const T& next) {
+ InternalBarrierMethods<T>::postBarrier(&this->value, prev, next);
+ }
+};
+
+// Incremental GC requires that weak pointers have read barriers. See the block
+// comment at the top of Barrier.h for a complete discussion of why.
+//
+// Note that this class also has post-barriers, so is safe to use with nursery
+// pointers. However, when used as a hashtable key, care must still be taken to
+// insert manual post-barriers on the table for rekeying if the key is based in
+// any way on the address of the object.
+template <typename T>
+class WeakHeapPtr : public ReadBarriered<T>,
+ public WrappedPtrOperations<T, WeakHeapPtr<T>> {
+ protected:
+ using ReadBarriered<T>::value;
+
+ public:
+ WeakHeapPtr() : ReadBarriered<T>(JS::SafelyInitialized<T>::create()) {}
+
+ // It is okay to add barriers implicitly.
+ MOZ_IMPLICIT WeakHeapPtr(const T& v) : ReadBarriered<T>(v) {
+ this->post(JS::SafelyInitialized<T>::create(), v);
+ }
+
+ // The copy constructor creates a new weak edge but the wrapped pointer does
+ // not escape, so no read barrier is necessary.
+ explicit WeakHeapPtr(const WeakHeapPtr& other) : ReadBarriered<T>(other) {
+ this->post(JS::SafelyInitialized<T>::create(), value);
+ }
+
+ // Move retains the lifetime status of the source edge, so does not fire
+ // the read barrier of the defunct edge.
+ WeakHeapPtr(WeakHeapPtr&& other) : ReadBarriered<T>(other.release()) {
+ this->post(JS::SafelyInitialized<T>::create(), value);
+ }
+
+ ~WeakHeapPtr() {
+ this->post(this->value, JS::SafelyInitialized<T>::create());
+ }
+
+ WeakHeapPtr& operator=(const WeakHeapPtr& v) {
+ AssertTargetIsNotGray(v.value);
+ T prior = this->value;
+ this->value = v.value;
+ this->post(prior, v.value);
+ return *this;
+ }
+
+ const T& get() const {
+ if (InternalBarrierMethods<T>::isMarkable(this->value)) {
+ this->read();
+ }
+ return this->value;
+ }
+
+ const T& unbarrieredGet() const { return this->value; }
+
+ explicit operator bool() const { return bool(this->value); }
+
+ operator const T&() const { return get(); }
+
+ const T& operator->() const { return get(); }
+
+ void set(const T& v) {
+ AssertTargetIsNotGray(v);
+ setUnchecked(v);
+ }
+
+ void unbarrieredSet(const T& v) {
+ AssertTargetIsNotGray(v);
+ this->value = v;
+ }
+
+ private:
+ void setUnchecked(const T& v) {
+ T tmp = this->value;
+ this->value = v;
+ this->post(tmp, v);
+ }
+
+ T release() {
+ T tmp = value;
+ set(JS::SafelyInitialized<T>::create());
+ return tmp;
+ }
+};
+
+// A wrapper for a bare pointer, with no barriers.
+//
+// This should only be necessary in a limited number of cases. Please don't add
+// more uses of this if at all possible.
+template <typename T>
+class UnsafeBarePtr : public BarrieredBase<T> {
+ public:
+ UnsafeBarePtr() : BarrieredBase<T>(JS::SafelyInitialized<T>::create()) {}
+ MOZ_IMPLICIT UnsafeBarePtr(T v) : BarrieredBase<T>(v) {}
+ const T& get() const { return this->value; }
+ void set(T newValue) { this->value = newValue; }
+ DECLARE_POINTER_CONSTREF_OPS(T);
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::WeakHeapPtr<T>> : std::true_type {
+ static const T& get(const js::WeakHeapPtr<T>& v) {
+ return v.unbarrieredGet();
+ }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+// A pre- and post-barriered Value that is specialized to be aware that it
+// resides in a slots or elements vector. This allows it to be relocated in
+// memory, but with substantially less overhead than a HeapPtr.
+class HeapSlot : public WriteBarriered<Value> {
+ public:
+ enum Kind { Slot = 0, Element = 1 };
+
+ void init(NativeObject* owner, Kind kind, uint32_t slot, const Value& v) {
+ value = v;
+ post(owner, kind, slot, v);
+ }
+
+ void initAsUndefined() { value.setUndefined(); }
+
+ void destroy() { pre(); }
+
+ void setUndefinedUnchecked() {
+ pre();
+ value.setUndefined();
+ }
+
+#ifdef DEBUG
+ bool preconditionForSet(NativeObject* owner, Kind kind, uint32_t slot) const;
+ void assertPreconditionForPostWriteBarrier(NativeObject* obj, Kind kind,
+ uint32_t slot,
+ const Value& target) const;
+#endif
+
+ MOZ_ALWAYS_INLINE void set(NativeObject* owner, Kind kind, uint32_t slot,
+ const Value& v) {
+ MOZ_ASSERT(preconditionForSet(owner, kind, slot));
+ pre();
+ value = v;
+ post(owner, kind, slot, v);
+ }
+
+ private:
+ void post(NativeObject* owner, Kind kind, uint32_t slot,
+ const Value& target) {
+#ifdef DEBUG
+ assertPreconditionForPostWriteBarrier(owner, kind, slot, target);
+#endif
+ if (this->value.isGCThing()) {
+ gc::Cell* cell = this->value.toGCThing();
+ if (cell->storeBuffer()) {
+ cell->storeBuffer()->putSlot(owner, kind, slot, 1);
+ }
+ }
+ }
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <>
+struct DefineComparisonOps<js::HeapSlot> : std::true_type {
+ static const Value& get(const js::HeapSlot& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+class HeapSlotArray {
+ HeapSlot* array;
+
+ public:
+ explicit HeapSlotArray(HeapSlot* array) : array(array) {}
+
+ HeapSlot* begin() const { return array; }
+
+ operator const Value*() const {
+ static_assert(sizeof(GCPtr<Value>) == sizeof(Value));
+ static_assert(sizeof(HeapSlot) == sizeof(Value));
+ return reinterpret_cast<const Value*>(array);
+ }
+ operator HeapSlot*() const { return begin(); }
+
+ HeapSlotArray operator+(int offset) const {
+ return HeapSlotArray(array + offset);
+ }
+ HeapSlotArray operator+(uint32_t offset) const {
+ return HeapSlotArray(array + offset);
+ }
+};
+
+/*
+ * This is a hack for RegExpStatics::updateFromMatch. It allows us to do two
+ * barriers with only one branch to check if we're in an incremental GC.
+ */
+template <class T1, class T2>
+static inline void BarrieredSetPair(Zone* zone, HeapPtr<T1*>& v1, T1* val1,
+ HeapPtr<T2*>& v2, T2* val2) {
+ AssertTargetIsNotGray(val1);
+ AssertTargetIsNotGray(val2);
+ if (T1::needPreWriteBarrier(zone)) {
+ v1.pre();
+ v2.pre();
+ }
+ v1.postBarrieredSet(val1);
+ v2.postBarrieredSet(val2);
+}
+
+/*
+ * ImmutableTenuredPtr is designed for one very narrow case: replacing
+ * immutable raw pointers to GC-managed things, implicitly converting to a
+ * handle type for ease of use. Pointers encapsulated by this type must:
+ *
+ * be immutable (no incremental write barriers),
+ * never point into the nursery (no generational write barriers), and
+ * be traced via MarkRuntime (we use fromMarkedLocation).
+ *
+ * In short: you *really* need to know what you're doing before you use this
+ * class!
+ */
+template <typename T>
+class MOZ_HEAP_CLASS ImmutableTenuredPtr {
+ T value;
+
+ public:
+ operator T() const { return value; }
+ T operator->() const { return value; }
+
+ // `ImmutableTenuredPtr<T>` is implicitly convertible to `Handle<T>`.
+ //
+ // In case you need to convert to `Handle<U>` where `U` is base class of `T`,
+ // convert this to `Handle<T>` by `toHandle()` and then use implicit
+ // conversion from `Handle<T>` to `Handle<U>`.
+ operator Handle<T>() const { return toHandle(); }
+ Handle<T> toHandle() const { return Handle<T>::fromMarkedLocation(&value); }
+
+ void init(T ptr) {
+ MOZ_ASSERT(ptr->isTenured());
+ AssertTargetIsNotGray(ptr);
+ value = ptr;
+ }
+
+ T get() const { return value; }
+ const T* address() { return &value; }
+};
+
+// Template to remove any barrier wrapper and get the underlying type.
+template <typename T>
+struct RemoveBarrier {
+ using Type = T;
+};
+template <typename T>
+struct RemoveBarrier<HeapPtr<T>> {
+ using Type = T;
+};
+template <typename T>
+struct RemoveBarrier<GCPtr<T>> {
+ using Type = T;
+};
+template <typename T>
+struct RemoveBarrier<PreBarriered<T>> {
+ using Type = T;
+};
+template <typename T>
+struct RemoveBarrier<WeakHeapPtr<T>> {
+ using Type = T;
+};
+
+#if MOZ_IS_GCC
+template struct JS_PUBLIC_API StableCellHasher<JSObject*>;
+#endif
+
+template <typename T>
+struct StableCellHasher<PreBarriered<T>> {
+ using Key = PreBarriered<T>;
+ using Lookup = T;
+
+ static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::maybeGetHash(l, hashOut);
+ }
+ static bool ensureHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::ensureHash(l, hashOut);
+ }
+ static HashNumber hash(const Lookup& l) {
+ return StableCellHasher<T>::hash(l);
+ }
+ static bool match(const Key& k, const Lookup& l) {
+ return StableCellHasher<T>::match(k, l);
+ }
+};
+
+template <typename T>
+struct StableCellHasher<HeapPtr<T>> {
+ using Key = HeapPtr<T>;
+ using Lookup = T;
+
+ static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::maybeGetHash(l, hashOut);
+ }
+ static bool ensureHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::ensureHash(l, hashOut);
+ }
+ static HashNumber hash(const Lookup& l) {
+ return StableCellHasher<T>::hash(l);
+ }
+ static bool match(const Key& k, const Lookup& l) {
+ return StableCellHasher<T>::match(k, l);
+ }
+};
+
+template <typename T>
+struct StableCellHasher<WeakHeapPtr<T>> {
+ using Key = WeakHeapPtr<T>;
+ using Lookup = T;
+
+ static bool maybeGetHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::maybeGetHash(l, hashOut);
+ }
+ static bool ensureHash(const Lookup& l, HashNumber* hashOut) {
+ return StableCellHasher<T>::ensureHash(l, hashOut);
+ }
+ static HashNumber hash(const Lookup& l) {
+ return StableCellHasher<T>::hash(l);
+ }
+ static bool match(const Key& k, const Lookup& l) {
+ return StableCellHasher<T>::match(k.unbarrieredGet(), l);
+ }
+};
+
+/* Useful for hashtables with a HeapPtr as key. */
+template <class T>
+struct HeapPtrHasher {
+ using Key = HeapPtr<T>;
+ using Lookup = T;
+
+ static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); }
+ static bool match(const Key& k, Lookup l) { return k.get() == l; }
+ static void rekey(Key& k, const Key& newKey) { k.unbarrieredSet(newKey); }
+};
+
+template <class T>
+struct PreBarrieredHasher {
+ using Key = PreBarriered<T>;
+ using Lookup = T;
+
+ static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); }
+ static bool match(const Key& k, Lookup l) { return k.get() == l; }
+ static void rekey(Key& k, const Key& newKey) { k.unbarrieredSet(newKey); }
+};
+
+/* Useful for hashtables with a WeakHeapPtr as key. */
+template <class T>
+struct WeakHeapPtrHasher {
+ using Key = WeakHeapPtr<T>;
+ using Lookup = T;
+
+ static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); }
+ static bool match(const Key& k, Lookup l) { return k.unbarrieredGet() == l; }
+ static void rekey(Key& k, const Key& newKey) {
+ k.set(newKey.unbarrieredGet());
+ }
+};
+
+template <class T>
+struct UnsafeBarePtrHasher {
+ using Key = UnsafeBarePtr<T>;
+ using Lookup = T;
+
+ static HashNumber hash(const Lookup& l) { return DefaultHasher<T>::hash(l); }
+ static bool match(const Key& k, Lookup l) { return k.get() == l; }
+ static void rekey(Key& k, const Key& newKey) { k.set(newKey.get()); }
+};
+
+} // namespace js
+
+namespace mozilla {
+
+template <class T>
+struct DefaultHasher<js::HeapPtr<T>> : js::HeapPtrHasher<T> {};
+
+template <class T>
+struct DefaultHasher<js::GCPtr<T>> {
+ // Not implemented. GCPtr can't be used as a hash table key because it has a
+ // post barrier but doesn't support relocation.
+};
+
+template <class T>
+struct DefaultHasher<js::PreBarriered<T>> : js::PreBarrieredHasher<T> {};
+
+template <class T>
+struct DefaultHasher<js::WeakHeapPtr<T>> : js::WeakHeapPtrHasher<T> {};
+
+template <class T>
+struct DefaultHasher<js::UnsafeBarePtr<T>> : js::UnsafeBarePtrHasher<T> {};
+
+} // namespace mozilla
+
+#endif /* gc_Barrier_h */
diff --git a/js/src/gc/Cell.h b/js/src/gc/Cell.h
new file mode 100644
index 0000000000..bc2da39b28
--- /dev/null
+++ b/js/src/gc/Cell.h
@@ -0,0 +1,930 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Cell_h
+#define gc_Cell_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/EndianUtils.h"
+
+#include <type_traits>
+
+#include "gc/GCContext.h"
+#include "gc/Heap.h"
+#include "gc/TraceKind.h"
+#include "js/GCAnnotations.h"
+#include "js/shadow/Zone.h" // JS::shadow::Zone
+#include "js/TypeDecls.h"
+
+namespace JS {
+enum class TraceKind;
+} /* namespace JS */
+
+namespace js {
+
+class JS_PUBLIC_API GenericPrinter;
+
+extern bool RuntimeFromMainThreadIsHeapMajorCollecting(
+ JS::shadow::Zone* shadowZone);
+
+#ifdef DEBUG
+// Barriers can't be triggered during backend Ion compilation, which may run on
+// a helper thread.
+extern bool CurrentThreadIsIonCompiling();
+#endif
+
+extern void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc,
+ gc::Cell** thingp,
+ const char* name);
+
+namespace gc {
+
+enum class AllocKind : uint8_t;
+class CellAllocator; // Declared so subtypes of Cell can friend it easily.
+class StoreBuffer;
+class TenuredCell;
+
+extern void PerformIncrementalReadBarrier(TenuredCell* cell);
+extern void PerformIncrementalPreWriteBarrier(TenuredCell* cell);
+extern void PerformIncrementalBarrierDuringFlattening(JSString* str);
+extern void UnmarkGrayGCThingRecursively(TenuredCell* cell);
+
+// Like gc::MarkColor but allows the possibility of the cell being unmarked.
+//
+// This class mimics an enum class, but supports operator overloading.
+class CellColor {
+ public:
+ enum Color { White = 0, Gray = 1, Black = 2 };
+
+ CellColor() : color(White) {}
+
+ MOZ_IMPLICIT CellColor(MarkColor markColor)
+ : color(markColor == MarkColor::Black ? Black : Gray) {}
+
+ MOZ_IMPLICIT constexpr CellColor(Color c) : color(c) {}
+
+ MarkColor asMarkColor() const {
+ MOZ_ASSERT(color != White);
+ return color == Black ? MarkColor::Black : MarkColor::Gray;
+ }
+
+ // Implement a total ordering for CellColor, with white being 'least marked'
+ // and black being 'most marked'.
+ bool operator<(const CellColor other) const { return color < other.color; }
+ bool operator>(const CellColor other) const { return color > other.color; }
+ bool operator<=(const CellColor other) const { return color <= other.color; }
+ bool operator>=(const CellColor other) const { return color >= other.color; }
+ bool operator!=(const CellColor other) const { return color != other.color; }
+ bool operator==(const CellColor other) const { return color == other.color; }
+ explicit operator bool() const { return color != White; }
+
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+ const char* name() const {
+ switch (color) {
+ case CellColor::White:
+ return "white";
+ case CellColor::Black:
+ return "black";
+ case CellColor::Gray:
+ return "gray";
+ default:
+ MOZ_CRASH("Unexpected cell color");
+ }
+ }
+#endif
+
+ private:
+ Color color;
+};
+
+// Cell header word. Stores GC flags and derived class data.
+//
+// Loads of GC flags + all stores are marked as (relaxed) atomic operations,
+// to deal with the following benign data race during compacting GC:
+//
+// - Thread 1 checks isForwarded (which is always false in this situation).
+// - Thread 2 updates the derived class data (without changing the forwarded
+// flag).
+//
+// To improve performance, we don't use atomic operations for get() because
+// atomic operations inhibit certain compiler optimizations: GCC and Clang are
+// unable to fold multiple loads even if they're both relaxed atomics. This is
+// especially a problem for chained loads such as obj->shape->base->clasp.
+class HeaderWord {
+ // Indicates whether the cell has been forwarded (moved) by generational or
+ // compacting GC and is now a RelocationOverlay.
+ static constexpr uintptr_t FORWARD_BIT = Bit(0);
+ // Bits 1 and 2 are reserved for future use by the GC.
+
+ uintptr_t value_;
+
+ void setAtomic(uintptr_t value) {
+ __atomic_store_n(&value_, value, __ATOMIC_RELAXED);
+ }
+
+ public:
+ static constexpr uintptr_t RESERVED_MASK =
+ BitMask(gc::CellFlagBitsReservedForGC);
+ static_assert(gc::CellFlagBitsReservedForGC >= 3,
+ "Not enough flag bits reserved for GC");
+
+ uintptr_t getAtomic() const {
+ return __atomic_load_n(&value_, __ATOMIC_RELAXED);
+ }
+
+ // Accessors for derived class data.
+ uintptr_t get() const {
+ // Note: non-atomic load. See class comment.
+ uintptr_t value = value_;
+ MOZ_ASSERT((value & RESERVED_MASK) == 0);
+ return value;
+ }
+ void set(uintptr_t value) {
+ MOZ_ASSERT((value & RESERVED_MASK) == 0);
+ setAtomic(value);
+ }
+
+ // Accessors for GC data.
+ uintptr_t flags() const { return getAtomic() & RESERVED_MASK; }
+ bool isForwarded() const { return flags() & FORWARD_BIT; }
+ void setForwardingAddress(uintptr_t ptr) {
+ MOZ_ASSERT((ptr & RESERVED_MASK) == 0);
+ setAtomic(ptr | FORWARD_BIT);
+ }
+ uintptr_t getForwardingAddress() const {
+ MOZ_ASSERT(isForwarded());
+ return getAtomic() & ~RESERVED_MASK;
+ }
+};
+
+// [SMDOC] GC Cell
+//
+// A GC cell is the ultimate base class for all GC things. All types allocated
+// on the GC heap extend either gc::Cell or gc::TenuredCell. If a type is always
+// tenured, prefer the TenuredCell class as base.
+//
+// The first word of Cell is a HeaderWord (a uintptr_t) that reserves the low
+// three bits for GC purposes. The remaining bits are available to sub-classes
+// and can be used store a pointer to another gc::Cell. To make use of the
+// remaining space, sub-classes derive from a helper class such as
+// TenuredCellWithNonGCPointer.
+//
+// During moving GC operation a Cell may be marked as forwarded. This indicates
+// that a gc::RelocationOverlay is currently stored in the Cell's memory and
+// should be used to find the new location of the Cell.
+struct Cell {
+ // Cell header word. Stores GC flags and derived class data.
+ HeaderWord header_;
+
+ public:
+ Cell() = default;
+
+ Cell(const Cell&) = delete;
+ void operator=(const Cell&) = delete;
+
+ bool isForwarded() const { return header_.isForwarded(); }
+ uintptr_t flags() const { return header_.flags(); }
+
+ MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); }
+ MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const;
+ MOZ_ALWAYS_INLINE TenuredCell& asTenured();
+
+ MOZ_ALWAYS_INLINE bool isMarkedAny() const;
+ MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
+ MOZ_ALWAYS_INLINE bool isMarkedGray() const;
+ MOZ_ALWAYS_INLINE bool isMarked(gc::MarkColor color) const;
+ MOZ_ALWAYS_INLINE bool isMarkedAtLeast(gc::MarkColor color) const;
+ MOZ_ALWAYS_INLINE CellColor color() const;
+
+ inline JSRuntime* runtimeFromMainThread() const;
+
+ // Note: Unrestricted access to the runtime of a GC thing from an arbitrary
+ // thread can easily lead to races. Use this method very carefully.
+ inline JSRuntime* runtimeFromAnyThread() const;
+
+ // May be overridden by GC thing kinds that have a compartment pointer.
+ inline JS::Compartment* maybeCompartment() const { return nullptr; }
+
+ // The StoreBuffer used to record incoming pointers from the tenured heap.
+ // This will return nullptr for a tenured cell.
+ inline StoreBuffer* storeBuffer() const;
+
+ inline JS::TraceKind getTraceKind() const;
+
+ static MOZ_ALWAYS_INLINE bool needPreWriteBarrier(JS::Zone* zone);
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline bool is() const {
+ return getTraceKind() == JS::MapTypeToTraceKind<T>::kind;
+ }
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline T* as() {
+ // |this|-qualify the |is| call below to avoid compile errors with even
+ // fairly recent versions of gcc, e.g. 7.1.1 according to bz.
+ MOZ_ASSERT(this->is<T>());
+ return static_cast<T*>(this);
+ }
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline const T* as() const {
+ // |this|-qualify the |is| call below to avoid compile errors with even
+ // fairly recent versions of gcc, e.g. 7.1.1 according to bz.
+ MOZ_ASSERT(this->is<T>());
+ return static_cast<const T*>(this);
+ }
+
+ inline JS::Zone* zone() const;
+ inline JS::Zone* zoneFromAnyThread() const;
+
+ // Get the zone for a cell known to be in the nursery.
+ inline JS::Zone* nurseryZone() const;
+ inline JS::Zone* nurseryZoneFromAnyThread() const;
+
+ // Default implementation for kinds that cannot be permanent. This may be
+ // overriden by derived classes.
+ MOZ_ALWAYS_INLINE bool isPermanentAndMayBeShared() const { return false; }
+
+#ifdef DEBUG
+ static inline void assertThingIsNotGray(Cell* cell);
+ inline bool isAligned() const;
+ void dump(GenericPrinter& out) const;
+ void dump() const;
+#endif
+
+ protected:
+ uintptr_t address() const;
+ inline ChunkBase* chunk() const;
+
+ private:
+ // Cells are destroyed by the GC. Do not delete them directly.
+ void operator delete(void*) = delete;
+} JS_HAZ_GC_THING;
+
+// A GC TenuredCell gets behaviors that are valid for things in the Tenured
+// heap, such as access to the arena and mark bits.
+class TenuredCell : public Cell {
+ public:
+ MOZ_ALWAYS_INLINE bool isTenured() const {
+ MOZ_ASSERT(!IsInsideNursery(this));
+ return true;
+ }
+
+ TenuredChunk* chunk() const {
+ return static_cast<TenuredChunk*>(Cell::chunk());
+ }
+
+ // Mark bit management.
+ MOZ_ALWAYS_INLINE bool isMarkedAny() const;
+ MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
+ MOZ_ALWAYS_INLINE bool isMarkedGray() const;
+ MOZ_ALWAYS_INLINE CellColor color() const;
+
+ // The return value indicates if the cell went from unmarked to marked.
+ MOZ_ALWAYS_INLINE bool markIfUnmarked(
+ MarkColor color = MarkColor::Black) const;
+ MOZ_ALWAYS_INLINE bool markIfUnmarkedAtomic(MarkColor color) const;
+ MOZ_ALWAYS_INLINE void markBlack() const;
+ MOZ_ALWAYS_INLINE void markBlackAtomic() const;
+ MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src);
+ MOZ_ALWAYS_INLINE void unmark();
+
+ // Access to the arena.
+ inline Arena* arena() const;
+ inline AllocKind getAllocKind() const;
+ inline JS::TraceKind getTraceKind() const;
+ inline JS::Zone* zone() const;
+ inline JS::Zone* zoneFromAnyThread() const;
+ inline bool isInsideZone(JS::Zone* zone) const;
+
+ MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const {
+ return JS::shadow::Zone::from(zone());
+ }
+ MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const {
+ return JS::shadow::Zone::from(zoneFromAnyThread());
+ }
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline bool is() const {
+ return getTraceKind() == JS::MapTypeToTraceKind<T>::kind;
+ }
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline T* as() {
+ // |this|-qualify the |is| call below to avoid compile errors with even
+ // fairly recent versions of gcc, e.g. 7.1.1 according to bz.
+ MOZ_ASSERT(this->is<T>());
+ return static_cast<T*>(this);
+ }
+
+ template <typename T, typename = std::enable_if_t<JS::IsBaseTraceType_v<T>>>
+ inline const T* as() const {
+ // |this|-qualify the |is| call below to avoid compile errors with even
+ // fairly recent versions of gcc, e.g. 7.1.1 according to bz.
+ MOZ_ASSERT(this->is<T>());
+ return static_cast<const T*>(this);
+ }
+
+ // Default implementation for kinds that don't require fixup.
+ void fixupAfterMovingGC() {}
+
+ static inline CellColor getColor(MarkBitmap* bitmap, const TenuredCell* cell);
+
+#ifdef DEBUG
+ inline bool isAligned() const;
+#endif
+};
+
+MOZ_ALWAYS_INLINE const TenuredCell& Cell::asTenured() const {
+ MOZ_ASSERT(isTenured());
+ return *static_cast<const TenuredCell*>(this);
+}
+
+MOZ_ALWAYS_INLINE TenuredCell& Cell::asTenured() {
+ MOZ_ASSERT(isTenured());
+ return *static_cast<TenuredCell*>(this);
+}
+
+MOZ_ALWAYS_INLINE bool Cell::isMarkedAny() const {
+ return !isTenured() || asTenured().isMarkedAny();
+}
+
+MOZ_ALWAYS_INLINE bool Cell::isMarkedBlack() const {
+ return !isTenured() || asTenured().isMarkedBlack();
+}
+
+MOZ_ALWAYS_INLINE bool Cell::isMarkedGray() const {
+ return isTenured() && asTenured().isMarkedGray();
+}
+
+MOZ_ALWAYS_INLINE bool Cell::isMarked(gc::MarkColor color) const {
+ return color == MarkColor::Gray ? isMarkedGray() : isMarkedBlack();
+}
+
+MOZ_ALWAYS_INLINE bool Cell::isMarkedAtLeast(gc::MarkColor color) const {
+ return color == MarkColor::Gray ? isMarkedAny() : isMarkedBlack();
+}
+
+MOZ_ALWAYS_INLINE CellColor Cell::color() const {
+ return isTenured() ? asTenured().color() : CellColor::Black;
+}
+
+inline JSRuntime* Cell::runtimeFromMainThread() const {
+ JSRuntime* rt = chunk()->runtime;
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ return rt;
+}
+
+inline JSRuntime* Cell::runtimeFromAnyThread() const {
+ return chunk()->runtime;
+}
+
+inline uintptr_t Cell::address() const {
+ uintptr_t addr = uintptr_t(this);
+ MOZ_ASSERT(addr % CellAlignBytes == 0);
+ MOZ_ASSERT(TenuredChunk::withinValidRange(addr));
+ return addr;
+}
+
+ChunkBase* Cell::chunk() const {
+ uintptr_t addr = uintptr_t(this);
+ MOZ_ASSERT(addr % CellAlignBytes == 0);
+ addr &= ~ChunkMask;
+ return reinterpret_cast<ChunkBase*>(addr);
+}
+
+inline StoreBuffer* Cell::storeBuffer() const { return chunk()->storeBuffer; }
+
+JS::Zone* Cell::zone() const {
+ if (isTenured()) {
+ return asTenured().zone();
+ }
+
+ return nurseryZone();
+}
+
+JS::Zone* Cell::zoneFromAnyThread() const {
+ if (isTenured()) {
+ return asTenured().zoneFromAnyThread();
+ }
+
+ return nurseryZoneFromAnyThread();
+}
+
+JS::Zone* Cell::nurseryZone() const {
+ JS::Zone* zone = nurseryZoneFromAnyThread();
+ MOZ_ASSERT(CurrentThreadIsGCMarking() || CurrentThreadCanAccessZone(zone));
+ return zone;
+}
+
+JS::Zone* Cell::nurseryZoneFromAnyThread() const {
+ return NurseryCellHeader::from(this)->zone();
+}
+
+#ifdef DEBUG
+extern Cell* UninlinedForwarded(const Cell* cell);
+#endif
+
+inline JS::TraceKind Cell::getTraceKind() const {
+ if (isTenured()) {
+ MOZ_ASSERT_IF(isForwarded(), UninlinedForwarded(this)->getTraceKind() ==
+ asTenured().getTraceKind());
+ return asTenured().getTraceKind();
+ }
+
+ return NurseryCellHeader::from(this)->traceKind();
+}
+
+/* static */ MOZ_ALWAYS_INLINE bool Cell::needPreWriteBarrier(JS::Zone* zone) {
+ return JS::shadow::Zone::from(zone)->needsIncrementalBarrier();
+}
+
+MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedAny() const {
+ MOZ_ASSERT(arena()->allocated());
+ return chunk()->markBits.isMarkedAny(this);
+}
+
+MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedBlack() const {
+ MOZ_ASSERT(arena()->allocated());
+ return chunk()->markBits.isMarkedBlack(this);
+}
+
+MOZ_ALWAYS_INLINE bool TenuredCell::isMarkedGray() const {
+ MOZ_ASSERT(arena()->allocated());
+ return chunk()->markBits.isMarkedGray(this);
+}
+
+MOZ_ALWAYS_INLINE CellColor TenuredCell::color() const {
+ return getColor(&chunk()->markBits, this);
+}
+
+/* static */
+inline CellColor TenuredCell::getColor(MarkBitmap* bitmap,
+ const TenuredCell* cell) {
+ // Note that this method isn't synchronised so may give surprising results if
+ // the mark bitmap is being modified concurrently.
+
+ if (bitmap->isMarkedBlack(cell)) {
+ return CellColor::Black;
+ }
+
+ if (bitmap->isMarkedGray(cell)) {
+ return CellColor::Gray;
+ }
+
+ return CellColor::White;
+}
+
+bool TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const {
+ return chunk()->markBits.markIfUnmarked(this, color);
+}
+
+bool TenuredCell::markIfUnmarkedAtomic(MarkColor color) const {
+ return chunk()->markBits.markIfUnmarkedAtomic(this, color);
+}
+
+void TenuredCell::markBlack() const { chunk()->markBits.markBlack(this); }
+void TenuredCell::markBlackAtomic() const {
+ chunk()->markBits.markBlackAtomic(this);
+}
+
+void TenuredCell::copyMarkBitsFrom(const TenuredCell* src) {
+ MarkBitmap& markBits = chunk()->markBits;
+ markBits.copyMarkBit(this, src, ColorBit::BlackBit);
+ markBits.copyMarkBit(this, src, ColorBit::GrayOrBlackBit);
+}
+
+void TenuredCell::unmark() { chunk()->markBits.unmark(this); }
+
+inline Arena* TenuredCell::arena() const {
+ MOZ_ASSERT(isTenured());
+ uintptr_t addr = address();
+ addr &= ~ArenaMask;
+ return reinterpret_cast<Arena*>(addr);
+}
+
+AllocKind TenuredCell::getAllocKind() const { return arena()->getAllocKind(); }
+
+JS::TraceKind TenuredCell::getTraceKind() const {
+ return MapAllocToTraceKind(getAllocKind());
+}
+
+JS::Zone* TenuredCell::zone() const {
+ JS::Zone* zone = arena()->zone;
+ MOZ_ASSERT(CurrentThreadIsGCMarking() || CurrentThreadCanAccessZone(zone));
+ return zone;
+}
+
+JS::Zone* TenuredCell::zoneFromAnyThread() const { return arena()->zone; }
+
+bool TenuredCell::isInsideZone(JS::Zone* zone) const {
+ return zone == arena()->zone;
+}
+
+// Read barrier and pre-write barrier implementation for GC cells.
+
+template <typename T>
+MOZ_ALWAYS_INLINE void ReadBarrier(T* thing) {
+ static_assert(std::is_base_of_v<Cell, T>);
+ static_assert(!std::is_same_v<Cell, T> && !std::is_same_v<TenuredCell, T>);
+
+ if (thing) {
+ ReadBarrierImpl(thing);
+ }
+}
+
+MOZ_ALWAYS_INLINE void ReadBarrierImpl(TenuredCell* thing) {
+ MOZ_ASSERT(CurrentThreadIsMainThread());
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+ MOZ_ASSERT(thing);
+
+ JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread();
+ if (shadowZone->needsIncrementalBarrier()) {
+ PerformIncrementalReadBarrier(thing);
+ return;
+ }
+
+ if (thing->isMarkedGray()) {
+ UnmarkGrayGCThingRecursively(thing);
+ }
+}
+
+MOZ_ALWAYS_INLINE void ReadBarrierImpl(Cell* thing) {
+ MOZ_ASSERT(!CurrentThreadIsGCMarking());
+ MOZ_ASSERT(thing);
+
+ if (thing->isTenured()) {
+ ReadBarrierImpl(&thing->asTenured());
+ }
+}
+
+MOZ_ALWAYS_INLINE void PreWriteBarrierImpl(TenuredCell* thing) {
+ MOZ_ASSERT(CurrentThreadIsMainThread() || CurrentThreadIsGCSweeping() ||
+ CurrentThreadIsGCFinalizing());
+ MOZ_ASSERT(thing);
+
+ // Barriers can be triggered on the main thread while collecting, but are
+ // disabled. For example, this happens when sweeping HeapPtr wrappers. See
+ // AutoDisableBarriers.
+
+ JS::shadow::Zone* zone = thing->shadowZoneFromAnyThread();
+ if (zone->needsIncrementalBarrier()) {
+ PerformIncrementalPreWriteBarrier(thing);
+ }
+}
+
+MOZ_ALWAYS_INLINE void PreWriteBarrierImpl(Cell* thing) {
+ MOZ_ASSERT(!CurrentThreadIsGCMarking());
+ MOZ_ASSERT(thing);
+
+ if (thing->isTenured()) {
+ PreWriteBarrierImpl(&thing->asTenured());
+ }
+}
+
+template <typename T>
+MOZ_ALWAYS_INLINE void PreWriteBarrier(T* thing) {
+ static_assert(std::is_base_of_v<Cell, T>);
+ static_assert(!std::is_same_v<Cell, T> && !std::is_same_v<TenuredCell, T>);
+
+ if (thing) {
+ PreWriteBarrierImpl(thing);
+ }
+}
+
+// Pre-write barrier implementation for structures containing GC cells, taking a
+// functor to trace the structure.
+template <typename T, typename F>
+MOZ_ALWAYS_INLINE void PreWriteBarrier(JS::Zone* zone, T* data,
+ const F& traceFn) {
+ MOZ_ASSERT(data);
+ MOZ_ASSERT(!CurrentThreadIsIonCompiling());
+ MOZ_ASSERT(!CurrentThreadIsGCMarking());
+
+ auto* shadowZone = JS::shadow::Zone::from(zone);
+ if (!shadowZone->needsIncrementalBarrier()) {
+ return;
+ }
+
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(shadowZone->runtimeFromAnyThread()));
+ MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone));
+
+ traceFn(shadowZone->barrierTracer(), data);
+}
+
+// Pre-write barrier implementation for structures containing GC cells. T must
+// support a |trace| method.
+template <typename T>
+MOZ_ALWAYS_INLINE void PreWriteBarrier(JS::Zone* zone, T* data) {
+ MOZ_ASSERT(data);
+ PreWriteBarrier(zone, data, [](JSTracer* trc, T* data) { data->trace(trc); });
+}
+
+#ifdef DEBUG
+
+/* static */ void Cell::assertThingIsNotGray(Cell* cell) {
+ JS::AssertCellIsNotGray(cell);
+}
+
+bool Cell::isAligned() const {
+ if (!isTenured()) {
+ return true;
+ }
+ return asTenured().isAligned();
+}
+
+bool TenuredCell::isAligned() const {
+ return Arena::isAligned(address(), arena()->getThingSize());
+}
+
+#endif
+
+// Base class for nusery-allocatable GC things that have 32-bit length and
+// 32-bit flags (currently JSString and BigInt).
+//
+// This tries to store both in Cell::header_, but if that isn't large enough the
+// length is stored separately.
+//
+// 32 0
+// ------------------
+// | Length | Flags |
+// ------------------
+//
+// The low bits of the flags word (see CellFlagBitsReservedForGC) are reserved
+// for GC. Derived classes must ensure they don't use these flags for non-GC
+// purposes.
+class alignas(gc::CellAlignBytes) CellWithLengthAndFlags : public Cell {
+#if JS_BITS_PER_WORD == 32
+ // Additional storage for length if |header_| is too small to fit both.
+ uint32_t length_;
+#endif
+
+ protected:
+ uint32_t headerLengthField() const {
+#if JS_BITS_PER_WORD == 32
+ return length_;
+#else
+ return uint32_t(header_.get() >> 32);
+#endif
+ }
+
+ uint32_t headerFlagsField() const { return uint32_t(header_.get()); }
+
+ void setHeaderFlagBit(uint32_t flag) {
+ header_.set(header_.get() | uintptr_t(flag));
+ }
+ void clearHeaderFlagBit(uint32_t flag) {
+ header_.set(header_.get() & ~uintptr_t(flag));
+ }
+ void toggleHeaderFlagBit(uint32_t flag) {
+ header_.set(header_.get() ^ uintptr_t(flag));
+ }
+
+ void setHeaderLengthAndFlags(uint32_t len, uint32_t flags) {
+#if JS_BITS_PER_WORD == 32
+ header_.set(flags);
+ length_ = len;
+#else
+ header_.set((uint64_t(len) << 32) | uint64_t(flags));
+#endif
+ }
+
+ public:
+ // Returns the offset of header_. JIT code should use offsetOfFlags
+ // below.
+ static constexpr size_t offsetOfRawHeaderFlagsField() {
+ return offsetof(CellWithLengthAndFlags, header_);
+ }
+
+ // Offsets for direct field from jit code. A number of places directly
+ // access 32-bit length and flags fields so do endian trickery here.
+#if JS_BITS_PER_WORD == 32
+ static constexpr size_t offsetOfHeaderFlags() {
+ return offsetof(CellWithLengthAndFlags, header_);
+ }
+ static constexpr size_t offsetOfHeaderLength() {
+ return offsetof(CellWithLengthAndFlags, length_);
+ }
+#elif MOZ_LITTLE_ENDIAN()
+ static constexpr size_t offsetOfHeaderFlags() {
+ return offsetof(CellWithLengthAndFlags, header_);
+ }
+ static constexpr size_t offsetOfHeaderLength() {
+ return offsetof(CellWithLengthAndFlags, header_) + sizeof(uint32_t);
+ }
+#else
+ static constexpr size_t offsetOfHeaderFlags() {
+ return offsetof(CellWithLengthAndFlags, header_) + sizeof(uint32_t);
+ }
+ static constexpr size_t offsetOfHeaderLength() {
+ return offsetof(CellWithLengthAndFlags, header_);
+ }
+#endif
+};
+
+// Base class for non-nursery-allocatable GC things that allows storing a non-GC
+// thing pointer in the first word.
+//
+// The low bits of the word (see CellFlagBitsReservedForGC) are reserved for GC.
+template <class PtrT>
+class alignas(gc::CellAlignBytes) TenuredCellWithNonGCPointer
+ : public TenuredCell {
+ static_assert(!std::is_pointer_v<PtrT>,
+ "PtrT should be the type of the referent, not of the pointer");
+ static_assert(
+ !std::is_base_of_v<Cell, PtrT>,
+ "Don't use TenuredCellWithNonGCPointer for pointers to GC things");
+
+ protected:
+ TenuredCellWithNonGCPointer() = default;
+ explicit TenuredCellWithNonGCPointer(PtrT* initial) {
+ uintptr_t data = uintptr_t(initial);
+ header_.set(data);
+ }
+
+ PtrT* headerPtr() const {
+ MOZ_ASSERT(flags() == 0);
+ return reinterpret_cast<PtrT*>(uintptr_t(header_.get()));
+ }
+
+ void setHeaderPtr(PtrT* newValue) {
+ // As above, no flags are expected to be set here.
+ uintptr_t data = uintptr_t(newValue);
+ MOZ_ASSERT(flags() == 0);
+ header_.set(data);
+ }
+
+ public:
+ static constexpr size_t offsetOfHeaderPtr() {
+ return offsetof(TenuredCellWithNonGCPointer, header_);
+ }
+};
+
+// Base class for non-nursery-allocatable GC things that allows storing flags
+// in the first word.
+//
+// The low bits of the flags word (see CellFlagBitsReservedForGC) are reserved
+// for GC.
+class alignas(gc::CellAlignBytes) TenuredCellWithFlags : public TenuredCell {
+ protected:
+ TenuredCellWithFlags() { header_.set(0); }
+ explicit TenuredCellWithFlags(uintptr_t initial) { header_.set(initial); }
+
+ uintptr_t headerFlagsField() const {
+ MOZ_ASSERT(flags() == 0);
+ return header_.get();
+ }
+
+ void setHeaderFlagBits(uintptr_t flags) {
+ header_.set(header_.get() | flags);
+ }
+ void clearHeaderFlagBits(uintptr_t flags) {
+ header_.set(header_.get() & ~flags);
+ }
+};
+
+// Base class for GC things that have a tenured GC pointer as their first word.
+//
+// The low bits of the first word (see CellFlagBitsReservedForGC) are reserved
+// for GC.
+//
+// This includes a pre write barrier when the pointer is update. No post barrier
+// is necessary as the pointer is always tenured.
+template <class BaseCell, class PtrT>
+class alignas(gc::CellAlignBytes) CellWithTenuredGCPointer : public BaseCell {
+ static void staticAsserts() {
+ // These static asserts are not in class scope because the PtrT may not be
+ // defined when this class template is instantiated.
+ static_assert(
+ std::is_same_v<BaseCell, Cell> || std::is_same_v<BaseCell, TenuredCell>,
+ "BaseCell must be either Cell or TenuredCell");
+ static_assert(
+ !std::is_pointer_v<PtrT>,
+ "PtrT should be the type of the referent, not of the pointer");
+ static_assert(
+ std::is_base_of_v<Cell, PtrT>,
+ "Only use CellWithTenuredGCPointer for pointers to GC things");
+ }
+
+ protected:
+ CellWithTenuredGCPointer() = default;
+ explicit CellWithTenuredGCPointer(PtrT* initial) { initHeaderPtr(initial); }
+
+ void initHeaderPtr(PtrT* initial) {
+ MOZ_ASSERT_IF(initial, !IsInsideNursery(initial));
+ uintptr_t data = uintptr_t(initial);
+ this->header_.set(data);
+ }
+
+ void setHeaderPtr(PtrT* newValue) {
+ // As above, no flags are expected to be set here.
+ MOZ_ASSERT_IF(newValue, !IsInsideNursery(newValue));
+ PreWriteBarrier(headerPtr());
+ unbarrieredSetHeaderPtr(newValue);
+ }
+
+ public:
+ PtrT* headerPtr() const {
+ staticAsserts();
+ MOZ_ASSERT(this->flags() == 0);
+ return reinterpret_cast<PtrT*>(uintptr_t(this->header_.get()));
+ }
+ PtrT* headerPtrAtomic() const {
+ staticAsserts();
+ MOZ_ASSERT(this->flags() == 0);
+ return reinterpret_cast<PtrT*>(uintptr_t(this->header_.getAtomic()));
+ }
+
+ void unbarrieredSetHeaderPtr(PtrT* newValue) {
+ uintptr_t data = uintptr_t(newValue);
+ MOZ_ASSERT(this->flags() == 0);
+ this->header_.set(data);
+ }
+
+ static constexpr size_t offsetOfHeaderPtr() {
+ return offsetof(CellWithTenuredGCPointer, header_);
+ }
+};
+
+void CellHeaderPostWriteBarrier(JSObject** ptr, JSObject* prev, JSObject* next);
+
+template <typename T>
+constexpr inline bool GCTypeIsTenured() {
+ static_assert(std::is_base_of_v<Cell, T>);
+ static_assert(!std::is_same_v<Cell, T> && !std::is_same_v<TenuredCell, T>);
+
+ return std::is_base_of_v<TenuredCell, T> || std::is_base_of_v<JSAtom, T>;
+}
+
+template <class PtrT>
+class alignas(gc::CellAlignBytes) TenuredCellWithGCPointer
+ : public TenuredCell {
+ static void staticAsserts() {
+ // These static asserts are not in class scope because the PtrT may not be
+ // defined when this class template is instantiated.
+ static_assert(
+ !std::is_pointer_v<PtrT>,
+ "PtrT should be the type of the referent, not of the pointer");
+ static_assert(
+ std::is_base_of_v<Cell, PtrT>,
+ "Only use TenuredCellWithGCPointer for pointers to GC things");
+ static_assert(
+ !GCTypeIsTenured<PtrT>,
+ "Don't use TenuredCellWithGCPointer for always-tenured GC things");
+ }
+
+ protected:
+ TenuredCellWithGCPointer() = default;
+ explicit TenuredCellWithGCPointer(PtrT* initial) { initHeaderPtr(initial); }
+
+ void initHeaderPtr(PtrT* initial) {
+ uintptr_t data = uintptr_t(initial);
+ this->header_.set(data);
+ if (initial && IsInsideNursery(initial)) {
+ CellHeaderPostWriteBarrier(headerPtrAddress(), nullptr, initial);
+ }
+ }
+
+ PtrT** headerPtrAddress() {
+ MOZ_ASSERT(this->flags() == 0);
+ return reinterpret_cast<PtrT**>(&this->header_);
+ }
+
+ public:
+ PtrT* headerPtr() const {
+ MOZ_ASSERT(this->flags() == 0);
+ return reinterpret_cast<PtrT*>(uintptr_t(this->header_.get()));
+ }
+
+ void unbarrieredSetHeaderPtr(PtrT* newValue) {
+ uintptr_t data = uintptr_t(newValue);
+ MOZ_ASSERT(this->flags() == 0);
+ this->header_.set(data);
+ }
+
+ static constexpr size_t offsetOfHeaderPtr() {
+ return offsetof(TenuredCellWithGCPointer, header_);
+ }
+};
+
+// Check whether a typed GC thing is marked at all. Doesn't check gray bits for
+// kinds that can't be marked gray.
+template <typename T>
+static inline bool TenuredThingIsMarkedAny(T* thing) {
+ using BaseT = typename BaseGCType<T>::type;
+ TenuredCell* cell = &thing->asTenured();
+ if constexpr (TraceKindCanBeGray<BaseT>::value) {
+ return cell->isMarkedAny();
+ } else {
+ MOZ_ASSERT(!cell->isMarkedGray());
+ return cell->isMarkedBlack();
+ }
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_Cell_h */
diff --git a/js/src/gc/ClearEdgesTracer.h b/js/src/gc/ClearEdgesTracer.h
new file mode 100644
index 0000000000..3e8f0309a3
--- /dev/null
+++ b/js/src/gc/ClearEdgesTracer.h
@@ -0,0 +1,27 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_ClearEdgesTracer_h
+#define gc_ClearEdgesTracer_h
+
+#include "js/TracingAPI.h"
+
+namespace js {
+namespace gc {
+
+struct ClearEdgesTracer final : public GenericTracerImpl<ClearEdgesTracer> {
+ explicit ClearEdgesTracer(JSRuntime* rt);
+
+ private:
+ template <typename T>
+ void onEdge(T** thingp, const char* name);
+ friend class GenericTracerImpl<ClearEdgesTracer>;
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_ClearEdgesTracer_h
diff --git a/js/src/gc/Compacting.cpp b/js/src/gc/Compacting.cpp
new file mode 100644
index 0000000000..6af0a98860
--- /dev/null
+++ b/js/src/gc/Compacting.cpp
@@ -0,0 +1,967 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Implementation of compacting GC.
+ */
+
+#include "mozilla/Maybe.h"
+
+#include "debugger/DebugAPI.h"
+#include "gc/ArenaList.h"
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/ParallelWork.h"
+#include "gc/Zone.h"
+#include "jit/JitCode.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "js/GCAPI.h"
+#include "vm/HelperThreads.h"
+#include "vm/Realm.h"
+#include "wasm/WasmGcObject.h"
+
+#include "gc/Heap-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/PrivateIterators-inl.h"
+#include "gc/StableCellHasher-inl.h"
+#include "gc/TraceMethods-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::Maybe;
+
+bool GCRuntime::canRelocateZone(Zone* zone) const {
+ return !zone->isAtomsZone();
+}
+
+void GCRuntime::beginCompactPhase() {
+ MOZ_ASSERT(!isBackgroundSweeping());
+ assertBackgroundSweepingFinished();
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT);
+
+ MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty());
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ if (canRelocateZone(zone)) {
+ zonesToMaybeCompact.ref().append(zone);
+ }
+ }
+
+ startedCompacting = true;
+ zonesCompacted = 0;
+
+#ifdef DEBUG
+ AutoLockGC lock(this);
+ MOZ_ASSERT(!relocatedArenasToRelease);
+#endif
+}
+
+IncrementalProgress GCRuntime::compactPhase(JS::GCReason reason,
+ SliceBudget& sliceBudget,
+ AutoGCSession& session) {
+ assertBackgroundSweepingFinished();
+ MOZ_ASSERT(startedCompacting);
+
+ AutoMajorGCProfilerEntry s(this);
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT);
+
+ // TODO: JSScripts can move. If the sampler interrupts the GC in the
+ // middle of relocating an arena, invalid JSScript pointers may be
+ // accessed. Suppress all sampling until a finer-grained solution can be
+ // found. See bug 1295775.
+ AutoSuppressProfilerSampling suppressSampling(rt->mainContextFromOwnThread());
+
+ ZoneList relocatedZones;
+ Arena* relocatedArenas = nullptr;
+ while (!zonesToMaybeCompact.ref().isEmpty()) {
+ Zone* zone = zonesToMaybeCompact.ref().front();
+ zonesToMaybeCompact.ref().removeFront();
+
+ MOZ_ASSERT(nursery().isEmpty());
+ zone->changeGCState(Zone::Finished, Zone::Compact);
+
+ if (relocateArenas(zone, reason, relocatedArenas, sliceBudget)) {
+ updateZonePointersToRelocatedCells(zone);
+ relocatedZones.append(zone);
+ zonesCompacted++;
+ } else {
+ zone->changeGCState(Zone::Compact, Zone::Finished);
+ }
+
+ if (sliceBudget.isOverBudget()) {
+ break;
+ }
+ }
+
+ if (!relocatedZones.isEmpty()) {
+ updateRuntimePointersToRelocatedCells(session);
+
+ do {
+ Zone* zone = relocatedZones.front();
+ relocatedZones.removeFront();
+ zone->changeGCState(Zone::Compact, Zone::Finished);
+ } while (!relocatedZones.isEmpty());
+ }
+
+ clearRelocatedArenas(relocatedArenas, reason);
+
+#ifdef DEBUG
+ protectOrReleaseRelocatedArenas(relocatedArenas, reason);
+#else
+ releaseRelocatedArenas(relocatedArenas);
+#endif
+
+ // Clear caches that can contain cell pointers.
+ rt->caches().purgeForCompaction();
+
+#ifdef DEBUG
+ checkHashTablesAfterMovingGC();
+#endif
+
+ return zonesToMaybeCompact.ref().isEmpty() ? Finished : NotFinished;
+}
+
+void GCRuntime::endCompactPhase() { startedCompacting = false; }
+
+static bool ShouldRelocateAllArenas(JS::GCReason reason) {
+ return reason == JS::GCReason::DEBUG_GC;
+}
+
+/*
+ * Choose which arenas to relocate all cells from. Return an arena cursor that
+ * can be passed to removeRemainingArenas().
+ */
+Arena** ArenaList::pickArenasToRelocate(size_t& arenaTotalOut,
+ size_t& relocTotalOut) {
+ // Relocate the greatest number of arenas such that the number of used cells
+ // in relocated arenas is less than or equal to the number of free cells in
+ // unrelocated arenas. In other words we only relocate cells we can move
+ // into existing arenas, and we choose the least full areans to relocate.
+ //
+ // This is made easier by the fact that the arena list has been sorted in
+ // descending order of number of used cells, so we will always relocate a
+ // tail of the arena list. All we need to do is find the point at which to
+ // start relocating.
+
+ check();
+
+ if (isCursorAtEnd()) {
+ return nullptr;
+ }
+
+ Arena** arenap = cursorp_; // Next arena to consider for relocation.
+ size_t previousFreeCells = 0; // Count of free cells before arenap.
+ size_t followingUsedCells = 0; // Count of used cells after arenap.
+ size_t fullArenaCount = 0; // Number of full arenas (not relocated).
+ size_t nonFullArenaCount =
+ 0; // Number of non-full arenas (considered for relocation).
+ size_t arenaIndex = 0; // Index of the next arena to consider.
+
+ for (Arena* arena = head_; arena != *cursorp_; arena = arena->next) {
+ fullArenaCount++;
+ }
+
+ for (Arena* arena = *cursorp_; arena; arena = arena->next) {
+ followingUsedCells += arena->countUsedCells();
+ nonFullArenaCount++;
+ }
+
+ mozilla::DebugOnly<size_t> lastFreeCells(0);
+ size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getAllocKind());
+
+ while (*arenap) {
+ Arena* arena = *arenap;
+ if (followingUsedCells <= previousFreeCells) {
+ break;
+ }
+
+ size_t freeCells = arena->countFreeCells();
+ size_t usedCells = cellsPerArena - freeCells;
+ followingUsedCells -= usedCells;
+#ifdef DEBUG
+ MOZ_ASSERT(freeCells >= lastFreeCells);
+ lastFreeCells = freeCells;
+#endif
+ previousFreeCells += freeCells;
+ arenap = &arena->next;
+ arenaIndex++;
+ }
+
+ size_t relocCount = nonFullArenaCount - arenaIndex;
+ MOZ_ASSERT(relocCount < nonFullArenaCount);
+ MOZ_ASSERT((relocCount == 0) == (!*arenap));
+ arenaTotalOut += fullArenaCount + nonFullArenaCount;
+ relocTotalOut += relocCount;
+
+ return arenap;
+}
+
+#ifdef DEBUG
+inline bool PtrIsInRange(const void* ptr, const void* start, size_t length) {
+ return uintptr_t(ptr) - uintptr_t(start) < length;
+}
+#endif
+
+static void RelocateCell(Zone* zone, TenuredCell* src, AllocKind thingKind,
+ size_t thingSize) {
+ JS::AutoSuppressGCAnalysis nogc;
+
+ // Allocate a new cell.
+ MOZ_ASSERT(zone == src->zone());
+ TenuredCell* dst =
+ reinterpret_cast<TenuredCell*>(AllocateCellInGC(zone, thingKind));
+
+ // Copy source cell contents to destination.
+ memcpy(dst, src, thingSize);
+
+ // Move any uid attached to the object.
+ gc::TransferUniqueId(dst, src);
+
+ if (IsObjectAllocKind(thingKind)) {
+ auto* srcObj = static_cast<JSObject*>(static_cast<Cell*>(src));
+ auto* dstObj = static_cast<JSObject*>(static_cast<Cell*>(dst));
+
+ if (srcObj->is<NativeObject>()) {
+ NativeObject* srcNative = &srcObj->as<NativeObject>();
+ NativeObject* dstNative = &dstObj->as<NativeObject>();
+
+ // Fixup the pointer to inline object elements if necessary.
+ if (srcNative->hasFixedElements()) {
+ uint32_t numShifted =
+ srcNative->getElementsHeader()->numShiftedElements();
+ dstNative->setFixedElements(numShifted);
+ }
+ } else if (srcObj->is<ProxyObject>()) {
+ if (srcObj->as<ProxyObject>().usingInlineValueArray()) {
+ dstObj->as<ProxyObject>().setInlineValueArray();
+ }
+ }
+
+ // Call object moved hook if present.
+ if (JSObjectMovedOp op = srcObj->getClass()->extObjectMovedOp()) {
+ op(dstObj, srcObj);
+ }
+
+ MOZ_ASSERT_IF(
+ dstObj->is<NativeObject>(),
+ !PtrIsInRange(
+ (const Value*)dstObj->as<NativeObject>().getDenseElements(), src,
+ thingSize));
+ }
+
+ // Copy the mark bits.
+ dst->copyMarkBitsFrom(src);
+
+ // Poison the source cell contents except for the forwarding flag and pointer
+ // which will be stored in the first word. We can't do this for native object
+ // with fixed elements because this would overwrite the element flags and
+ // these are needed when updating COW elements referred to by other objects.
+#ifdef DEBUG
+ JSObject* srcObj = IsObjectAllocKind(thingKind)
+ ? static_cast<JSObject*>(static_cast<Cell*>(src))
+ : nullptr;
+ if (!srcObj || !srcObj->is<NativeObject>() ||
+ !srcObj->as<NativeObject>().hasFixedElements()) {
+ AlwaysPoison(reinterpret_cast<uint8_t*>(src) + sizeof(uintptr_t),
+ JS_MOVED_TENURED_PATTERN, thingSize - sizeof(uintptr_t),
+ MemCheckKind::MakeNoAccess);
+ }
+#endif
+
+ // Mark source cell as forwarded and leave a pointer to the destination.
+ RelocationOverlay::forwardCell(src, dst);
+}
+
+static void RelocateArena(Arena* arena, SliceBudget& sliceBudget) {
+ MOZ_ASSERT(arena->allocated());
+ MOZ_ASSERT(!arena->onDelayedMarkingList());
+ MOZ_ASSERT(arena->bufferedCells()->isEmpty());
+
+ Zone* zone = arena->zone;
+
+ AllocKind thingKind = arena->getAllocKind();
+ size_t thingSize = arena->getThingSize();
+
+ for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+ RelocateCell(zone, cell, thingKind, thingSize);
+ sliceBudget.step();
+ }
+
+#ifdef DEBUG
+ for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+ TenuredCell* src = cell;
+ MOZ_ASSERT(src->isForwarded());
+ TenuredCell* dest = Forwarded(src);
+ MOZ_ASSERT(src->isMarkedBlack() == dest->isMarkedBlack());
+ MOZ_ASSERT(src->isMarkedGray() == dest->isMarkedGray());
+ }
+#endif
+}
+
+/*
+ * Relocate all arenas identified by pickArenasToRelocate: for each arena,
+ * relocate each cell within it, then add it to a list of relocated arenas.
+ */
+Arena* ArenaList::relocateArenas(Arena* toRelocate, Arena* relocated,
+ SliceBudget& sliceBudget,
+ gcstats::Statistics& stats) {
+ check();
+
+ while (Arena* arena = toRelocate) {
+ toRelocate = arena->next;
+ RelocateArena(arena, sliceBudget);
+ // Prepend to list of relocated arenas
+ arena->next = relocated;
+ relocated = arena;
+ stats.count(gcstats::COUNT_ARENA_RELOCATED);
+ }
+
+ check();
+
+ return relocated;
+}
+
+// Skip compacting zones unless we can free a certain proportion of their GC
+// heap memory.
+static const float MIN_ZONE_RECLAIM_PERCENT = 2.0;
+
+static bool ShouldRelocateZone(size_t arenaCount, size_t relocCount,
+ JS::GCReason reason) {
+ if (relocCount == 0) {
+ return false;
+ }
+
+ if (IsOOMReason(reason)) {
+ return true;
+ }
+
+ return (relocCount * 100.0f) / arenaCount >= MIN_ZONE_RECLAIM_PERCENT;
+}
+
+static AllocKinds CompactingAllocKinds() {
+ AllocKinds result;
+ for (AllocKind kind : AllAllocKinds()) {
+ if (IsCompactingKind(kind)) {
+ result += kind;
+ }
+ }
+ return result;
+}
+
+bool ArenaLists::relocateArenas(Arena*& relocatedListOut, JS::GCReason reason,
+ SliceBudget& sliceBudget,
+ gcstats::Statistics& stats) {
+ // This is only called from the main thread while we are doing a GC, so
+ // there is no need to lock.
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+ MOZ_ASSERT(runtime()->gc.isHeapCompacting());
+ MOZ_ASSERT(!runtime()->gc.isBackgroundSweeping());
+
+ // Relocate all compatible kinds
+ AllocKinds allocKindsToRelocate = CompactingAllocKinds();
+
+ // Clear all the free lists.
+ clearFreeLists();
+
+ if (ShouldRelocateAllArenas(reason)) {
+ zone_->prepareForCompacting();
+ for (auto kind : allocKindsToRelocate) {
+ ArenaList& al = arenaList(kind);
+ Arena* allArenas = al.head();
+ al.clear();
+ relocatedListOut =
+ al.relocateArenas(allArenas, relocatedListOut, sliceBudget, stats);
+ }
+ } else {
+ size_t arenaCount = 0;
+ size_t relocCount = 0;
+ AllAllocKindArray<Arena**> toRelocate;
+
+ for (auto kind : allocKindsToRelocate) {
+ toRelocate[kind] =
+ arenaList(kind).pickArenasToRelocate(arenaCount, relocCount);
+ }
+
+ if (!ShouldRelocateZone(arenaCount, relocCount, reason)) {
+ return false;
+ }
+
+ zone_->prepareForCompacting();
+ for (auto kind : allocKindsToRelocate) {
+ if (toRelocate[kind]) {
+ ArenaList& al = arenaList(kind);
+ Arena* arenas = al.removeRemainingArenas(toRelocate[kind]);
+ relocatedListOut =
+ al.relocateArenas(arenas, relocatedListOut, sliceBudget, stats);
+ }
+ }
+ }
+
+ return true;
+}
+
+bool GCRuntime::relocateArenas(Zone* zone, JS::GCReason reason,
+ Arena*& relocatedListOut,
+ SliceBudget& sliceBudget) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_MOVE);
+
+ MOZ_ASSERT(!zone->isPreservingCode());
+ MOZ_ASSERT(canRelocateZone(zone));
+
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Compact);
+
+ if (!zone->arenas.relocateArenas(relocatedListOut, reason, sliceBudget,
+ stats())) {
+ return false;
+ }
+
+#ifdef DEBUG
+ // Check that we did as much compaction as we should have. There
+ // should always be less than one arena's worth of free cells.
+ for (auto kind : CompactingAllocKinds()) {
+ ArenaList& al = zone->arenas.arenaList(kind);
+ size_t freeCells = 0;
+ for (Arena* arena = al.arenaAfterCursor(); arena; arena = arena->next) {
+ freeCells += arena->countFreeCells();
+ }
+ MOZ_ASSERT(freeCells < Arena::thingsPerArena(kind));
+ }
+#endif
+
+ return true;
+}
+
+MovingTracer::MovingTracer(JSRuntime* rt)
+ : GenericTracerImpl(rt, JS::TracerKind::Moving,
+ JS::WeakMapTraceAction::TraceKeysAndValues) {}
+
+template <typename T>
+inline void MovingTracer::onEdge(T** thingp, const char* name) {
+ T* thing = *thingp;
+ if (thing->runtimeFromAnyThread() == runtime() && IsForwarded(thing)) {
+ *thingp = Forwarded(thing);
+ }
+}
+
+void Zone::prepareForCompacting() {
+ JS::GCContext* gcx = runtimeFromMainThread()->gcContext();
+ discardJitCode(gcx);
+}
+
+void GCRuntime::sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone) {
+ MOZ_ASSERT(zone->isGCCompacting());
+
+ zone->traceWeakMaps(trc);
+
+ traceWeakFinalizationObserverEdges(trc, zone);
+
+ for (auto* cache : zone->weakCaches()) {
+ cache->traceWeak(trc, nullptr);
+ }
+
+ if (jit::JitZone* jitZone = zone->jitZone()) {
+ jitZone->traceWeak(trc);
+ }
+
+ for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
+ c->traceWeakNativeIterators(trc);
+
+ for (RealmsInCompartmentIter r(c); !r.done(); r.next()) {
+ r->traceWeakRegExps(trc);
+ r->traceWeakSavedStacks(trc);
+ r->traceWeakGlobalEdge(trc);
+ r->traceWeakDebugEnvironmentEdges(trc);
+ r->traceWeakEdgesInJitRealm(trc);
+ }
+ }
+}
+
+template <typename T>
+static inline void UpdateCellPointers(MovingTracer* trc, T* cell) {
+ // We only update unmoved GC things or the new copy of moved GC things, never
+ // the old copy. If this happened it could clear the forwarded flag which
+ // could lead to pointers to the old copy not being updated.
+ MOZ_ASSERT(!cell->isForwarded());
+
+ cell->fixupAfterMovingGC();
+ cell->traceChildren(trc);
+}
+
+template <typename T>
+static void UpdateArenaPointersTyped(MovingTracer* trc, Arena* arena) {
+ for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+ UpdateCellPointers(trc, cell.as<T>());
+ }
+}
+
+static bool CanUpdateKindInBackground(AllocKind kind) {
+ // We try to update as many GC things in parallel as we can, but there are
+ // kinds for which this might not be safe:
+ // - we assume JSObjects that are foreground finalized are not safe to
+ // update in parallel
+ // - updating a SharedPropMap touches child maps in
+ // SharedPropMap::fixupAfterMovingGC
+ return js::gc::IsBackgroundFinalized(kind) && !IsShapeAllocKind(kind) &&
+ kind != AllocKind::BASE_SHAPE;
+}
+
+/*
+ * Update the internal pointers for all cells in an arena.
+ */
+static void UpdateArenaPointers(MovingTracer* trc, Arena* arena) {
+ AllocKind kind = arena->getAllocKind();
+
+ MOZ_ASSERT_IF(!CanUpdateKindInBackground(kind),
+ CurrentThreadCanAccessRuntime(trc->runtime()));
+
+ switch (kind) {
+#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery, \
+ compact) \
+ case AllocKind::allocKind: \
+ UpdateArenaPointersTyped<type>(trc, arena); \
+ return;
+ FOR_EACH_ALLOCKIND(EXPAND_CASE)
+#undef EXPAND_CASE
+
+ default:
+ MOZ_CRASH("Invalid alloc kind for UpdateArenaPointers");
+ }
+}
+
+struct ArenaListSegment {
+ Arena* begin;
+ Arena* end;
+};
+
+/*
+ * Update the internal pointers for all arenas in a segment of an arena list.
+ *
+ * Returns the number of steps to count against the slice budget.
+ */
+static size_t UpdateArenaListSegmentPointers(GCRuntime* gc,
+ const ArenaListSegment& arenas) {
+ MOZ_ASSERT(arenas.begin);
+ MovingTracer trc(gc->rt);
+ size_t count = 0;
+ for (Arena* arena = arenas.begin; arena != arenas.end; arena = arena->next) {
+ UpdateArenaPointers(&trc, arena);
+ count++;
+ }
+ return count * 256;
+}
+
+class ArenasToUpdate {
+ // Maximum number of arenas to update in one block.
+#ifdef DEBUG
+ static const unsigned MaxArenasToProcess = 16;
+#else
+ static const unsigned MaxArenasToProcess = 256;
+#endif
+
+ public:
+ explicit ArenasToUpdate(Zone* zone);
+ ArenasToUpdate(Zone* zone, const AllocKinds& kinds);
+
+ bool done() const { return !segmentBegin; }
+
+ ArenaListSegment get() const {
+ MOZ_ASSERT(!done());
+ return {segmentBegin, segmentEnd};
+ }
+
+ void next();
+
+ private:
+ Maybe<AllocKinds> kinds; // Selects which thing kinds to update.
+ Zone* zone; // Zone to process.
+ AllocKind kind = AllocKind::FIRST; // Current alloc kind to process.
+ Arena* segmentBegin = nullptr;
+ Arena* segmentEnd = nullptr;
+
+ static AllocKind nextAllocKind(AllocKind i) {
+ return AllocKind(uint8_t(i) + 1);
+ }
+
+ void settle();
+ void findSegmentEnd();
+};
+
+ArenasToUpdate::ArenasToUpdate(Zone* zone) : zone(zone) { settle(); }
+
+ArenasToUpdate::ArenasToUpdate(Zone* zone, const AllocKinds& kinds)
+ : kinds(Some(kinds)), zone(zone) {
+ settle();
+}
+
+void ArenasToUpdate::settle() {
+ // Called when we have set |kind| to a new kind. Sets |arena| to the next
+ // arena or null if there are no more arenas to update.
+
+ MOZ_ASSERT(!segmentBegin);
+
+ for (; kind < AllocKind::LIMIT; kind = nextAllocKind(kind)) {
+ if (kinds && !kinds.ref().contains(kind)) {
+ continue;
+ }
+
+ Arena* arena = zone->arenas.getFirstArena(kind);
+ if (arena) {
+ segmentBegin = arena;
+ findSegmentEnd();
+ break;
+ }
+ }
+}
+
+void ArenasToUpdate::findSegmentEnd() {
+ // Take up to MaxArenasToProcess arenas from the list starting at
+ // |segmentBegin| and set |segmentEnd|.
+ Arena* arena = segmentBegin;
+ for (size_t i = 0; arena && i < MaxArenasToProcess; i++) {
+ arena = arena->next;
+ }
+ segmentEnd = arena;
+}
+
+void ArenasToUpdate::next() {
+ MOZ_ASSERT(!done());
+
+ segmentBegin = segmentEnd;
+ if (segmentBegin) {
+ findSegmentEnd();
+ return;
+ }
+
+ kind = nextAllocKind(kind);
+ settle();
+}
+
+static AllocKinds ForegroundUpdateKinds(AllocKinds kinds) {
+ AllocKinds result;
+ for (AllocKind kind : kinds) {
+ if (!CanUpdateKindInBackground(kind)) {
+ result += kind;
+ }
+ }
+ return result;
+}
+
+void GCRuntime::updateCellPointers(Zone* zone, AllocKinds kinds) {
+ AllocKinds fgKinds = ForegroundUpdateKinds(kinds);
+ AllocKinds bgKinds = kinds - fgKinds;
+
+ ArenasToUpdate fgArenas(zone, fgKinds);
+ ArenasToUpdate bgArenas(zone, bgKinds);
+
+ AutoLockHelperThreadState lock;
+
+ AutoRunParallelWork bgTasks(this, UpdateArenaListSegmentPointers,
+ gcstats::PhaseKind::COMPACT_UPDATE_CELLS,
+ GCUse::Unspecified, bgArenas,
+ SliceBudget::unlimited(), lock);
+
+ AutoUnlockHelperThreadState unlock(lock);
+
+ for (; !fgArenas.done(); fgArenas.next()) {
+ UpdateArenaListSegmentPointers(this, fgArenas.get());
+ }
+}
+
+// After cells have been relocated any pointers to a cell's old locations must
+// be updated to point to the new location. This happens by iterating through
+// all cells in heap and tracing their children (non-recursively) to update
+// them.
+//
+// This is complicated by the fact that updating a GC thing sometimes depends on
+// making use of other GC things. After a moving GC these things may not be in
+// a valid state since they may contain pointers which have not been updated
+// yet.
+//
+// The main dependencies are:
+//
+// - Updating a JSObject makes use of its shape
+// - Updating a typed object makes use of its type descriptor object
+//
+// This means we require at least three phases for update:
+//
+// 1) shapes
+// 2) typed object type descriptor objects
+// 3) all other objects
+//
+// Also, there can be data races calling IsForwarded() on the new location of a
+// cell whose first word is being updated in parallel on another thread. This
+// easiest way to avoid this is to not store a GC pointer in the first word of a
+// cell. Otherwise this can be avoided by updating different kinds of cell in
+// different phases.
+//
+// Since we want to minimize the number of phases, arrange kinds into three
+// arbitrary phases.
+
+static constexpr AllocKinds UpdatePhaseOne{AllocKind::SCRIPT,
+ AllocKind::BASE_SHAPE,
+ AllocKind::SHAPE,
+ AllocKind::STRING,
+ AllocKind::JITCODE,
+ AllocKind::REGEXP_SHARED,
+ AllocKind::SCOPE,
+ AllocKind::GETTER_SETTER,
+ AllocKind::COMPACT_PROP_MAP,
+ AllocKind::NORMAL_PROP_MAP,
+ AllocKind::DICT_PROP_MAP};
+
+// UpdatePhaseTwo is typed object descriptor objects.
+
+static constexpr AllocKinds UpdatePhaseThree{AllocKind::FUNCTION,
+ AllocKind::FUNCTION_EXTENDED,
+ AllocKind::OBJECT0,
+ AllocKind::OBJECT0_BACKGROUND,
+ AllocKind::OBJECT2,
+ AllocKind::OBJECT2_BACKGROUND,
+ AllocKind::ARRAYBUFFER4,
+ AllocKind::OBJECT4,
+ AllocKind::OBJECT4_BACKGROUND,
+ AllocKind::ARRAYBUFFER8,
+ AllocKind::OBJECT8,
+ AllocKind::OBJECT8_BACKGROUND,
+ AllocKind::ARRAYBUFFER12,
+ AllocKind::OBJECT12,
+ AllocKind::OBJECT12_BACKGROUND,
+ AllocKind::ARRAYBUFFER16,
+ AllocKind::OBJECT16,
+ AllocKind::OBJECT16_BACKGROUND};
+
+void GCRuntime::updateAllCellPointers(MovingTracer* trc, Zone* zone) {
+ updateCellPointers(zone, UpdatePhaseOne);
+
+ updateCellPointers(zone, UpdatePhaseThree);
+}
+
+/*
+ * Update pointers to relocated cells in a single zone by doing a traversal of
+ * that zone's arenas and calling per-zone sweep hooks.
+ *
+ * The latter is necessary to update weak references which are not marked as
+ * part of the traversal.
+ */
+void GCRuntime::updateZonePointersToRelocatedCells(Zone* zone) {
+ MOZ_ASSERT(!rt->isBeingDestroyed());
+ MOZ_ASSERT(zone->isGCCompacting());
+
+ AutoTouchingGrayThings tgt;
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_UPDATE);
+ MovingTracer trc(rt);
+
+ zone->fixupAfterMovingGC();
+ zone->fixupScriptMapsAfterMovingGC(&trc);
+
+ // Fixup compartment global pointers as these get accessed during marking.
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ comp->fixupAfterMovingGC(&trc);
+ }
+
+ zone->externalStringCache().purge();
+ zone->functionToStringCache().purge();
+ zone->shapeZone().purgeShapeCaches(rt->gcContext());
+ rt->caches().stringToAtomCache.purge();
+
+ // Iterate through all cells that can contain relocatable pointers to update
+ // them. Since updating each cell is independent we try to parallelize this
+ // as much as possible.
+ updateAllCellPointers(&trc, zone);
+
+ // Sweep everything to fix up weak pointers.
+ sweepZoneAfterCompacting(&trc, zone);
+
+ // Call callbacks to get the rest of the system to fixup other untraced
+ // pointers.
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ callWeakPointerCompartmentCallbacks(&trc, comp);
+ }
+}
+
+/*
+ * Update runtime-wide pointers to relocated cells.
+ */
+void GCRuntime::updateRuntimePointersToRelocatedCells(AutoGCSession& session) {
+ MOZ_ASSERT(!rt->isBeingDestroyed());
+
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::COMPACT_UPDATE);
+ MovingTracer trc(rt);
+
+ Zone::fixupAllCrossCompartmentWrappersAfterMovingGC(&trc);
+
+ rt->geckoProfiler().fixupStringsMapAfterMovingGC();
+
+ // Mark roots to update them.
+
+ traceRuntimeForMajorGC(&trc, session);
+
+ {
+ gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::MARK_ROOTS);
+ DebugAPI::traceAllForMovingGC(&trc);
+ DebugAPI::traceCrossCompartmentEdges(&trc);
+
+ // Mark all gray roots.
+ traceEmbeddingGrayRoots(&trc);
+ Compartment::traceIncomingCrossCompartmentEdgesForZoneGC(
+ &trc, Compartment::GrayEdges);
+ }
+
+ // Sweep everything to fix up weak pointers.
+ jit::JitRuntime::TraceWeakJitcodeGlobalTable(rt, &trc);
+ for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) {
+ cache->traceWeak(&trc, nullptr);
+ }
+
+ if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) {
+ rt->jitRuntime()->getInterpreterEntryMap()->updateScriptsAfterMovingGC();
+ }
+
+ // Type inference may put more blocks here to free.
+ {
+ AutoLockHelperThreadState lock;
+ lifoBlocksToFree.ref().freeAll();
+ }
+
+ // Call callbacks to get the rest of the system to fixup other untraced
+ // pointers.
+ callWeakPointerZonesCallbacks(&trc);
+}
+
+void GCRuntime::clearRelocatedArenas(Arena* arenaList, JS::GCReason reason) {
+ AutoLockGC lock(this);
+ clearRelocatedArenasWithoutUnlocking(arenaList, reason, lock);
+}
+
+void GCRuntime::clearRelocatedArenasWithoutUnlocking(Arena* arenaList,
+ JS::GCReason reason,
+ const AutoLockGC& lock) {
+ // Clear the relocated arenas, now containing only forwarding pointers
+ while (arenaList) {
+ Arena* arena = arenaList;
+ arenaList = arenaList->next;
+
+ // Clear the mark bits
+ arena->unmarkAll();
+
+ // Mark arena as empty
+ arena->setAsFullyUnused();
+
+#ifdef DEBUG
+ // The cell contents have been partially marked no access in RelocateCell,
+ // so we need to mark the region as undefined again so we can poison it.
+ SetMemCheckKind(reinterpret_cast<void*>(arena->thingsStart()),
+ arena->getThingsSpan(), MemCheckKind::MakeUndefined);
+#endif
+
+ AlwaysPoison(reinterpret_cast<void*>(arena->thingsStart()),
+ JS_MOVED_TENURED_PATTERN, arena->getThingsSpan(),
+ MemCheckKind::MakeNoAccess);
+
+ // Don't count emptied arenas as being freed by the current GC:
+ // - if we purposely moved everything to new arenas, as that will already
+ // have allocated a similar number of arenas. (This only happens for
+ // collections triggered by GC zeal.)
+ // - if they were allocated since the start of the GC.
+ bool allArenasRelocated = ShouldRelocateAllArenas(reason);
+ bool updateRetainedSize = !allArenasRelocated && !arena->isNewlyCreated();
+ arena->zone->gcHeapSize.removeBytes(ArenaSize, updateRetainedSize,
+ heapSize);
+
+ // Release the arena but don't return it to the chunk yet.
+ arena->release(lock);
+ }
+}
+
+#ifdef DEBUG
+
+// In debug mode we don't always release relocated arenas straight away.
+// Sometimes protect them instead and hold onto them until the next GC sweep
+// phase to catch any pointers to them that didn't get forwarded.
+
+static inline bool CanProtectArenas() {
+ // On some systems the page size is larger than the size of an arena so we
+ // can't change the mapping permissions per arena.
+ return SystemPageSize() <= ArenaSize;
+}
+
+static inline bool ShouldProtectRelocatedArenas(JS::GCReason reason) {
+ // For zeal mode collections we don't release the relocated arenas
+ // immediately. Instead we protect them and keep them around until the next
+ // collection so we can catch any stray accesses to them.
+ return reason == JS::GCReason::DEBUG_GC && CanProtectArenas();
+}
+
+void GCRuntime::protectOrReleaseRelocatedArenas(Arena* arenaList,
+ JS::GCReason reason) {
+ if (ShouldProtectRelocatedArenas(reason)) {
+ protectAndHoldArenas(arenaList);
+ return;
+ }
+
+ releaseRelocatedArenas(arenaList);
+}
+
+void GCRuntime::protectAndHoldArenas(Arena* arenaList) {
+ for (Arena* arena = arenaList; arena;) {
+ MOZ_ASSERT(!arena->allocated());
+ Arena* next = arena->next;
+ if (!next) {
+ // Prepend to hold list before we protect the memory.
+ AutoLockGC lock(this);
+ arena->next = relocatedArenasToRelease;
+ relocatedArenasToRelease = arenaList;
+ }
+ ProtectPages(arena, ArenaSize);
+ arena = next;
+ }
+}
+
+void GCRuntime::unprotectHeldRelocatedArenas(const AutoLockGC& lock) {
+ for (Arena* arena = relocatedArenasToRelease; arena; arena = arena->next) {
+ UnprotectPages(arena, ArenaSize);
+ MOZ_ASSERT(!arena->allocated());
+ }
+}
+
+void GCRuntime::releaseHeldRelocatedArenas() {
+ AutoLockGC lock(this);
+ unprotectHeldRelocatedArenas(lock);
+ Arena* arenas = relocatedArenasToRelease;
+ relocatedArenasToRelease = nullptr;
+ releaseRelocatedArenasWithoutUnlocking(arenas, lock);
+}
+
+void GCRuntime::releaseHeldRelocatedArenasWithoutUnlocking(
+ const AutoLockGC& lock) {
+ unprotectHeldRelocatedArenas(lock);
+ releaseRelocatedArenasWithoutUnlocking(relocatedArenasToRelease, lock);
+ relocatedArenasToRelease = nullptr;
+}
+
+#endif
+
+void GCRuntime::releaseRelocatedArenas(Arena* arenaList) {
+ AutoLockGC lock(this);
+ releaseRelocatedArenasWithoutUnlocking(arenaList, lock);
+}
+
+void GCRuntime::releaseRelocatedArenasWithoutUnlocking(Arena* arenaList,
+ const AutoLockGC& lock) {
+ // Release relocated arenas previously cleared with clearRelocatedArenas().
+ while (arenaList) {
+ Arena* arena = arenaList;
+ arenaList = arenaList->next;
+
+ // We already updated the memory accounting so just call
+ // Chunk::releaseArena.
+ arena->chunk()->releaseArena(this, arena, lock);
+ }
+}
diff --git a/js/src/gc/FinalizationObservers.cpp b/js/src/gc/FinalizationObservers.cpp
new file mode 100644
index 0000000000..3a7a114645
--- /dev/null
+++ b/js/src/gc/FinalizationObservers.cpp
@@ -0,0 +1,509 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC support for FinalizationRegistry and WeakRef objects.
+ */
+
+#include "gc/FinalizationObservers.h"
+
+#include "mozilla/ScopeExit.h"
+
+#include "builtin/FinalizationRegistryObject.h"
+#include "builtin/WeakRefObject.h"
+#include "gc/GCRuntime.h"
+#include "gc/Zone.h"
+#include "vm/JSContext.h"
+
+#include "gc/WeakMap-inl.h"
+#include "vm/JSObject-inl.h"
+#include "vm/NativeObject-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+FinalizationObservers::FinalizationObservers(Zone* zone)
+ : zone(zone),
+ registries(zone),
+ recordMap(zone),
+ crossZoneRecords(zone),
+ weakRefMap(zone),
+ crossZoneWeakRefs(zone) {}
+
+FinalizationObservers::~FinalizationObservers() {
+ MOZ_ASSERT(registries.empty());
+ MOZ_ASSERT(recordMap.empty());
+ MOZ_ASSERT(crossZoneRecords.empty());
+ MOZ_ASSERT(crossZoneWeakRefs.empty());
+}
+
+bool GCRuntime::addFinalizationRegistry(
+ JSContext* cx, Handle<FinalizationRegistryObject*> registry) {
+ if (!cx->zone()->ensureFinalizationObservers() ||
+ !cx->zone()->finalizationObservers()->addRegistry(registry)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+bool FinalizationObservers::addRegistry(
+ Handle<FinalizationRegistryObject*> registry) {
+ return registries.put(registry);
+}
+
+bool GCRuntime::registerWithFinalizationRegistry(JSContext* cx,
+ HandleObject target,
+ HandleObject record) {
+ MOZ_ASSERT(!IsCrossCompartmentWrapper(target));
+ MOZ_ASSERT(
+ UncheckedUnwrapWithoutExpose(record)->is<FinalizationRecordObject>());
+ MOZ_ASSERT(target->compartment() == record->compartment());
+
+ Zone* zone = cx->zone();
+ if (!zone->ensureFinalizationObservers() ||
+ !zone->finalizationObservers()->addRecord(target, record)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+bool FinalizationObservers::addRecord(HandleObject target,
+ HandleObject record) {
+ // Add a record to the record map and clean up on failure.
+ //
+ // The following must be updated and kept in sync:
+ // - the zone's recordMap (to observe the target)
+ // - the registry's global objects's recordSet (to trace the record)
+ // - the count of cross zone records (to calculate sweep groups)
+
+ MOZ_ASSERT(target->zone() == zone);
+
+ FinalizationRecordObject* unwrappedRecord =
+ &UncheckedUnwrapWithoutExpose(record)->as<FinalizationRecordObject>();
+
+ Zone* registryZone = unwrappedRecord->zone();
+ bool crossZone = registryZone != zone;
+ if (crossZone && !addCrossZoneWrapper(crossZoneRecords, record)) {
+ return false;
+ }
+ auto wrapperGuard = mozilla::MakeScopeExit([&] {
+ if (crossZone) {
+ removeCrossZoneWrapper(crossZoneRecords, record);
+ }
+ });
+
+ GlobalObject* registryGlobal = &unwrappedRecord->global();
+ auto* globalData = registryGlobal->getOrCreateFinalizationRegistryData();
+ if (!globalData || !globalData->addRecord(unwrappedRecord)) {
+ return false;
+ }
+ auto globalDataGuard = mozilla::MakeScopeExit(
+ [&] { globalData->removeRecord(unwrappedRecord); });
+
+ auto ptr = recordMap.lookupForAdd(target);
+ if (!ptr && !recordMap.add(ptr, target, RecordVector(zone))) {
+ return false;
+ }
+
+ if (!ptr->value().append(record)) {
+ return false;
+ }
+
+ unwrappedRecord->setInRecordMap(true);
+
+ globalDataGuard.release();
+ wrapperGuard.release();
+ return true;
+}
+
+bool FinalizationObservers::addCrossZoneWrapper(WrapperWeakSet& weakSet,
+ JSObject* wrapper) {
+ MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper));
+ MOZ_ASSERT(UncheckedUnwrapWithoutExpose(wrapper)->zone() != zone);
+
+ auto ptr = weakSet.lookupForAdd(wrapper);
+ MOZ_ASSERT(!ptr);
+ return weakSet.add(ptr, wrapper, UndefinedValue());
+}
+
+void FinalizationObservers::removeCrossZoneWrapper(WrapperWeakSet& weakSet,
+ JSObject* wrapper) {
+ MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper));
+ MOZ_ASSERT(UncheckedUnwrapWithoutExpose(wrapper)->zone() != zone);
+
+ auto ptr = weakSet.lookupForAdd(wrapper);
+ MOZ_ASSERT(ptr);
+ weakSet.remove(ptr);
+}
+
+static FinalizationRecordObject* UnwrapFinalizationRecord(JSObject* obj) {
+ obj = UncheckedUnwrapWithoutExpose(obj);
+ if (!obj->is<FinalizationRecordObject>()) {
+ MOZ_ASSERT(JS_IsDeadWrapper(obj));
+ // CCWs between the compartments have been nuked. The
+ // FinalizationRegistry's callback doesn't run in this case.
+ return nullptr;
+ }
+ return &obj->as<FinalizationRecordObject>();
+}
+
+void FinalizationObservers::clearRecords() {
+ // Clear table entries related to FinalizationRecordObjects, which are not
+ // processed after the start of shutdown.
+ //
+ // WeakRefs are still updated during shutdown to avoid the possibility of
+ // stale or dangling pointers.
+
+#ifdef DEBUG
+ checkTables();
+#endif
+
+ recordMap.clear();
+ crossZoneRecords.clear();
+}
+
+void GCRuntime::traceWeakFinalizationObserverEdges(JSTracer* trc, Zone* zone) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(trc->runtime()));
+ FinalizationObservers* observers = zone->finalizationObservers();
+ if (observers) {
+ observers->traceWeakEdges(trc);
+ }
+}
+
+void FinalizationObservers::traceRoots(JSTracer* trc) {
+ // The cross-zone wrapper weak maps are traced as roots; this does not keep
+ // any of their entries alive by itself.
+ crossZoneRecords.trace(trc);
+ crossZoneWeakRefs.trace(trc);
+}
+
+void FinalizationObservers::traceWeakEdges(JSTracer* trc) {
+ // Removing dead pointers from vectors may reorder live pointers to gray
+ // things in the vector. This is OK.
+ AutoTouchingGrayThings atgt;
+
+ traceWeakWeakRefEdges(trc);
+ traceWeakFinalizationRegistryEdges(trc);
+}
+
+void FinalizationObservers::traceWeakFinalizationRegistryEdges(JSTracer* trc) {
+ // Sweep finalization registry data and queue finalization records for cleanup
+ // for any entries whose target is dying and remove them from the map.
+
+ GCRuntime* gc = &trc->runtime()->gc;
+
+ for (RegistrySet::Enum e(registries); !e.empty(); e.popFront()) {
+ auto result = TraceWeakEdge(trc, &e.mutableFront(), "FinalizationRegistry");
+ if (result.isDead()) {
+ auto* registry =
+ &result.initialTarget()->as<FinalizationRegistryObject>();
+ registry->queue()->setHasRegistry(false);
+ e.removeFront();
+ } else {
+ result.finalTarget()->as<FinalizationRegistryObject>().traceWeak(trc);
+ }
+ }
+
+ for (RecordMap::Enum e(recordMap); !e.empty(); e.popFront()) {
+ RecordVector& records = e.front().value();
+
+ // Sweep finalization records, updating any pointers moved by the GC and
+ // remove if necessary.
+ records.mutableEraseIf([&](HeapPtr<JSObject*>& heapPtr) {
+ auto result = TraceWeakEdge(trc, &heapPtr, "FinalizationRecord");
+ JSObject* obj =
+ result.isLive() ? result.finalTarget() : result.initialTarget();
+ FinalizationRecordObject* record = UnwrapFinalizationRecord(obj);
+ MOZ_ASSERT_IF(record, record->isInRecordMap());
+
+ bool shouldRemove = !result.isLive() || shouldRemoveRecord(record);
+ if (shouldRemove && record && record->isInRecordMap()) {
+ updateForRemovedRecord(obj, record);
+ }
+
+ return shouldRemove;
+ });
+
+#ifdef DEBUG
+ for (JSObject* obj : records) {
+ MOZ_ASSERT(UnwrapFinalizationRecord(obj)->isInRecordMap());
+ }
+#endif
+
+ // Queue finalization records for targets that are dying.
+ if (!TraceWeakEdge(trc, &e.front().mutableKey(),
+ "FinalizationRecord target")) {
+ for (JSObject* obj : records) {
+ FinalizationRecordObject* record = UnwrapFinalizationRecord(obj);
+ FinalizationQueueObject* queue = record->queue();
+ updateForRemovedRecord(obj, record);
+ queue->queueRecordToBeCleanedUp(record);
+ gc->queueFinalizationRegistryForCleanup(queue);
+ }
+ e.removeFront();
+ }
+ }
+}
+
+// static
+bool FinalizationObservers::shouldRemoveRecord(
+ FinalizationRecordObject* record) {
+ // Records are removed from the target's vector for the following reasons:
+ return !record || // Nuked CCW to record.
+ !record->isRegistered() || // Unregistered record.
+ !record->queue()->hasRegistry(); // Dead finalization registry.
+}
+
+void FinalizationObservers::updateForRemovedRecord(
+ JSObject* wrapper, FinalizationRecordObject* record) {
+ // Remove other references to a record when it has been removed from the
+ // zone's record map. See addRecord().
+ MOZ_ASSERT(record->isInRecordMap());
+
+ Zone* registryZone = record->zone();
+ if (registryZone != zone) {
+ removeCrossZoneWrapper(crossZoneRecords, wrapper);
+ }
+
+ GlobalObject* registryGlobal = &record->global();
+ auto* globalData = registryGlobal->maybeFinalizationRegistryData();
+ globalData->removeRecord(record);
+
+ // The removed record may be gray, and that's OK.
+ AutoTouchingGrayThings atgt;
+
+ record->setInRecordMap(false);
+}
+
+void GCRuntime::nukeFinalizationRecordWrapper(
+ JSObject* wrapper, FinalizationRecordObject* record) {
+ if (record->isInRecordMap()) {
+ FinalizationRegistryObject::unregisterRecord(record);
+ FinalizationObservers* observers = wrapper->zone()->finalizationObservers();
+ observers->updateForRemovedRecord(wrapper, record);
+ }
+}
+
+void GCRuntime::queueFinalizationRegistryForCleanup(
+ FinalizationQueueObject* queue) {
+ // Prod the embedding to call us back later to run the finalization callbacks,
+ // if necessary.
+
+ if (queue->isQueuedForCleanup()) {
+ return;
+ }
+
+ // Derive the incumbent global by unwrapping the incumbent global object and
+ // then getting its global.
+ JSObject* object = UncheckedUnwrapWithoutExpose(queue->incumbentObject());
+ MOZ_ASSERT(object);
+ GlobalObject* incumbentGlobal = &object->nonCCWGlobal();
+
+ callHostCleanupFinalizationRegistryCallback(queue->doCleanupFunction(),
+ incumbentGlobal);
+
+ // The queue object may be gray, and that's OK.
+ AutoTouchingGrayThings atgt;
+
+ queue->setQueuedForCleanup(true);
+}
+
+// Insert a target -> weakRef mapping in the target's Zone so that a dying
+// target will clear out the weakRef's target. If the weakRef is in a different
+// Zone, then the crossZoneWeakRefs table will keep the weakRef alive. If the
+// weakRef is in the same Zone, then it must be the actual WeakRefObject and
+// not a cross-compartment wrapper, since nothing would keep that alive.
+bool GCRuntime::registerWeakRef(HandleObject target, HandleObject weakRef) {
+ MOZ_ASSERT(!IsCrossCompartmentWrapper(target));
+ MOZ_ASSERT(UncheckedUnwrap(weakRef)->is<WeakRefObject>());
+ MOZ_ASSERT_IF(target->zone() != weakRef->zone(),
+ target->compartment() == weakRef->compartment());
+
+ Zone* zone = target->zone();
+ return zone->ensureFinalizationObservers() &&
+ zone->finalizationObservers()->addWeakRefTarget(target, weakRef);
+}
+
+bool FinalizationObservers::addWeakRefTarget(HandleObject target,
+ HandleObject weakRef) {
+ WeakRefObject* unwrappedWeakRef =
+ &UncheckedUnwrapWithoutExpose(weakRef)->as<WeakRefObject>();
+
+ Zone* weakRefZone = unwrappedWeakRef->zone();
+ bool crossZone = weakRefZone != zone;
+ if (crossZone && !addCrossZoneWrapper(crossZoneWeakRefs, weakRef)) {
+ return false;
+ }
+ auto wrapperGuard = mozilla::MakeScopeExit([&] {
+ if (crossZone) {
+ removeCrossZoneWrapper(crossZoneWeakRefs, weakRef);
+ }
+ });
+
+ auto ptr = weakRefMap.lookupForAdd(target);
+ if (!ptr && !weakRefMap.add(ptr, target, WeakRefHeapPtrVector(zone))) {
+ return false;
+ }
+
+ if (!ptr->value().emplaceBack(weakRef)) {
+ return false;
+ }
+
+ wrapperGuard.release();
+ return true;
+}
+
+static WeakRefObject* UnwrapWeakRef(JSObject* obj) {
+ MOZ_ASSERT(!JS_IsDeadWrapper(obj));
+ obj = UncheckedUnwrapWithoutExpose(obj);
+ return &obj->as<WeakRefObject>();
+}
+
+void FinalizationObservers::removeWeakRefTarget(
+ Handle<JSObject*> target, Handle<WeakRefObject*> weakRef) {
+ MOZ_ASSERT(target);
+
+ WeakRefHeapPtrVector& weakRefs = weakRefMap.lookup(target)->value();
+ JSObject* wrapper = nullptr;
+ weakRefs.eraseIf([weakRef, &wrapper](JSObject* obj) {
+ if (UnwrapWeakRef(obj) == weakRef) {
+ wrapper = obj;
+ return true;
+ }
+ return false;
+ });
+
+ MOZ_ASSERT(wrapper);
+ updateForRemovedWeakRef(wrapper, weakRef);
+}
+
+void GCRuntime::nukeWeakRefWrapper(JSObject* wrapper, WeakRefObject* weakRef) {
+ // WeakRef wrappers can exist independently of the ones we create for the
+ // weakRefMap so don't assume |wrapper| is in the same zone as the WeakRef
+ // target.
+ JSObject* target = weakRef->target();
+ if (!target) {
+ return;
+ }
+
+ FinalizationObservers* observers = target->zone()->finalizationObservers();
+ if (observers) {
+ observers->unregisterWeakRefWrapper(wrapper, weakRef);
+ }
+}
+
+void FinalizationObservers::unregisterWeakRefWrapper(JSObject* wrapper,
+ WeakRefObject* weakRef) {
+ JSObject* target = weakRef->target();
+ MOZ_ASSERT(target);
+
+ bool removed = false;
+ WeakRefHeapPtrVector& weakRefs = weakRefMap.lookup(target)->value();
+ weakRefs.eraseIf([wrapper, &removed](JSObject* obj) {
+ bool remove = obj == wrapper;
+ if (remove) {
+ removed = true;
+ }
+ return remove;
+ });
+
+ if (removed) {
+ updateForRemovedWeakRef(wrapper, weakRef);
+ }
+}
+
+void FinalizationObservers::updateForRemovedWeakRef(JSObject* wrapper,
+ WeakRefObject* weakRef) {
+ weakRef->clearTarget();
+
+ Zone* weakRefZone = weakRef->zone();
+ if (weakRefZone != zone) {
+ removeCrossZoneWrapper(crossZoneWeakRefs, wrapper);
+ }
+}
+
+void FinalizationObservers::traceWeakWeakRefEdges(JSTracer* trc) {
+ for (WeakRefMap::Enum e(weakRefMap); !e.empty(); e.popFront()) {
+ // If target is dying, clear the target field of all weakRefs, and remove
+ // the entry from the map.
+ auto result = TraceWeakEdge(trc, &e.front().mutableKey(), "WeakRef target");
+ if (result.isDead()) {
+ for (JSObject* obj : e.front().value()) {
+ updateForRemovedWeakRef(obj, UnwrapWeakRef(obj));
+ }
+ e.removeFront();
+ } else {
+ // Update the target field after compacting.
+ traceWeakWeakRefVector(trc, e.front().value(), result.finalTarget());
+ }
+ }
+}
+
+void FinalizationObservers::traceWeakWeakRefVector(
+ JSTracer* trc, WeakRefHeapPtrVector& weakRefs, JSObject* target) {
+ weakRefs.mutableEraseIf([&](HeapPtr<JSObject*>& obj) -> bool {
+ auto result = TraceWeakEdge(trc, &obj, "WeakRef");
+ if (result.isDead()) {
+ JSObject* wrapper = result.initialTarget();
+ updateForRemovedWeakRef(wrapper, UnwrapWeakRef(wrapper));
+ } else {
+ UnwrapWeakRef(result.finalTarget())->setTargetUnbarriered(target);
+ }
+ return result.isDead();
+ });
+}
+
+#ifdef DEBUG
+void FinalizationObservers::checkTables() const {
+ // Check all cross-zone wrappers are present in the appropriate table.
+ size_t recordCount = 0;
+ for (auto r = recordMap.all(); !r.empty(); r.popFront()) {
+ for (JSObject* object : r.front().value()) {
+ FinalizationRecordObject* record = UnwrapFinalizationRecord(object);
+ if (record && record->isInRecordMap() && record->zone() != zone) {
+ MOZ_ASSERT(crossZoneRecords.has(object));
+ recordCount++;
+ }
+ }
+ }
+ MOZ_ASSERT(crossZoneRecords.count() == recordCount);
+
+ size_t weakRefCount = 0;
+ for (auto r = weakRefMap.all(); !r.empty(); r.popFront()) {
+ for (JSObject* object : r.front().value()) {
+ WeakRefObject* weakRef = UnwrapWeakRef(object);
+ if (weakRef && weakRef->zone() != zone) {
+ MOZ_ASSERT(crossZoneWeakRefs.has(object));
+ weakRefCount++;
+ }
+ }
+ }
+ MOZ_ASSERT(crossZoneWeakRefs.count() == weakRefCount);
+}
+#endif
+
+FinalizationRegistryGlobalData::FinalizationRegistryGlobalData(Zone* zone)
+ : recordSet(zone) {}
+
+bool FinalizationRegistryGlobalData::addRecord(
+ FinalizationRecordObject* record) {
+ return recordSet.putNew(record);
+}
+
+void FinalizationRegistryGlobalData::removeRecord(
+ FinalizationRecordObject* record) {
+ MOZ_ASSERT_IF(!record->runtimeFromMainThread()->gc.isShuttingDown(),
+ recordSet.has(record));
+ recordSet.remove(record);
+}
+
+void FinalizationRegistryGlobalData::trace(JSTracer* trc) {
+ recordSet.trace(trc);
+}
diff --git a/js/src/gc/FinalizationObservers.h b/js/src/gc/FinalizationObservers.h
new file mode 100644
index 0000000000..eee8ef14e0
--- /dev/null
+++ b/js/src/gc/FinalizationObservers.h
@@ -0,0 +1,129 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_FinalizationObservers_h
+#define gc_FinalizationObservers_h
+
+#include "gc/Barrier.h"
+#include "gc/WeakMap.h"
+#include "gc/ZoneAllocator.h"
+#include "js/GCHashTable.h"
+#include "js/GCVector.h"
+
+namespace js {
+
+class FinalizationRegistryObject;
+class FinalizationRecordObject;
+class WeakRefObject;
+
+namespace gc {
+
+// Per-zone data structures to support FinalizationRegistry and WeakRef.
+class FinalizationObservers {
+ Zone* const zone;
+
+ // The set of all finalization registries in the associated zone. These are
+ // direct pointers and are not wrapped.
+ using RegistrySet =
+ GCHashSet<HeapPtr<JSObject*>, StableCellHasher<HeapPtr<JSObject*>>,
+ ZoneAllocPolicy>;
+ RegistrySet registries;
+
+ // A vector of FinalizationRecord objects, or CCWs to them.
+ using RecordVector = GCVector<HeapPtr<JSObject*>, 1, ZoneAllocPolicy>;
+
+ // A map from finalization registry targets to a vector of finalization
+ // records representing registries that the target is registered with and
+ // their associated held values. The records may be in other zones and are
+ // wrapped appropriately.
+ using RecordMap =
+ GCHashMap<HeapPtr<JSObject*>, RecordVector,
+ StableCellHasher<HeapPtr<JSObject*>>, ZoneAllocPolicy>;
+ RecordMap recordMap;
+
+ // A weak map used as a set of cross-zone wrappers. This is used for both
+ // finalization registries and weak refs. For the former it has wrappers to
+ // finalization record objects and for the latter wrappers to weak refs.
+ //
+ // The weak map marking rules keep the wrappers alive while their targets are
+ // alive and ensure that they are both swept in the same sweep group.
+ using WrapperWeakSet = ObjectValueWeakMap;
+ WrapperWeakSet crossZoneRecords;
+
+ // A map of weak ref targets to a vector of weak refs that are observing the
+ // target. The weak refs may be in other zones and are wrapped appropriately.
+ using WeakRefHeapPtrVector =
+ GCVector<js::HeapPtr<JSObject*>, 1, js::ZoneAllocPolicy>;
+ using WeakRefMap =
+ GCHashMap<HeapPtr<JSObject*>, WeakRefHeapPtrVector,
+ StableCellHasher<HeapPtr<JSObject*>>, ZoneAllocPolicy>;
+ WeakRefMap weakRefMap;
+
+ // A weak map used as a set of cross-zone weak refs wrappers.
+ WrapperWeakSet crossZoneWeakRefs;
+
+ public:
+ explicit FinalizationObservers(Zone* zone);
+ ~FinalizationObservers();
+
+ // FinalizationRegistry support:
+ bool addRegistry(Handle<FinalizationRegistryObject*> registry);
+ bool addRecord(HandleObject target, HandleObject record);
+ void clearRecords();
+
+ void updateForRemovedRecord(JSObject* wrapper,
+ FinalizationRecordObject* record);
+
+ // WeakRef support:
+ bool addWeakRefTarget(Handle<JSObject*> target, Handle<JSObject*> weakRef);
+ void removeWeakRefTarget(Handle<JSObject*> target,
+ Handle<WeakRefObject*> weakRef);
+
+ void unregisterWeakRefWrapper(JSObject* wrapper, WeakRefObject* weakRef);
+
+ void traceRoots(JSTracer* trc);
+ void traceWeakEdges(JSTracer* trc);
+
+#ifdef DEBUG
+ void checkTables() const;
+#endif
+
+ private:
+ bool addCrossZoneWrapper(WrapperWeakSet& weakSet, JSObject* wrapper);
+ void removeCrossZoneWrapper(WrapperWeakSet& weakSet, JSObject* wrapper);
+
+ void updateForRemovedWeakRef(JSObject* wrapper, WeakRefObject* weakRef);
+
+ void traceWeakFinalizationRegistryEdges(JSTracer* trc);
+ void traceWeakWeakRefEdges(JSTracer* trc);
+ void traceWeakWeakRefVector(JSTracer* trc, WeakRefHeapPtrVector& weakRefs,
+ JSObject* target);
+
+ static bool shouldRemoveRecord(FinalizationRecordObject* record);
+};
+
+// Per-global data structures to support FinalizationRegistry.
+class FinalizationRegistryGlobalData {
+ // Set of finalization records for finalization registries in this
+ // realm. These are traced as part of the realm's global.
+ using RecordSet =
+ GCHashSet<HeapPtr<JSObject*>, StableCellHasher<HeapPtr<JSObject*>>,
+ ZoneAllocPolicy>;
+ RecordSet recordSet;
+
+ public:
+ explicit FinalizationRegistryGlobalData(Zone* zone);
+
+ bool addRecord(FinalizationRecordObject* record);
+ void removeRecord(FinalizationRecordObject* record);
+
+ void trace(JSTracer* trc);
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_FinalizationObservers_h
diff --git a/js/src/gc/FindSCCs.h b/js/src/gc/FindSCCs.h
new file mode 100644
index 0000000000..97069bcd09
--- /dev/null
+++ b/js/src/gc/FindSCCs.h
@@ -0,0 +1,207 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_FindSCCs_h
+#define gc_FindSCCs_h
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+
+#include <algorithm> // std::min
+
+#include "js/AllocPolicy.h" // js::SystemAllocPolicy
+#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit
+#include "js/HashTable.h" // js::HashSet, js::DefaultHasher
+
+namespace js {
+namespace gc {
+
+template <typename Node>
+struct GraphNodeBase {
+ using NodeSet =
+ js::HashSet<Node*, js::DefaultHasher<Node*>, js::SystemAllocPolicy>;
+
+ NodeSet gcGraphEdges;
+ Node* gcNextGraphNode = nullptr;
+ Node* gcNextGraphComponent = nullptr;
+ unsigned gcDiscoveryTime = 0;
+ unsigned gcLowLink = 0;
+
+ Node* nextNodeInGroup() const {
+ if (gcNextGraphNode &&
+ gcNextGraphNode->gcNextGraphComponent == gcNextGraphComponent) {
+ return gcNextGraphNode;
+ }
+ return nullptr;
+ }
+
+ Node* nextGroup() const { return gcNextGraphComponent; }
+};
+
+/*
+ * Find the strongly connected components of a graph using Tarjan's algorithm,
+ * and return them in topological order.
+ *
+ * Nodes derive from GraphNodeBase and add target edge pointers to
+ * sourceNode.gcGraphEdges to describe the graph:
+ *
+ * struct MyGraphNode : public GraphNodeBase<MyGraphNode>
+ * {
+ * ...
+ * }
+ *
+ * MyGraphNode node1, node2, node3;
+ * node1.gcGraphEdges.put(node2); // Error checking elided.
+ * node2.gcGraphEdges.put(node3);
+ * node3.gcGraphEdges.put(node2);
+ *
+ * ComponentFinder<MyGraphNode> finder;
+ * finder.addNode(node1);
+ * finder.addNode(node2);
+ * finder.addNode(node3);
+ * MyGraphNode* result = finder.getResultsList();
+ */
+
+template <typename Node>
+class ComponentFinder {
+ public:
+ explicit ComponentFinder(JSContext* cx) : cx(cx) {}
+
+ ~ComponentFinder() {
+ MOZ_ASSERT(!stack);
+ MOZ_ASSERT(!firstComponent);
+ }
+
+ /* Forces all nodes to be added to a single component. */
+ void useOneComponent() { stackFull = true; }
+
+ void addNode(Node* v) {
+ if (v->gcDiscoveryTime == Undefined) {
+ MOZ_ASSERT(v->gcLowLink == Undefined);
+ processNode(v);
+ }
+ }
+
+ Node* getResultsList() {
+ if (stackFull) {
+ /*
+ * All nodes after the stack overflow are in |stack|. Put them all in
+ * one big component of their own.
+ */
+ Node* firstGoodComponent = firstComponent;
+ for (Node* v = stack; v; v = stack) {
+ stack = v->gcNextGraphNode;
+ v->gcNextGraphComponent = firstGoodComponent;
+ v->gcNextGraphNode = firstComponent;
+ firstComponent = v;
+ }
+ stackFull = false;
+ }
+
+ MOZ_ASSERT(!stack);
+
+ Node* result = firstComponent;
+ firstComponent = nullptr;
+
+ for (Node* v = result; v; v = v->gcNextGraphNode) {
+ v->gcDiscoveryTime = Undefined;
+ v->gcLowLink = Undefined;
+ }
+
+ return result;
+ }
+
+ static void mergeGroups(Node* first) {
+ for (Node* v = first; v; v = v->gcNextGraphNode) {
+ v->gcNextGraphComponent = nullptr;
+ }
+ }
+
+ private:
+ // Constant used to indicate an unprocessed vertex.
+ static const unsigned Undefined = 0;
+
+ // Constant used to indicate a processed vertex that is no longer on the
+ // stack.
+ static const unsigned Finished = (unsigned)-1;
+
+ void addEdgeTo(Node* w) {
+ if (w->gcDiscoveryTime == Undefined) {
+ processNode(w);
+ cur->gcLowLink = std::min(cur->gcLowLink, w->gcLowLink);
+ } else if (w->gcDiscoveryTime != Finished) {
+ cur->gcLowLink = std::min(cur->gcLowLink, w->gcDiscoveryTime);
+ }
+ }
+
+ void processNode(Node* v) {
+ v->gcDiscoveryTime = clock;
+ v->gcLowLink = clock;
+ ++clock;
+
+ v->gcNextGraphNode = stack;
+ stack = v;
+
+ if (stackFull) {
+ return;
+ }
+
+ AutoCheckRecursionLimit recursion(cx);
+ if (!recursion.checkSystemDontReport(cx)) {
+ stackFull = true;
+ return;
+ }
+
+ Node* old = cur;
+ cur = v;
+ for (auto r = cur->gcGraphEdges.all(); !r.empty(); r.popFront()) {
+ addEdgeTo(r.front());
+ }
+ cur = old;
+
+ if (stackFull) {
+ return;
+ }
+
+ if (v->gcLowLink == v->gcDiscoveryTime) {
+ Node* nextComponent = firstComponent;
+ Node* w;
+ do {
+ MOZ_ASSERT(stack);
+ w = stack;
+ stack = w->gcNextGraphNode;
+
+ /*
+ * Record that the element is no longer on the stack by setting the
+ * discovery time to a special value that's not Undefined.
+ */
+ w->gcDiscoveryTime = Finished;
+
+ /* Figure out which group we're in. */
+ w->gcNextGraphComponent = nextComponent;
+
+ /*
+ * Prepend the component to the beginning of the output list to
+ * reverse the list and achieve the desired order.
+ */
+ w->gcNextGraphNode = firstComponent;
+ firstComponent = w;
+ } while (w != v);
+ }
+ }
+
+ private:
+ unsigned clock = 1;
+ Node* stack = nullptr;
+ Node* firstComponent = nullptr;
+ Node* cur = nullptr;
+ JSContext* cx;
+ bool stackFull = false;
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_FindSCCs_h */
diff --git a/js/src/gc/GC-inl.h b/js/src/gc/GC-inl.h
new file mode 100644
index 0000000000..1c177f51ab
--- /dev/null
+++ b/js/src/gc/GC-inl.h
@@ -0,0 +1,344 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GC_inl_h
+#define gc_GC_inl_h
+
+#include "gc/GC.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include "gc/IteratorUtils.h"
+#include "gc/Marking.h"
+#include "gc/Zone.h"
+#include "vm/Runtime.h"
+
+#include "gc/ArenaList-inl.h"
+
+namespace js {
+namespace gc {
+
+class AutoAssertEmptyNursery;
+
+class ArenaListIter {
+ Arena* arena;
+
+ public:
+ explicit ArenaListIter(Arena* head) : arena(head) {}
+ bool done() const { return !arena; }
+ Arena* get() const {
+ MOZ_ASSERT(!done());
+ return arena;
+ }
+ void next() {
+ MOZ_ASSERT(!done());
+ arena = arena->next;
+ }
+
+ operator Arena*() const { return get(); }
+ Arena* operator->() const { return get(); }
+};
+
+class ArenaIter : public ChainedIterator<ArenaListIter, 3> {
+ public:
+ ArenaIter(JS::Zone* zone, AllocKind kind)
+ : ChainedIterator(zone->arenas.getFirstArena(kind),
+ zone->arenas.getFirstCollectingArena(kind),
+ zone->arenas.getFirstSweptArena(kind)) {}
+};
+
+class ArenaCellIter {
+ size_t firstThingOffset;
+ size_t thingSize;
+ Arena* arenaAddr;
+ FreeSpan span;
+ uint_fast16_t thing;
+ mozilla::DebugOnly<JS::TraceKind> traceKind;
+
+ // Upon entry, |thing| points to any thing (free or used) and finds the
+ // first used thing, which may be |thing|.
+ void settle() {
+ MOZ_ASSERT(!done());
+ MOZ_ASSERT(thing);
+ // Note: if |span| is empty, this test will fail, which is what we want
+ // -- |span| being empty means that we're past the end of the last free
+ // thing, all the remaining things in the arena are used, and we'll
+ // never need to move forward.
+ if (thing == span.first) {
+ thing = span.last + thingSize;
+ span = *span.nextSpan(arenaAddr);
+ }
+ }
+
+ public:
+ explicit ArenaCellIter(Arena* arena) {
+ MOZ_ASSERT(arena);
+ AllocKind kind = arena->getAllocKind();
+ firstThingOffset = Arena::firstThingOffset(kind);
+ thingSize = Arena::thingSize(kind);
+ traceKind = MapAllocToTraceKind(kind);
+ arenaAddr = arena;
+ span = *arena->getFirstFreeSpan();
+ thing = firstThingOffset;
+ settle();
+ }
+
+ bool done() const {
+ MOZ_ASSERT(thing <= ArenaSize);
+ return thing == ArenaSize;
+ }
+
+ TenuredCell* get() const {
+ MOZ_ASSERT(!done());
+ return reinterpret_cast<TenuredCell*>(uintptr_t(arenaAddr) + thing);
+ }
+
+ template <typename T>
+ T* as() const {
+ MOZ_ASSERT(!done());
+ MOZ_ASSERT(JS::MapTypeToTraceKind<T>::kind == traceKind);
+ return reinterpret_cast<T*>(get());
+ }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ thing += thingSize;
+ if (thing < ArenaSize) {
+ settle();
+ }
+ }
+
+ operator TenuredCell*() const { return get(); }
+ TenuredCell* operator->() const { return get(); }
+};
+
+template <typename T>
+class ZoneAllCellIter;
+
+template <>
+class ZoneAllCellIter<TenuredCell> {
+ mozilla::Maybe<NestedIterator<ArenaIter, ArenaCellIter>> iter;
+ mozilla::Maybe<JS::AutoAssertNoGC> nogc;
+
+ protected:
+ // For use when a subclass wants to insert some setup before init().
+ ZoneAllCellIter() = default;
+
+ void init(JS::Zone* zone, AllocKind kind) {
+ MOZ_ASSERT_IF(IsNurseryAllocable(kind),
+ (zone->isAtomsZone() ||
+ zone->runtimeFromMainThread()->gc.nursery().isEmpty()));
+ initForTenuredIteration(zone, kind);
+ }
+
+ void initForTenuredIteration(JS::Zone* zone, AllocKind kind) {
+ JSRuntime* rt = zone->runtimeFromAnyThread();
+
+ // If called from outside a GC, ensure that the heap is in a state
+ // that allows us to iterate.
+ if (!JS::RuntimeHeapIsBusy()) {
+ // Assert that no GCs can occur while a ZoneAllCellIter is live.
+ nogc.emplace();
+ }
+
+ // We have a single-threaded runtime, so there's no need to protect
+ // against other threads iterating or allocating. However, we do have
+ // background finalization; we may have to wait for this to finish if
+ // it's currently active.
+ if (IsBackgroundFinalized(kind) &&
+ zone->arenas.needBackgroundFinalizeWait(kind)) {
+ rt->gc.waitBackgroundSweepEnd();
+ }
+ iter.emplace(zone, kind);
+ }
+
+ public:
+ ZoneAllCellIter(JS::Zone* zone, AllocKind kind) {
+ // If we are iterating a nursery-allocated kind then we need to
+ // evict first so that we can see all things.
+ if (IsNurseryAllocable(kind)) {
+ zone->runtimeFromMainThread()->gc.evictNursery();
+ }
+
+ init(zone, kind);
+ }
+
+ ZoneAllCellIter(JS::Zone* zone, AllocKind kind,
+ const js::gc::AutoAssertEmptyNursery&) {
+ // No need to evict the nursery. (This constructor is known statically
+ // to not GC.)
+ init(zone, kind);
+ }
+
+ bool done() const { return iter->done(); }
+
+ template <typename T>
+ T* get() const {
+ return iter->ref().as<T>();
+ }
+
+ TenuredCell* getCell() const { return iter->get(); }
+
+ void next() { iter->next(); }
+};
+
+/* clang-format off */
+//
+// Iterator over the cells in a Zone, where the GC type (JSString, JSObject) is
+// known, for a single AllocKind. Example usages:
+//
+// for (auto obj = zone->cellIter<JSObject>(AllocKind::OBJECT0); !obj.done(); obj.next()) {
+// ...
+// }
+//
+// for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next()) {
+// f(script->code());
+// }
+//
+// As this code demonstrates, you can use 'script' as if it were a JSScript*.
+// Its actual type is ZoneAllCellIter<JSScript>, but for most purposes it will
+// autoconvert to JSScript*.
+//
+// Note that in the JSScript case, ZoneAllCellIter is able to infer the AllocKind
+// from the type 'JSScript', whereas in the JSObject case, the kind must be
+// given (because there are multiple AllocKinds for objects).
+//
+// Also, the static rooting hazard analysis knows that the JSScript case will
+// not GC during construction. The JSObject case needs to GC, or more precisely
+// to empty the nursery and clear out the store buffer, so that it can see all
+// objects to iterate over (the nursery is not iterable) and remove the
+// possibility of having pointers from the store buffer to data hanging off
+// stuff we're iterating over that we are going to delete. (The latter should
+// not be a problem, since such instances should be using RelocatablePtr do
+// remove themselves from the store buffer on deletion, but currently for
+// subtle reasons that isn't good enough.)
+//
+// If the iterator is used within a GC, then there is no need to evict the
+// nursery (again). You may select a variant that will skip the eviction either
+// by specializing on a GCType that is never allocated in the nursery, or
+// explicitly by passing in a trailing AutoAssertEmptyNursery argument.
+//
+// NOTE: This class can return items that are about to be swept/finalized.
+// You must not keep pointers to such items across GCs. Use
+// ZoneCellIter below to filter these out.
+//
+// NOTE: This class also does not read barrier returned items, so may return
+// gray cells. You must not store such items anywhere on the heap without
+// gray-unmarking them. Use ZoneCellIter to automatically unmark them.
+//
+/* clang-format on */
+template <typename GCType>
+class ZoneAllCellIter : public ZoneAllCellIter<TenuredCell> {
+ public:
+ // Non-nursery allocated (equivalent to having an entry in
+ // MapTypeToAllocKind). The template declaration here is to discard this
+ // constructor overload if MapTypeToAllocKind<GCType>::kind does not
+ // exist. Note that there will be no remaining overloads that will work, which
+ // makes sense given that you haven't specified which of the AllocKinds to use
+ // for GCType.
+ //
+ // If we later add a nursery allocable GCType with a single AllocKind, we will
+ // want to add an overload of this constructor that does the right thing (ie,
+ // it empties the nursery before iterating.)
+ explicit ZoneAllCellIter(JS::Zone* zone) : ZoneAllCellIter<TenuredCell>() {
+ init(zone, MapTypeToAllocKind<GCType>::kind);
+ }
+
+ // Non-nursery allocated, nursery is known to be empty: same behavior as
+ // above.
+ ZoneAllCellIter(JS::Zone* zone, const js::gc::AutoAssertEmptyNursery&)
+ : ZoneAllCellIter(zone) {}
+
+ // Arbitrary kind, which will be assumed to be nursery allocable (and
+ // therefore the nursery will be emptied before iterating.)
+ ZoneAllCellIter(JS::Zone* zone, AllocKind kind)
+ : ZoneAllCellIter<TenuredCell>(zone, kind) {}
+
+ // Arbitrary kind, which will be assumed to be nursery allocable, but the
+ // nursery is known to be empty already: same behavior as non-nursery types.
+ ZoneAllCellIter(JS::Zone* zone, AllocKind kind,
+ const js::gc::AutoAssertEmptyNursery& empty)
+ : ZoneAllCellIter<TenuredCell>(zone, kind, empty) {}
+
+ GCType* get() const { return ZoneAllCellIter<TenuredCell>::get<GCType>(); }
+ operator GCType*() const { return get(); }
+ GCType* operator->() const { return get(); }
+};
+
+// Like the above class but filter out cells that are about to be finalized.
+// Also, read barrier all cells returned (unless the Unbarriered variants are
+// used) to prevent gray cells from escaping.
+template <typename T>
+class ZoneCellIter : protected ZoneAllCellIter<T> {
+ using Base = ZoneAllCellIter<T>;
+
+ public:
+ /*
+ * The same constructors as above.
+ */
+ explicit ZoneCellIter(JS::Zone* zone) : ZoneAllCellIter<T>(zone) {
+ skipDying();
+ }
+ ZoneCellIter(JS::Zone* zone, const js::gc::AutoAssertEmptyNursery& empty)
+ : ZoneAllCellIter<T>(zone, empty) {
+ skipDying();
+ }
+ ZoneCellIter(JS::Zone* zone, AllocKind kind)
+ : ZoneAllCellIter<T>(zone, kind) {
+ skipDying();
+ }
+ ZoneCellIter(JS::Zone* zone, AllocKind kind,
+ const js::gc::AutoAssertEmptyNursery& empty)
+ : ZoneAllCellIter<T>(zone, kind, empty) {
+ skipDying();
+ }
+
+ using Base::done;
+
+ void next() {
+ ZoneAllCellIter<T>::next();
+ skipDying();
+ }
+
+ TenuredCell* getCell() const {
+ TenuredCell* cell = Base::getCell();
+
+ // This can result in a new reference being created to an object that an
+ // ongoing incremental GC may find to be unreachable, so we may need a
+ // barrier here.
+ JSRuntime* rt = cell->runtimeFromAnyThread();
+ if (!JS::RuntimeHeapIsCollecting(rt->heapState())) {
+ JS::TraceKind traceKind = JS::MapTypeToTraceKind<T>::kind;
+ ExposeGCThingToActiveJS(JS::GCCellPtr(cell, traceKind));
+ }
+
+ return cell;
+ }
+
+ T* get() const { return reinterpret_cast<T*>(getCell()); }
+
+ TenuredCell* unbarrieredGetCell() const { return Base::getCell(); }
+ T* unbarrieredGet() const { return Base::get(); }
+ operator T*() const { return get(); }
+ T* operator->() const { return get(); }
+
+ private:
+ void skipDying() {
+ while (!ZoneAllCellIter<T>::done()) {
+ T* current = ZoneAllCellIter<T>::get();
+ if (!IsAboutToBeFinalizedUnbarriered(current)) {
+ return;
+ }
+ ZoneAllCellIter<T>::next();
+ }
+ }
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_GC_inl_h */
diff --git a/js/src/gc/GC.cpp b/js/src/gc/GC.cpp
new file mode 100644
index 0000000000..b8e1d21f2a
--- /dev/null
+++ b/js/src/gc/GC.cpp
@@ -0,0 +1,5101 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * [SMDOC] Garbage Collector
+ *
+ * This code implements an incremental mark-and-sweep garbage collector, with
+ * most sweeping carried out in the background on a parallel thread.
+ *
+ * Full vs. zone GC
+ * ----------------
+ *
+ * The collector can collect all zones at once, or a subset. These types of
+ * collection are referred to as a full GC and a zone GC respectively.
+ *
+ * It is possible for an incremental collection that started out as a full GC to
+ * become a zone GC if new zones are created during the course of the
+ * collection.
+ *
+ * Incremental collection
+ * ----------------------
+ *
+ * For a collection to be carried out incrementally the following conditions
+ * must be met:
+ * - the collection must be run by calling js::GCSlice() rather than js::GC()
+ * - the GC parameter JSGC_INCREMENTAL_GC_ENABLED must be true.
+ *
+ * The last condition is an engine-internal mechanism to ensure that incremental
+ * collection is not carried out without the correct barriers being implemented.
+ * For more information see 'Incremental marking' below.
+ *
+ * If the collection is not incremental, all foreground activity happens inside
+ * a single call to GC() or GCSlice(). However the collection is not complete
+ * until the background sweeping activity has finished.
+ *
+ * An incremental collection proceeds as a series of slices, interleaved with
+ * mutator activity, i.e. running JavaScript code. Slices are limited by a time
+ * budget. The slice finishes as soon as possible after the requested time has
+ * passed.
+ *
+ * Collector states
+ * ----------------
+ *
+ * The collector proceeds through the following states, the current state being
+ * held in JSRuntime::gcIncrementalState:
+ *
+ * - Prepare - unmarks GC things, discards JIT code and other setup
+ * - MarkRoots - marks the stack and other roots
+ * - Mark - incrementally marks reachable things
+ * - Sweep - sweeps zones in groups and continues marking unswept zones
+ * - Finalize - performs background finalization, concurrent with mutator
+ * - Compact - incrementally compacts by zone
+ * - Decommit - performs background decommit and chunk removal
+ *
+ * Roots are marked in the first MarkRoots slice; this is the start of the GC
+ * proper. The following states can take place over one or more slices.
+ *
+ * In other words an incremental collection proceeds like this:
+ *
+ * Slice 1: Prepare: Starts background task to unmark GC things
+ *
+ * ... JS code runs, background unmarking finishes ...
+ *
+ * Slice 2: MarkRoots: Roots are pushed onto the mark stack.
+ * Mark: The mark stack is processed by popping an element,
+ * marking it, and pushing its children.
+ *
+ * ... JS code runs ...
+ *
+ * Slice 3: Mark: More mark stack processing.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n-1: Mark: More mark stack processing.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n: Mark: Mark stack is completely drained.
+ * Sweep: Select first group of zones to sweep and sweep them.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n+1: Sweep: Mark objects in unswept zones that were newly
+ * identified as alive (see below). Then sweep more zone
+ * sweep groups.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n+2: Sweep: Mark objects in unswept zones that were newly
+ * identified as alive. Then sweep more zones.
+ *
+ * ... JS code runs ...
+ *
+ * Slice m: Sweep: Sweeping is finished, and background sweeping
+ * started on the helper thread.
+ *
+ * ... JS code runs, remaining sweeping done on background thread ...
+ *
+ * When background sweeping finishes the GC is complete.
+ *
+ * Incremental marking
+ * -------------------
+ *
+ * Incremental collection requires close collaboration with the mutator (i.e.,
+ * JS code) to guarantee correctness.
+ *
+ * - During an incremental GC, if a memory location (except a root) is written
+ * to, then the value it previously held must be marked. Write barriers
+ * ensure this.
+ *
+ * - Any object that is allocated during incremental GC must start out marked.
+ *
+ * - Roots are marked in the first slice and hence don't need write barriers.
+ * Roots are things like the C stack and the VM stack.
+ *
+ * The problem that write barriers solve is that between slices the mutator can
+ * change the object graph. We must ensure that it cannot do this in such a way
+ * that makes us fail to mark a reachable object (marking an unreachable object
+ * is tolerable).
+ *
+ * We use a snapshot-at-the-beginning algorithm to do this. This means that we
+ * promise to mark at least everything that is reachable at the beginning of
+ * collection. To implement it we mark the old contents of every non-root memory
+ * location written to by the mutator while the collection is in progress, using
+ * write barriers. This is described in gc/Barrier.h.
+ *
+ * Incremental sweeping
+ * --------------------
+ *
+ * Sweeping is difficult to do incrementally because object finalizers must be
+ * run at the start of sweeping, before any mutator code runs. The reason is
+ * that some objects use their finalizers to remove themselves from caches. If
+ * mutator code was allowed to run after the start of sweeping, it could observe
+ * the state of the cache and create a new reference to an object that was just
+ * about to be destroyed.
+ *
+ * Sweeping all finalizable objects in one go would introduce long pauses, so
+ * instead sweeping broken up into groups of zones. Zones which are not yet
+ * being swept are still marked, so the issue above does not apply.
+ *
+ * The order of sweeping is restricted by cross compartment pointers - for
+ * example say that object |a| from zone A points to object |b| in zone B and
+ * neither object was marked when we transitioned to the Sweep phase. Imagine we
+ * sweep B first and then return to the mutator. It's possible that the mutator
+ * could cause |a| to become alive through a read barrier (perhaps it was a
+ * shape that was accessed via a shape table). Then we would need to mark |b|,
+ * which |a| points to, but |b| has already been swept.
+ *
+ * So if there is such a pointer then marking of zone B must not finish before
+ * marking of zone A. Pointers which form a cycle between zones therefore
+ * restrict those zones to being swept at the same time, and these are found
+ * using Tarjan's algorithm for finding the strongly connected components of a
+ * graph.
+ *
+ * GC things without finalizers, and things with finalizers that are able to run
+ * in the background, are swept on the background thread. This accounts for most
+ * of the sweeping work.
+ *
+ * Reset
+ * -----
+ *
+ * During incremental collection it is possible, although unlikely, for
+ * conditions to change such that incremental collection is no longer safe. In
+ * this case, the collection is 'reset' by resetIncrementalGC(). If we are in
+ * the mark state, this just stops marking, but if we have started sweeping
+ * already, we continue non-incrementally until we have swept the current sweep
+ * group. Following a reset, a new collection is started.
+ *
+ * Compacting GC
+ * -------------
+ *
+ * Compacting GC happens at the end of a major GC as part of the last slice.
+ * There are three parts:
+ *
+ * - Arenas are selected for compaction.
+ * - The contents of those arenas are moved to new arenas.
+ * - All references to moved things are updated.
+ *
+ * Collecting Atoms
+ * ----------------
+ *
+ * Atoms are collected differently from other GC things. They are contained in
+ * a special zone and things in other zones may have pointers to them that are
+ * not recorded in the cross compartment pointer map. Each zone holds a bitmap
+ * with the atoms it might be keeping alive, and atoms are only collected if
+ * they are not included in any zone's atom bitmap. See AtomMarking.cpp for how
+ * this bitmap is managed.
+ */
+
+#include "gc/GC-inl.h"
+
+#include "mozilla/Range.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/TextUtils.h"
+#include "mozilla/TimeStamp.h"
+
+#include <algorithm>
+#include <initializer_list>
+#include <iterator>
+#include <stdlib.h>
+#include <string.h>
+#include <utility>
+
+#include "jsapi.h" // JS_AbortIfWrongThread
+#include "jstypes.h"
+
+#include "debugger/DebugAPI.h"
+#include "gc/ClearEdgesTracer.h"
+#include "gc/GCContext.h"
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/GCProbes.h"
+#include "gc/Memory.h"
+#include "gc/ParallelMarking.h"
+#include "gc/ParallelWork.h"
+#include "gc/WeakMap.h"
+#include "jit/ExecutableAllocator.h"
+#include "jit/JitCode.h"
+#include "jit/JitRealm.h"
+#include "jit/JitRuntime.h"
+#include "jit/ProcessExecutableMemory.h"
+#include "js/HeapAPI.h" // JS::GCCellPtr
+#include "js/Printer.h"
+#include "js/SliceBudget.h"
+#include "util/DifferentialTesting.h"
+#include "vm/BigIntType.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/GetterSetter.h"
+#include "vm/HelperThreadState.h"
+#include "vm/JitActivation.h"
+#include "vm/JSObject.h"
+#include "vm/JSScript.h"
+#include "vm/PropMap.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+#include "vm/StringType.h"
+#include "vm/SymbolType.h"
+#include "vm/Time.h"
+
+#include "gc/Heap-inl.h"
+#include "gc/Nursery-inl.h"
+#include "gc/ObjectKind-inl.h"
+#include "gc/PrivateIterators-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSContext-inl.h"
+#include "vm/Realm-inl.h"
+#include "vm/Stack-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::MakeScopeExit;
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+using JS::AutoGCRooter;
+
+const AllocKind gc::slotsToThingKind[] = {
+ // clang-format off
+ /* 0 */ AllocKind::OBJECT0, AllocKind::OBJECT2, AllocKind::OBJECT2, AllocKind::OBJECT4,
+ /* 4 */ AllocKind::OBJECT4, AllocKind::OBJECT8, AllocKind::OBJECT8, AllocKind::OBJECT8,
+ /* 8 */ AllocKind::OBJECT8, AllocKind::OBJECT12, AllocKind::OBJECT12, AllocKind::OBJECT12,
+ /* 12 */ AllocKind::OBJECT12, AllocKind::OBJECT16, AllocKind::OBJECT16, AllocKind::OBJECT16,
+ /* 16 */ AllocKind::OBJECT16
+ // clang-format on
+};
+
+static_assert(std::size(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT,
+ "We have defined a slot count for each kind.");
+
+MOZ_THREAD_LOCAL(JS::GCContext*) js::TlsGCContext;
+
+JS::GCContext::GCContext(JSRuntime* runtime) : runtime_(runtime) {}
+
+JS::GCContext::~GCContext() {
+ MOZ_ASSERT(!hasJitCodeToPoison());
+ MOZ_ASSERT(!isCollecting());
+ MOZ_ASSERT(gcUse() == GCUse::None);
+ MOZ_ASSERT(!gcSweepZone());
+ MOZ_ASSERT(!isTouchingGrayThings());
+}
+
+void JS::GCContext::poisonJitCode() {
+ if (hasJitCodeToPoison()) {
+ jit::ExecutableAllocator::poisonCode(runtime(), jitPoisonRanges);
+ jitPoisonRanges.clearAndFree();
+ }
+}
+
+#ifdef DEBUG
+void GCRuntime::verifyAllChunks() {
+ AutoLockGC lock(this);
+ fullChunks(lock).verifyChunks();
+ availableChunks(lock).verifyChunks();
+ emptyChunks(lock).verifyChunks();
+}
+#endif
+
+void GCRuntime::setMinEmptyChunkCount(uint32_t value, const AutoLockGC& lock) {
+ minEmptyChunkCount_ = value;
+ if (minEmptyChunkCount_ > maxEmptyChunkCount_) {
+ maxEmptyChunkCount_ = minEmptyChunkCount_;
+ }
+ MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_);
+}
+
+void GCRuntime::setMaxEmptyChunkCount(uint32_t value, const AutoLockGC& lock) {
+ maxEmptyChunkCount_ = value;
+ if (minEmptyChunkCount_ > maxEmptyChunkCount_) {
+ minEmptyChunkCount_ = maxEmptyChunkCount_;
+ }
+ MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_);
+}
+
+inline bool GCRuntime::tooManyEmptyChunks(const AutoLockGC& lock) {
+ return emptyChunks(lock).count() > minEmptyChunkCount(lock);
+}
+
+ChunkPool GCRuntime::expireEmptyChunkPool(const AutoLockGC& lock) {
+ MOZ_ASSERT(emptyChunks(lock).verify());
+ MOZ_ASSERT(minEmptyChunkCount(lock) <= maxEmptyChunkCount(lock));
+
+ ChunkPool expired;
+ while (tooManyEmptyChunks(lock)) {
+ TenuredChunk* chunk = emptyChunks(lock).pop();
+ prepareToFreeChunk(chunk->info);
+ expired.push(chunk);
+ }
+
+ MOZ_ASSERT(expired.verify());
+ MOZ_ASSERT(emptyChunks(lock).verify());
+ MOZ_ASSERT(emptyChunks(lock).count() <= maxEmptyChunkCount(lock));
+ MOZ_ASSERT(emptyChunks(lock).count() <= minEmptyChunkCount(lock));
+ return expired;
+}
+
+static void FreeChunkPool(ChunkPool& pool) {
+ for (ChunkPool::Iter iter(pool); !iter.done();) {
+ TenuredChunk* chunk = iter.get();
+ iter.next();
+ pool.remove(chunk);
+ MOZ_ASSERT(chunk->unused());
+ UnmapPages(static_cast<void*>(chunk), ChunkSize);
+ }
+ MOZ_ASSERT(pool.count() == 0);
+}
+
+void GCRuntime::freeEmptyChunks(const AutoLockGC& lock) {
+ FreeChunkPool(emptyChunks(lock));
+}
+
+inline void GCRuntime::prepareToFreeChunk(TenuredChunkInfo& info) {
+ MOZ_ASSERT(numArenasFreeCommitted >= info.numArenasFreeCommitted);
+ numArenasFreeCommitted -= info.numArenasFreeCommitted;
+ stats().count(gcstats::COUNT_DESTROY_CHUNK);
+#ifdef DEBUG
+ /*
+ * Let FreeChunkPool detect a missing prepareToFreeChunk call before it
+ * frees chunk.
+ */
+ info.numArenasFreeCommitted = 0;
+#endif
+}
+
+void GCRuntime::releaseArena(Arena* arena, const AutoLockGC& lock) {
+ MOZ_ASSERT(arena->allocated());
+ MOZ_ASSERT(!arena->onDelayedMarkingList());
+ MOZ_ASSERT(TlsGCContext.get()->isFinalizing());
+
+ arena->zone->gcHeapSize.removeGCArena(heapSize);
+ arena->release(lock);
+ arena->chunk()->releaseArena(this, arena, lock);
+}
+
+GCRuntime::GCRuntime(JSRuntime* rt)
+ : rt(rt),
+ systemZone(nullptr),
+ mainThreadContext(rt),
+ heapState_(JS::HeapState::Idle),
+ stats_(this),
+ sweepingTracer(rt),
+ fullGCRequested(false),
+ helperThreadRatio(TuningDefaults::HelperThreadRatio),
+ maxHelperThreads(TuningDefaults::MaxHelperThreads),
+ helperThreadCount(1),
+ createBudgetCallback(nullptr),
+ minEmptyChunkCount_(TuningDefaults::MinEmptyChunkCount),
+ maxEmptyChunkCount_(TuningDefaults::MaxEmptyChunkCount),
+ rootsHash(256),
+ nextCellUniqueId_(LargestTaggedNullCellPointer +
+ 1), // Ensure disjoint from null tagged pointers.
+ numArenasFreeCommitted(0),
+ verifyPreData(nullptr),
+ lastGCStartTime_(TimeStamp::Now()),
+ lastGCEndTime_(TimeStamp::Now()),
+ incrementalGCEnabled(TuningDefaults::IncrementalGCEnabled),
+ perZoneGCEnabled(TuningDefaults::PerZoneGCEnabled),
+ numActiveZoneIters(0),
+ cleanUpEverything(false),
+ grayBitsValid(true),
+ majorGCTriggerReason(JS::GCReason::NO_REASON),
+ minorGCNumber(0),
+ majorGCNumber(0),
+ number(0),
+ sliceNumber(0),
+ isFull(false),
+ incrementalState(gc::State::NotActive),
+ initialState(gc::State::NotActive),
+ useZeal(false),
+ lastMarkSlice(false),
+ safeToYield(true),
+ markOnBackgroundThreadDuringSweeping(false),
+ useBackgroundThreads(false),
+#ifdef DEBUG
+ hadShutdownGC(false),
+#endif
+ requestSliceAfterBackgroundTask(false),
+ lifoBlocksToFree((size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+ lifoBlocksToFreeAfterMinorGC(
+ (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+ sweepGroupIndex(0),
+ sweepGroups(nullptr),
+ currentSweepGroup(nullptr),
+ sweepZone(nullptr),
+ abortSweepAfterCurrentGroup(false),
+ sweepMarkResult(IncrementalProgress::NotFinished),
+#ifdef DEBUG
+ testMarkQueue(rt),
+#endif
+ startedCompacting(false),
+ zonesCompacted(0),
+#ifdef DEBUG
+ relocatedArenasToRelease(nullptr),
+#endif
+#ifdef JS_GC_ZEAL
+ markingValidator(nullptr),
+#endif
+ defaultTimeBudgetMS_(TuningDefaults::DefaultTimeBudgetMS),
+ incrementalAllowed(true),
+ compactingEnabled(TuningDefaults::CompactingEnabled),
+ parallelMarkingEnabled(TuningDefaults::ParallelMarkingEnabled),
+ rootsRemoved(false),
+#ifdef JS_GC_ZEAL
+ zealModeBits(0),
+ zealFrequency(0),
+ nextScheduled(0),
+ deterministicOnly(false),
+ zealSliceBudget(0),
+ selectedForMarking(rt),
+#endif
+ fullCompartmentChecks(false),
+ gcCallbackDepth(0),
+ alwaysPreserveCode(false),
+ lowMemoryState(false),
+ lock(mutexid::GCLock),
+ delayedMarkingLock(mutexid::GCDelayedMarkingLock),
+ allocTask(this, emptyChunks_.ref()),
+ unmarkTask(this),
+ markTask(this),
+ sweepTask(this),
+ freeTask(this),
+ decommitTask(this),
+ nursery_(this),
+ storeBuffer_(rt, nursery()),
+ lastAllocRateUpdateTime(TimeStamp::Now()) {
+}
+
+using CharRange = mozilla::Range<const char>;
+using CharRangeVector = Vector<CharRange, 0, SystemAllocPolicy>;
+
+static bool SplitStringBy(CharRange text, char delimiter,
+ CharRangeVector* result) {
+ auto start = text.begin();
+ for (auto ptr = start; ptr != text.end(); ptr++) {
+ if (*ptr == delimiter) {
+ if (!result->emplaceBack(start, ptr)) {
+ return false;
+ }
+ start = ptr + 1;
+ }
+ }
+
+ return result->emplaceBack(start, text.end());
+}
+
+static bool ParseTimeDuration(CharRange text, TimeDuration* durationOut) {
+ const char* str = text.begin().get();
+ char* end;
+ *durationOut = TimeDuration::FromMilliseconds(strtol(str, &end, 10));
+ return str != end && end == text.end().get();
+}
+
+static void PrintProfileHelpAndExit(const char* envName, const char* helpText) {
+ fprintf(stderr, "%s=N[,(main|all)]\n", envName);
+ fprintf(stderr, "%s", helpText);
+ exit(0);
+}
+
+void js::gc::ReadProfileEnv(const char* envName, const char* helpText,
+ bool* enableOut, bool* workersOut,
+ TimeDuration* thresholdOut) {
+ *enableOut = false;
+ *workersOut = false;
+ *thresholdOut = TimeDuration();
+
+ const char* env = getenv(envName);
+ if (!env) {
+ return;
+ }
+
+ if (strcmp(env, "help") == 0) {
+ PrintProfileHelpAndExit(envName, helpText);
+ }
+
+ CharRangeVector parts;
+ auto text = CharRange(env, strlen(env));
+ if (!SplitStringBy(text, ',', &parts)) {
+ MOZ_CRASH("OOM parsing environment variable");
+ }
+
+ if (parts.length() == 0 || parts.length() > 2) {
+ PrintProfileHelpAndExit(envName, helpText);
+ }
+
+ *enableOut = true;
+
+ if (!ParseTimeDuration(parts[0], thresholdOut)) {
+ PrintProfileHelpAndExit(envName, helpText);
+ }
+
+ if (parts.length() == 2) {
+ const char* threads = parts[1].begin().get();
+ if (strcmp(threads, "all") == 0) {
+ *workersOut = true;
+ } else if (strcmp(threads, "main") != 0) {
+ PrintProfileHelpAndExit(envName, helpText);
+ }
+ }
+}
+
+bool js::gc::ShouldPrintProfile(JSRuntime* runtime, bool enable,
+ bool profileWorkers, TimeDuration threshold,
+ TimeDuration duration) {
+ return enable && (runtime->isMainRuntime() || profileWorkers) &&
+ duration >= threshold;
+}
+
+#ifdef JS_GC_ZEAL
+
+void GCRuntime::getZealBits(uint32_t* zealBits, uint32_t* frequency,
+ uint32_t* scheduled) {
+ *zealBits = zealModeBits;
+ *frequency = zealFrequency;
+ *scheduled = nextScheduled;
+}
+
+const char gc::ZealModeHelpText[] =
+ " Specifies how zealous the garbage collector should be. Some of these "
+ "modes can\n"
+ " be set simultaneously, by passing multiple level options, e.g. \"2;4\" "
+ "will activate\n"
+ " both modes 2 and 4. Modes can be specified by name or number.\n"
+ " \n"
+ " Values:\n"
+ " 0: (None) Normal amount of collection (resets all modes)\n"
+ " 1: (RootsChange) Collect when roots are added or removed\n"
+ " 2: (Alloc) Collect when every N allocations (default: 100)\n"
+ " 4: (VerifierPre) Verify pre write barriers between instructions\n"
+ " 6: (YieldBeforeRootMarking) Incremental GC in two slices that yields "
+ "before root marking\n"
+ " 7: (GenerationalGC) Collect the nursery every N nursery allocations\n"
+ " 8: (YieldBeforeMarking) Incremental GC in two slices that yields "
+ "between\n"
+ " the root marking and marking phases\n"
+ " 9: (YieldBeforeSweeping) Incremental GC in two slices that yields "
+ "between\n"
+ " the marking and sweeping phases\n"
+ " 10: (IncrementalMultipleSlices) Incremental GC in many slices\n"
+ " 11: (IncrementalMarkingValidator) Verify incremental marking\n"
+ " 12: (ElementsBarrier) Use the individual element post-write barrier\n"
+ " regardless of elements size\n"
+ " 13: (CheckHashTablesOnMinorGC) Check internal hashtables on minor GC\n"
+ " 14: (Compact) Perform a shrinking collection every N allocations\n"
+ " 15: (CheckHeapAfterGC) Walk the heap to check its integrity after "
+ "every GC\n"
+ " 17: (YieldBeforeSweepingAtoms) Incremental GC in two slices that "
+ "yields\n"
+ " before sweeping the atoms table\n"
+ " 18: (CheckGrayMarking) Check gray marking invariants after every GC\n"
+ " 19: (YieldBeforeSweepingCaches) Incremental GC in two slices that "
+ "yields\n"
+ " before sweeping weak caches\n"
+ " 21: (YieldBeforeSweepingObjects) Incremental GC in two slices that "
+ "yields\n"
+ " before sweeping foreground finalized objects\n"
+ " 22: (YieldBeforeSweepingNonObjects) Incremental GC in two slices that "
+ "yields\n"
+ " before sweeping non-object GC things\n"
+ " 23: (YieldBeforeSweepingPropMapTrees) Incremental GC in two slices "
+ "that "
+ "yields\n"
+ " before sweeping shape trees\n"
+ " 24: (CheckWeakMapMarking) Check weak map marking invariants after "
+ "every GC\n"
+ " 25: (YieldWhileGrayMarking) Incremental GC in two slices that yields\n"
+ " during gray marking\n";
+
+// The set of zeal modes that control incremental slices. These modes are
+// mutually exclusive.
+static const mozilla::EnumSet<ZealMode> IncrementalSliceZealModes = {
+ ZealMode::YieldBeforeRootMarking,
+ ZealMode::YieldBeforeMarking,
+ ZealMode::YieldBeforeSweeping,
+ ZealMode::IncrementalMultipleSlices,
+ ZealMode::YieldBeforeSweepingAtoms,
+ ZealMode::YieldBeforeSweepingCaches,
+ ZealMode::YieldBeforeSweepingObjects,
+ ZealMode::YieldBeforeSweepingNonObjects,
+ ZealMode::YieldBeforeSweepingPropMapTrees};
+
+void GCRuntime::setZeal(uint8_t zeal, uint32_t frequency) {
+ MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit));
+
+ if (verifyPreData) {
+ VerifyBarriers(rt, PreBarrierVerifier);
+ }
+
+ if (zeal == 0) {
+ if (hasZealMode(ZealMode::GenerationalGC)) {
+ evictNursery(JS::GCReason::DEBUG_GC);
+ nursery().leaveZealMode();
+ }
+
+ if (isIncrementalGCInProgress()) {
+ finishGC(JS::GCReason::DEBUG_GC);
+ }
+ }
+
+ ZealMode zealMode = ZealMode(zeal);
+ if (zealMode == ZealMode::GenerationalGC) {
+ evictNursery(JS::GCReason::DEBUG_GC);
+ nursery().enterZealMode();
+ }
+
+ // Some modes are mutually exclusive. If we're setting one of those, we
+ // first reset all of them.
+ if (IncrementalSliceZealModes.contains(zealMode)) {
+ for (auto mode : IncrementalSliceZealModes) {
+ clearZealMode(mode);
+ }
+ }
+
+ bool schedule = zealMode >= ZealMode::Alloc;
+ if (zeal != 0) {
+ zealModeBits |= 1 << unsigned(zeal);
+ } else {
+ zealModeBits = 0;
+ }
+ zealFrequency = frequency;
+ nextScheduled = schedule ? frequency : 0;
+}
+
+void GCRuntime::unsetZeal(uint8_t zeal) {
+ MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit));
+ ZealMode zealMode = ZealMode(zeal);
+
+ if (!hasZealMode(zealMode)) {
+ return;
+ }
+
+ if (verifyPreData) {
+ VerifyBarriers(rt, PreBarrierVerifier);
+ }
+
+ if (zealMode == ZealMode::GenerationalGC) {
+ evictNursery(JS::GCReason::DEBUG_GC);
+ nursery().leaveZealMode();
+ }
+
+ clearZealMode(zealMode);
+
+ if (zealModeBits == 0) {
+ if (isIncrementalGCInProgress()) {
+ finishGC(JS::GCReason::DEBUG_GC);
+ }
+
+ zealFrequency = 0;
+ nextScheduled = 0;
+ }
+}
+
+void GCRuntime::setNextScheduled(uint32_t count) { nextScheduled = count; }
+
+static bool ParseZealModeName(CharRange text, uint32_t* modeOut) {
+ struct ModeInfo {
+ const char* name;
+ size_t length;
+ uint32_t value;
+ };
+
+ static const ModeInfo zealModes[] = {{"None", 0},
+# define ZEAL_MODE(name, value) {#name, strlen(#name), value},
+ JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE)
+# undef ZEAL_MODE
+ };
+
+ for (auto mode : zealModes) {
+ if (text.length() == mode.length &&
+ memcmp(text.begin().get(), mode.name, mode.length) == 0) {
+ *modeOut = mode.value;
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static bool ParseZealModeNumericParam(CharRange text, uint32_t* paramOut) {
+ if (text.length() == 0) {
+ return false;
+ }
+
+ for (auto c : text) {
+ if (!mozilla::IsAsciiDigit(c)) {
+ return false;
+ }
+ }
+
+ *paramOut = atoi(text.begin().get());
+ return true;
+}
+
+static bool PrintZealHelpAndFail() {
+ fprintf(stderr, "Format: JS_GC_ZEAL=level(;level)*[,N]\n");
+ fputs(ZealModeHelpText, stderr);
+ return false;
+}
+
+bool GCRuntime::parseAndSetZeal(const char* str) {
+ // Set the zeal mode from a string consisting of one or more mode specifiers
+ // separated by ';', optionally followed by a ',' and the trigger frequency.
+ // The mode specifiers can by a mode name or its number.
+
+ auto text = CharRange(str, strlen(str));
+
+ CharRangeVector parts;
+ if (!SplitStringBy(text, ',', &parts)) {
+ return false;
+ }
+
+ if (parts.length() == 0 || parts.length() > 2) {
+ return PrintZealHelpAndFail();
+ }
+
+ uint32_t frequency = JS_DEFAULT_ZEAL_FREQ;
+ if (parts.length() == 2 && !ParseZealModeNumericParam(parts[1], &frequency)) {
+ return PrintZealHelpAndFail();
+ }
+
+ CharRangeVector modes;
+ if (!SplitStringBy(parts[0], ';', &modes)) {
+ return false;
+ }
+
+ for (const auto& descr : modes) {
+ uint32_t mode;
+ if (!ParseZealModeName(descr, &mode) &&
+ !(ParseZealModeNumericParam(descr, &mode) &&
+ mode <= unsigned(ZealMode::Limit))) {
+ return PrintZealHelpAndFail();
+ }
+
+ setZeal(mode, frequency);
+ }
+
+ return true;
+}
+
+const char* js::gc::AllocKindName(AllocKind kind) {
+ static const char* const names[] = {
+# define EXPAND_THING_NAME(allocKind, _1, _2, _3, _4, _5, _6) #allocKind,
+ FOR_EACH_ALLOCKIND(EXPAND_THING_NAME)
+# undef EXPAND_THING_NAME
+ };
+ static_assert(std::size(names) == AllocKindCount,
+ "names array should have an entry for every AllocKind");
+
+ size_t i = size_t(kind);
+ MOZ_ASSERT(i < std::size(names));
+ return names[i];
+}
+
+void js::gc::DumpArenaInfo() {
+ fprintf(stderr, "Arena header size: %zu\n\n", ArenaHeaderSize);
+
+ fprintf(stderr, "GC thing kinds:\n");
+ fprintf(stderr, "%25s %8s %8s %8s\n",
+ "AllocKind:", "Size:", "Count:", "Padding:");
+ for (auto kind : AllAllocKinds()) {
+ fprintf(stderr, "%25s %8zu %8zu %8zu\n", AllocKindName(kind),
+ Arena::thingSize(kind), Arena::thingsPerArena(kind),
+ Arena::firstThingOffset(kind) - ArenaHeaderSize);
+ }
+}
+
+#endif // JS_GC_ZEAL
+
+bool GCRuntime::init(uint32_t maxbytes) {
+ MOZ_ASSERT(!wasInitialized());
+
+ MOZ_ASSERT(SystemPageSize());
+ Arena::checkLookupTables();
+
+ if (!TlsGCContext.init()) {
+ return false;
+ }
+ TlsGCContext.set(&mainThreadContext.ref());
+
+ updateHelperThreadCount();
+
+#ifdef JS_GC_ZEAL
+ const char* size = getenv("JSGC_MARK_STACK_LIMIT");
+ if (size) {
+ maybeMarkStackLimit = atoi(size);
+ }
+#endif
+
+ if (!updateMarkersVector()) {
+ return false;
+ }
+
+ {
+ AutoLockGCBgAlloc lock(this);
+
+ MOZ_ALWAYS_TRUE(tunables.setParameter(JSGC_MAX_BYTES, maxbytes));
+
+ if (!nursery().init(lock)) {
+ return false;
+ }
+
+ const char* pretenureThresholdStr = getenv("JSGC_PRETENURE_THRESHOLD");
+ if (pretenureThresholdStr && pretenureThresholdStr[0]) {
+ char* last;
+ long pretenureThreshold = strtol(pretenureThresholdStr, &last, 10);
+ if (last[0] || !tunables.setParameter(JSGC_PRETENURE_THRESHOLD,
+ pretenureThreshold)) {
+ fprintf(stderr, "Invalid value for JSGC_PRETENURE_THRESHOLD: %s\n",
+ pretenureThresholdStr);
+ }
+ }
+ }
+
+#ifdef JS_GC_ZEAL
+ const char* zealSpec = getenv("JS_GC_ZEAL");
+ if (zealSpec && zealSpec[0] && !parseAndSetZeal(zealSpec)) {
+ return false;
+ }
+#endif
+
+ for (auto& marker : markers) {
+ if (!marker->init()) {
+ return false;
+ }
+ }
+
+ if (!initSweepActions()) {
+ return false;
+ }
+
+ UniquePtr<Zone> zone = MakeUnique<Zone>(rt, Zone::AtomsZone);
+ if (!zone || !zone->init()) {
+ return false;
+ }
+
+ // The atoms zone is stored as the first element of the zones vector.
+ MOZ_ASSERT(zone->isAtomsZone());
+ MOZ_ASSERT(zones().empty());
+ MOZ_ALWAYS_TRUE(zones().reserve(1)); // ZonesVector has inline capacity 4.
+ zones().infallibleAppend(zone.release());
+
+ gcprobes::Init(this);
+
+ initialized = true;
+ return true;
+}
+
+void GCRuntime::finish() {
+ MOZ_ASSERT(inPageLoadCount == 0);
+ MOZ_ASSERT(!sharedAtomsZone_);
+
+ // Wait for nursery background free to end and disable it to release memory.
+ if (nursery().isEnabled()) {
+ nursery().disable();
+ }
+
+ // Wait until the background finalization and allocation stops and the
+ // helper thread shuts down before we forcefully release any remaining GC
+ // memory.
+ sweepTask.join();
+ markTask.join();
+ freeTask.join();
+ allocTask.cancelAndWait();
+ decommitTask.cancelAndWait();
+
+#ifdef JS_GC_ZEAL
+ // Free memory associated with GC verification.
+ finishVerifier();
+#endif
+
+ // Delete all remaining zones.
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ AutoSetThreadIsSweeping threadIsSweeping(rt->gcContext(), zone);
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ for (RealmsInCompartmentIter realm(comp); !realm.done(); realm.next()) {
+ js_delete(realm.get());
+ }
+ comp->realms().clear();
+ js_delete(comp.get());
+ }
+ zone->compartments().clear();
+ js_delete(zone.get());
+ }
+
+ zones().clear();
+
+ FreeChunkPool(fullChunks_.ref());
+ FreeChunkPool(availableChunks_.ref());
+ FreeChunkPool(emptyChunks_.ref());
+
+ TlsGCContext.set(nullptr);
+
+ gcprobes::Finish(this);
+
+ nursery().printTotalProfileTimes();
+ stats().printTotalProfileTimes();
+}
+
+bool GCRuntime::freezeSharedAtomsZone() {
+ // This is called just after permanent atoms and well-known symbols have been
+ // created. At this point all existing atoms and symbols are permanent.
+ //
+ // This method makes the current atoms zone into a shared atoms zone and
+ // removes it from the zones list. Everything in it is marked black. A new
+ // empty atoms zone is created, where all atoms local to this runtime will
+ // live.
+ //
+ // The shared atoms zone will not be collected until shutdown when it is
+ // returned to the zone list by restoreSharedAtomsZone().
+
+ MOZ_ASSERT(rt->isMainRuntime());
+ MOZ_ASSERT(!sharedAtomsZone_);
+ MOZ_ASSERT(zones().length() == 1);
+ MOZ_ASSERT(atomsZone());
+ MOZ_ASSERT(!atomsZone()->wasGCStarted());
+ MOZ_ASSERT(!atomsZone()->needsIncrementalBarrier());
+
+ AutoAssertEmptyNursery nurseryIsEmpty(rt->mainContextFromOwnThread());
+
+ atomsZone()->arenas.clearFreeLists();
+
+ for (auto kind : AllAllocKinds()) {
+ for (auto thing =
+ atomsZone()->cellIterUnsafe<TenuredCell>(kind, nurseryIsEmpty);
+ !thing.done(); thing.next()) {
+ TenuredCell* cell = thing.getCell();
+ MOZ_ASSERT((cell->is<JSString>() &&
+ cell->as<JSString>()->isPermanentAndMayBeShared()) ||
+ (cell->is<JS::Symbol>() &&
+ cell->as<JS::Symbol>()->isPermanentAndMayBeShared()));
+ cell->markBlack();
+ }
+ }
+
+ sharedAtomsZone_ = atomsZone();
+ zones().clear();
+
+ UniquePtr<Zone> zone = MakeUnique<Zone>(rt, Zone::AtomsZone);
+ if (!zone || !zone->init()) {
+ return false;
+ }
+
+ MOZ_ASSERT(zone->isAtomsZone());
+ zones().infallibleAppend(zone.release());
+
+ return true;
+}
+
+void GCRuntime::restoreSharedAtomsZone() {
+ // Return the shared atoms zone to the zone list. This allows the contents of
+ // the shared atoms zone to be collected when the parent runtime is shut down.
+
+ if (!sharedAtomsZone_) {
+ return;
+ }
+
+ MOZ_ASSERT(rt->isMainRuntime());
+ MOZ_ASSERT(rt->childRuntimeCount == 0);
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zones().append(sharedAtomsZone_)) {
+ oomUnsafe.crash("restoreSharedAtomsZone");
+ }
+
+ sharedAtomsZone_ = nullptr;
+}
+
+bool GCRuntime::setParameter(JSContext* cx, JSGCParamKey key, uint32_t value) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ AutoStopVerifyingBarriers pauseVerification(rt, false);
+ FinishGC(cx);
+ waitBackgroundSweepEnd();
+
+ AutoLockGC lock(this);
+ return setParameter(key, value, lock);
+}
+
+static bool IsGCThreadParameter(JSGCParamKey key) {
+ return key == JSGC_HELPER_THREAD_RATIO || key == JSGC_MAX_HELPER_THREADS ||
+ key == JSGC_MARKING_THREAD_COUNT;
+}
+
+bool GCRuntime::setParameter(JSGCParamKey key, uint32_t value,
+ AutoLockGC& lock) {
+ switch (key) {
+ case JSGC_SLICE_TIME_BUDGET_MS:
+ defaultTimeBudgetMS_ = value;
+ break;
+ case JSGC_INCREMENTAL_GC_ENABLED:
+ setIncrementalGCEnabled(value != 0);
+ break;
+ case JSGC_PER_ZONE_GC_ENABLED:
+ perZoneGCEnabled = value != 0;
+ break;
+ case JSGC_COMPACTING_ENABLED:
+ compactingEnabled = value != 0;
+ break;
+ case JSGC_PARALLEL_MARKING_ENABLED:
+ // Not supported on workers.
+ parallelMarkingEnabled = rt->isMainRuntime() && value != 0;
+ updateMarkersVector();
+ break;
+ case JSGC_INCREMENTAL_WEAKMAP_ENABLED:
+ for (auto& marker : markers) {
+ marker->incrementalWeakMapMarkingEnabled = value != 0;
+ }
+ break;
+ case JSGC_MIN_EMPTY_CHUNK_COUNT:
+ setMinEmptyChunkCount(value, lock);
+ break;
+ case JSGC_MAX_EMPTY_CHUNK_COUNT:
+ setMaxEmptyChunkCount(value, lock);
+ break;
+ default:
+ if (IsGCThreadParameter(key)) {
+ return setThreadParameter(key, value, lock);
+ }
+
+ if (!tunables.setParameter(key, value)) {
+ return false;
+ }
+ updateAllGCStartThresholds();
+ }
+
+ return true;
+}
+
+bool GCRuntime::setThreadParameter(JSGCParamKey key, uint32_t value,
+ AutoLockGC& lock) {
+ if (rt->parentRuntime) {
+ // Don't allow these to be set for worker runtimes.
+ return false;
+ }
+
+ switch (key) {
+ case JSGC_HELPER_THREAD_RATIO:
+ if (value == 0) {
+ return false;
+ }
+ helperThreadRatio = double(value) / 100.0;
+ break;
+ case JSGC_MAX_HELPER_THREADS:
+ if (value == 0) {
+ return false;
+ }
+ maxHelperThreads = value;
+ break;
+ case JSGC_MARKING_THREAD_COUNT:
+ markingThreadCount = std::min(size_t(value), MaxParallelWorkers);
+ break;
+ default:
+ MOZ_CRASH("Unexpected parameter key");
+ }
+
+ updateHelperThreadCount();
+ updateMarkersVector();
+
+ return true;
+}
+
+void GCRuntime::resetParameter(JSContext* cx, JSGCParamKey key) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ AutoStopVerifyingBarriers pauseVerification(rt, false);
+ FinishGC(cx);
+ waitBackgroundSweepEnd();
+
+ AutoLockGC lock(this);
+ resetParameter(key, lock);
+}
+
+void GCRuntime::resetParameter(JSGCParamKey key, AutoLockGC& lock) {
+ switch (key) {
+ case JSGC_SLICE_TIME_BUDGET_MS:
+ defaultTimeBudgetMS_ = TuningDefaults::DefaultTimeBudgetMS;
+ break;
+ case JSGC_INCREMENTAL_GC_ENABLED:
+ setIncrementalGCEnabled(TuningDefaults::IncrementalGCEnabled);
+ break;
+ case JSGC_PER_ZONE_GC_ENABLED:
+ perZoneGCEnabled = TuningDefaults::PerZoneGCEnabled;
+ break;
+ case JSGC_COMPACTING_ENABLED:
+ compactingEnabled = TuningDefaults::CompactingEnabled;
+ break;
+ case JSGC_PARALLEL_MARKING_ENABLED:
+ parallelMarkingEnabled = TuningDefaults::ParallelMarkingEnabled;
+ updateMarkersVector();
+ break;
+ case JSGC_INCREMENTAL_WEAKMAP_ENABLED:
+ for (auto& marker : markers) {
+ marker->incrementalWeakMapMarkingEnabled =
+ TuningDefaults::IncrementalWeakMapMarkingEnabled;
+ }
+ break;
+ case JSGC_MIN_EMPTY_CHUNK_COUNT:
+ setMinEmptyChunkCount(TuningDefaults::MinEmptyChunkCount, lock);
+ break;
+ case JSGC_MAX_EMPTY_CHUNK_COUNT:
+ setMaxEmptyChunkCount(TuningDefaults::MaxEmptyChunkCount, lock);
+ break;
+ default:
+ if (IsGCThreadParameter(key)) {
+ resetThreadParameter(key, lock);
+ return;
+ }
+
+ tunables.resetParameter(key);
+ updateAllGCStartThresholds();
+ }
+}
+
+void GCRuntime::resetThreadParameter(JSGCParamKey key, AutoLockGC& lock) {
+ if (rt->parentRuntime) {
+ return;
+ }
+
+ switch (key) {
+ case JSGC_HELPER_THREAD_RATIO:
+ helperThreadRatio = TuningDefaults::HelperThreadRatio;
+ break;
+ case JSGC_MAX_HELPER_THREADS:
+ maxHelperThreads = TuningDefaults::MaxHelperThreads;
+ break;
+ case JSGC_MARKING_THREAD_COUNT:
+ markingThreadCount = 0;
+ break;
+ default:
+ MOZ_CRASH("Unexpected parameter key");
+ }
+
+ updateHelperThreadCount();
+ updateMarkersVector();
+}
+
+uint32_t GCRuntime::getParameter(JSGCParamKey key) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ AutoLockGC lock(this);
+ return getParameter(key, lock);
+}
+
+uint32_t GCRuntime::getParameter(JSGCParamKey key, const AutoLockGC& lock) {
+ switch (key) {
+ case JSGC_BYTES:
+ return uint32_t(heapSize.bytes());
+ case JSGC_NURSERY_BYTES:
+ return nursery().capacity();
+ case JSGC_NUMBER:
+ return uint32_t(number);
+ case JSGC_MAJOR_GC_NUMBER:
+ return uint32_t(majorGCNumber);
+ case JSGC_MINOR_GC_NUMBER:
+ return uint32_t(minorGCNumber);
+ case JSGC_INCREMENTAL_GC_ENABLED:
+ return incrementalGCEnabled;
+ case JSGC_PER_ZONE_GC_ENABLED:
+ return perZoneGCEnabled;
+ case JSGC_UNUSED_CHUNKS:
+ return uint32_t(emptyChunks(lock).count());
+ case JSGC_TOTAL_CHUNKS:
+ return uint32_t(fullChunks(lock).count() + availableChunks(lock).count() +
+ emptyChunks(lock).count());
+ case JSGC_SLICE_TIME_BUDGET_MS:
+ MOZ_RELEASE_ASSERT(defaultTimeBudgetMS_ >= 0);
+ MOZ_RELEASE_ASSERT(defaultTimeBudgetMS_ <= UINT32_MAX);
+ return uint32_t(defaultTimeBudgetMS_);
+ case JSGC_MIN_EMPTY_CHUNK_COUNT:
+ return minEmptyChunkCount(lock);
+ case JSGC_MAX_EMPTY_CHUNK_COUNT:
+ return maxEmptyChunkCount(lock);
+ case JSGC_COMPACTING_ENABLED:
+ return compactingEnabled;
+ case JSGC_PARALLEL_MARKING_ENABLED:
+ return parallelMarkingEnabled;
+ case JSGC_INCREMENTAL_WEAKMAP_ENABLED:
+ return marker().incrementalWeakMapMarkingEnabled;
+ case JSGC_CHUNK_BYTES:
+ return ChunkSize;
+ case JSGC_HELPER_THREAD_RATIO:
+ MOZ_ASSERT(helperThreadRatio > 0.0);
+ return uint32_t(helperThreadRatio * 100.0);
+ case JSGC_MAX_HELPER_THREADS:
+ MOZ_ASSERT(maxHelperThreads <= UINT32_MAX);
+ return maxHelperThreads;
+ case JSGC_HELPER_THREAD_COUNT:
+ return helperThreadCount;
+ case JSGC_MARKING_THREAD_COUNT:
+ return markingThreadCount;
+ case JSGC_SYSTEM_PAGE_SIZE_KB:
+ return SystemPageSize() / 1024;
+ default:
+ return tunables.getParameter(key);
+ }
+}
+
+#ifdef JS_GC_ZEAL
+void GCRuntime::setMarkStackLimit(size_t limit, AutoLockGC& lock) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+
+ maybeMarkStackLimit = limit;
+
+ AutoUnlockGC unlock(lock);
+ AutoStopVerifyingBarriers pauseVerification(rt, false);
+ for (auto& marker : markers) {
+ marker->setMaxCapacity(limit);
+ }
+}
+#endif
+
+void GCRuntime::setIncrementalGCEnabled(bool enabled) {
+ incrementalGCEnabled = enabled;
+}
+
+void GCRuntime::updateHelperThreadCount() {
+ if (!CanUseExtraThreads()) {
+ // startTask will run the work on the main thread if the count is 1.
+ MOZ_ASSERT(helperThreadCount == 1);
+ return;
+ }
+
+ // Number of extra threads required during parallel marking to ensure we can
+ // start the necessary marking tasks. Background free and background
+ // allocation may already be running and we want to avoid these tasks blocking
+ // marking. In real configurations there will be enough threads that this
+ // won't affect anything.
+ static constexpr size_t SpareThreadsDuringParallelMarking = 2;
+
+ // The count of helper threads used for GC tasks is process wide. Don't set it
+ // for worker JS runtimes.
+ if (rt->parentRuntime) {
+ helperThreadCount = rt->parentRuntime->gc.helperThreadCount;
+ return;
+ }
+
+ // Calculate the target thread count for GC parallel tasks.
+ double cpuCount = GetHelperThreadCPUCount();
+ helperThreadCount = std::clamp(size_t(cpuCount * helperThreadRatio.ref()),
+ size_t(1), maxHelperThreads.ref());
+
+ // Calculate the overall target thread count taking into account the separate
+ // parameter for parallel marking threads. Add spare threads to avoid blocking
+ // parallel marking when there is other GC work happening.
+ size_t targetCount =
+ std::max(helperThreadCount.ref(),
+ markingThreadCount.ref() + SpareThreadsDuringParallelMarking);
+
+ // Attempt to create extra threads if possible. This is not supported when
+ // using an external thread pool.
+ AutoLockHelperThreadState lock;
+ (void)HelperThreadState().ensureThreadCount(targetCount, lock);
+
+ // Limit all thread counts based on the number of threads available, which may
+ // be fewer than requested.
+ size_t availableThreadCount = GetHelperThreadCount();
+ MOZ_ASSERT(availableThreadCount != 0);
+ targetCount = std::min(targetCount, availableThreadCount);
+ helperThreadCount = std::min(helperThreadCount.ref(), availableThreadCount);
+ markingThreadCount =
+ std::min(markingThreadCount.ref(),
+ availableThreadCount - SpareThreadsDuringParallelMarking);
+
+ // Update the maximum number of threads that will be used for GC work.
+ HelperThreadState().setGCParallelThreadCount(targetCount, lock);
+}
+
+size_t GCRuntime::markingWorkerCount() const {
+ if (!CanUseExtraThreads() || !parallelMarkingEnabled) {
+ return 1;
+ }
+
+ if (markingThreadCount) {
+ return markingThreadCount;
+ }
+
+ // Limit parallel marking to use at most two threads initially.
+ return 2;
+}
+
+#ifdef DEBUG
+void GCRuntime::assertNoMarkingWork() const {
+ for (auto& marker : markers) {
+ MOZ_ASSERT(marker->isDrained());
+ }
+ MOZ_ASSERT(!hasDelayedMarking());
+}
+#endif
+
+static size_t GetGCParallelThreadCount() {
+ AutoLockHelperThreadState lock;
+ return HelperThreadState().getGCParallelThreadCount(lock);
+}
+
+bool GCRuntime::updateMarkersVector() {
+ MOZ_ASSERT(helperThreadCount >= 1,
+ "There must always be at least one mark task");
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ assertNoMarkingWork();
+
+ // Limit worker count to number of GC parallel tasks that can run
+ // concurrently, otherwise one thread can deadlock waiting on another.
+ size_t targetCount =
+ std::min(markingWorkerCount(), GetGCParallelThreadCount());
+
+ if (markers.length() > targetCount) {
+ return markers.resize(targetCount);
+ }
+
+ while (markers.length() < targetCount) {
+ auto marker = MakeUnique<GCMarker>(rt);
+ if (!marker) {
+ return false;
+ }
+
+#ifdef JS_GC_ZEAL
+ if (maybeMarkStackLimit) {
+ marker->setMaxCapacity(maybeMarkStackLimit);
+ }
+#endif
+
+ if (!marker->init()) {
+ return false;
+ }
+
+ if (!markers.emplaceBack(std::move(marker))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool GCRuntime::addBlackRootsTracer(JSTraceDataOp traceOp, void* data) {
+ AssertHeapIsIdle();
+ return !!blackRootTracers.ref().append(
+ Callback<JSTraceDataOp>(traceOp, data));
+}
+
+void GCRuntime::removeBlackRootsTracer(JSTraceDataOp traceOp, void* data) {
+ // Can be called from finalizers
+ for (size_t i = 0; i < blackRootTracers.ref().length(); i++) {
+ Callback<JSTraceDataOp>* e = &blackRootTracers.ref()[i];
+ if (e->op == traceOp && e->data == data) {
+ blackRootTracers.ref().erase(e);
+ break;
+ }
+ }
+}
+
+void GCRuntime::setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data) {
+ AssertHeapIsIdle();
+ grayRootTracer.ref() = {traceOp, data};
+}
+
+void GCRuntime::clearBlackAndGrayRootTracers() {
+ MOZ_ASSERT(rt->isBeingDestroyed());
+ blackRootTracers.ref().clear();
+ setGrayRootsTracer(nullptr, nullptr);
+}
+
+void GCRuntime::setGCCallback(JSGCCallback callback, void* data) {
+ gcCallback.ref() = {callback, data};
+}
+
+void GCRuntime::callGCCallback(JSGCStatus status, JS::GCReason reason) const {
+ const auto& callback = gcCallback.ref();
+ MOZ_ASSERT(callback.op);
+ callback.op(rt->mainContextFromOwnThread(), status, reason, callback.data);
+}
+
+void GCRuntime::setObjectsTenuredCallback(JSObjectsTenuredCallback callback,
+ void* data) {
+ tenuredCallback.ref() = {callback, data};
+}
+
+void GCRuntime::callObjectsTenuredCallback() {
+ JS::AutoSuppressGCAnalysis nogc;
+ const auto& callback = tenuredCallback.ref();
+ if (callback.op) {
+ callback.op(rt->mainContextFromOwnThread(), callback.data);
+ }
+}
+
+bool GCRuntime::addFinalizeCallback(JSFinalizeCallback callback, void* data) {
+ return finalizeCallbacks.ref().append(
+ Callback<JSFinalizeCallback>(callback, data));
+}
+
+template <typename F>
+static void EraseCallback(CallbackVector<F>& vector, F callback) {
+ for (Callback<F>* p = vector.begin(); p != vector.end(); p++) {
+ if (p->op == callback) {
+ vector.erase(p);
+ return;
+ }
+ }
+}
+
+void GCRuntime::removeFinalizeCallback(JSFinalizeCallback callback) {
+ EraseCallback(finalizeCallbacks.ref(), callback);
+}
+
+void GCRuntime::callFinalizeCallbacks(JS::GCContext* gcx,
+ JSFinalizeStatus status) const {
+ for (auto& p : finalizeCallbacks.ref()) {
+ p.op(gcx, status, p.data);
+ }
+}
+
+void GCRuntime::setHostCleanupFinalizationRegistryCallback(
+ JSHostCleanupFinalizationRegistryCallback callback, void* data) {
+ hostCleanupFinalizationRegistryCallback.ref() = {callback, data};
+}
+
+void GCRuntime::callHostCleanupFinalizationRegistryCallback(
+ JSFunction* doCleanup, GlobalObject* incumbentGlobal) {
+ JS::AutoSuppressGCAnalysis nogc;
+ const auto& callback = hostCleanupFinalizationRegistryCallback.ref();
+ if (callback.op) {
+ callback.op(doCleanup, incumbentGlobal, callback.data);
+ }
+}
+
+bool GCRuntime::addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback,
+ void* data) {
+ return updateWeakPointerZonesCallbacks.ref().append(
+ Callback<JSWeakPointerZonesCallback>(callback, data));
+}
+
+void GCRuntime::removeWeakPointerZonesCallback(
+ JSWeakPointerZonesCallback callback) {
+ EraseCallback(updateWeakPointerZonesCallbacks.ref(), callback);
+}
+
+void GCRuntime::callWeakPointerZonesCallbacks(JSTracer* trc) const {
+ for (auto const& p : updateWeakPointerZonesCallbacks.ref()) {
+ p.op(trc, p.data);
+ }
+}
+
+bool GCRuntime::addWeakPointerCompartmentCallback(
+ JSWeakPointerCompartmentCallback callback, void* data) {
+ return updateWeakPointerCompartmentCallbacks.ref().append(
+ Callback<JSWeakPointerCompartmentCallback>(callback, data));
+}
+
+void GCRuntime::removeWeakPointerCompartmentCallback(
+ JSWeakPointerCompartmentCallback callback) {
+ EraseCallback(updateWeakPointerCompartmentCallbacks.ref(), callback);
+}
+
+void GCRuntime::callWeakPointerCompartmentCallbacks(
+ JSTracer* trc, JS::Compartment* comp) const {
+ for (auto const& p : updateWeakPointerCompartmentCallbacks.ref()) {
+ p.op(trc, comp, p.data);
+ }
+}
+
+JS::GCSliceCallback GCRuntime::setSliceCallback(JS::GCSliceCallback callback) {
+ return stats().setSliceCallback(callback);
+}
+
+JS::GCNurseryCollectionCallback GCRuntime::setNurseryCollectionCallback(
+ JS::GCNurseryCollectionCallback callback) {
+ return stats().setNurseryCollectionCallback(callback);
+}
+
+JS::DoCycleCollectionCallback GCRuntime::setDoCycleCollectionCallback(
+ JS::DoCycleCollectionCallback callback) {
+ const auto prior = gcDoCycleCollectionCallback.ref();
+ gcDoCycleCollectionCallback.ref() = {callback, nullptr};
+ return prior.op;
+}
+
+void GCRuntime::callDoCycleCollectionCallback(JSContext* cx) {
+ const auto& callback = gcDoCycleCollectionCallback.ref();
+ if (callback.op) {
+ callback.op(cx);
+ }
+}
+
+bool GCRuntime::addRoot(Value* vp, const char* name) {
+ /*
+ * Sometimes Firefox will hold weak references to objects and then convert
+ * them to strong references by calling AddRoot (e.g., via PreserveWrapper,
+ * or ModifyBusyCount in workers). We need a read barrier to cover these
+ * cases.
+ */
+ MOZ_ASSERT(vp);
+ Value value = *vp;
+ if (value.isGCThing()) {
+ ValuePreWriteBarrier(value);
+ }
+
+ return rootsHash.ref().put(vp, name);
+}
+
+void GCRuntime::removeRoot(Value* vp) {
+ rootsHash.ref().remove(vp);
+ notifyRootsRemoved();
+}
+
+/* Compacting GC */
+
+bool js::gc::IsCurrentlyAnimating(const TimeStamp& lastAnimationTime,
+ const TimeStamp& currentTime) {
+ // Assume that we're currently animating if js::NotifyAnimationActivity has
+ // been called in the last second.
+ static const auto oneSecond = TimeDuration::FromSeconds(1);
+ return !lastAnimationTime.IsNull() &&
+ currentTime < (lastAnimationTime + oneSecond);
+}
+
+static bool DiscardedCodeRecently(Zone* zone, const TimeStamp& currentTime) {
+ static const auto thirtySeconds = TimeDuration::FromSeconds(30);
+ return !zone->lastDiscardedCodeTime().IsNull() &&
+ currentTime < (zone->lastDiscardedCodeTime() + thirtySeconds);
+}
+
+bool GCRuntime::shouldCompact() {
+ // Compact on shrinking GC if enabled. Skip compacting in incremental GCs
+ // if we are currently animating, unless the user is inactive or we're
+ // responding to memory pressure.
+
+ if (!isShrinkingGC() || !isCompactingGCEnabled()) {
+ return false;
+ }
+
+ if (initialReason == JS::GCReason::USER_INACTIVE ||
+ initialReason == JS::GCReason::MEM_PRESSURE) {
+ return true;
+ }
+
+ return !isIncremental ||
+ !IsCurrentlyAnimating(rt->lastAnimationTime, TimeStamp::Now());
+}
+
+bool GCRuntime::isCompactingGCEnabled() const {
+ return compactingEnabled &&
+ rt->mainContextFromOwnThread()->compactingDisabledCount == 0;
+}
+
+JS_PUBLIC_API void JS::SetCreateGCSliceBudgetCallback(
+ JSContext* cx, JS::CreateSliceBudgetCallback cb) {
+ cx->runtime()->gc.createBudgetCallback = cb;
+}
+
+void TimeBudget::setDeadlineFromNow() { deadline = TimeStamp::Now() + budget; }
+
+SliceBudget::SliceBudget(TimeBudget time, InterruptRequestFlag* interrupt)
+ : budget(TimeBudget(time)),
+ interruptRequested(interrupt),
+ counter(StepsPerExpensiveCheck) {
+ budget.as<TimeBudget>().setDeadlineFromNow();
+}
+
+SliceBudget::SliceBudget(WorkBudget work)
+ : budget(work), interruptRequested(nullptr), counter(work.budget) {}
+
+int SliceBudget::describe(char* buffer, size_t maxlen) const {
+ if (isUnlimited()) {
+ return snprintf(buffer, maxlen, "unlimited");
+ } else if (isWorkBudget()) {
+ return snprintf(buffer, maxlen, "work(%" PRId64 ")", workBudget());
+ } else {
+ const char* interruptStr = "";
+ if (interruptRequested) {
+ interruptStr = interrupted ? "INTERRUPTED " : "interruptible ";
+ }
+ const char* extra = "";
+ if (idle) {
+ extra = extended ? " (started idle but extended)" : " (idle)";
+ }
+ return snprintf(buffer, maxlen, "%s%" PRId64 "ms%s", interruptStr,
+ timeBudget(), extra);
+ }
+}
+
+bool SliceBudget::checkOverBudget() {
+ MOZ_ASSERT(counter <= 0);
+ MOZ_ASSERT(!isUnlimited());
+
+ if (isWorkBudget()) {
+ return true;
+ }
+
+ if (interruptRequested && *interruptRequested) {
+ *interruptRequested = false;
+ interrupted = true;
+ }
+
+ if (interrupted) {
+ return true;
+ }
+
+ if (TimeStamp::Now() >= budget.as<TimeBudget>().deadline) {
+ return true;
+ }
+
+ counter = StepsPerExpensiveCheck;
+ return false;
+}
+
+void GCRuntime::requestMajorGC(JS::GCReason reason) {
+ MOZ_ASSERT_IF(reason != JS::GCReason::BG_TASK_FINISHED,
+ !CurrentThreadIsPerformingGC());
+
+ if (majorGCRequested()) {
+ return;
+ }
+
+ majorGCTriggerReason = reason;
+ rt->mainContextFromAnyThread()->requestInterrupt(InterruptReason::GC);
+}
+
+void Nursery::requestMinorGC(JS::GCReason reason) const {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+
+ if (minorGCRequested()) {
+ return;
+ }
+
+ minorGCTriggerReason_ = reason;
+ runtime()->mainContextFromOwnThread()->requestInterrupt(InterruptReason::GC);
+}
+
+bool GCRuntime::triggerGC(JS::GCReason reason) {
+ /*
+ * Don't trigger GCs if this is being called off the main thread from
+ * onTooMuchMalloc().
+ */
+ if (!CurrentThreadCanAccessRuntime(rt)) {
+ return false;
+ }
+
+ /* GC is already running. */
+ if (JS::RuntimeHeapIsCollecting()) {
+ return false;
+ }
+
+ JS::PrepareForFullGC(rt->mainContextFromOwnThread());
+ requestMajorGC(reason);
+ return true;
+}
+
+void GCRuntime::maybeTriggerGCAfterAlloc(Zone* zone) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+
+ TriggerResult trigger =
+ checkHeapThreshold(zone, zone->gcHeapSize, zone->gcHeapThreshold);
+
+ if (trigger.shouldTrigger) {
+ // Start or continue an in progress incremental GC. We do this to try to
+ // avoid performing non-incremental GCs on zones which allocate a lot of
+ // data, even when incremental slices can't be triggered via scheduling in
+ // the event loop.
+ triggerZoneGC(zone, JS::GCReason::ALLOC_TRIGGER, trigger.usedBytes,
+ trigger.thresholdBytes);
+ }
+}
+
+void js::gc::MaybeMallocTriggerZoneGC(JSRuntime* rt, ZoneAllocator* zoneAlloc,
+ const HeapSize& heap,
+ const HeapThreshold& threshold,
+ JS::GCReason reason) {
+ rt->gc.maybeTriggerGCAfterMalloc(Zone::from(zoneAlloc), heap, threshold,
+ reason);
+}
+
+void GCRuntime::maybeTriggerGCAfterMalloc(Zone* zone) {
+ if (maybeTriggerGCAfterMalloc(zone, zone->mallocHeapSize,
+ zone->mallocHeapThreshold,
+ JS::GCReason::TOO_MUCH_MALLOC)) {
+ return;
+ }
+
+ maybeTriggerGCAfterMalloc(zone, zone->jitHeapSize, zone->jitHeapThreshold,
+ JS::GCReason::TOO_MUCH_JIT_CODE);
+}
+
+bool GCRuntime::maybeTriggerGCAfterMalloc(Zone* zone, const HeapSize& heap,
+ const HeapThreshold& threshold,
+ JS::GCReason reason) {
+ // Ignore malloc during sweeping, for example when we resize hash tables.
+ if (heapState() != JS::HeapState::Idle) {
+ return false;
+ }
+
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ TriggerResult trigger = checkHeapThreshold(zone, heap, threshold);
+ if (!trigger.shouldTrigger) {
+ return false;
+ }
+
+ // Trigger a zone GC. budgetIncrementalGC() will work out whether to do an
+ // incremental or non-incremental collection.
+ triggerZoneGC(zone, reason, trigger.usedBytes, trigger.thresholdBytes);
+ return true;
+}
+
+TriggerResult GCRuntime::checkHeapThreshold(
+ Zone* zone, const HeapSize& heapSize, const HeapThreshold& heapThreshold) {
+ MOZ_ASSERT_IF(heapThreshold.hasSliceThreshold(), zone->wasGCStarted());
+
+ size_t usedBytes = heapSize.bytes();
+ size_t thresholdBytes = heapThreshold.hasSliceThreshold()
+ ? heapThreshold.sliceBytes()
+ : heapThreshold.startBytes();
+
+ // The incremental limit will be checked if we trigger a GC slice.
+ MOZ_ASSERT(thresholdBytes <= heapThreshold.incrementalLimitBytes());
+
+ return TriggerResult{usedBytes >= thresholdBytes, usedBytes, thresholdBytes};
+}
+
+bool GCRuntime::triggerZoneGC(Zone* zone, JS::GCReason reason, size_t used,
+ size_t threshold) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ /* GC is already running. */
+ if (JS::RuntimeHeapIsBusy()) {
+ return false;
+ }
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::Alloc)) {
+ MOZ_RELEASE_ASSERT(triggerGC(reason));
+ return true;
+ }
+#endif
+
+ if (zone->isAtomsZone()) {
+ stats().recordTrigger(used, threshold);
+ MOZ_RELEASE_ASSERT(triggerGC(reason));
+ return true;
+ }
+
+ stats().recordTrigger(used, threshold);
+ zone->scheduleGC();
+ requestMajorGC(reason);
+ return true;
+}
+
+void GCRuntime::maybeGC() {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::RootsChange)) {
+ JS::PrepareForFullGC(rt->mainContextFromOwnThread());
+ gc(JS::GCOptions::Normal, JS::GCReason::DEBUG_GC);
+ return;
+ }
+#endif
+
+ (void)gcIfRequestedImpl(/* eagerOk = */ true);
+}
+
+JS::GCReason GCRuntime::wantMajorGC(bool eagerOk) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ if (majorGCRequested()) {
+ return majorGCTriggerReason;
+ }
+
+ if (isIncrementalGCInProgress() || !eagerOk) {
+ return JS::GCReason::NO_REASON;
+ }
+
+ JS::GCReason reason = JS::GCReason::NO_REASON;
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ if (checkEagerAllocTrigger(zone->gcHeapSize, zone->gcHeapThreshold) ||
+ checkEagerAllocTrigger(zone->mallocHeapSize,
+ zone->mallocHeapThreshold)) {
+ zone->scheduleGC();
+ reason = JS::GCReason::EAGER_ALLOC_TRIGGER;
+ }
+ }
+
+ return reason;
+}
+
+bool GCRuntime::checkEagerAllocTrigger(const HeapSize& size,
+ const HeapThreshold& threshold) {
+ double thresholdBytes =
+ threshold.eagerAllocTrigger(schedulingState.inHighFrequencyGCMode());
+ double usedBytes = size.bytes();
+ if (usedBytes <= 1024 * 1024 || usedBytes < thresholdBytes) {
+ return false;
+ }
+
+ stats().recordTrigger(usedBytes, thresholdBytes);
+ return true;
+}
+
+bool GCRuntime::shouldDecommit() const {
+ // If we're doing a shrinking GC we always decommit to release as much memory
+ // as possible.
+ if (cleanUpEverything) {
+ return true;
+ }
+
+ // If we are allocating heavily enough to trigger "high frequency" GC then
+ // skip decommit so that we do not compete with the mutator.
+ return !schedulingState.inHighFrequencyGCMode();
+}
+
+void GCRuntime::startDecommit() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::DECOMMIT);
+
+#ifdef DEBUG
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(decommitTask.isIdle());
+
+ {
+ AutoLockGC lock(this);
+ MOZ_ASSERT(fullChunks(lock).verify());
+ MOZ_ASSERT(availableChunks(lock).verify());
+ MOZ_ASSERT(emptyChunks(lock).verify());
+
+ // Verify that all entries in the empty chunks pool are unused.
+ for (ChunkPool::Iter chunk(emptyChunks(lock)); !chunk.done();
+ chunk.next()) {
+ MOZ_ASSERT(chunk->unused());
+ }
+ }
+#endif
+
+ if (!shouldDecommit()) {
+ return;
+ }
+
+ {
+ AutoLockGC lock(this);
+ if (availableChunks(lock).empty() && !tooManyEmptyChunks(lock) &&
+ emptyChunks(lock).empty()) {
+ return; // Nothing to do.
+ }
+ }
+
+#ifdef DEBUG
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(!requestSliceAfterBackgroundTask);
+ }
+#endif
+
+ if (useBackgroundThreads) {
+ decommitTask.start();
+ return;
+ }
+
+ decommitTask.runFromMainThread();
+}
+
+BackgroundDecommitTask::BackgroundDecommitTask(GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::DECOMMIT) {}
+
+void js::gc::BackgroundDecommitTask::run(AutoLockHelperThreadState& lock) {
+ {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ ChunkPool emptyChunksToFree;
+ {
+ AutoLockGC gcLock(gc);
+ emptyChunksToFree = gc->expireEmptyChunkPool(gcLock);
+ }
+
+ FreeChunkPool(emptyChunksToFree);
+
+ {
+ AutoLockGC gcLock(gc);
+
+ // To help minimize the total number of chunks needed over time, sort the
+ // available chunks list so that we allocate into more-used chunks first.
+ gc->availableChunks(gcLock).sort();
+
+ if (DecommitEnabled()) {
+ gc->decommitEmptyChunks(cancel_, gcLock);
+ gc->decommitFreeArenas(cancel_, gcLock);
+ }
+ }
+ }
+
+ gc->maybeRequestGCAfterBackgroundTask(lock);
+}
+
+static inline bool CanDecommitWholeChunk(TenuredChunk* chunk) {
+ return chunk->unused() && chunk->info.numArenasFreeCommitted != 0;
+}
+
+// Called from a background thread to decommit free arenas. Releases the GC
+// lock.
+void GCRuntime::decommitEmptyChunks(const bool& cancel, AutoLockGC& lock) {
+ Vector<TenuredChunk*, 0, SystemAllocPolicy> chunksToDecommit;
+ for (ChunkPool::Iter chunk(emptyChunks(lock)); !chunk.done(); chunk.next()) {
+ if (CanDecommitWholeChunk(chunk) && !chunksToDecommit.append(chunk)) {
+ onOutOfMallocMemory(lock);
+ return;
+ }
+ }
+
+ for (TenuredChunk* chunk : chunksToDecommit) {
+ if (cancel) {
+ break;
+ }
+
+ // Check whether something used the chunk while lock was released.
+ if (!CanDecommitWholeChunk(chunk)) {
+ continue;
+ }
+
+ // Temporarily remove the chunk while decommitting its memory so that the
+ // mutator doesn't start allocating from it when we drop the lock.
+ emptyChunks(lock).remove(chunk);
+
+ {
+ AutoUnlockGC unlock(lock);
+ chunk->decommitAllArenas();
+ MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0);
+ }
+
+ emptyChunks(lock).push(chunk);
+ }
+}
+
+// Called from a background thread to decommit free arenas. Releases the GC
+// lock.
+void GCRuntime::decommitFreeArenas(const bool& cancel, AutoLockGC& lock) {
+ MOZ_ASSERT(DecommitEnabled());
+
+ // Since we release the GC lock while doing the decommit syscall below,
+ // it is dangerous to iterate the available list directly, as the active
+ // thread could modify it concurrently. Instead, we build and pass an
+ // explicit Vector containing the Chunks we want to visit.
+ Vector<TenuredChunk*, 0, SystemAllocPolicy> chunksToDecommit;
+ for (ChunkPool::Iter chunk(availableChunks(lock)); !chunk.done();
+ chunk.next()) {
+ if (chunk->info.numArenasFreeCommitted != 0 &&
+ !chunksToDecommit.append(chunk)) {
+ onOutOfMallocMemory(lock);
+ return;
+ }
+ }
+
+ for (TenuredChunk* chunk : chunksToDecommit) {
+ chunk->decommitFreeArenas(this, cancel, lock);
+ }
+}
+
+// Do all possible decommit immediately from the current thread without
+// releasing the GC lock or allocating any memory.
+void GCRuntime::decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock) {
+ MOZ_ASSERT(DecommitEnabled());
+ for (ChunkPool::Iter chunk(availableChunks(lock)); !chunk.done();
+ chunk.next()) {
+ chunk->decommitFreeArenasWithoutUnlocking(lock);
+ }
+ MOZ_ASSERT(availableChunks(lock).verify());
+}
+
+void GCRuntime::maybeRequestGCAfterBackgroundTask(
+ const AutoLockHelperThreadState& lock) {
+ if (requestSliceAfterBackgroundTask) {
+ // Trigger a slice so the main thread can continue the collection
+ // immediately.
+ requestSliceAfterBackgroundTask = false;
+ requestMajorGC(JS::GCReason::BG_TASK_FINISHED);
+ }
+}
+
+void GCRuntime::cancelRequestedGCAfterBackgroundTask() {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+#ifdef DEBUG
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(!requestSliceAfterBackgroundTask);
+ }
+#endif
+
+ majorGCTriggerReason.compareExchange(JS::GCReason::BG_TASK_FINISHED,
+ JS::GCReason::NO_REASON);
+}
+
+bool GCRuntime::isWaitingOnBackgroundTask() const {
+ AutoLockHelperThreadState lock;
+ return requestSliceAfterBackgroundTask;
+}
+
+void GCRuntime::queueUnusedLifoBlocksForFree(LifoAlloc* lifo) {
+ MOZ_ASSERT(JS::RuntimeHeapIsBusy());
+ AutoLockHelperThreadState lock;
+ lifoBlocksToFree.ref().transferUnusedFrom(lifo);
+}
+
+void GCRuntime::queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo) {
+ lifoBlocksToFreeAfterMinorGC.ref().transferFrom(lifo);
+}
+
+void GCRuntime::queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers) {
+ AutoLockHelperThreadState lock;
+
+ if (!buffersToFreeAfterMinorGC.ref().empty()) {
+ // In the rare case that this hasn't processed the buffers from a previous
+ // minor GC we have to wait here.
+ MOZ_ASSERT(!freeTask.isIdle(lock));
+ freeTask.joinWithLockHeld(lock);
+ }
+
+ MOZ_ASSERT(buffersToFreeAfterMinorGC.ref().empty());
+ std::swap(buffersToFreeAfterMinorGC.ref(), buffers);
+}
+
+void Realm::destroy(JS::GCContext* gcx) {
+ JSRuntime* rt = gcx->runtime();
+ if (auto callback = rt->destroyRealmCallback) {
+ callback(gcx, this);
+ }
+ if (principals()) {
+ JS_DropPrincipals(rt->mainContextFromOwnThread(), principals());
+ }
+ // Bug 1560019: Malloc memory associated with a zone but not with a specific
+ // GC thing is not currently tracked.
+ gcx->deleteUntracked(this);
+}
+
+void Compartment::destroy(JS::GCContext* gcx) {
+ JSRuntime* rt = gcx->runtime();
+ if (auto callback = rt->destroyCompartmentCallback) {
+ callback(gcx, this);
+ }
+ // Bug 1560019: Malloc memory associated with a zone but not with a specific
+ // GC thing is not currently tracked.
+ gcx->deleteUntracked(this);
+ rt->gc.stats().sweptCompartment();
+}
+
+void Zone::destroy(JS::GCContext* gcx) {
+ MOZ_ASSERT(compartments().empty());
+ JSRuntime* rt = gcx->runtime();
+ if (auto callback = rt->destroyZoneCallback) {
+ callback(gcx, this);
+ }
+ // Bug 1560019: Malloc memory associated with a zone but not with a specific
+ // GC thing is not currently tracked.
+ gcx->deleteUntracked(this);
+ gcx->runtime()->gc.stats().sweptZone();
+}
+
+/*
+ * It's simpler if we preserve the invariant that every zone (except atoms
+ * zones) has at least one compartment, and every compartment has at least one
+ * realm. If we know we're deleting the entire zone, then sweepCompartments is
+ * allowed to delete all compartments. In this case, |keepAtleastOne| is false.
+ * If any cells remain alive in the zone, set |keepAtleastOne| true to prohibit
+ * sweepCompartments from deleting every compartment. Instead, it preserves an
+ * arbitrary compartment in the zone.
+ */
+void Zone::sweepCompartments(JS::GCContext* gcx, bool keepAtleastOne,
+ bool destroyingRuntime) {
+ MOZ_ASSERT_IF(!isAtomsZone(), !compartments().empty());
+ MOZ_ASSERT_IF(destroyingRuntime, !keepAtleastOne);
+
+ Compartment** read = compartments().begin();
+ Compartment** end = compartments().end();
+ Compartment** write = read;
+ while (read < end) {
+ Compartment* comp = *read++;
+
+ /*
+ * Don't delete the last compartment and realm if keepAtleastOne is
+ * still true, meaning all the other compartments were deleted.
+ */
+ bool keepAtleastOneRealm = read == end && keepAtleastOne;
+ comp->sweepRealms(gcx, keepAtleastOneRealm, destroyingRuntime);
+
+ if (!comp->realms().empty()) {
+ *write++ = comp;
+ keepAtleastOne = false;
+ } else {
+ comp->destroy(gcx);
+ }
+ }
+ compartments().shrinkTo(write - compartments().begin());
+ MOZ_ASSERT_IF(keepAtleastOne, !compartments().empty());
+ MOZ_ASSERT_IF(destroyingRuntime, compartments().empty());
+}
+
+void Compartment::sweepRealms(JS::GCContext* gcx, bool keepAtleastOne,
+ bool destroyingRuntime) {
+ MOZ_ASSERT(!realms().empty());
+ MOZ_ASSERT_IF(destroyingRuntime, !keepAtleastOne);
+
+ Realm** read = realms().begin();
+ Realm** end = realms().end();
+ Realm** write = read;
+ while (read < end) {
+ Realm* realm = *read++;
+
+ /*
+ * Don't delete the last realm if keepAtleastOne is still true, meaning
+ * all the other realms were deleted.
+ */
+ bool dontDelete = read == end && keepAtleastOne;
+ if ((realm->marked() || dontDelete) && !destroyingRuntime) {
+ *write++ = realm;
+ keepAtleastOne = false;
+ } else {
+ realm->destroy(gcx);
+ }
+ }
+ realms().shrinkTo(write - realms().begin());
+ MOZ_ASSERT_IF(keepAtleastOne, !realms().empty());
+ MOZ_ASSERT_IF(destroyingRuntime, realms().empty());
+}
+
+void GCRuntime::sweepZones(JS::GCContext* gcx, bool destroyingRuntime) {
+ MOZ_ASSERT_IF(destroyingRuntime, numActiveZoneIters == 0);
+
+ if (numActiveZoneIters) {
+ return;
+ }
+
+ assertBackgroundSweepingFinished();
+
+ // Sweep zones following the atoms zone.
+ MOZ_ASSERT(zones()[0]->isAtomsZone());
+ Zone** read = zones().begin() + 1;
+ Zone** end = zones().end();
+ Zone** write = read;
+
+ while (read < end) {
+ Zone* zone = *read++;
+
+ if (zone->wasGCStarted()) {
+ MOZ_ASSERT(!zone->isQueuedForBackgroundSweep());
+ AutoSetThreadIsSweeping threadIsSweeping(zone);
+ const bool zoneIsDead =
+ zone->arenas.arenaListsAreEmpty() && !zone->hasMarkedRealms();
+ MOZ_ASSERT_IF(destroyingRuntime, zoneIsDead);
+ if (zoneIsDead) {
+ zone->arenas.checkEmptyFreeLists();
+ zone->sweepCompartments(gcx, false, destroyingRuntime);
+ MOZ_ASSERT(zone->compartments().empty());
+ zone->destroy(gcx);
+ continue;
+ }
+ zone->sweepCompartments(gcx, true, destroyingRuntime);
+ }
+ *write++ = zone;
+ }
+ zones().shrinkTo(write - zones().begin());
+}
+
+void ArenaLists::checkEmptyArenaList(AllocKind kind) {
+ MOZ_ASSERT(arenaList(kind).isEmpty());
+}
+
+void GCRuntime::purgeRuntimeForMinorGC() {
+ // If external strings become nursery allocable, remember to call
+ // zone->externalStringCache().purge() (and delete this assert.)
+ MOZ_ASSERT(!IsNurseryAllocable(AllocKind::EXTERNAL_STRING));
+
+ for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) {
+ zone->functionToStringCache().purge();
+ }
+}
+
+void GCRuntime::purgeRuntime() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE);
+
+ for (GCRealmsIter realm(rt); !realm.done(); realm.next()) {
+ realm->purge();
+ }
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->purgeAtomCache();
+ zone->externalStringCache().purge();
+ zone->functionToStringCache().purge();
+ zone->boundPrefixCache().clearAndCompact();
+ zone->shapeZone().purgeShapeCaches(rt->gcContext());
+ }
+
+ JSContext* cx = rt->mainContextFromOwnThread();
+ queueUnusedLifoBlocksForFree(&cx->tempLifoAlloc());
+ cx->interpreterStack().purge(rt);
+ cx->frontendCollectionPool().purge();
+
+ rt->caches().purge();
+
+ if (rt->isMainRuntime()) {
+ SharedImmutableStringsCache::getSingleton().purge();
+ }
+
+ MOZ_ASSERT(marker().unmarkGrayStack.empty());
+ marker().unmarkGrayStack.clearAndFree();
+
+ // If we're the main runtime, tell helper threads to free their unused
+ // memory when they are next idle.
+ if (!rt->parentRuntime) {
+ HelperThreadState().triggerFreeUnusedMemory();
+ }
+}
+
+bool GCRuntime::shouldPreserveJITCode(Realm* realm,
+ const TimeStamp& currentTime,
+ JS::GCReason reason,
+ bool canAllocateMoreCode,
+ bool isActiveCompartment) {
+ if (cleanUpEverything) {
+ return false;
+ }
+ if (!canAllocateMoreCode) {
+ return false;
+ }
+
+ if (isActiveCompartment) {
+ return true;
+ }
+ if (alwaysPreserveCode) {
+ return true;
+ }
+ if (realm->preserveJitCode()) {
+ return true;
+ }
+ if (IsCurrentlyAnimating(realm->lastAnimationTime, currentTime) &&
+ DiscardedCodeRecently(realm->zone(), currentTime)) {
+ return true;
+ }
+ if (reason == JS::GCReason::DEBUG_GC) {
+ return true;
+ }
+
+ return false;
+}
+
+#ifdef DEBUG
+class CompartmentCheckTracer final : public JS::CallbackTracer {
+ void onChild(JS::GCCellPtr thing, const char* name) override;
+ bool edgeIsInCrossCompartmentMap(JS::GCCellPtr dst);
+
+ public:
+ explicit CompartmentCheckTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt, JS::TracerKind::CompartmentCheck,
+ JS::WeakEdgeTraceAction::Skip),
+ src(nullptr),
+ zone(nullptr),
+ compartment(nullptr) {}
+
+ Cell* src;
+ JS::TraceKind srcKind;
+ Zone* zone;
+ Compartment* compartment;
+};
+
+static bool InCrossCompartmentMap(JSRuntime* rt, JSObject* src,
+ JS::GCCellPtr dst) {
+ // Cross compartment edges are either in the cross compartment map or in a
+ // debugger weakmap.
+
+ Compartment* srccomp = src->compartment();
+
+ if (dst.is<JSObject>()) {
+ if (ObjectWrapperMap::Ptr p = srccomp->lookupWrapper(&dst.as<JSObject>())) {
+ if (*p->value().unsafeGet() == src) {
+ return true;
+ }
+ }
+ }
+
+ if (DebugAPI::edgeIsInDebuggerWeakmap(rt, src, dst)) {
+ return true;
+ }
+
+ return false;
+}
+
+void CompartmentCheckTracer::onChild(JS::GCCellPtr thing, const char* name) {
+ Compartment* comp =
+ MapGCThingTyped(thing, [](auto t) { return t->maybeCompartment(); });
+ if (comp && compartment) {
+ MOZ_ASSERT(comp == compartment || edgeIsInCrossCompartmentMap(thing));
+ } else {
+ TenuredCell* tenured = &thing.asCell()->asTenured();
+ Zone* thingZone = tenured->zoneFromAnyThread();
+ MOZ_ASSERT(thingZone == zone || thingZone->isAtomsZone());
+ }
+}
+
+bool CompartmentCheckTracer::edgeIsInCrossCompartmentMap(JS::GCCellPtr dst) {
+ return srcKind == JS::TraceKind::Object &&
+ InCrossCompartmentMap(runtime(), static_cast<JSObject*>(src), dst);
+}
+
+static bool IsPartiallyInitializedObject(Cell* cell) {
+ if (!cell->is<JSObject>()) {
+ return false;
+ }
+
+ JSObject* obj = cell->as<JSObject>();
+ if (!obj->is<NativeObject>()) {
+ return false;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+
+ // Check for failed allocation of dynamic slots in
+ // NativeObject::allocateInitialSlots.
+ size_t nDynamicSlots = NativeObject::calculateDynamicSlots(
+ nobj->numFixedSlots(), nobj->slotSpan(), nobj->getClass());
+ return nDynamicSlots != 0 && !nobj->hasDynamicSlots();
+}
+
+void GCRuntime::checkForCompartmentMismatches() {
+ JSContext* cx = rt->mainContextFromOwnThread();
+ if (cx->disableStrictProxyCheckingCount) {
+ return;
+ }
+
+ CompartmentCheckTracer trc(rt);
+ AutoAssertEmptyNursery empty(cx);
+ for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) {
+ trc.zone = zone;
+ for (auto thingKind : AllAllocKinds()) {
+ for (auto i = zone->cellIterUnsafe<TenuredCell>(thingKind, empty);
+ !i.done(); i.next()) {
+ // We may encounter partially initialized objects. These are unreachable
+ // and it's safe to ignore them.
+ if (IsPartiallyInitializedObject(i.getCell())) {
+ continue;
+ }
+
+ trc.src = i.getCell();
+ trc.srcKind = MapAllocToTraceKind(thingKind);
+ trc.compartment = MapGCThingTyped(
+ trc.src, trc.srcKind, [](auto t) { return t->maybeCompartment(); });
+ JS::TraceChildren(&trc, JS::GCCellPtr(trc.src, trc.srcKind));
+ }
+ }
+ }
+}
+#endif
+
+static bool ShouldCleanUpEverything(JS::GCOptions options) {
+ // During shutdown, we must clean everything up, for the sake of leak
+ // detection. When a runtime has no contexts, or we're doing a GC before a
+ // shutdown CC, those are strong indications that we're shutting down.
+ return options == JS::GCOptions::Shutdown || options == JS::GCOptions::Shrink;
+}
+
+static bool ShouldUseBackgroundThreads(bool isIncremental,
+ JS::GCReason reason) {
+ bool shouldUse = isIncremental && CanUseExtraThreads();
+ MOZ_ASSERT_IF(reason == JS::GCReason::DESTROY_RUNTIME, !shouldUse);
+ return shouldUse;
+}
+
+void GCRuntime::startCollection(JS::GCReason reason) {
+ checkGCStateNotInUse();
+ MOZ_ASSERT_IF(
+ isShuttingDown(),
+ isShutdownGC() ||
+ reason == JS::GCReason::XPCONNECT_SHUTDOWN /* Bug 1650075 */);
+
+ initialReason = reason;
+ cleanUpEverything = ShouldCleanUpEverything(gcOptions());
+ isCompacting = shouldCompact();
+ rootsRemoved = false;
+ sweepGroupIndex = 0;
+ lastGCStartTime_ = TimeStamp::Now();
+
+#ifdef DEBUG
+ if (isShutdownGC()) {
+ hadShutdownGC = true;
+ }
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->gcSweepGroupIndex = 0;
+ }
+#endif
+}
+
+static void RelazifyFunctions(Zone* zone, AllocKind kind) {
+ MOZ_ASSERT(kind == AllocKind::FUNCTION ||
+ kind == AllocKind::FUNCTION_EXTENDED);
+
+ JSRuntime* rt = zone->runtimeFromMainThread();
+ AutoAssertEmptyNursery empty(rt->mainContextFromOwnThread());
+
+ for (auto i = zone->cellIterUnsafe<JSObject>(kind, empty); !i.done();
+ i.next()) {
+ JSFunction* fun = &i->as<JSFunction>();
+ // When iterating over the GC-heap, we may encounter function objects that
+ // are incomplete (missing a BaseScript when we expect one). We must check
+ // for this case before we can call JSFunction::hasBytecode().
+ if (fun->isIncomplete()) {
+ continue;
+ }
+ if (fun->hasBytecode()) {
+ fun->maybeRelazify(rt);
+ }
+ }
+}
+
+static bool ShouldCollectZone(Zone* zone, JS::GCReason reason) {
+ // If we are repeating a GC because we noticed dead compartments haven't
+ // been collected, then only collect zones containing those compartments.
+ if (reason == JS::GCReason::COMPARTMENT_REVIVED) {
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ if (comp->gcState.scheduledForDestruction) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ // Otherwise we only collect scheduled zones.
+ return zone->isGCScheduled();
+}
+
+bool GCRuntime::prepareZonesForCollection(JS::GCReason reason,
+ bool* isFullOut) {
+#ifdef DEBUG
+ /* Assert that zone state is as we expect */
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ MOZ_ASSERT(!zone->isCollecting());
+ MOZ_ASSERT_IF(!zone->isAtomsZone(), !zone->compartments().empty());
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT(zone->arenas.collectingArenaList(i).isEmpty());
+ }
+ }
+#endif
+
+ *isFullOut = true;
+ bool any = false;
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ /* Set up which zones will be collected. */
+ bool shouldCollect = ShouldCollectZone(zone, reason);
+ if (shouldCollect) {
+ any = true;
+ zone->changeGCState(Zone::NoGC, Zone::Prepare);
+ } else {
+ *isFullOut = false;
+ }
+
+ zone->setWasCollected(shouldCollect);
+ }
+
+ /* Check that at least one zone is scheduled for collection. */
+ return any;
+}
+
+void GCRuntime::discardJITCodeForGC() {
+ size_t nurserySiteResetCount = 0;
+ size_t pretenuredSiteResetCount = 0;
+
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Prepare);
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_DISCARD_CODE);
+
+ // We may need to reset allocation sites and discard JIT code to recover if
+ // we find object lifetimes have changed.
+ PretenuringZone& pz = zone->pretenuring;
+ bool resetNurserySites = pz.shouldResetNurseryAllocSites();
+ bool resetPretenuredSites = pz.shouldResetPretenuredAllocSites();
+
+ if (!zone->isPreservingCode()) {
+ Zone::DiscardOptions options;
+ options.discardBaselineCode = true;
+ options.discardJitScripts = true;
+ options.resetNurseryAllocSites = resetNurserySites;
+ options.resetPretenuredAllocSites = resetPretenuredSites;
+ zone->discardJitCode(rt->gcContext(), options);
+ } else if (resetNurserySites || resetPretenuredSites) {
+ zone->resetAllocSitesAndInvalidate(resetNurserySites,
+ resetPretenuredSites);
+ }
+
+ if (resetNurserySites) {
+ nurserySiteResetCount++;
+ }
+ if (resetPretenuredSites) {
+ pretenuredSiteResetCount++;
+ }
+ }
+
+ if (nursery().reportPretenuring()) {
+ if (nurserySiteResetCount) {
+ fprintf(
+ stderr,
+ "GC reset nursery alloc sites and invalidated code in %zu zones\n",
+ nurserySiteResetCount);
+ }
+ if (pretenuredSiteResetCount) {
+ fprintf(
+ stderr,
+ "GC reset pretenured alloc sites and invalidated code in %zu zones\n",
+ pretenuredSiteResetCount);
+ }
+ }
+}
+
+void GCRuntime::relazifyFunctionsForShrinkingGC() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::RELAZIFY_FUNCTIONS);
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ RelazifyFunctions(zone, AllocKind::FUNCTION);
+ RelazifyFunctions(zone, AllocKind::FUNCTION_EXTENDED);
+ }
+}
+
+void GCRuntime::purgePropMapTablesForShrinkingGC() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE_PROP_MAP_TABLES);
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ if (!canRelocateZone(zone) || zone->keepPropMapTables()) {
+ continue;
+ }
+
+ // Note: CompactPropMaps never have a table.
+ for (auto map = zone->cellIterUnsafe<NormalPropMap>(); !map.done();
+ map.next()) {
+ if (map->asLinked()->hasTable()) {
+ map->asLinked()->purgeTable(rt->gcContext());
+ }
+ }
+ for (auto map = zone->cellIterUnsafe<DictionaryPropMap>(); !map.done();
+ map.next()) {
+ if (map->asLinked()->hasTable()) {
+ map->asLinked()->purgeTable(rt->gcContext());
+ }
+ }
+ }
+}
+
+// The debugger keeps track of the URLs for the sources of each realm's scripts.
+// These URLs are purged on shrinking GCs.
+void GCRuntime::purgeSourceURLsForShrinkingGC() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PURGE_SOURCE_URLS);
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ // URLs are not tracked for realms in the system zone.
+ if (!canRelocateZone(zone) || zone->isSystemZone()) {
+ continue;
+ }
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ for (RealmsInCompartmentIter realm(comp); !realm.done(); realm.next()) {
+ GlobalObject* global = realm.get()->unsafeUnbarrieredMaybeGlobal();
+ if (global) {
+ global->clearSourceURLSHolder();
+ }
+ }
+ }
+ }
+}
+
+void GCRuntime::unmarkWeakMaps() {
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ /* Unmark all weak maps in the zones being collected. */
+ WeakMapBase::unmarkZone(zone);
+ }
+}
+
+bool GCRuntime::beginPreparePhase(JS::GCReason reason, AutoGCSession& session) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PREPARE);
+
+ if (!prepareZonesForCollection(reason, &isFull.ref())) {
+ return false;
+ }
+
+ /*
+ * Start a parallel task to clear all mark state for the zones we are
+ * collecting. This is linear in the size of the heap we are collecting and so
+ * can be slow. This usually happens concurrently with the mutator and GC
+ * proper does not start until this is complete.
+ */
+ unmarkTask.initZones();
+ if (useBackgroundThreads) {
+ unmarkTask.start();
+ } else {
+ unmarkTask.runFromMainThread();
+ }
+
+ /*
+ * Process any queued source compressions during the start of a major
+ * GC.
+ *
+ * Bug 1650075: When we start passing GCOptions::Shutdown for
+ * GCReason::XPCONNECT_SHUTDOWN GCs we can remove the extra check.
+ */
+ if (!isShutdownGC() && reason != JS::GCReason::XPCONNECT_SHUTDOWN) {
+ StartHandlingCompressionsOnGC(rt);
+ }
+
+ return true;
+}
+
+BackgroundUnmarkTask::BackgroundUnmarkTask(GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::UNMARK) {}
+
+void BackgroundUnmarkTask::initZones() {
+ MOZ_ASSERT(isIdle());
+ MOZ_ASSERT(zones.empty());
+ MOZ_ASSERT(!isCancelled());
+
+ // We can't safely iterate the zones vector from another thread so we copy the
+ // zones to be collected into another vector.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ if (!zones.append(zone.get())) {
+ oomUnsafe.crash("BackgroundUnmarkTask::initZones");
+ }
+
+ zone->arenas.clearFreeLists();
+ zone->arenas.moveArenasToCollectingLists();
+ }
+}
+
+void BackgroundUnmarkTask::run(AutoLockHelperThreadState& helperTheadLock) {
+ AutoUnlockHelperThreadState unlock(helperTheadLock);
+
+ for (Zone* zone : zones) {
+ for (auto kind : AllAllocKinds()) {
+ ArenaList& arenas = zone->arenas.collectingArenaList(kind);
+ for (ArenaListIter arena(arenas.head()); !arena.done(); arena.next()) {
+ arena->unmarkAll();
+ if (isCancelled()) {
+ break;
+ }
+ }
+ }
+ }
+
+ zones.clear();
+}
+
+void GCRuntime::endPreparePhase(JS::GCReason reason) {
+ MOZ_ASSERT(unmarkTask.isIdle());
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ /*
+ * In an incremental GC, clear the area free lists to ensure that subsequent
+ * allocations refill them and end up marking new cells back. See
+ * arenaAllocatedDuringGC().
+ */
+ zone->arenas.clearFreeLists();
+
+ zone->markedStrings = 0;
+ zone->finalizedStrings = 0;
+
+ zone->setPreservingCode(false);
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::YieldBeforeRootMarking)) {
+ for (auto kind : AllAllocKinds()) {
+ for (ArenaIter arena(zone, kind); !arena.done(); arena.next()) {
+ arena->checkNoMarkedCells();
+ }
+ }
+ }
+#endif
+ }
+
+ // Discard JIT code more aggressively if the process is approaching its
+ // executable code limit.
+ bool canAllocateMoreCode = jit::CanLikelyAllocateMoreExecutableMemory();
+ auto currentTime = TimeStamp::Now();
+
+ Compartment* activeCompartment = nullptr;
+ jit::JitActivationIterator activation(rt->mainContextFromOwnThread());
+ if (!activation.done()) {
+ activeCompartment = activation->compartment();
+ }
+
+ for (CompartmentsIter c(rt); !c.done(); c.next()) {
+ c->gcState.scheduledForDestruction = false;
+ c->gcState.maybeAlive = false;
+ c->gcState.hasEnteredRealm = false;
+ bool isActiveCompartment = c == activeCompartment;
+ for (RealmsInCompartmentIter r(c); !r.done(); r.next()) {
+ if (r->shouldTraceGlobal() || !r->zone()->isGCScheduled()) {
+ c->gcState.maybeAlive = true;
+ }
+ if (shouldPreserveJITCode(r, currentTime, reason, canAllocateMoreCode,
+ isActiveCompartment)) {
+ r->zone()->setPreservingCode(true);
+ }
+ if (r->hasBeenEnteredIgnoringJit()) {
+ c->gcState.hasEnteredRealm = true;
+ }
+ }
+ }
+
+ /*
+ * Perform remaining preparation work that must take place in the first true
+ * GC slice.
+ */
+
+ {
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::PREPARE);
+
+ AutoLockHelperThreadState helperLock;
+
+ /* Clear mark state for WeakMaps in parallel with other work. */
+ AutoRunParallelTask unmarkWeakMaps(this, &GCRuntime::unmarkWeakMaps,
+ gcstats::PhaseKind::UNMARK_WEAKMAPS,
+ GCUse::Unspecified, helperLock);
+
+ AutoUnlockHelperThreadState unlock(helperLock);
+
+ // Discard JIT code. For incremental collections, the sweep phase will
+ // also discard JIT code.
+ discardJITCodeForGC();
+
+ /*
+ * Relazify functions after discarding JIT code (we can't relazify
+ * functions with JIT code) and before the actual mark phase, so that
+ * the current GC can collect the JSScripts we're unlinking here. We do
+ * this only when we're performing a shrinking GC, as too much
+ * relazification can cause performance issues when we have to reparse
+ * the same functions over and over.
+ */
+ if (isShrinkingGC()) {
+ relazifyFunctionsForShrinkingGC();
+ purgePropMapTablesForShrinkingGC();
+ purgeSourceURLsForShrinkingGC();
+ }
+
+ /*
+ * We must purge the runtime at the beginning of an incremental GC. The
+ * danger if we purge later is that the snapshot invariant of
+ * incremental GC will be broken, as follows. If some object is
+ * reachable only through some cache (say the dtoaCache) then it will
+ * not be part of the snapshot. If we purge after root marking, then
+ * the mutator could obtain a pointer to the object and start using
+ * it. This object might never be marked, so a GC hazard would exist.
+ */
+ purgeRuntime();
+
+ startBackgroundFreeAfterMinorGC();
+
+ if (isShutdownGC()) {
+ /* Clear any engine roots that may hold external data live. */
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->clearRootsForShutdownGC();
+ }
+
+#ifdef DEBUG
+ testMarkQueue.clear();
+ queuePos = 0;
+#endif
+ }
+ }
+
+#ifdef DEBUG
+ if (fullCompartmentChecks) {
+ checkForCompartmentMismatches();
+ }
+#endif
+}
+
+AutoUpdateLiveCompartments::AutoUpdateLiveCompartments(GCRuntime* gc) : gc(gc) {
+ for (GCCompartmentsIter c(gc->rt); !c.done(); c.next()) {
+ c->gcState.hasMarkedCells = false;
+ }
+}
+
+AutoUpdateLiveCompartments::~AutoUpdateLiveCompartments() {
+ for (GCCompartmentsIter c(gc->rt); !c.done(); c.next()) {
+ if (c->gcState.hasMarkedCells) {
+ c->gcState.maybeAlive = true;
+ }
+ }
+}
+
+Zone::GCState Zone::initialMarkingState() const {
+ if (isAtomsZone()) {
+ // Don't delay gray marking in the atoms zone like we do in other zones.
+ return MarkBlackAndGray;
+ }
+
+ return MarkBlackOnly;
+}
+
+void GCRuntime::beginMarkPhase(AutoGCSession& session) {
+ /*
+ * Mark phase.
+ */
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+
+ // This is the slice we actually start collecting. The number can be used to
+ // check whether a major GC has started so we must not increment it until we
+ // get here.
+ incMajorGcNumber();
+
+ MOZ_ASSERT(!hasDelayedMarking());
+ for (auto& marker : markers) {
+ marker->start();
+ }
+
+#ifdef DEBUG
+ queuePos = 0;
+ queueMarkColor.reset();
+#endif
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ // Incremental marking barriers are enabled at this point.
+ zone->changeGCState(Zone::Prepare, zone->initialMarkingState());
+
+ // Merge arenas allocated during the prepare phase, then move all arenas to
+ // the collecting arena lists.
+ zone->arenas.mergeArenasFromCollectingLists();
+ zone->arenas.moveArenasToCollectingLists();
+
+ for (RealmsInZoneIter realm(zone); !realm.done(); realm.next()) {
+ realm->clearAllocatedDuringGC();
+ }
+ }
+
+ if (rt->isBeingDestroyed()) {
+ checkNoRuntimeRoots(session);
+ } else {
+ AutoUpdateLiveCompartments updateLive(this);
+ marker().setRootMarkingMode(true);
+ traceRuntimeForMajorGC(marker().tracer(), session);
+ marker().setRootMarkingMode(false);
+ }
+
+ updateSchedulingStateOnGCStart();
+ stats().measureInitialHeapSize();
+}
+
+void GCRuntime::findDeadCompartments() {
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::FIND_DEAD_COMPARTMENTS);
+
+ /*
+ * This code ensures that if a compartment is "dead", then it will be
+ * collected in this GC. A compartment is considered dead if its maybeAlive
+ * flag is false. The maybeAlive flag is set if:
+ *
+ * (1) the compartment has been entered (set in beginMarkPhase() above)
+ * (2) the compartment's zone is not being collected (set in
+ * beginMarkPhase() above)
+ * (3) an object in the compartment was marked during root marking, either
+ * as a black root or a gray root. This is arranged by
+ * SetCompartmentHasMarkedCells and AutoUpdateLiveCompartments.
+ * (4) the compartment has incoming cross-compartment edges from another
+ * compartment that has maybeAlive set (set by this method).
+ *
+ * If the maybeAlive is false, then we set the scheduledForDestruction flag.
+ * At the end of the GC, we look for compartments where
+ * scheduledForDestruction is true. These are compartments that were somehow
+ * "revived" during the incremental GC. If any are found, we do a special,
+ * non-incremental GC of those compartments to try to collect them.
+ *
+ * Compartments can be revived for a variety of reasons. On reason is bug
+ * 811587, where a reflector that was dead can be revived by DOM code that
+ * still refers to the underlying DOM node.
+ *
+ * Read barriers and allocations can also cause revival. This might happen
+ * during a function like JS_TransplantObject, which iterates over all
+ * compartments, live or dead, and operates on their objects. See bug 803376
+ * for details on this problem. To avoid the problem, we try to avoid
+ * allocation and read barriers during JS_TransplantObject and the like.
+ */
+
+ // Propagate the maybeAlive flag via cross-compartment edges.
+
+ Vector<Compartment*, 0, js::SystemAllocPolicy> workList;
+
+ for (CompartmentsIter comp(rt); !comp.done(); comp.next()) {
+ if (comp->gcState.maybeAlive) {
+ if (!workList.append(comp)) {
+ return;
+ }
+ }
+ }
+
+ while (!workList.empty()) {
+ Compartment* comp = workList.popCopy();
+ for (Compartment::WrappedObjectCompartmentEnum e(comp); !e.empty();
+ e.popFront()) {
+ Compartment* dest = e.front();
+ if (!dest->gcState.maybeAlive) {
+ dest->gcState.maybeAlive = true;
+ if (!workList.append(dest)) {
+ return;
+ }
+ }
+ }
+ }
+
+ // Set scheduledForDestruction based on maybeAlive.
+
+ for (GCCompartmentsIter comp(rt); !comp.done(); comp.next()) {
+ MOZ_ASSERT(!comp->gcState.scheduledForDestruction);
+ if (!comp->gcState.maybeAlive) {
+ comp->gcState.scheduledForDestruction = true;
+ }
+ }
+}
+
+void GCRuntime::updateSchedulingStateOnGCStart() {
+ heapSize.updateOnGCStart();
+
+ // Update memory counters for the zones we are collecting.
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->updateSchedulingStateOnGCStart();
+ }
+}
+
+inline bool GCRuntime::canMarkInParallel() const {
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+ // OOM testing limits the engine to using a single helper thread.
+ if (oom::simulator.targetThread() == THREAD_TYPE_GCPARALLEL) {
+ return false;
+ }
+#endif
+
+ return markers.length() > 1 && stats().initialCollectedBytes() >=
+ tunables.parallelMarkingThresholdBytes();
+}
+
+IncrementalProgress GCRuntime::markUntilBudgetExhausted(
+ SliceBudget& sliceBudget, ParallelMarking allowParallelMarking,
+ ShouldReportMarkTime reportTime) {
+ // Run a marking slice and return whether the stack is now empty.
+
+ AutoMajorGCProfilerEntry s(this);
+
+ if (processTestMarkQueue() == QueueYielded) {
+ return NotFinished;
+ }
+
+ if (allowParallelMarking && canMarkInParallel()) {
+ MOZ_ASSERT(parallelMarkingEnabled);
+ MOZ_ASSERT(reportTime);
+ MOZ_ASSERT(!isBackgroundMarking());
+
+ ParallelMarker pm(this);
+ if (!pm.mark(sliceBudget)) {
+ return NotFinished;
+ }
+
+ assertNoMarkingWork();
+ return Finished;
+ }
+
+#ifdef DEBUG
+ AutoSetThreadIsMarking threadIsMarking;
+#endif // DEBUG
+
+ return marker().markUntilBudgetExhausted(sliceBudget, reportTime)
+ ? Finished
+ : NotFinished;
+}
+
+void GCRuntime::drainMarkStack() {
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(marker().markUntilBudgetExhausted(unlimited));
+}
+
+#ifdef DEBUG
+
+const GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>&
+GCRuntime::getTestMarkQueue() const {
+ return testMarkQueue.get();
+}
+
+bool GCRuntime::appendTestMarkQueue(const JS::Value& value) {
+ return testMarkQueue.append(value);
+}
+
+void GCRuntime::clearTestMarkQueue() {
+ testMarkQueue.clear();
+ queuePos = 0;
+}
+
+size_t GCRuntime::testMarkQueuePos() const { return queuePos; }
+
+#endif
+
+GCRuntime::MarkQueueProgress GCRuntime::processTestMarkQueue() {
+#ifdef DEBUG
+ if (testMarkQueue.empty()) {
+ return QueueComplete;
+ }
+
+ if (queueMarkColor == mozilla::Some(MarkColor::Gray) &&
+ state() != State::Sweep) {
+ return QueueSuspended;
+ }
+
+ // If the queue wants to be gray marking, but we've pushed a black object
+ // since set-color-gray was processed, then we can't switch to gray and must
+ // again wait until gray marking is possible.
+ //
+ // Remove this code if the restriction against marking gray during black is
+ // relaxed.
+ if (queueMarkColor == mozilla::Some(MarkColor::Gray) &&
+ marker().hasBlackEntries()) {
+ return QueueSuspended;
+ }
+
+ // If the queue wants to be marking a particular color, switch to that color.
+ // In any case, restore the mark color to whatever it was when we entered
+ // this function.
+ bool willRevertToGray = marker().markColor() == MarkColor::Gray;
+ AutoSetMarkColor autoRevertColor(
+ marker(), queueMarkColor.valueOr(marker().markColor()));
+
+ // Process the mark queue by taking each object in turn, pushing it onto the
+ // mark stack, and processing just the top element with processMarkStackTop
+ // without recursing into reachable objects.
+ while (queuePos < testMarkQueue.length()) {
+ Value val = testMarkQueue[queuePos++].get();
+ if (val.isObject()) {
+ JSObject* obj = &val.toObject();
+ JS::Zone* zone = obj->zone();
+ if (!zone->isGCMarking() || obj->isMarkedAtLeast(marker().markColor())) {
+ continue;
+ }
+
+ // If we have started sweeping, obey sweep group ordering. But note that
+ // we will first be called during the initial sweep slice, when the sweep
+ // group indexes have not yet been computed. In that case, we can mark
+ // freely.
+ if (state() == State::Sweep && initialState != State::Sweep) {
+ if (zone->gcSweepGroupIndex < getCurrentSweepGroupIndex()) {
+ // Too late. This must have been added after we started collecting,
+ // and we've already processed its sweep group. Skip it.
+ continue;
+ }
+ if (zone->gcSweepGroupIndex > getCurrentSweepGroupIndex()) {
+ // Not ready yet. Wait until we reach the object's sweep group.
+ queuePos--;
+ return QueueSuspended;
+ }
+ }
+
+ if (marker().markColor() == MarkColor::Gray &&
+ zone->isGCMarkingBlackOnly()) {
+ // Have not yet reached the point where we can mark this object, so
+ // continue with the GC.
+ queuePos--;
+ return QueueSuspended;
+ }
+
+ if (marker().markColor() == MarkColor::Black && willRevertToGray) {
+ // If we put any black objects on the stack, we wouldn't be able to
+ // return to gray marking. So delay the marking until we're back to
+ // black marking.
+ queuePos--;
+ return QueueSuspended;
+ }
+
+ // Mark the object and push it onto the stack.
+ size_t oldPosition = marker().stack.position();
+ marker().markAndTraverse<NormalMarkingOptions>(obj);
+
+ // If we overflow the stack here and delay marking, then we won't be
+ // testing what we think we're testing.
+ if (marker().stack.position() == oldPosition) {
+ MOZ_ASSERT(obj->asTenured().arena()->onDelayedMarkingList());
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ oomUnsafe.crash("Overflowed stack while marking test queue");
+ }
+
+ SliceBudget unlimited = SliceBudget::unlimited();
+ marker().processMarkStackTop<NormalMarkingOptions>(unlimited);
+ } else if (val.isString()) {
+ JSLinearString* str = &val.toString()->asLinear();
+ if (js::StringEqualsLiteral(str, "yield") && isIncrementalGc()) {
+ return QueueYielded;
+ } else if (js::StringEqualsLiteral(str, "enter-weak-marking-mode") ||
+ js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
+ if (marker().isRegularMarking()) {
+ // We can't enter weak marking mode at just any time, so instead
+ // we'll stop processing the queue and continue on with the GC. Once
+ // we enter weak marking mode, we can continue to the rest of the
+ // queue. Note that we will also suspend for aborting, and then abort
+ // the earliest following weak marking mode.
+ queuePos--;
+ return QueueSuspended;
+ }
+ if (js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
+ marker().abortLinearWeakMarking();
+ }
+ } else if (js::StringEqualsLiteral(str, "drain")) {
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(
+ marker().markUntilBudgetExhausted(unlimited, DontReportMarkTime));
+ } else if (js::StringEqualsLiteral(str, "set-color-gray")) {
+ queueMarkColor = mozilla::Some(MarkColor::Gray);
+ if (state() != State::Sweep || marker().hasBlackEntries()) {
+ // Cannot mark gray yet, so continue with the GC.
+ queuePos--;
+ return QueueSuspended;
+ }
+ marker().setMarkColor(MarkColor::Gray);
+ } else if (js::StringEqualsLiteral(str, "set-color-black")) {
+ queueMarkColor = mozilla::Some(MarkColor::Black);
+ marker().setMarkColor(MarkColor::Black);
+ } else if (js::StringEqualsLiteral(str, "unset-color")) {
+ queueMarkColor.reset();
+ }
+ }
+ }
+#endif
+
+ return QueueComplete;
+}
+
+static bool IsEmergencyGC(JS::GCReason reason) {
+ return reason == JS::GCReason::LAST_DITCH ||
+ reason == JS::GCReason::MEM_PRESSURE;
+}
+
+void GCRuntime::finishCollection(JS::GCReason reason) {
+ assertBackgroundSweepingFinished();
+
+ MOZ_ASSERT(!hasDelayedMarking());
+ for (auto& marker : markers) {
+ marker->stop();
+ }
+
+ maybeStopPretenuring();
+
+ if (IsEmergencyGC(reason)) {
+ waitBackgroundFreeEnd();
+ }
+
+ TimeStamp currentTime = TimeStamp::Now();
+
+ updateSchedulingStateAfterCollection(currentTime);
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->changeGCState(Zone::Finished, Zone::NoGC);
+ zone->notifyObservingDebuggers();
+ }
+
+#ifdef JS_GC_ZEAL
+ clearSelectedForMarking();
+#endif
+
+ schedulingState.updateHighFrequencyMode(lastGCEndTime_, currentTime,
+ tunables);
+ lastGCEndTime_ = currentTime;
+
+ checkGCStateNotInUse();
+}
+
+void GCRuntime::checkGCStateNotInUse() {
+#ifdef DEBUG
+ for (auto& marker : markers) {
+ MOZ_ASSERT(!marker->isActive());
+ MOZ_ASSERT(marker->isDrained());
+ }
+ MOZ_ASSERT(!hasDelayedMarking());
+
+ MOZ_ASSERT(!lastMarkSlice);
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->wasCollected()) {
+ zone->arenas.checkGCStateNotInUse();
+ }
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ MOZ_ASSERT(!zone->isOnList());
+ }
+
+ MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty());
+ MOZ_ASSERT(cellsToAssertNotGray.ref().empty());
+
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(!requestSliceAfterBackgroundTask);
+ MOZ_ASSERT(unmarkTask.isIdle(lock));
+ MOZ_ASSERT(markTask.isIdle(lock));
+ MOZ_ASSERT(sweepTask.isIdle(lock));
+ MOZ_ASSERT(decommitTask.isIdle(lock));
+#endif
+}
+
+void GCRuntime::maybeStopPretenuring() {
+ nursery().maybeStopPretenuring(this);
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ if (!zone->nurseryStringsDisabled) {
+ continue;
+ }
+
+ // Count the number of strings before the major GC.
+ size_t numStrings = zone->markedStrings + zone->finalizedStrings;
+ double rate = double(zone->finalizedStrings) / double(numStrings);
+ if (rate > tunables.stopPretenureStringThreshold()) {
+ zone->markedStrings = 0;
+ zone->finalizedStrings = 0;
+ zone->nurseryStringsDisabled = false;
+ nursery().updateAllocFlagsForZone(zone);
+ }
+ }
+}
+
+void GCRuntime::updateSchedulingStateAfterCollection(TimeStamp currentTime) {
+ TimeDuration totalGCTime = stats().totalGCTime();
+ size_t totalInitialBytes = stats().initialCollectedBytes();
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ if (tunables.balancedHeapLimitsEnabled() && totalInitialBytes != 0) {
+ zone->updateCollectionRate(totalGCTime, totalInitialBytes);
+ }
+ zone->clearGCSliceThresholds();
+ zone->updateGCStartThresholds(*this);
+ }
+}
+
+void GCRuntime::updateAllGCStartThresholds() {
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->updateGCStartThresholds(*this);
+ }
+}
+
+void GCRuntime::updateAllocationRates() {
+ // Calculate mutator time since the last update. This ignores the fact that
+ // the zone could have been created since the last update.
+
+ TimeStamp currentTime = TimeStamp::Now();
+ TimeDuration totalTime = currentTime - lastAllocRateUpdateTime;
+ if (collectorTimeSinceAllocRateUpdate >= totalTime) {
+ // It shouldn't happen but occasionally we see collector time being larger
+ // than total time. Skip the update in that case.
+ return;
+ }
+
+ TimeDuration mutatorTime = totalTime - collectorTimeSinceAllocRateUpdate;
+
+ for (AllZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->updateAllocationRate(mutatorTime);
+ zone->updateGCStartThresholds(*this);
+ }
+
+ lastAllocRateUpdateTime = currentTime;
+ collectorTimeSinceAllocRateUpdate = TimeDuration();
+}
+
+static const char* GCHeapStateToLabel(JS::HeapState heapState) {
+ switch (heapState) {
+ case JS::HeapState::MinorCollecting:
+ return "js::Nursery::collect";
+ case JS::HeapState::MajorCollecting:
+ return "js::GCRuntime::collect";
+ default:
+ MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame");
+ }
+ MOZ_ASSERT_UNREACHABLE("Should have exhausted every JS::HeapState variant!");
+ return nullptr;
+}
+
+static JS::ProfilingCategoryPair GCHeapStateToProfilingCategory(
+ JS::HeapState heapState) {
+ return heapState == JS::HeapState::MinorCollecting
+ ? JS::ProfilingCategoryPair::GCCC_MinorGC
+ : JS::ProfilingCategoryPair::GCCC_MajorGC;
+}
+
+/* Start a new heap session. */
+AutoHeapSession::AutoHeapSession(GCRuntime* gc, JS::HeapState heapState)
+ : gc(gc), prevState(gc->heapState_) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt));
+ MOZ_ASSERT(prevState == JS::HeapState::Idle ||
+ (prevState == JS::HeapState::MajorCollecting &&
+ heapState == JS::HeapState::MinorCollecting));
+ MOZ_ASSERT(heapState != JS::HeapState::Idle);
+
+ gc->heapState_ = heapState;
+
+ if (heapState == JS::HeapState::MinorCollecting ||
+ heapState == JS::HeapState::MajorCollecting) {
+ profilingStackFrame.emplace(gc->rt->mainContextFromOwnThread(),
+ GCHeapStateToLabel(heapState),
+ GCHeapStateToProfilingCategory(heapState));
+ }
+}
+
+AutoHeapSession::~AutoHeapSession() {
+ MOZ_ASSERT(JS::RuntimeHeapIsBusy());
+ gc->heapState_ = prevState;
+}
+
+static const char* MajorGCStateToLabel(State state) {
+ switch (state) {
+ case State::Mark:
+ return "js::GCRuntime::markUntilBudgetExhausted";
+ case State::Sweep:
+ return "js::GCRuntime::performSweepActions";
+ case State::Compact:
+ return "js::GCRuntime::compactPhase";
+ default:
+ MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame");
+ }
+
+ MOZ_ASSERT_UNREACHABLE("Should have exhausted every State variant!");
+ return nullptr;
+}
+
+static JS::ProfilingCategoryPair MajorGCStateToProfilingCategory(State state) {
+ switch (state) {
+ case State::Mark:
+ return JS::ProfilingCategoryPair::GCCC_MajorGC_Mark;
+ case State::Sweep:
+ return JS::ProfilingCategoryPair::GCCC_MajorGC_Sweep;
+ case State::Compact:
+ return JS::ProfilingCategoryPair::GCCC_MajorGC_Compact;
+ default:
+ MOZ_CRASH("Unexpected heap state when pushing GC profiling stack frame");
+ }
+}
+
+AutoMajorGCProfilerEntry::AutoMajorGCProfilerEntry(GCRuntime* gc)
+ : AutoGeckoProfilerEntry(gc->rt->mainContextFromAnyThread(),
+ MajorGCStateToLabel(gc->state()),
+ MajorGCStateToProfilingCategory(gc->state())) {
+ MOZ_ASSERT(gc->heapState() == JS::HeapState::MajorCollecting);
+}
+
+GCRuntime::IncrementalResult GCRuntime::resetIncrementalGC(
+ GCAbortReason reason) {
+ MOZ_ASSERT(reason != GCAbortReason::None);
+
+ // Drop as much work as possible from an ongoing incremental GC so
+ // we can start a new GC after it has finished.
+ if (incrementalState == State::NotActive) {
+ return IncrementalResult::Ok;
+ }
+
+ AutoGCSession session(this, JS::HeapState::MajorCollecting);
+
+ switch (incrementalState) {
+ case State::NotActive:
+ case State::MarkRoots:
+ case State::Finish:
+ MOZ_CRASH("Unexpected GC state in resetIncrementalGC");
+ break;
+
+ case State::Prepare:
+ unmarkTask.cancelAndWait();
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->changeGCState(Zone::Prepare, Zone::NoGC);
+ zone->clearGCSliceThresholds();
+ zone->arenas.clearFreeLists();
+ zone->arenas.mergeArenasFromCollectingLists();
+ }
+
+ incrementalState = State::NotActive;
+ checkGCStateNotInUse();
+ break;
+
+ case State::Mark: {
+ // Cancel any ongoing marking.
+ for (auto& marker : markers) {
+ marker->reset();
+ }
+ resetDelayedMarking();
+
+ for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
+ resetGrayList(c);
+ }
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->changeGCState(zone->initialMarkingState(), Zone::NoGC);
+ zone->clearGCSliceThresholds();
+ zone->arenas.unmarkPreMarkedFreeCells();
+ zone->arenas.mergeArenasFromCollectingLists();
+ }
+
+ {
+ AutoLockHelperThreadState lock;
+ lifoBlocksToFree.ref().freeAll();
+ }
+
+ lastMarkSlice = false;
+ incrementalState = State::Finish;
+
+#ifdef DEBUG
+ for (auto& marker : markers) {
+ MOZ_ASSERT(!marker->shouldCheckCompartments());
+ }
+#endif
+
+ break;
+ }
+
+ case State::Sweep: {
+ // Finish sweeping the current sweep group, then abort.
+ for (CompartmentsIter c(rt); !c.done(); c.next()) {
+ c->gcState.scheduledForDestruction = false;
+ }
+
+ abortSweepAfterCurrentGroup = true;
+ isCompacting = false;
+
+ break;
+ }
+
+ case State::Finalize: {
+ isCompacting = false;
+ break;
+ }
+
+ case State::Compact: {
+ // Skip any remaining zones that would have been compacted.
+ MOZ_ASSERT(isCompacting);
+ startedCompacting = true;
+ zonesToMaybeCompact.ref().clear();
+ break;
+ }
+
+ case State::Decommit: {
+ break;
+ }
+ }
+
+ stats().reset(reason);
+
+ return IncrementalResult::ResetIncremental;
+}
+
+AutoDisableBarriers::AutoDisableBarriers(GCRuntime* gc) : gc(gc) {
+ /*
+ * Clear needsIncrementalBarrier early so we don't do any write barriers
+ * during sweeping.
+ */
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ if (zone->isGCMarking()) {
+ MOZ_ASSERT(zone->needsIncrementalBarrier());
+ zone->setNeedsIncrementalBarrier(false);
+ }
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ }
+}
+
+AutoDisableBarriers::~AutoDisableBarriers() {
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ if (zone->isGCMarking()) {
+ zone->setNeedsIncrementalBarrier(true);
+ }
+ }
+}
+
+static bool NeedToCollectNursery(GCRuntime* gc) {
+ return !gc->nursery().isEmpty() || !gc->storeBuffer().isEmpty();
+}
+
+#ifdef DEBUG
+static const char* DescribeBudget(const SliceBudget& budget) {
+ MOZ_ASSERT(TlsContext.get()->isMainThreadContext());
+ constexpr size_t length = 32;
+ static char buffer[length];
+ budget.describe(buffer, length);
+ return buffer;
+}
+#endif
+
+static bool ShouldPauseMutatorWhileWaiting(const SliceBudget& budget,
+ JS::GCReason reason,
+ bool budgetWasIncreased) {
+ // When we're nearing the incremental limit at which we will finish the
+ // collection synchronously, pause the main thread if there is only background
+ // GC work happening. This allows the GC to catch up and avoid hitting the
+ // limit.
+ return budget.isTimeBudget() &&
+ (reason == JS::GCReason::ALLOC_TRIGGER ||
+ reason == JS::GCReason::TOO_MUCH_MALLOC) &&
+ budgetWasIncreased;
+}
+
+void GCRuntime::incrementalSlice(SliceBudget& budget, JS::GCReason reason,
+ bool budgetWasIncreased) {
+ MOZ_ASSERT_IF(isIncrementalGCInProgress(), isIncremental);
+
+ AutoSetThreadIsPerformingGC performingGC(rt->gcContext());
+
+ AutoGCSession session(this, JS::HeapState::MajorCollecting);
+
+ bool destroyingRuntime = (reason == JS::GCReason::DESTROY_RUNTIME);
+
+ initialState = incrementalState;
+ isIncremental = !budget.isUnlimited();
+ useBackgroundThreads = ShouldUseBackgroundThreads(isIncremental, reason);
+
+#ifdef JS_GC_ZEAL
+ // Do the incremental collection type specified by zeal mode if the collection
+ // was triggered by runDebugGC() and incremental GC has not been cancelled by
+ // resetIncrementalGC().
+ useZeal = isIncremental && reason == JS::GCReason::DEBUG_GC;
+#endif
+
+#ifdef DEBUG
+ stats().log(
+ "Incremental: %d, lastMarkSlice: %d, useZeal: %d, budget: %s, "
+ "budgetWasIncreased: %d",
+ bool(isIncremental), bool(lastMarkSlice), bool(useZeal),
+ DescribeBudget(budget), budgetWasIncreased);
+#endif
+
+ if (useZeal && hasIncrementalTwoSliceZealMode()) {
+ // Yields between slices occurs at predetermined points in these modes; the
+ // budget is not used. |isIncremental| is still true.
+ stats().log("Using unlimited budget for two-slice zeal mode");
+ budget = SliceBudget::unlimited();
+ }
+
+ bool shouldPauseMutator =
+ ShouldPauseMutatorWhileWaiting(budget, reason, budgetWasIncreased);
+
+ switch (incrementalState) {
+ case State::NotActive:
+ startCollection(reason);
+
+ incrementalState = State::Prepare;
+ if (!beginPreparePhase(reason, session)) {
+ incrementalState = State::NotActive;
+ break;
+ }
+
+ if (useZeal && hasZealMode(ZealMode::YieldBeforeRootMarking)) {
+ break;
+ }
+
+ [[fallthrough]];
+
+ case State::Prepare:
+ if (waitForBackgroundTask(unmarkTask, budget, shouldPauseMutator,
+ DontTriggerSliceWhenFinished) == NotFinished) {
+ break;
+ }
+
+ incrementalState = State::MarkRoots;
+ [[fallthrough]];
+
+ case State::MarkRoots:
+ if (NeedToCollectNursery(this)) {
+ collectNurseryFromMajorGC(reason);
+ }
+
+ endPreparePhase(reason);
+ beginMarkPhase(session);
+ incrementalState = State::Mark;
+
+ if (useZeal && hasZealMode(ZealMode::YieldBeforeMarking) &&
+ isIncremental) {
+ break;
+ }
+
+ [[fallthrough]];
+
+ case State::Mark:
+ if (mightSweepInThisSlice(budget.isUnlimited())) {
+ // Trace wrapper rooters before marking if we might start sweeping in
+ // this slice.
+ rt->mainContextFromOwnThread()->traceWrapperGCRooters(
+ marker().tracer());
+ }
+
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+ if (markUntilBudgetExhausted(budget, AllowParallelMarking) ==
+ NotFinished) {
+ break;
+ }
+ }
+
+ assertNoMarkingWork();
+
+ /*
+ * There are a number of reasons why we break out of collection here,
+ * either ending the slice or to run a new interation of the loop in
+ * GCRuntime::collect()
+ */
+
+ /*
+ * In incremental GCs where we have already performed more than one
+ * slice we yield after marking with the aim of starting the sweep in
+ * the next slice, since the first slice of sweeping can be expensive.
+ *
+ * This is modified by the various zeal modes. We don't yield in
+ * YieldBeforeMarking mode and we always yield in YieldBeforeSweeping
+ * mode.
+ *
+ * We will need to mark anything new on the stack when we resume, so
+ * we stay in Mark state.
+ */
+ if (isIncremental && !lastMarkSlice) {
+ if ((initialState == State::Mark &&
+ !(useZeal && hasZealMode(ZealMode::YieldBeforeMarking))) ||
+ (useZeal && hasZealMode(ZealMode::YieldBeforeSweeping))) {
+ lastMarkSlice = true;
+ stats().log("Yielding before starting sweeping");
+ break;
+ }
+ }
+
+ incrementalState = State::Sweep;
+ lastMarkSlice = false;
+
+ beginSweepPhase(reason, session);
+
+ [[fallthrough]];
+
+ case State::Sweep:
+ if (storeBuffer().mayHavePointersToDeadCells()) {
+ collectNurseryFromMajorGC(reason);
+ }
+
+ if (initialState == State::Sweep) {
+ rt->mainContextFromOwnThread()->traceWrapperGCRooters(
+ marker().tracer());
+ }
+
+ if (performSweepActions(budget) == NotFinished) {
+ break;
+ }
+
+ endSweepPhase(destroyingRuntime);
+
+ incrementalState = State::Finalize;
+
+ [[fallthrough]];
+
+ case State::Finalize:
+ if (waitForBackgroundTask(sweepTask, budget, shouldPauseMutator,
+ TriggerSliceWhenFinished) == NotFinished) {
+ break;
+ }
+
+ assertBackgroundSweepingFinished();
+
+ {
+ // Sweep the zones list now that background finalization is finished to
+ // remove and free dead zones, compartments and realms.
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::SWEEP);
+ gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::DESTROY);
+ sweepZones(rt->gcContext(), destroyingRuntime);
+ }
+
+ MOZ_ASSERT(!startedCompacting);
+ incrementalState = State::Compact;
+
+ // Always yield before compacting since it is not incremental.
+ if (isCompacting && !budget.isUnlimited()) {
+ break;
+ }
+
+ [[fallthrough]];
+
+ case State::Compact:
+ if (isCompacting) {
+ if (NeedToCollectNursery(this)) {
+ collectNurseryFromMajorGC(reason);
+ }
+
+ storeBuffer().checkEmpty();
+ if (!startedCompacting) {
+ beginCompactPhase();
+ }
+
+ if (compactPhase(reason, budget, session) == NotFinished) {
+ break;
+ }
+
+ endCompactPhase();
+ }
+
+ startDecommit();
+ incrementalState = State::Decommit;
+
+ [[fallthrough]];
+
+ case State::Decommit:
+ if (waitForBackgroundTask(decommitTask, budget, shouldPauseMutator,
+ TriggerSliceWhenFinished) == NotFinished) {
+ break;
+ }
+
+ incrementalState = State::Finish;
+
+ [[fallthrough]];
+
+ case State::Finish:
+ finishCollection(reason);
+ incrementalState = State::NotActive;
+ break;
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(safeToYield);
+ for (auto& marker : markers) {
+ MOZ_ASSERT(marker->markColor() == MarkColor::Black);
+ }
+ MOZ_ASSERT(!rt->gcContext()->hasJitCodeToPoison());
+#endif
+}
+
+void GCRuntime::collectNurseryFromMajorGC(JS::GCReason reason) {
+ collectNursery(gcOptions(), reason,
+ gcstats::PhaseKind::EVICT_NURSERY_FOR_MAJOR_GC);
+}
+
+bool GCRuntime::hasForegroundWork() const {
+ switch (incrementalState) {
+ case State::NotActive:
+ // Incremental GC is not running and no work is pending.
+ return false;
+ case State::Prepare:
+ // We yield in the Prepare state after starting unmarking.
+ return !unmarkTask.wasStarted();
+ case State::Finalize:
+ // We yield in the Finalize state to wait for background sweeping.
+ return !isBackgroundSweeping();
+ case State::Decommit:
+ // We yield in the Decommit state to wait for background decommit.
+ return !decommitTask.wasStarted();
+ default:
+ // In all other states there is still work to do.
+ return true;
+ }
+}
+
+IncrementalProgress GCRuntime::waitForBackgroundTask(
+ GCParallelTask& task, const SliceBudget& budget, bool shouldPauseMutator,
+ ShouldTriggerSliceWhenFinished triggerSlice) {
+ // Wait here in non-incremental collections, or if we want to pause the
+ // mutator to let the GC catch up.
+ if (budget.isUnlimited() || shouldPauseMutator) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::WAIT_BACKGROUND_THREAD);
+ Maybe<TimeStamp> deadline;
+ if (budget.isTimeBudget()) {
+ deadline.emplace(budget.deadline());
+ }
+ task.join(deadline);
+ }
+
+ // In incremental collections, yield if the task has not finished and
+ // optionally request a slice to notify us when this happens.
+ if (!budget.isUnlimited()) {
+ AutoLockHelperThreadState lock;
+ if (task.wasStarted(lock)) {
+ if (triggerSlice) {
+ requestSliceAfterBackgroundTask = true;
+ }
+ return NotFinished;
+ }
+
+ task.joinWithLockHeld(lock);
+ }
+
+ MOZ_ASSERT(task.isIdle());
+
+ if (triggerSlice) {
+ cancelRequestedGCAfterBackgroundTask();
+ }
+
+ return Finished;
+}
+
+GCAbortReason gc::IsIncrementalGCUnsafe(JSRuntime* rt) {
+ MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
+
+ if (!rt->gc.isIncrementalGCAllowed()) {
+ return GCAbortReason::IncrementalDisabled;
+ }
+
+ return GCAbortReason::None;
+}
+
+inline void GCRuntime::checkZoneIsScheduled(Zone* zone, JS::GCReason reason,
+ const char* trigger) {
+#ifdef DEBUG
+ if (zone->isGCScheduled()) {
+ return;
+ }
+
+ fprintf(stderr,
+ "checkZoneIsScheduled: Zone %p not scheduled as expected in %s GC "
+ "for %s trigger\n",
+ zone, JS::ExplainGCReason(reason), trigger);
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ fprintf(stderr, " Zone %p:%s%s\n", zone.get(),
+ zone->isAtomsZone() ? " atoms" : "",
+ zone->isGCScheduled() ? " scheduled" : "");
+ }
+ fflush(stderr);
+ MOZ_CRASH("Zone not scheduled");
+#endif
+}
+
+GCRuntime::IncrementalResult GCRuntime::budgetIncrementalGC(
+ bool nonincrementalByAPI, JS::GCReason reason, SliceBudget& budget) {
+ if (nonincrementalByAPI) {
+ stats().nonincremental(GCAbortReason::NonIncrementalRequested);
+ budget = SliceBudget::unlimited();
+
+ // Reset any in progress incremental GC if this was triggered via the
+ // API. This isn't required for correctness, but sometimes during tests
+ // the caller expects this GC to collect certain objects, and we need
+ // to make sure to collect everything possible.
+ if (reason != JS::GCReason::ALLOC_TRIGGER) {
+ return resetIncrementalGC(GCAbortReason::NonIncrementalRequested);
+ }
+
+ return IncrementalResult::Ok;
+ }
+
+ if (reason == JS::GCReason::ABORT_GC) {
+ budget = SliceBudget::unlimited();
+ stats().nonincremental(GCAbortReason::AbortRequested);
+ return resetIncrementalGC(GCAbortReason::AbortRequested);
+ }
+
+ if (!budget.isUnlimited()) {
+ GCAbortReason unsafeReason = IsIncrementalGCUnsafe(rt);
+ if (unsafeReason == GCAbortReason::None) {
+ if (reason == JS::GCReason::COMPARTMENT_REVIVED) {
+ unsafeReason = GCAbortReason::CompartmentRevived;
+ } else if (!incrementalGCEnabled) {
+ unsafeReason = GCAbortReason::ModeChange;
+ }
+ }
+
+ if (unsafeReason != GCAbortReason::None) {
+ budget = SliceBudget::unlimited();
+ stats().nonincremental(unsafeReason);
+ return resetIncrementalGC(unsafeReason);
+ }
+ }
+
+ GCAbortReason resetReason = GCAbortReason::None;
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->gcHeapSize.bytes() >=
+ zone->gcHeapThreshold.incrementalLimitBytes()) {
+ checkZoneIsScheduled(zone, reason, "GC bytes");
+ budget = SliceBudget::unlimited();
+ stats().nonincremental(GCAbortReason::GCBytesTrigger);
+ if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) {
+ resetReason = GCAbortReason::GCBytesTrigger;
+ }
+ }
+
+ if (zone->mallocHeapSize.bytes() >=
+ zone->mallocHeapThreshold.incrementalLimitBytes()) {
+ checkZoneIsScheduled(zone, reason, "malloc bytes");
+ budget = SliceBudget::unlimited();
+ stats().nonincremental(GCAbortReason::MallocBytesTrigger);
+ if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) {
+ resetReason = GCAbortReason::MallocBytesTrigger;
+ }
+ }
+
+ if (zone->jitHeapSize.bytes() >=
+ zone->jitHeapThreshold.incrementalLimitBytes()) {
+ checkZoneIsScheduled(zone, reason, "JIT code bytes");
+ budget = SliceBudget::unlimited();
+ stats().nonincremental(GCAbortReason::JitCodeBytesTrigger);
+ if (zone->wasGCStarted() && zone->gcState() > Zone::Sweep) {
+ resetReason = GCAbortReason::JitCodeBytesTrigger;
+ }
+ }
+
+ if (isIncrementalGCInProgress() &&
+ zone->isGCScheduled() != zone->wasGCStarted()) {
+ budget = SliceBudget::unlimited();
+ resetReason = GCAbortReason::ZoneChange;
+ }
+ }
+
+ if (resetReason != GCAbortReason::None) {
+ return resetIncrementalGC(resetReason);
+ }
+
+ return IncrementalResult::Ok;
+}
+
+bool GCRuntime::maybeIncreaseSliceBudget(SliceBudget& budget) {
+ if (js::SupportDifferentialTesting()) {
+ return false;
+ }
+
+ if (!budget.isTimeBudget() || !isIncrementalGCInProgress()) {
+ return false;
+ }
+
+ bool wasIncreasedForLongCollections =
+ maybeIncreaseSliceBudgetForLongCollections(budget);
+ bool wasIncreasedForUgentCollections =
+ maybeIncreaseSliceBudgetForUrgentCollections(budget);
+
+ return wasIncreasedForLongCollections || wasIncreasedForUgentCollections;
+}
+
+// Return true if the budget is actually extended after rounding.
+static bool ExtendBudget(SliceBudget& budget, double newDuration) {
+ long newDurationMS = lround(newDuration);
+ if (newDurationMS <= budget.timeBudget()) {
+ return false;
+ }
+
+ bool idleTriggered = budget.idle;
+ budget = SliceBudget(TimeBudget(newDuration), nullptr); // Uninterruptible.
+ budget.idle = idleTriggered;
+ budget.extended = true;
+ return true;
+}
+
+bool GCRuntime::maybeIncreaseSliceBudgetForLongCollections(
+ SliceBudget& budget) {
+ // For long-running collections, enforce a minimum time budget that increases
+ // linearly with time up to a maximum.
+
+ // All times are in milliseconds.
+ struct BudgetAtTime {
+ double time;
+ double budget;
+ };
+ const BudgetAtTime MinBudgetStart{1500, 0.0};
+ const BudgetAtTime MinBudgetEnd{2500, 100.0};
+
+ double totalTime = (TimeStamp::Now() - lastGCStartTime()).ToMilliseconds();
+
+ double minBudget =
+ LinearInterpolate(totalTime, MinBudgetStart.time, MinBudgetStart.budget,
+ MinBudgetEnd.time, MinBudgetEnd.budget);
+
+ return ExtendBudget(budget, minBudget);
+}
+
+bool GCRuntime::maybeIncreaseSliceBudgetForUrgentCollections(
+ SliceBudget& budget) {
+ // Enforce a minimum time budget based on how close we are to the incremental
+ // limit.
+
+ size_t minBytesRemaining = SIZE_MAX;
+ for (AllZonesIter zone(this); !zone.done(); zone.next()) {
+ if (!zone->wasGCStarted()) {
+ continue;
+ }
+ size_t gcBytesRemaining =
+ zone->gcHeapThreshold.incrementalBytesRemaining(zone->gcHeapSize);
+ minBytesRemaining = std::min(minBytesRemaining, gcBytesRemaining);
+ size_t mallocBytesRemaining =
+ zone->mallocHeapThreshold.incrementalBytesRemaining(
+ zone->mallocHeapSize);
+ minBytesRemaining = std::min(minBytesRemaining, mallocBytesRemaining);
+ }
+
+ if (minBytesRemaining < tunables.urgentThresholdBytes() &&
+ minBytesRemaining != 0) {
+ // Increase budget based on the reciprocal of the fraction remaining.
+ double fractionRemaining =
+ double(minBytesRemaining) / double(tunables.urgentThresholdBytes());
+ double minBudget = double(defaultSliceBudgetMS()) / fractionRemaining;
+ return ExtendBudget(budget, minBudget);
+ }
+
+ return false;
+}
+
+static void ScheduleZones(GCRuntime* gc, JS::GCReason reason) {
+ for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) {
+ // Re-check heap threshold for alloc-triggered zones that were not
+ // previously collected. Now we have allocation rate data, the heap limit
+ // may have been increased beyond the current size.
+ if (gc->tunables.balancedHeapLimitsEnabled() && zone->isGCScheduled() &&
+ zone->smoothedCollectionRate.ref().isNothing() &&
+ reason == JS::GCReason::ALLOC_TRIGGER &&
+ zone->gcHeapSize.bytes() < zone->gcHeapThreshold.startBytes()) {
+ zone->unscheduleGC(); // May still be re-scheduled below.
+ }
+
+ if (gc->isShutdownGC()) {
+ zone->scheduleGC();
+ }
+
+ if (!gc->isPerZoneGCEnabled()) {
+ zone->scheduleGC();
+ }
+
+ // To avoid resets, continue to collect any zones that were being
+ // collected in a previous slice.
+ if (gc->isIncrementalGCInProgress() && zone->wasGCStarted()) {
+ zone->scheduleGC();
+ }
+
+ // This is a heuristic to reduce the total number of collections.
+ bool inHighFrequencyMode = gc->schedulingState.inHighFrequencyGCMode();
+ if (zone->gcHeapSize.bytes() >=
+ zone->gcHeapThreshold.eagerAllocTrigger(inHighFrequencyMode) ||
+ zone->mallocHeapSize.bytes() >=
+ zone->mallocHeapThreshold.eagerAllocTrigger(inHighFrequencyMode) ||
+ zone->jitHeapSize.bytes() >= zone->jitHeapThreshold.startBytes()) {
+ zone->scheduleGC();
+ }
+ }
+}
+
+static void UnscheduleZones(GCRuntime* gc) {
+ for (ZonesIter zone(gc->rt, WithAtoms); !zone.done(); zone.next()) {
+ zone->unscheduleGC();
+ }
+}
+
+class js::gc::AutoCallGCCallbacks {
+ GCRuntime& gc_;
+ JS::GCReason reason_;
+
+ public:
+ explicit AutoCallGCCallbacks(GCRuntime& gc, JS::GCReason reason)
+ : gc_(gc), reason_(reason) {
+ gc_.maybeCallGCCallback(JSGC_BEGIN, reason);
+ }
+ ~AutoCallGCCallbacks() { gc_.maybeCallGCCallback(JSGC_END, reason_); }
+};
+
+void GCRuntime::maybeCallGCCallback(JSGCStatus status, JS::GCReason reason) {
+ if (!gcCallback.ref().op) {
+ return;
+ }
+
+ if (isIncrementalGCInProgress()) {
+ return;
+ }
+
+ if (gcCallbackDepth == 0) {
+ // Save scheduled zone information in case the callback clears it.
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->gcScheduledSaved_ = zone->gcScheduled_;
+ }
+ }
+
+ // Save and clear GC options and state in case the callback reenters GC.
+ JS::GCOptions options = gcOptions();
+ maybeGcOptions = Nothing();
+ bool savedFullGCRequested = fullGCRequested;
+ fullGCRequested = false;
+
+ gcCallbackDepth++;
+
+ callGCCallback(status, reason);
+
+ MOZ_ASSERT(gcCallbackDepth != 0);
+ gcCallbackDepth--;
+
+ // Restore the original GC options.
+ maybeGcOptions = Some(options);
+
+ // At the end of a GC, clear out the fullGCRequested state. At the start,
+ // restore the previous setting.
+ fullGCRequested = (status == JSGC_END) ? false : savedFullGCRequested;
+
+ if (gcCallbackDepth == 0) {
+ // Ensure any zone that was originally scheduled stays scheduled.
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->gcScheduled_ = zone->gcScheduled_ || zone->gcScheduledSaved_;
+ }
+ }
+}
+
+/*
+ * We disable inlining to ensure that the bottom of the stack with possible GC
+ * roots recorded in MarkRuntime excludes any pointers we use during the marking
+ * implementation.
+ */
+MOZ_NEVER_INLINE GCRuntime::IncrementalResult GCRuntime::gcCycle(
+ bool nonincrementalByAPI, const SliceBudget& budgetArg,
+ JS::GCReason reason) {
+ // Assert if this is a GC unsafe region.
+ rt->mainContextFromOwnThread()->verifyIsSafeToGC();
+
+ // It's ok if threads other than the main thread have suppressGC set, as
+ // they are operating on zones which will not be collected from here.
+ MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
+
+ // This reason is used internally. See below.
+ MOZ_ASSERT(reason != JS::GCReason::RESET);
+
+ // Background finalization and decommit are finished by definition before we
+ // can start a new major GC. Background allocation may still be running, but
+ // that's OK because chunk pools are protected by the GC lock.
+ if (!isIncrementalGCInProgress()) {
+ assertBackgroundSweepingFinished();
+ MOZ_ASSERT(decommitTask.isIdle());
+ }
+
+ // Note that GC callbacks are allowed to re-enter GC.
+ AutoCallGCCallbacks callCallbacks(*this, reason);
+
+ // Increase slice budget for long running collections before it is recorded by
+ // AutoGCSlice.
+ SliceBudget budget(budgetArg);
+ bool budgetWasIncreased = maybeIncreaseSliceBudget(budget);
+
+ ScheduleZones(this, reason);
+
+ auto updateCollectorTime = MakeScopeExit([&] {
+ if (const gcstats::Statistics::SliceData* slice = stats().lastSlice()) {
+ collectorTimeSinceAllocRateUpdate += slice->duration();
+ }
+ });
+
+ gcstats::AutoGCSlice agc(stats(), scanZonesBeforeGC(), gcOptions(), budget,
+ reason, budgetWasIncreased);
+
+ IncrementalResult result =
+ budgetIncrementalGC(nonincrementalByAPI, reason, budget);
+ if (result == IncrementalResult::ResetIncremental) {
+ if (incrementalState == State::NotActive) {
+ // The collection was reset and has finished.
+ return result;
+ }
+
+ // The collection was reset but we must finish up some remaining work.
+ reason = JS::GCReason::RESET;
+ }
+
+ majorGCTriggerReason = JS::GCReason::NO_REASON;
+ MOZ_ASSERT(!stats().hasTrigger());
+
+ incGcNumber();
+ incGcSliceNumber();
+
+ gcprobes::MajorGCStart();
+ incrementalSlice(budget, reason, budgetWasIncreased);
+ gcprobes::MajorGCEnd();
+
+ MOZ_ASSERT_IF(result == IncrementalResult::ResetIncremental,
+ !isIncrementalGCInProgress());
+ return result;
+}
+
+inline bool GCRuntime::mightSweepInThisSlice(bool nonIncremental) {
+ MOZ_ASSERT(incrementalState < State::Sweep);
+ return nonIncremental || lastMarkSlice || hasIncrementalTwoSliceZealMode();
+}
+
+#ifdef JS_GC_ZEAL
+static bool IsDeterministicGCReason(JS::GCReason reason) {
+ switch (reason) {
+ case JS::GCReason::API:
+ case JS::GCReason::DESTROY_RUNTIME:
+ case JS::GCReason::LAST_DITCH:
+ case JS::GCReason::TOO_MUCH_MALLOC:
+ case JS::GCReason::TOO_MUCH_WASM_MEMORY:
+ case JS::GCReason::TOO_MUCH_JIT_CODE:
+ case JS::GCReason::ALLOC_TRIGGER:
+ case JS::GCReason::DEBUG_GC:
+ case JS::GCReason::CC_FORCED:
+ case JS::GCReason::SHUTDOWN_CC:
+ case JS::GCReason::ABORT_GC:
+ case JS::GCReason::DISABLE_GENERATIONAL_GC:
+ case JS::GCReason::FINISH_GC:
+ case JS::GCReason::PREPARE_FOR_TRACING:
+ return true;
+
+ default:
+ return false;
+ }
+}
+#endif
+
+gcstats::ZoneGCStats GCRuntime::scanZonesBeforeGC() {
+ gcstats::ZoneGCStats zoneStats;
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zoneStats.zoneCount++;
+ zoneStats.compartmentCount += zone->compartments().length();
+ if (zone->isGCScheduled()) {
+ zoneStats.collectedZoneCount++;
+ zoneStats.collectedCompartmentCount += zone->compartments().length();
+ }
+ }
+
+ return zoneStats;
+}
+
+// The GC can only clean up scheduledForDestruction realms that were marked live
+// by a barrier (e.g. by RemapWrappers from a navigation event). It is also
+// common to have realms held live because they are part of a cycle in gecko,
+// e.g. involving the HTMLDocument wrapper. In this case, we need to run the
+// CycleCollector in order to remove these edges before the realm can be freed.
+void GCRuntime::maybeDoCycleCollection() {
+ const static float ExcessiveGrayRealms = 0.8f;
+ const static size_t LimitGrayRealms = 200;
+
+ size_t realmsTotal = 0;
+ size_t realmsGray = 0;
+ for (RealmsIter realm(rt); !realm.done(); realm.next()) {
+ ++realmsTotal;
+ GlobalObject* global = realm->unsafeUnbarrieredMaybeGlobal();
+ if (global && global->isMarkedGray()) {
+ ++realmsGray;
+ }
+ }
+ float grayFraction = float(realmsGray) / float(realmsTotal);
+ if (grayFraction > ExcessiveGrayRealms || realmsGray > LimitGrayRealms) {
+ callDoCycleCollectionCallback(rt->mainContextFromOwnThread());
+ }
+}
+
+void GCRuntime::checkCanCallAPI() {
+ MOZ_RELEASE_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ /* If we attempt to invoke the GC while we are running in the GC, assert. */
+ MOZ_RELEASE_ASSERT(!JS::RuntimeHeapIsBusy());
+}
+
+bool GCRuntime::checkIfGCAllowedInCurrentState(JS::GCReason reason) {
+ if (rt->mainContextFromOwnThread()->suppressGC) {
+ return false;
+ }
+
+ // Only allow shutdown GCs when we're destroying the runtime. This keeps
+ // the GC callback from triggering a nested GC and resetting global state.
+ if (rt->isBeingDestroyed() && !isShutdownGC()) {
+ return false;
+ }
+
+#ifdef JS_GC_ZEAL
+ if (deterministicOnly && !IsDeterministicGCReason(reason)) {
+ return false;
+ }
+#endif
+
+ return true;
+}
+
+bool GCRuntime::shouldRepeatForDeadZone(JS::GCReason reason) {
+ MOZ_ASSERT_IF(reason == JS::GCReason::COMPARTMENT_REVIVED, !isIncremental);
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+
+ if (!isIncremental) {
+ return false;
+ }
+
+ for (CompartmentsIter c(rt); !c.done(); c.next()) {
+ if (c->gcState.scheduledForDestruction) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+struct MOZ_RAII AutoSetZoneSliceThresholds {
+ explicit AutoSetZoneSliceThresholds(GCRuntime* gc) : gc(gc) {
+ // On entry, zones that are already collecting should have a slice threshold
+ // set.
+ for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->wasGCStarted() ==
+ zone->gcHeapThreshold.hasSliceThreshold());
+ MOZ_ASSERT(zone->wasGCStarted() ==
+ zone->mallocHeapThreshold.hasSliceThreshold());
+ }
+ }
+
+ ~AutoSetZoneSliceThresholds() {
+ // On exit, update the thresholds for all collecting zones.
+ bool waitingOnBGTask = gc->isWaitingOnBackgroundTask();
+ for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->wasGCStarted()) {
+ zone->setGCSliceThresholds(*gc, waitingOnBGTask);
+ } else {
+ MOZ_ASSERT(!zone->gcHeapThreshold.hasSliceThreshold());
+ MOZ_ASSERT(!zone->mallocHeapThreshold.hasSliceThreshold());
+ }
+ }
+ }
+
+ GCRuntime* gc;
+};
+
+void GCRuntime::collect(bool nonincrementalByAPI, const SliceBudget& budget,
+ JS::GCReason reason) {
+ TimeStamp startTime = TimeStamp::Now();
+ auto timer = MakeScopeExit([&] {
+ if (Realm* realm = rt->mainContextFromOwnThread()->realm()) {
+ realm->timers.gcTime += TimeStamp::Now() - startTime;
+ }
+ });
+
+ auto clearGCOptions = MakeScopeExit([&] {
+ if (!isIncrementalGCInProgress()) {
+ maybeGcOptions = Nothing();
+ }
+ });
+
+ MOZ_ASSERT(reason != JS::GCReason::NO_REASON);
+
+ // Checks run for each request, even if we do not actually GC.
+ checkCanCallAPI();
+
+ // Check if we are allowed to GC at this time before proceeding.
+ if (!checkIfGCAllowedInCurrentState(reason)) {
+ return;
+ }
+
+ stats().log("GC slice starting in state %s", StateName(incrementalState));
+
+ AutoStopVerifyingBarriers av(rt, isShutdownGC());
+ AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread());
+ AutoSetZoneSliceThresholds sliceThresholds(this);
+
+ schedulingState.updateHighFrequencyModeForReason(reason);
+
+ if (!isIncrementalGCInProgress() && tunables.balancedHeapLimitsEnabled()) {
+ updateAllocationRates();
+ }
+
+ bool repeat;
+ do {
+ IncrementalResult cycleResult =
+ gcCycle(nonincrementalByAPI, budget, reason);
+
+ if (reason == JS::GCReason::ABORT_GC) {
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ stats().log("GC aborted by request");
+ break;
+ }
+
+ /*
+ * Sometimes when we finish a GC we need to immediately start a new one.
+ * This happens in the following cases:
+ * - when we reset the current GC
+ * - when finalizers drop roots during shutdown
+ * - when zones that we thought were dead at the start of GC are
+ * not collected (see the large comment in beginMarkPhase)
+ */
+ repeat = false;
+ if (!isIncrementalGCInProgress()) {
+ if (cycleResult == ResetIncremental) {
+ repeat = true;
+ } else if (rootsRemoved && isShutdownGC()) {
+ /* Need to re-schedule all zones for GC. */
+ JS::PrepareForFullGC(rt->mainContextFromOwnThread());
+ repeat = true;
+ reason = JS::GCReason::ROOTS_REMOVED;
+ } else if (shouldRepeatForDeadZone(reason)) {
+ repeat = true;
+ reason = JS::GCReason::COMPARTMENT_REVIVED;
+ }
+ }
+ } while (repeat);
+
+ if (reason == JS::GCReason::COMPARTMENT_REVIVED) {
+ maybeDoCycleCollection();
+ }
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::CheckHeapAfterGC)) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::TRACE_HEAP);
+ CheckHeapAfterGC(rt);
+ }
+ if (hasZealMode(ZealMode::CheckGrayMarking) && !isIncrementalGCInProgress()) {
+ MOZ_RELEASE_ASSERT(CheckGrayMarkingState(rt));
+ }
+#endif
+ stats().log("GC slice ending in state %s", StateName(incrementalState));
+
+ UnscheduleZones(this);
+}
+
+SliceBudget GCRuntime::defaultBudget(JS::GCReason reason, int64_t millis) {
+ // millis == 0 means use internal GC scheduling logic to come up with
+ // a duration for the slice budget. This may end up still being zero
+ // based on preferences.
+ if (millis == 0) {
+ millis = defaultSliceBudgetMS();
+ }
+
+ // If the embedding has registered a callback for creating SliceBudgets,
+ // then use it.
+ if (createBudgetCallback) {
+ return createBudgetCallback(reason, millis);
+ }
+
+ // Otherwise, the preference can request an unlimited duration slice.
+ if (millis == 0) {
+ return SliceBudget::unlimited();
+ }
+
+ return SliceBudget(TimeBudget(millis));
+}
+
+void GCRuntime::gc(JS::GCOptions options, JS::GCReason reason) {
+ if (!isIncrementalGCInProgress()) {
+ setGCOptions(options);
+ }
+
+ collect(true, SliceBudget::unlimited(), reason);
+}
+
+void GCRuntime::startGC(JS::GCOptions options, JS::GCReason reason,
+ const js::SliceBudget& budget) {
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ setGCOptions(options);
+
+ if (!JS::IsIncrementalGCEnabled(rt->mainContextFromOwnThread())) {
+ collect(true, SliceBudget::unlimited(), reason);
+ return;
+ }
+
+ collect(false, budget, reason);
+}
+
+void GCRuntime::setGCOptions(JS::GCOptions options) {
+ MOZ_ASSERT(maybeGcOptions == Nothing());
+ maybeGcOptions = Some(options);
+}
+
+void GCRuntime::gcSlice(JS::GCReason reason, const js::SliceBudget& budget) {
+ MOZ_ASSERT(isIncrementalGCInProgress());
+ collect(false, budget, reason);
+}
+
+void GCRuntime::finishGC(JS::GCReason reason) {
+ MOZ_ASSERT(isIncrementalGCInProgress());
+
+ // If we're not collecting because we're out of memory then skip the
+ // compacting phase if we need to finish an ongoing incremental GC
+ // non-incrementally to avoid janking the browser.
+ if (!IsOOMReason(initialReason)) {
+ if (incrementalState == State::Compact) {
+ abortGC();
+ return;
+ }
+
+ isCompacting = false;
+ }
+
+ collect(false, SliceBudget::unlimited(), reason);
+}
+
+void GCRuntime::abortGC() {
+ MOZ_ASSERT(isIncrementalGCInProgress());
+ checkCanCallAPI();
+ MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
+
+ collect(false, SliceBudget::unlimited(), JS::GCReason::ABORT_GC);
+}
+
+static bool ZonesSelected(GCRuntime* gc) {
+ for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isGCScheduled()) {
+ return true;
+ }
+ }
+ return false;
+}
+
+void GCRuntime::startDebugGC(JS::GCOptions options, const SliceBudget& budget) {
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ setGCOptions(options);
+
+ if (!ZonesSelected(this)) {
+ JS::PrepareForFullGC(rt->mainContextFromOwnThread());
+ }
+
+ collect(false, budget, JS::GCReason::DEBUG_GC);
+}
+
+void GCRuntime::debugGCSlice(const SliceBudget& budget) {
+ MOZ_ASSERT(isIncrementalGCInProgress());
+
+ if (!ZonesSelected(this)) {
+ JS::PrepareForIncrementalGC(rt->mainContextFromOwnThread());
+ }
+
+ collect(false, budget, JS::GCReason::DEBUG_GC);
+}
+
+/* Schedule a full GC unless a zone will already be collected. */
+void js::PrepareForDebugGC(JSRuntime* rt) {
+ if (!ZonesSelected(&rt->gc)) {
+ JS::PrepareForFullGC(rt->mainContextFromOwnThread());
+ }
+}
+
+void GCRuntime::onOutOfMallocMemory() {
+ // Stop allocating new chunks.
+ allocTask.cancelAndWait();
+
+ // Make sure we release anything queued for release.
+ decommitTask.join();
+ nursery().joinDecommitTask();
+
+ // Wait for background free of nursery huge slots to finish.
+ sweepTask.join();
+
+ AutoLockGC lock(this);
+ onOutOfMallocMemory(lock);
+}
+
+void GCRuntime::onOutOfMallocMemory(const AutoLockGC& lock) {
+#ifdef DEBUG
+ // Release any relocated arenas we may be holding on to, without releasing
+ // the GC lock.
+ releaseHeldRelocatedArenasWithoutUnlocking(lock);
+#endif
+
+ // Throw away any excess chunks we have lying around.
+ freeEmptyChunks(lock);
+
+ // Immediately decommit as many arenas as possible in the hopes that this
+ // might let the OS scrape together enough pages to satisfy the failing
+ // malloc request.
+ if (DecommitEnabled()) {
+ decommitFreeArenasWithoutUnlocking(lock);
+ }
+}
+
+void GCRuntime::minorGC(JS::GCReason reason, gcstats::PhaseKind phase) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+
+ MOZ_ASSERT_IF(reason == JS::GCReason::EVICT_NURSERY,
+ !rt->mainContextFromOwnThread()->suppressGC);
+ if (rt->mainContextFromOwnThread()->suppressGC) {
+ return;
+ }
+
+ incGcNumber();
+
+ collectNursery(JS::GCOptions::Normal, reason, phase);
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::CheckHeapAfterGC)) {
+ gcstats::AutoPhase ap(stats(), phase);
+ CheckHeapAfterGC(rt);
+ }
+#endif
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ maybeTriggerGCAfterAlloc(zone);
+ maybeTriggerGCAfterMalloc(zone);
+ }
+}
+
+void GCRuntime::collectNursery(JS::GCOptions options, JS::GCReason reason,
+ gcstats::PhaseKind phase) {
+ AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread());
+
+ uint32_t numAllocs = 0;
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ numAllocs += zone->getAndResetTenuredAllocsSinceMinorGC();
+ }
+ stats().setAllocsSinceMinorGCTenured(numAllocs);
+
+ gcstats::AutoPhase ap(stats(), phase);
+
+ nursery().clearMinorGCRequest();
+ nursery().collect(options, reason);
+ MOZ_ASSERT(nursery().isEmpty());
+
+ startBackgroundFreeAfterMinorGC();
+}
+
+void GCRuntime::startBackgroundFreeAfterMinorGC() {
+ MOZ_ASSERT(nursery().isEmpty());
+
+ {
+ AutoLockHelperThreadState lock;
+
+ lifoBlocksToFree.ref().transferFrom(&lifoBlocksToFreeAfterMinorGC.ref());
+
+ if (lifoBlocksToFree.ref().isEmpty() &&
+ buffersToFreeAfterMinorGC.ref().empty()) {
+ return;
+ }
+ }
+
+ startBackgroundFree();
+}
+
+bool GCRuntime::gcIfRequestedImpl(bool eagerOk) {
+ // This method returns whether a major GC was performed.
+
+ if (nursery().minorGCRequested()) {
+ minorGC(nursery().minorGCTriggerReason());
+ }
+
+ JS::GCReason reason = wantMajorGC(eagerOk);
+ if (reason == JS::GCReason::NO_REASON) {
+ return false;
+ }
+
+ SliceBudget budget = defaultBudget(reason, 0);
+ if (!isIncrementalGCInProgress()) {
+ startGC(JS::GCOptions::Normal, reason, budget);
+ } else {
+ gcSlice(reason, budget);
+ }
+ return true;
+}
+
+void js::gc::FinishGC(JSContext* cx, JS::GCReason reason) {
+ // Calling this when GC is suppressed won't have any effect.
+ MOZ_ASSERT(!cx->suppressGC);
+
+ // GC callbacks may run arbitrary code, including JS. Check this regardless of
+ // whether we GC for this invocation.
+ MOZ_ASSERT(cx->isNurseryAllocAllowed());
+
+ if (JS::IsIncrementalGCInProgress(cx)) {
+ JS::PrepareForIncrementalGC(cx);
+ JS::FinishIncrementalGC(cx, reason);
+ }
+}
+
+void js::gc::WaitForBackgroundTasks(JSContext* cx) {
+ cx->runtime()->gc.waitForBackgroundTasks();
+}
+
+void GCRuntime::waitForBackgroundTasks() {
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ MOZ_ASSERT(sweepTask.isIdle());
+ MOZ_ASSERT(decommitTask.isIdle());
+ MOZ_ASSERT(markTask.isIdle());
+
+ allocTask.join();
+ freeTask.join();
+ nursery().joinDecommitTask();
+}
+
+Realm* js::NewRealm(JSContext* cx, JSPrincipals* principals,
+ const JS::RealmOptions& options) {
+ JSRuntime* rt = cx->runtime();
+ JS_AbortIfWrongThread(cx);
+
+ UniquePtr<Zone> zoneHolder;
+ UniquePtr<Compartment> compHolder;
+
+ Compartment* comp = nullptr;
+ Zone* zone = nullptr;
+ JS::CompartmentSpecifier compSpec =
+ options.creationOptions().compartmentSpecifier();
+ switch (compSpec) {
+ case JS::CompartmentSpecifier::NewCompartmentInSystemZone:
+ // systemZone might be null here, in which case we'll make a zone and
+ // set this field below.
+ zone = rt->gc.systemZone;
+ break;
+ case JS::CompartmentSpecifier::NewCompartmentInExistingZone:
+ zone = options.creationOptions().zone();
+ MOZ_ASSERT(zone);
+ break;
+ case JS::CompartmentSpecifier::ExistingCompartment:
+ comp = options.creationOptions().compartment();
+ zone = comp->zone();
+ break;
+ case JS::CompartmentSpecifier::NewCompartmentAndZone:
+ break;
+ }
+
+ if (!zone) {
+ Zone::Kind kind = Zone::NormalZone;
+ const JSPrincipals* trusted = rt->trustedPrincipals();
+ if (compSpec == JS::CompartmentSpecifier::NewCompartmentInSystemZone ||
+ (principals && principals == trusted)) {
+ kind = Zone::SystemZone;
+ }
+
+ zoneHolder = MakeUnique<Zone>(cx->runtime(), kind);
+ if (!zoneHolder || !zoneHolder->init()) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ zone = zoneHolder.get();
+ }
+
+ bool invisibleToDebugger = options.creationOptions().invisibleToDebugger();
+ if (comp) {
+ // Debugger visibility is per-compartment, not per-realm, so make sure the
+ // new realm's visibility matches its compartment's.
+ MOZ_ASSERT(comp->invisibleToDebugger() == invisibleToDebugger);
+ } else {
+ compHolder = cx->make_unique<JS::Compartment>(zone, invisibleToDebugger);
+ if (!compHolder) {
+ return nullptr;
+ }
+
+ comp = compHolder.get();
+ }
+
+ UniquePtr<Realm> realm(cx->new_<Realm>(comp, options));
+ if (!realm) {
+ return nullptr;
+ }
+ realm->init(cx, principals);
+
+ // Make sure we don't put system and non-system realms in the same
+ // compartment.
+ if (!compHolder) {
+ MOZ_RELEASE_ASSERT(realm->isSystem() == IsSystemCompartment(comp));
+ }
+
+ AutoLockGC lock(rt);
+
+ // Reserve space in the Vectors before we start mutating them.
+ if (!comp->realms().reserve(comp->realms().length() + 1) ||
+ (compHolder &&
+ !zone->compartments().reserve(zone->compartments().length() + 1)) ||
+ (zoneHolder && !rt->gc.zones().reserve(rt->gc.zones().length() + 1))) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ // After this everything must be infallible.
+
+ comp->realms().infallibleAppend(realm.get());
+
+ if (compHolder) {
+ zone->compartments().infallibleAppend(compHolder.release());
+ }
+
+ if (zoneHolder) {
+ rt->gc.zones().infallibleAppend(zoneHolder.release());
+
+ // Lazily set the runtime's system zone.
+ if (compSpec == JS::CompartmentSpecifier::NewCompartmentInSystemZone) {
+ MOZ_RELEASE_ASSERT(!rt->gc.systemZone);
+ MOZ_ASSERT(zone->isSystemZone());
+ rt->gc.systemZone = zone;
+ }
+ }
+
+ return realm.release();
+}
+
+void GCRuntime::runDebugGC() {
+#ifdef JS_GC_ZEAL
+ if (rt->mainContextFromOwnThread()->suppressGC) {
+ return;
+ }
+
+ if (hasZealMode(ZealMode::GenerationalGC)) {
+ return minorGC(JS::GCReason::DEBUG_GC);
+ }
+
+ PrepareForDebugGC(rt);
+
+ auto budget = SliceBudget::unlimited();
+ if (hasZealMode(ZealMode::IncrementalMultipleSlices)) {
+ /*
+ * Start with a small slice limit and double it every slice. This
+ * ensure that we get multiple slices, and collection runs to
+ * completion.
+ */
+ if (!isIncrementalGCInProgress()) {
+ zealSliceBudget = zealFrequency / 2;
+ } else {
+ zealSliceBudget *= 2;
+ }
+ budget = SliceBudget(WorkBudget(zealSliceBudget));
+
+ js::gc::State initialState = incrementalState;
+ if (!isIncrementalGCInProgress()) {
+ setGCOptions(JS::GCOptions::Shrink);
+ }
+ collect(false, budget, JS::GCReason::DEBUG_GC);
+
+ /* Reset the slice size when we get to the sweep or compact phases. */
+ if ((initialState == State::Mark && incrementalState == State::Sweep) ||
+ (initialState == State::Sweep && incrementalState == State::Compact)) {
+ zealSliceBudget = zealFrequency / 2;
+ }
+ } else if (hasIncrementalTwoSliceZealMode()) {
+ // These modes trigger incremental GC that happens in two slices and the
+ // supplied budget is ignored by incrementalSlice.
+ budget = SliceBudget(WorkBudget(1));
+
+ if (!isIncrementalGCInProgress()) {
+ setGCOptions(JS::GCOptions::Normal);
+ }
+ collect(false, budget, JS::GCReason::DEBUG_GC);
+ } else if (hasZealMode(ZealMode::Compact)) {
+ gc(JS::GCOptions::Shrink, JS::GCReason::DEBUG_GC);
+ } else {
+ gc(JS::GCOptions::Normal, JS::GCReason::DEBUG_GC);
+ }
+
+#endif
+}
+
+void GCRuntime::setFullCompartmentChecks(bool enabled) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+ fullCompartmentChecks = enabled;
+}
+
+void GCRuntime::notifyRootsRemoved() {
+ rootsRemoved = true;
+
+#ifdef JS_GC_ZEAL
+ /* Schedule a GC to happen "soon". */
+ if (hasZealMode(ZealMode::RootsChange)) {
+ nextScheduled = 1;
+ }
+#endif
+}
+
+#ifdef JS_GC_ZEAL
+bool GCRuntime::selectForMarking(JSObject* object) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+ return selectedForMarking.ref().get().append(object);
+}
+
+void GCRuntime::clearSelectedForMarking() {
+ selectedForMarking.ref().get().clearAndFree();
+}
+
+void GCRuntime::setDeterministic(bool enabled) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+ deterministicOnly = enabled;
+}
+#endif
+
+#ifdef DEBUG
+
+AutoAssertNoNurseryAlloc::AutoAssertNoNurseryAlloc() {
+ TlsContext.get()->disallowNurseryAlloc();
+}
+
+AutoAssertNoNurseryAlloc::~AutoAssertNoNurseryAlloc() {
+ TlsContext.get()->allowNurseryAlloc();
+}
+
+#endif // DEBUG
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void GCRuntime::checkHashTablesAfterMovingGC() {
+ /*
+ * Check that internal hash tables no longer have any pointers to things
+ * that have been moved.
+ */
+ rt->geckoProfiler().checkStringsMapAfterMovingGC();
+ if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) {
+ rt->jitRuntime()->getInterpreterEntryMap()->checkScriptsAfterMovingGC();
+ }
+ for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) {
+ zone->checkUniqueIdTableAfterMovingGC();
+ zone->shapeZone().checkTablesAfterMovingGC();
+ zone->checkAllCrossCompartmentWrappersAfterMovingGC();
+ zone->checkScriptMapsAfterMovingGC();
+
+ // Note: CompactPropMaps never have a table.
+ JS::AutoCheckCannotGC nogc;
+ for (auto map = zone->cellIterUnsafe<NormalPropMap>(); !map.done();
+ map.next()) {
+ if (PropMapTable* table = map->asLinked()->maybeTable(nogc)) {
+ table->checkAfterMovingGC();
+ }
+ }
+ for (auto map = zone->cellIterUnsafe<DictionaryPropMap>(); !map.done();
+ map.next()) {
+ if (PropMapTable* table = map->asLinked()->maybeTable(nogc)) {
+ table->checkAfterMovingGC();
+ }
+ }
+ }
+
+ for (CompartmentsIter c(this); !c.done(); c.next()) {
+ for (RealmsInCompartmentIter r(c); !r.done(); r.next()) {
+ r->dtoaCache.checkCacheAfterMovingGC();
+ if (r->debugEnvs()) {
+ r->debugEnvs()->checkHashTablesAfterMovingGC();
+ }
+ }
+ }
+}
+#endif
+
+#ifdef DEBUG
+bool GCRuntime::hasZone(Zone* target) {
+ for (AllZonesIter zone(this); !zone.done(); zone.next()) {
+ if (zone == target) {
+ return true;
+ }
+ }
+ return false;
+}
+#endif
+
+void AutoAssertEmptyNursery::checkCondition(JSContext* cx) {
+ if (!noAlloc) {
+ noAlloc.emplace();
+ }
+ this->cx = cx;
+ MOZ_ASSERT(cx->nursery().isEmpty());
+}
+
+AutoEmptyNursery::AutoEmptyNursery(JSContext* cx) : AutoAssertEmptyNursery() {
+ MOZ_ASSERT(!cx->suppressGC);
+ cx->runtime()->gc.stats().suspendPhases();
+ cx->runtime()->gc.evictNursery(JS::GCReason::EVICT_NURSERY);
+ cx->runtime()->gc.stats().resumePhases();
+ checkCondition(cx);
+}
+
+#ifdef DEBUG
+
+namespace js {
+
+// We don't want jsfriendapi.h to depend on GenericPrinter,
+// so these functions are declared directly in the cpp.
+
+extern JS_PUBLIC_API void DumpString(JSString* str, js::GenericPrinter& out);
+
+} // namespace js
+
+void js::gc::Cell::dump(js::GenericPrinter& out) const {
+ switch (getTraceKind()) {
+ case JS::TraceKind::Object:
+ reinterpret_cast<const JSObject*>(this)->dump(out);
+ break;
+
+ case JS::TraceKind::String:
+ js::DumpString(reinterpret_cast<JSString*>(const_cast<Cell*>(this)), out);
+ break;
+
+ case JS::TraceKind::Shape:
+ reinterpret_cast<const Shape*>(this)->dump(out);
+ break;
+
+ default:
+ out.printf("%s(%p)\n", JS::GCTraceKindToAscii(getTraceKind()),
+ (void*)this);
+ }
+}
+
+// For use in a debugger.
+void js::gc::Cell::dump() const {
+ js::Fprinter out(stderr);
+ dump(out);
+}
+#endif
+
+JS_PUBLIC_API bool js::gc::detail::CanCheckGrayBits(const TenuredCell* cell) {
+ // We do not check the gray marking state of cells in the following cases:
+ //
+ // 1) When OOM has caused us to clear the gcGrayBitsValid_ flag.
+ //
+ // 2) When we are in an incremental GC and examine a cell that is in a zone
+ // that is not being collected. Gray targets of CCWs that are marked black
+ // by a barrier will eventually be marked black in a later GC slice.
+ //
+ // 3) When mark bits are being cleared concurrently by a helper thread.
+
+ MOZ_ASSERT(cell);
+
+ auto runtime = cell->runtimeFromAnyThread();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime));
+
+ if (!runtime->gc.areGrayBitsValid()) {
+ return false;
+ }
+
+ JS::Zone* zone = cell->zone();
+
+ if (runtime->gc.isIncrementalGCInProgress() && !zone->wasGCStarted()) {
+ return false;
+ }
+
+ return !zone->isGCPreparing();
+}
+
+JS_PUBLIC_API bool js::gc::detail::CellIsMarkedGrayIfKnown(
+ const TenuredCell* cell) {
+ MOZ_ASSERT_IF(cell->isPermanentAndMayBeShared(), cell->isMarkedBlack());
+ if (!cell->isMarkedGray()) {
+ return false;
+ }
+
+ return CanCheckGrayBits(cell);
+}
+
+#ifdef DEBUG
+
+JS_PUBLIC_API void js::gc::detail::AssertCellIsNotGray(const Cell* cell) {
+ if (!cell->isTenured()) {
+ return;
+ }
+
+ // Check that a cell is not marked gray.
+ //
+ // Since this is a debug-only check, take account of the eventual mark state
+ // of cells that will be marked black by the next GC slice in an incremental
+ // GC. For performance reasons we don't do this in CellIsMarkedGrayIfKnown.
+
+ auto tc = &cell->asTenured();
+ if (!tc->isMarkedGray() || !CanCheckGrayBits(tc)) {
+ return;
+ }
+
+ // TODO: I'd like to AssertHeapIsIdle() here, but this ends up getting
+ // called during GC and while iterating the heap for memory reporting.
+ MOZ_ASSERT(!JS::RuntimeHeapIsCycleCollecting());
+
+ if (tc->zone()->isGCMarkingBlackAndGray()) {
+ // We are doing gray marking in the cell's zone. Even if the cell is
+ // currently marked gray it may eventually be marked black. Delay checking
+ // non-black cells until we finish gray marking.
+
+ if (!tc->isMarkedBlack()) {
+ JSRuntime* rt = tc->zone()->runtimeFromMainThread();
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!rt->gc.cellsToAssertNotGray.ref().append(cell)) {
+ oomUnsafe.crash("Can't append to delayed gray checks list");
+ }
+ }
+ return;
+ }
+
+ MOZ_ASSERT(!tc->isMarkedGray());
+}
+
+extern JS_PUBLIC_API bool js::gc::detail::ObjectIsMarkedBlack(
+ const JSObject* obj) {
+ return obj->isMarkedBlack();
+}
+
+#endif
+
+js::gc::ClearEdgesTracer::ClearEdgesTracer(JSRuntime* rt)
+ : GenericTracerImpl(rt, JS::TracerKind::ClearEdges,
+ JS::WeakMapTraceAction::TraceKeysAndValues) {}
+
+template <typename T>
+void js::gc::ClearEdgesTracer::onEdge(T** thingp, const char* name) {
+ // We don't handle removing pointers to nursery edges from the store buffer
+ // with this tracer. Check that this doesn't happen.
+ T* thing = *thingp;
+ MOZ_ASSERT(!IsInsideNursery(thing));
+
+ // Fire the pre-barrier since we're removing an edge from the graph.
+ InternalBarrierMethods<T*>::preBarrier(thing);
+
+ *thingp = nullptr;
+}
+
+void GCRuntime::setPerformanceHint(PerformanceHint hint) {
+ if (hint == PerformanceHint::InPageLoad) {
+ inPageLoadCount++;
+ } else {
+ MOZ_ASSERT(inPageLoadCount);
+ inPageLoadCount--;
+ }
+}
diff --git a/js/src/gc/GC.h b/js/src/gc/GC.h
new file mode 100644
index 0000000000..58475bc74a
--- /dev/null
+++ b/js/src/gc/GC.h
@@ -0,0 +1,243 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * JS engine garbage collector API.
+ */
+
+#ifndef gc_GC_h
+#define gc_GC_h
+
+#include "gc/GCEnum.h"
+#include "js/GCAPI.h"
+#include "js/HeapAPI.h"
+#include "js/RealmIterators.h"
+#include "js/TraceKind.h"
+
+class JSTracer;
+
+namespace JS {
+class RealmOptions;
+}
+
+namespace js {
+
+class Nursery;
+
+namespace gc {
+
+class Arena;
+class TenuredChunk;
+
+} /* namespace gc */
+
+// Define name, key and writability for the GC parameters.
+#define FOR_EACH_GC_PARAM(_) \
+ _("maxBytes", JSGC_MAX_BYTES, true) \
+ _("minNurseryBytes", JSGC_MIN_NURSERY_BYTES, true) \
+ _("maxNurseryBytes", JSGC_MAX_NURSERY_BYTES, true) \
+ _("gcBytes", JSGC_BYTES, false) \
+ _("nurseryBytes", JSGC_NURSERY_BYTES, false) \
+ _("gcNumber", JSGC_NUMBER, false) \
+ _("majorGCNumber", JSGC_MAJOR_GC_NUMBER, false) \
+ _("minorGCNumber", JSGC_MINOR_GC_NUMBER, false) \
+ _("incrementalGCEnabled", JSGC_INCREMENTAL_GC_ENABLED, true) \
+ _("perZoneGCEnabled", JSGC_PER_ZONE_GC_ENABLED, true) \
+ _("unusedChunks", JSGC_UNUSED_CHUNKS, false) \
+ _("totalChunks", JSGC_TOTAL_CHUNKS, false) \
+ _("sliceTimeBudgetMS", JSGC_SLICE_TIME_BUDGET_MS, true) \
+ _("highFrequencyTimeLimit", JSGC_HIGH_FREQUENCY_TIME_LIMIT, true) \
+ _("smallHeapSizeMax", JSGC_SMALL_HEAP_SIZE_MAX, true) \
+ _("largeHeapSizeMin", JSGC_LARGE_HEAP_SIZE_MIN, true) \
+ _("highFrequencySmallHeapGrowth", JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH, \
+ true) \
+ _("highFrequencyLargeHeapGrowth", JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH, \
+ true) \
+ _("lowFrequencyHeapGrowth", JSGC_LOW_FREQUENCY_HEAP_GROWTH, true) \
+ _("balancedHeapLimitsEnabled", JSGC_BALANCED_HEAP_LIMITS_ENABLED, true) \
+ _("heapGrowthFactor", JSGC_HEAP_GROWTH_FACTOR, true) \
+ _("allocationThreshold", JSGC_ALLOCATION_THRESHOLD, true) \
+ _("smallHeapIncrementalLimit", JSGC_SMALL_HEAP_INCREMENTAL_LIMIT, true) \
+ _("largeHeapIncrementalLimit", JSGC_LARGE_HEAP_INCREMENTAL_LIMIT, true) \
+ _("minEmptyChunkCount", JSGC_MIN_EMPTY_CHUNK_COUNT, true) \
+ _("maxEmptyChunkCount", JSGC_MAX_EMPTY_CHUNK_COUNT, true) \
+ _("compactingEnabled", JSGC_COMPACTING_ENABLED, true) \
+ _("parallelMarkingEnabled", JSGC_PARALLEL_MARKING_ENABLED, true) \
+ _("parallelMarkingThresholdKB", JSGC_PARALLEL_MARKING_THRESHOLD_KB, true) \
+ _("minLastDitchGCPeriod", JSGC_MIN_LAST_DITCH_GC_PERIOD, true) \
+ _("nurseryFreeThresholdForIdleCollection", \
+ JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION, true) \
+ _("nurseryFreeThresholdForIdleCollectionPercent", \
+ JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_PERCENT, true) \
+ _("nurseryTimeoutForIdleCollectionMS", \
+ JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS, true) \
+ _("pretenureThreshold", JSGC_PRETENURE_THRESHOLD, true) \
+ _("zoneAllocDelayKB", JSGC_ZONE_ALLOC_DELAY_KB, true) \
+ _("mallocThresholdBase", JSGC_MALLOC_THRESHOLD_BASE, true) \
+ _("urgentThreshold", JSGC_URGENT_THRESHOLD_MB, true) \
+ _("chunkBytes", JSGC_CHUNK_BYTES, false) \
+ _("helperThreadRatio", JSGC_HELPER_THREAD_RATIO, true) \
+ _("maxHelperThreads", JSGC_MAX_HELPER_THREADS, true) \
+ _("helperThreadCount", JSGC_HELPER_THREAD_COUNT, false) \
+ _("markingThreadCount", JSGC_MARKING_THREAD_COUNT, true) \
+ _("systemPageSizeKB", JSGC_SYSTEM_PAGE_SIZE_KB, false)
+
+// Get the key and writability give a GC parameter name.
+extern bool GetGCParameterInfo(const char* name, JSGCParamKey* keyOut,
+ bool* writableOut);
+
+extern void TraceRuntime(JSTracer* trc);
+
+// Trace roots but don't evict the nursery first; used from DumpHeap.
+extern void TraceRuntimeWithoutEviction(JSTracer* trc);
+
+extern void ReleaseAllJITCode(JS::GCContext* gcx);
+
+extern void PrepareForDebugGC(JSRuntime* rt);
+
+/* Functions for managing cross compartment gray pointers. */
+
+extern void NotifyGCNukeWrapper(JSContext* cx, JSObject* wrapper);
+
+extern unsigned NotifyGCPreSwap(JSObject* a, JSObject* b);
+
+extern void NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned preResult);
+
+using IterateChunkCallback = void (*)(JSRuntime*, void*, gc::TenuredChunk*,
+ const JS::AutoRequireNoGC&);
+using IterateZoneCallback = void (*)(JSRuntime*, void*, JS::Zone*,
+ const JS::AutoRequireNoGC&);
+using IterateArenaCallback = void (*)(JSRuntime*, void*, gc::Arena*,
+ JS::TraceKind, size_t,
+ const JS::AutoRequireNoGC&);
+using IterateCellCallback = void (*)(JSRuntime*, void*, JS::GCCellPtr, size_t,
+ const JS::AutoRequireNoGC&);
+
+/*
+ * This function calls |zoneCallback| on every zone, |realmCallback| on
+ * every realm, |arenaCallback| on every in-use arena, and |cellCallback|
+ * on every in-use cell in the GC heap.
+ *
+ * Note that no read barrier is triggered on the cells passed to cellCallback,
+ * so no these pointers must not escape the callback.
+ */
+extern void IterateHeapUnbarriered(JSContext* cx, void* data,
+ IterateZoneCallback zoneCallback,
+ JS::IterateRealmCallback realmCallback,
+ IterateArenaCallback arenaCallback,
+ IterateCellCallback cellCallback);
+
+/*
+ * This function is like IterateHeapUnbarriered, but does it for a single zone.
+ */
+extern void IterateHeapUnbarrieredForZone(
+ JSContext* cx, JS::Zone* zone, void* data, IterateZoneCallback zoneCallback,
+ JS::IterateRealmCallback realmCallback, IterateArenaCallback arenaCallback,
+ IterateCellCallback cellCallback);
+
+/*
+ * Invoke chunkCallback on every in-use chunk.
+ */
+extern void IterateChunks(JSContext* cx, void* data,
+ IterateChunkCallback chunkCallback);
+
+using IterateScriptCallback = void (*)(JSRuntime*, void*, BaseScript*,
+ const JS::AutoRequireNoGC&);
+
+/*
+ * Invoke scriptCallback on every in-use script for the given realm or for all
+ * realms if it is null. The scripts may or may not have bytecode.
+ */
+extern void IterateScripts(JSContext* cx, JS::Realm* realm, void* data,
+ IterateScriptCallback scriptCallback);
+
+JS::Realm* NewRealm(JSContext* cx, JSPrincipals* principals,
+ const JS::RealmOptions& options);
+
+namespace gc {
+
+void FinishGC(JSContext* cx, JS::GCReason = JS::GCReason::FINISH_GC);
+
+void WaitForBackgroundTasks(JSContext* cx);
+
+enum VerifierType { PreBarrierVerifier };
+
+#ifdef JS_GC_ZEAL
+
+extern const char ZealModeHelpText[];
+
+/* Check that write barriers have been used correctly. See gc/Verifier.cpp. */
+void VerifyBarriers(JSRuntime* rt, VerifierType type);
+
+void MaybeVerifyBarriers(JSContext* cx, bool always = false);
+
+void DumpArenaInfo();
+
+#else
+
+static inline void VerifyBarriers(JSRuntime* rt, VerifierType type) {}
+
+static inline void MaybeVerifyBarriers(JSContext* cx, bool always = false) {}
+
+#endif
+
+/*
+ * Instances of this class prevent GC from happening while they are live. If an
+ * allocation causes a heap threshold to be exceeded, no GC will be performed
+ * and the allocation will succeed. Allocation may still fail for other reasons.
+ *
+ * Use of this class is highly discouraged, since without GC system memory can
+ * become exhausted and this can cause crashes at places where we can't handle
+ * allocation failure.
+ *
+ * Use of this is permissible in situations where it would be impossible (or at
+ * least very difficult) to tolerate GC and where only a fixed number of objects
+ * are allocated, such as:
+ *
+ * - error reporting
+ * - JIT bailout handling
+ * - brain transplants (JSObject::swap)
+ * - debugging utilities not exposed to the browser
+ *
+ * This works by updating the |JSContext::suppressGC| counter which is checked
+ * at the start of GC.
+ */
+class MOZ_RAII JS_HAZ_GC_SUPPRESSED AutoSuppressGC
+ : public JS::AutoRequireNoGC {
+ int32_t& suppressGC_;
+
+ public:
+ explicit AutoSuppressGC(JSContext* cx);
+
+ ~AutoSuppressGC() { suppressGC_--; }
+};
+
+const char* StateName(State state);
+
+} /* namespace gc */
+
+/* Use this to avoid assertions when manipulating the wrapper map. */
+class MOZ_RAII AutoDisableProxyCheck {
+ public:
+#ifdef DEBUG
+ AutoDisableProxyCheck();
+ ~AutoDisableProxyCheck();
+#else
+ AutoDisableProxyCheck() {}
+#endif
+};
+
+struct MOZ_RAII AutoDisableCompactingGC {
+ explicit AutoDisableCompactingGC(JSContext* cx);
+ ~AutoDisableCompactingGC();
+
+ private:
+ JSContext* cx;
+};
+
+} /* namespace js */
+
+#endif /* gc_GC_h */
diff --git a/js/src/gc/GCAPI.cpp b/js/src/gc/GCAPI.cpp
new file mode 100644
index 0000000000..7911baaf93
--- /dev/null
+++ b/js/src/gc/GCAPI.cpp
@@ -0,0 +1,798 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * API functions and methods used by the rest of SpiderMonkey and by embeddings.
+ */
+
+#include "mozilla/TimeStamp.h"
+
+#include "jsapi.h"
+#include "jsfriendapi.h"
+
+#include "gc/GC.h"
+#include "gc/PublicIterators.h"
+#include "jit/JitRealm.h"
+#include "js/HeapAPI.h"
+#include "js/Value.h"
+#include "util/DifferentialTesting.h"
+#include "vm/HelperThreads.h"
+#include "vm/Realm.h"
+#include "vm/Scope.h"
+
+#include "gc/Marking-inl.h"
+#include "gc/StableCellHasher-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSContext-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::TimeStamp;
+
+extern JS_PUBLIC_API bool js::AddRawValueRoot(JSContext* cx, Value* vp,
+ const char* name) {
+ MOZ_ASSERT(vp);
+ MOZ_ASSERT(name);
+ bool ok = cx->runtime()->gc.addRoot(vp, name);
+ if (!ok) {
+ JS_ReportOutOfMemory(cx);
+ }
+ return ok;
+}
+
+extern JS_PUBLIC_API void js::RemoveRawValueRoot(JSContext* cx, Value* vp) {
+ cx->runtime()->gc.removeRoot(vp);
+}
+
+JS_PUBLIC_API JS::HeapState JS::RuntimeHeapState() {
+ return TlsContext.get()->runtime()->gc.heapState();
+}
+
+JS::AutoDisableGenerationalGC::AutoDisableGenerationalGC(JSContext* cx)
+ : cx(cx) {
+ if (!cx->generationalDisabled) {
+ cx->runtime()->gc.evictNursery(JS::GCReason::DISABLE_GENERATIONAL_GC);
+ cx->nursery().disable();
+ }
+ ++cx->generationalDisabled;
+ MOZ_ASSERT(cx->nursery().isEmpty());
+}
+
+JS::AutoDisableGenerationalGC::~AutoDisableGenerationalGC() {
+ if (--cx->generationalDisabled == 0 &&
+ cx->runtime()->gc.tunables.gcMaxNurseryBytes() > 0) {
+ cx->nursery().enable();
+ }
+}
+
+JS_PUBLIC_API bool JS::IsGenerationalGCEnabled(JSRuntime* rt) {
+ return !rt->mainContextFromOwnThread()->generationalDisabled;
+}
+
+AutoDisableCompactingGC::AutoDisableCompactingGC(JSContext* cx) : cx(cx) {
+ ++cx->compactingDisabledCount;
+ if (cx->runtime()->gc.isIncrementalGCInProgress() &&
+ cx->runtime()->gc.isCompactingGc()) {
+ FinishGC(cx);
+ }
+}
+
+AutoDisableCompactingGC::~AutoDisableCompactingGC() {
+ MOZ_ASSERT(cx->compactingDisabledCount > 0);
+ --cx->compactingDisabledCount;
+}
+
+#ifdef DEBUG
+
+/* Should only be called manually under gdb */
+void PreventGCDuringInteractiveDebug() { TlsContext.get()->suppressGC++; }
+
+#endif
+
+void js::ReleaseAllJITCode(JS::GCContext* gcx) {
+ js::CancelOffThreadIonCompile(gcx->runtime());
+
+ for (ZonesIter zone(gcx->runtime(), SkipAtoms); !zone.done(); zone.next()) {
+ zone->forceDiscardJitCode(gcx);
+ }
+
+ for (RealmsIter realm(gcx->runtime()); !realm.done(); realm.next()) {
+ if (jit::JitRealm* jitRealm = realm->jitRealm()) {
+ jitRealm->discardStubs();
+ }
+ }
+}
+
+AutoSuppressGC::AutoSuppressGC(JSContext* cx)
+ : suppressGC_(cx->suppressGC.ref()) {
+ suppressGC_++;
+}
+
+#ifdef DEBUG
+AutoDisableProxyCheck::AutoDisableProxyCheck() {
+ TlsContext.get()->disableStrictProxyChecking();
+}
+
+AutoDisableProxyCheck::~AutoDisableProxyCheck() {
+ TlsContext.get()->enableStrictProxyChecking();
+}
+
+JS_PUBLIC_API void JS::AssertGCThingMustBeTenured(JSObject* obj) {
+ MOZ_ASSERT(obj->isTenured());
+ MOZ_ASSERT(obj->getClass()->hasFinalize() &&
+ !(obj->getClass()->flags & JSCLASS_SKIP_NURSERY_FINALIZE));
+}
+
+JS_PUBLIC_API void JS::AssertGCThingIsNotNurseryAllocable(Cell* cell) {
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(!cell->is<JSObject>() && !cell->is<JSString>() &&
+ !cell->is<JS::BigInt>());
+}
+
+JS_PUBLIC_API void js::gc::AssertGCThingHasType(js::gc::Cell* cell,
+ JS::TraceKind kind) {
+ if (!cell) {
+ MOZ_ASSERT(kind == JS::TraceKind::Null);
+ return;
+ }
+
+ MOZ_ASSERT(IsCellPointerValid(cell));
+ MOZ_ASSERT(cell->getTraceKind() == kind);
+}
+#endif
+
+#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+
+JS::AutoAssertNoGC::AutoAssertNoGC(JSContext* maybecx) {
+ if (maybecx) {
+ cx_ = maybecx;
+ } else if (TlsContext.initialized()) {
+ cx_ = TlsContext.get();
+ } else {
+ cx_ = nullptr;
+ }
+ if (cx_) {
+ cx_->inUnsafeRegion++;
+ }
+}
+
+JS::AutoAssertNoGC::~AutoAssertNoGC() {
+ if (cx_) {
+ MOZ_ASSERT(cx_->inUnsafeRegion > 0);
+ cx_->inUnsafeRegion--;
+ }
+}
+
+#endif // MOZ_DIAGNOSTIC_ASSERT_ENABLED
+
+#ifdef DEBUG
+
+JS::AutoEnterCycleCollection::AutoEnterCycleCollection(JSRuntime* rt)
+ : runtime_(rt) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+ runtime_->gc.heapState_ = HeapState::CycleCollecting;
+}
+
+JS::AutoEnterCycleCollection::~AutoEnterCycleCollection() {
+ MOZ_ASSERT(JS::RuntimeHeapIsCycleCollecting());
+ runtime_->gc.heapState_ = HeapState::Idle;
+}
+
+JS::AutoAssertGCCallback::AutoAssertGCCallback() : AutoSuppressGCAnalysis() {
+ MOZ_ASSERT(JS::RuntimeHeapIsCollecting());
+}
+
+#endif // DEBUG
+
+JS_PUBLIC_API const char* JS::GCTraceKindToAscii(JS::TraceKind kind) {
+ switch (kind) {
+#define MAP_NAME(name, _0, _1, _2) \
+ case JS::TraceKind::name: \
+ return "JS " #name;
+ JS_FOR_EACH_TRACEKIND(MAP_NAME);
+#undef MAP_NAME
+ default:
+ return "Invalid";
+ }
+}
+
+JS_PUBLIC_API size_t JS::GCTraceKindSize(JS::TraceKind kind) {
+ switch (kind) {
+#define MAP_SIZE(name, type, _0, _1) \
+ case JS::TraceKind::name: \
+ return sizeof(type);
+ JS_FOR_EACH_TRACEKIND(MAP_SIZE);
+#undef MAP_SIZE
+ default:
+ return 0;
+ }
+}
+
+JS::GCCellPtr::GCCellPtr(const Value& v)
+ : GCCellPtr(v.toGCThing(), v.traceKind()) {}
+
+JS::TraceKind JS::GCCellPtr::outOfLineKind() const {
+ MOZ_ASSERT((ptr & OutOfLineTraceKindMask) == OutOfLineTraceKindMask);
+ MOZ_ASSERT(asCell()->isTenured());
+ return MapAllocToTraceKind(asCell()->asTenured().getAllocKind());
+}
+
+JS_PUBLIC_API void JS::PrepareZoneForGC(JSContext* cx, Zone* zone) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ // If we got the zone from a shared atom, we may have the wrong atoms zone
+ // here.
+ if (zone->isAtomsZone()) {
+ zone = cx->runtime()->atomsZone();
+ }
+
+ MOZ_ASSERT(cx->runtime()->gc.hasZone(zone));
+ zone->scheduleGC();
+}
+
+JS_PUBLIC_API void JS::PrepareForFullGC(JSContext* cx) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ cx->runtime()->gc.fullGCRequested = true;
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ zone->scheduleGC();
+ }
+}
+
+JS_PUBLIC_API void JS::PrepareForIncrementalGC(JSContext* cx) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ if (!JS::IsIncrementalGCInProgress(cx)) {
+ return;
+ }
+
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ if (zone->wasGCStarted()) {
+ zone->scheduleGC();
+ }
+ }
+}
+
+JS_PUBLIC_API bool JS::IsGCScheduled(JSContext* cx) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isGCScheduled()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+JS_PUBLIC_API void JS::SkipZoneForGC(JSContext* cx, Zone* zone) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+ MOZ_ASSERT(cx->runtime()->gc.hasZone(zone));
+
+ cx->runtime()->gc.fullGCRequested = false;
+ zone->unscheduleGC();
+}
+
+static inline void CheckGCOptions(JS::GCOptions options) {
+ MOZ_ASSERT(options == JS::GCOptions::Normal ||
+ options == JS::GCOptions::Shrink ||
+ options == JS::GCOptions::Shutdown);
+}
+
+JS_PUBLIC_API void JS::NonIncrementalGC(JSContext* cx, JS::GCOptions options,
+ GCReason reason) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+ CheckGCOptions(options);
+
+ cx->runtime()->gc.gc(options, reason);
+
+ MOZ_ASSERT(!IsIncrementalGCInProgress(cx));
+}
+
+JS_PUBLIC_API void JS::StartIncrementalGC(JSContext* cx, JS::GCOptions options,
+ GCReason reason,
+ const js::SliceBudget& budget) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+ CheckGCOptions(options);
+
+ cx->runtime()->gc.startGC(options, reason, budget);
+}
+
+JS_PUBLIC_API void JS::IncrementalGCSlice(JSContext* cx, GCReason reason,
+ const js::SliceBudget& budget) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ cx->runtime()->gc.gcSlice(reason, budget);
+}
+
+JS_PUBLIC_API bool JS::IncrementalGCHasForegroundWork(JSContext* cx) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ return cx->runtime()->gc.hasForegroundWork();
+}
+
+JS_PUBLIC_API void JS::FinishIncrementalGC(JSContext* cx, GCReason reason) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ cx->runtime()->gc.finishGC(reason);
+}
+
+JS_PUBLIC_API void JS::AbortIncrementalGC(JSContext* cx) {
+ AssertHeapIsIdle();
+ CHECK_THREAD(cx);
+
+ if (IsIncrementalGCInProgress(cx)) {
+ cx->runtime()->gc.abortGC();
+ }
+}
+
+char16_t* JS::GCDescription::formatSliceMessage(JSContext* cx) const {
+ UniqueChars cstr = cx->runtime()->gc.stats().formatCompactSliceMessage();
+
+ size_t nchars = strlen(cstr.get());
+ UniqueTwoByteChars out(js_pod_malloc<char16_t>(nchars + 1));
+ if (!out) {
+ return nullptr;
+ }
+ out.get()[nchars] = 0;
+
+ CopyAndInflateChars(out.get(), cstr.get(), nchars);
+ return out.release();
+}
+
+char16_t* JS::GCDescription::formatSummaryMessage(JSContext* cx) const {
+ UniqueChars cstr = cx->runtime()->gc.stats().formatCompactSummaryMessage();
+
+ size_t nchars = strlen(cstr.get());
+ UniqueTwoByteChars out(js_pod_malloc<char16_t>(nchars + 1));
+ if (!out) {
+ return nullptr;
+ }
+ out.get()[nchars] = 0;
+
+ CopyAndInflateChars(out.get(), cstr.get(), nchars);
+ return out.release();
+}
+
+JS::dbg::GarbageCollectionEvent::Ptr JS::GCDescription::toGCEvent(
+ JSContext* cx) const {
+ return JS::dbg::GarbageCollectionEvent::Create(
+ cx->runtime(), cx->runtime()->gc.stats(),
+ cx->runtime()->gc.majorGCCount());
+}
+
+TimeStamp JS::GCDescription::startTime(JSContext* cx) const {
+ return cx->runtime()->gc.stats().start();
+}
+
+TimeStamp JS::GCDescription::endTime(JSContext* cx) const {
+ return cx->runtime()->gc.stats().end();
+}
+
+TimeStamp JS::GCDescription::lastSliceStart(JSContext* cx) const {
+ return cx->runtime()->gc.stats().slices().back().start;
+}
+
+TimeStamp JS::GCDescription::lastSliceEnd(JSContext* cx) const {
+ return cx->runtime()->gc.stats().slices().back().end;
+}
+
+JS::UniqueChars JS::GCDescription::sliceToJSONProfiler(JSContext* cx) const {
+ size_t slices = cx->runtime()->gc.stats().slices().length();
+ MOZ_ASSERT(slices > 0);
+ return cx->runtime()->gc.stats().renderJsonSlice(slices - 1);
+}
+
+JS::UniqueChars JS::GCDescription::formatJSONProfiler(JSContext* cx) const {
+ return cx->runtime()->gc.stats().renderJsonMessage();
+}
+
+JS_PUBLIC_API JS::UniqueChars JS::MinorGcToJSON(JSContext* cx) {
+ JSRuntime* rt = cx->runtime();
+ return rt->gc.stats().renderNurseryJson();
+}
+
+JS_PUBLIC_API JS::GCSliceCallback JS::SetGCSliceCallback(
+ JSContext* cx, GCSliceCallback callback) {
+ return cx->runtime()->gc.setSliceCallback(callback);
+}
+
+JS_PUBLIC_API JS::DoCycleCollectionCallback JS::SetDoCycleCollectionCallback(
+ JSContext* cx, JS::DoCycleCollectionCallback callback) {
+ return cx->runtime()->gc.setDoCycleCollectionCallback(callback);
+}
+
+JS_PUBLIC_API JS::GCNurseryCollectionCallback
+JS::SetGCNurseryCollectionCallback(JSContext* cx,
+ GCNurseryCollectionCallback callback) {
+ return cx->runtime()->gc.setNurseryCollectionCallback(callback);
+}
+
+JS_PUBLIC_API void JS::SetLowMemoryState(JSContext* cx, bool newState) {
+ return cx->runtime()->gc.setLowMemoryState(newState);
+}
+
+JS_PUBLIC_API void JS::DisableIncrementalGC(JSContext* cx) {
+ cx->runtime()->gc.disallowIncrementalGC();
+}
+
+JS_PUBLIC_API bool JS::IsIncrementalGCEnabled(JSContext* cx) {
+ GCRuntime& gc = cx->runtime()->gc;
+ return gc.isIncrementalGCEnabled() && gc.isIncrementalGCAllowed();
+}
+
+JS_PUBLIC_API bool JS::IsIncrementalGCInProgress(JSContext* cx) {
+ return cx->runtime()->gc.isIncrementalGCInProgress();
+}
+
+JS_PUBLIC_API bool JS::IsIncrementalGCInProgress(JSRuntime* rt) {
+ return rt->gc.isIncrementalGCInProgress() &&
+ !rt->gc.isVerifyPreBarriersEnabled();
+}
+
+JS_PUBLIC_API bool JS::IsIncrementalBarrierNeeded(JSContext* cx) {
+ if (JS::RuntimeHeapIsBusy()) {
+ return false;
+ }
+
+ auto state = cx->runtime()->gc.state();
+ return state != gc::State::NotActive && state <= gc::State::Sweep;
+}
+
+JS_PUBLIC_API void JS::IncrementalPreWriteBarrier(JSObject* obj) {
+ if (!obj) {
+ return;
+ }
+
+ AutoGeckoProfilerEntry profilingStackFrame(
+ TlsContext.get(), "IncrementalPreWriteBarrier(JSObject*)",
+ JS::ProfilingCategoryPair::GCCC_Barrier);
+ PreWriteBarrier(obj);
+}
+
+JS_PUBLIC_API void JS::IncrementalPreWriteBarrier(GCCellPtr thing) {
+ if (!thing) {
+ return;
+ }
+
+ AutoGeckoProfilerEntry profilingStackFrame(
+ TlsContext.get(), "IncrementalPreWriteBarrier(GCCellPtr)",
+ JS::ProfilingCategoryPair::GCCC_Barrier);
+ CellPtrPreWriteBarrier(thing);
+}
+
+JS_PUBLIC_API bool JS::WasIncrementalGC(JSRuntime* rt) {
+ return rt->gc.isIncrementalGc();
+}
+
+bool js::gc::CreateUniqueIdForNativeObject(NativeObject* nobj, uint64_t* uidp) {
+ JSRuntime* runtime = nobj->runtimeFromMainThread();
+ *uidp = NextCellUniqueId(runtime);
+ JSContext* cx = runtime->mainContextFromOwnThread();
+ return nobj->setUniqueId(cx, *uidp);
+}
+
+bool js::gc::CreateUniqueIdForNonNativeObject(Cell* cell,
+ UniqueIdMap::AddPtr ptr,
+ uint64_t* uidp) {
+ // If the cell is in the nursery, hopefully unlikely, then we need to tell the
+ // nursery about it so that it can sweep the uid if the thing does not get
+ // tenured.
+ JSRuntime* runtime = cell->runtimeFromMainThread();
+ if (IsInsideNursery(cell) &&
+ !runtime->gc.nursery().addedUniqueIdToCell(cell)) {
+ return false;
+ }
+
+ // Set a new uid on the cell.
+ *uidp = NextCellUniqueId(runtime);
+ return cell->zone()->uniqueIds().add(ptr, cell, *uidp);
+}
+
+uint64_t js::gc::NextCellUniqueId(JSRuntime* rt) {
+ return rt->gc.nextCellUniqueId();
+}
+
+namespace js {
+
+static const struct GCParamInfo {
+ const char* name;
+ JSGCParamKey key;
+ bool writable;
+} GCParameters[] = {
+#define DEFINE_PARAM_INFO(name, key, writable) {name, key, writable},
+ FOR_EACH_GC_PARAM(DEFINE_PARAM_INFO)
+#undef DEFINE_PARAM_INFO
+};
+
+bool GetGCParameterInfo(const char* name, JSGCParamKey* keyOut,
+ bool* writableOut) {
+ MOZ_ASSERT(keyOut);
+ MOZ_ASSERT(writableOut);
+
+ for (const GCParamInfo& info : GCParameters) {
+ if (strcmp(name, info.name) == 0) {
+ *keyOut = info.key;
+ *writableOut = info.writable;
+ return true;
+ }
+ }
+
+ return false;
+}
+
+namespace gc {
+namespace MemInfo {
+
+static bool GCBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.heapSize.bytes()));
+ return true;
+}
+
+static bool MallocBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ double bytes = 0;
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ bytes += zone->mallocHeapSize.bytes();
+ }
+ args.rval().setNumber(bytes);
+ return true;
+}
+
+static bool GCMaxBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.tunables.gcMaxBytes()));
+ return true;
+}
+
+static bool GCHighFreqGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setBoolean(
+ cx->runtime()->gc.schedulingState.inHighFrequencyGCMode());
+ return true;
+}
+
+static bool GCNumberGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.gcNumber()));
+ return true;
+}
+
+static bool MajorGCCountGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.majorGCCount()));
+ return true;
+}
+
+static bool MinorGCCountGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.minorGCCount()));
+ return true;
+}
+
+static bool GCSliceCountGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.gcSliceCount()));
+ return true;
+}
+
+static bool GCCompartmentCount(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ size_t count = 0;
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ count += zone->compartments().length();
+ }
+
+ args.rval().setNumber(double(count));
+ return true;
+}
+
+static bool GCLastStartReason(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ const char* reason = ExplainGCReason(cx->runtime()->gc.lastStartReason());
+ RootedString str(cx, JS_NewStringCopyZ(cx, reason));
+ if (!str) {
+ return false;
+ }
+
+ args.rval().setString(str);
+ return true;
+}
+
+static bool ZoneGCBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcHeapSize.bytes()));
+ return true;
+}
+
+static bool ZoneGCTriggerBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcHeapThreshold.startBytes()));
+ return true;
+}
+
+static bool ZoneGCAllocTriggerGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ bool highFrequency =
+ cx->runtime()->gc.schedulingState.inHighFrequencyGCMode();
+ args.rval().setNumber(
+ double(cx->zone()->gcHeapThreshold.eagerAllocTrigger(highFrequency)));
+ return true;
+}
+
+static bool ZoneMallocBytesGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->mallocHeapSize.bytes()));
+ return true;
+}
+
+static bool ZoneMallocTriggerBytesGetter(JSContext* cx, unsigned argc,
+ Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->mallocHeapThreshold.startBytes()));
+ return true;
+}
+
+static bool ZoneGCNumberGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.gcNumber()));
+ return true;
+}
+
+#ifdef DEBUG
+static bool DummyGetter(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setUndefined();
+ return true;
+}
+#endif
+
+} /* namespace MemInfo */
+
+JSObject* NewMemoryInfoObject(JSContext* cx) {
+ RootedObject obj(cx, JS_NewObject(cx, nullptr));
+ if (!obj) {
+ return nullptr;
+ }
+
+ using namespace MemInfo;
+ struct NamedGetter {
+ const char* name;
+ JSNative getter;
+ } getters[] = {{"gcBytes", GCBytesGetter},
+ {"gcMaxBytes", GCMaxBytesGetter},
+ {"mallocBytes", MallocBytesGetter},
+ {"gcIsHighFrequencyMode", GCHighFreqGetter},
+ {"gcNumber", GCNumberGetter},
+ {"majorGCCount", MajorGCCountGetter},
+ {"minorGCCount", MinorGCCountGetter},
+ {"sliceCount", GCSliceCountGetter},
+ {"compartmentCount", GCCompartmentCount},
+ {"lastStartReason", GCLastStartReason}};
+
+ for (auto pair : getters) {
+ JSNative getter = pair.getter;
+
+#ifdef DEBUG
+ if (js::SupportDifferentialTesting()) {
+ getter = DummyGetter;
+ }
+#endif
+
+ if (!JS_DefineProperty(cx, obj, pair.name, getter, nullptr,
+ JSPROP_ENUMERATE)) {
+ return nullptr;
+ }
+ }
+
+ RootedObject zoneObj(cx, JS_NewObject(cx, nullptr));
+ if (!zoneObj) {
+ return nullptr;
+ }
+
+ if (!JS_DefineProperty(cx, obj, "zone", zoneObj, JSPROP_ENUMERATE)) {
+ return nullptr;
+ }
+
+ struct NamedZoneGetter {
+ const char* name;
+ JSNative getter;
+ } zoneGetters[] = {{"gcBytes", ZoneGCBytesGetter},
+ {"gcTriggerBytes", ZoneGCTriggerBytesGetter},
+ {"gcAllocTrigger", ZoneGCAllocTriggerGetter},
+ {"mallocBytes", ZoneMallocBytesGetter},
+ {"mallocTriggerBytes", ZoneMallocTriggerBytesGetter},
+ {"gcNumber", ZoneGCNumberGetter}};
+
+ for (auto pair : zoneGetters) {
+ JSNative getter = pair.getter;
+
+#ifdef DEBUG
+ if (js::SupportDifferentialTesting()) {
+ getter = DummyGetter;
+ }
+#endif
+
+ if (!JS_DefineProperty(cx, zoneObj, pair.name, getter, nullptr,
+ JSPROP_ENUMERATE)) {
+ return nullptr;
+ }
+ }
+
+ return obj;
+}
+
+const char* StateName(State state) {
+ switch (state) {
+#define MAKE_CASE(name) \
+ case State::name: \
+ return #name;
+ GCSTATES(MAKE_CASE)
+#undef MAKE_CASE
+ }
+ MOZ_CRASH("Invalid gc::State enum value");
+}
+
+const char* StateName(JS::Zone::GCState state) {
+ switch (state) {
+ case JS::Zone::NoGC:
+ return "NoGC";
+ case JS::Zone::Prepare:
+ return "Prepare";
+ case JS::Zone::MarkBlackOnly:
+ return "MarkBlackOnly";
+ case JS::Zone::MarkBlackAndGray:
+ return "MarkBlackAndGray";
+ case JS::Zone::Sweep:
+ return "Sweep";
+ case JS::Zone::Finished:
+ return "Finished";
+ case JS::Zone::Compact:
+ return "Compact";
+ case JS::Zone::VerifyPreBarriers:
+ return "VerifyPreBarriers";
+ case JS::Zone::Limit:
+ break;
+ }
+ MOZ_CRASH("Invalid Zone::GCState enum value");
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+JS_PUBLIC_API void js::gc::FinalizeDeadNurseryObject(JSContext* cx,
+ JSObject* obj) {
+ CHECK_THREAD(cx);
+ MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(IsInsideNursery(obj));
+ MOZ_ASSERT(!IsForwarded(obj));
+
+ const JSClass* jsClass = JS::GetClass(obj);
+ jsClass->doFinalize(cx->gcContext(), obj);
+}
+
+JS_PUBLIC_API void js::gc::SetPerformanceHint(JSContext* cx,
+ PerformanceHint hint) {
+ CHECK_THREAD(cx);
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+
+ cx->runtime()->gc.setPerformanceHint(hint);
+}
diff --git a/js/src/gc/GCContext-inl.h b/js/src/gc/GCContext-inl.h
new file mode 100644
index 0000000000..7b53d50b77
--- /dev/null
+++ b/js/src/gc/GCContext-inl.h
@@ -0,0 +1,40 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCContext_inl_h
+#define gc_GCContext_inl_h
+
+#include "gc/GCContext.h"
+
+#include "gc/ZoneAllocator.h"
+
+inline void JS::GCContext::free_(Cell* cell, void* p, size_t nbytes,
+ MemoryUse use) {
+ if (p) {
+ removeCellMemory(cell, nbytes, use);
+ js_free(p);
+ }
+}
+
+template <class T>
+inline void JS::GCContext::release(Cell* cell, T* p, size_t nbytes,
+ MemoryUse use) {
+ if (p) {
+ removeCellMemory(cell, nbytes, use);
+ p->Release();
+ }
+}
+
+inline void JS::GCContext::removeCellMemory(Cell* cell, size_t nbytes,
+ MemoryUse use) {
+ // This may or may not be called as part of GC.
+ if (nbytes && cell->isTenured()) {
+ auto zone = js::ZoneAllocator::from(cell->asTenured().zoneFromAnyThread());
+ zone->removeCellMemory(cell, nbytes, use, isFinalizing());
+ }
+}
+
+#endif // gc_GCContext_inl_h
diff --git a/js/src/gc/GCContext.h b/js/src/gc/GCContext.h
new file mode 100644
index 0000000000..8c74a81e0c
--- /dev/null
+++ b/js/src/gc/GCContext.h
@@ -0,0 +1,257 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCContext_h
+#define gc_GCContext_h
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+#include "mozilla/ThreadLocal.h"
+
+#include "jspubtd.h"
+#include "jstypes.h" // JS_PUBLIC_API
+#include "gc/GCEnum.h" // js::MemoryUse
+#include "jit/ExecutableAllocator.h" // jit::JitPoisonRangeVector
+#include "js/Utility.h" // js_free
+
+struct JS_PUBLIC_API JSRuntime;
+
+namespace js {
+
+class AutoTouchingGrayThings;
+
+namespace gc {
+
+class AutoSetThreadGCUse;
+class AutoSetThreadIsSweeping;
+
+enum class GCUse {
+ // This thread is not running in the garbage collector.
+ None,
+
+ // This thread is currently collecting. Used when no finer detail is known.
+ Unspecified,
+
+ // This thread is currently marking GC things. This thread could be the main
+ // thread or a helper thread doing sweep-marking.
+ Marking,
+
+ // This thread is currently sweeping GC things. This thread could be the
+ // main thread or a helper thread while the main thread is running the
+ // mutator.
+ Sweeping,
+
+ // Whether this thread is currently finalizing GC things. This thread could
+ // be the main thread or a helper thread doing finalization while the main
+ // thread is running the mutator.
+ Finalizing
+};
+
+} // namespace gc
+} // namespace js
+
+namespace JS {
+
+/*
+ * GCContext is by GC operations that can run on or off the main thread.
+ *
+ * Its main function is to provide methods to free memory and update memory
+ * accounting. For convenience, it also has delete_ convenience methods that
+ * also call destructors.
+ *
+ * It is passed to finalizers and other sweep-phase hooks as JSContext is not
+ * available off the main thread.
+ */
+class GCContext {
+ using Cell = js::gc::Cell;
+ using MemoryUse = js::MemoryUse;
+
+ JSRuntime* const runtime_;
+
+ js::jit::JitPoisonRangeVector jitPoisonRanges;
+
+ // Which part of the garbage collector this context is running at the moment.
+ js::gc::GCUse gcUse_ = js::gc::GCUse::None;
+ friend class js::gc::AutoSetThreadGCUse;
+ friend class js::gc::AutoSetThreadIsSweeping;
+
+#ifdef DEBUG
+ // The specific zone currently being swept, if any.
+ Zone* gcSweepZone_ = nullptr;
+
+ // Whether this thread is currently manipulating possibly-gray GC things.
+ size_t isTouchingGrayThings_ = false;
+ friend class js::AutoTouchingGrayThings;
+#endif
+
+ public:
+ explicit GCContext(JSRuntime* maybeRuntime);
+ ~GCContext();
+
+ JSRuntime* runtime() const {
+ MOZ_ASSERT(onMainThread());
+ return runtimeFromAnyThread();
+ }
+ JSRuntime* runtimeFromAnyThread() const {
+ MOZ_ASSERT(runtime_);
+ return runtime_;
+ }
+
+ js::gc::GCUse gcUse() const { return gcUse_; }
+ bool isCollecting() const { return gcUse() != js::gc::GCUse::None; }
+ bool isFinalizing() const { return gcUse_ == js::gc::GCUse::Finalizing; }
+
+#ifdef DEBUG
+ bool onMainThread() const {
+ return js::CurrentThreadCanAccessRuntime(runtime_);
+ }
+
+ Zone* gcSweepZone() const { return gcSweepZone_; }
+ bool isTouchingGrayThings() const { return isTouchingGrayThings_; }
+#endif
+
+ // Deprecated. Where possible, memory should be tracked against the owning GC
+ // thing by calling js::AddCellMemory and the memory freed with free_() below.
+ void freeUntracked(void* p) { js_free(p); }
+
+ // Free memory associated with a GC thing and update the memory accounting.
+ //
+ // The memory should have been associated with the GC thing using
+ // js::InitReservedSlot or js::InitObjectPrivate, or possibly
+ // js::AddCellMemory.
+ void free_(Cell* cell, void* p, size_t nbytes, MemoryUse use);
+
+ bool appendJitPoisonRange(const js::jit::JitPoisonRange& range) {
+ return jitPoisonRanges.append(range);
+ }
+ bool hasJitCodeToPoison() const { return !jitPoisonRanges.empty(); }
+ void poisonJitCode();
+
+ // Deprecated. Where possible, memory should be tracked against the owning GC
+ // thing by calling js::AddCellMemory and the memory freed with delete_()
+ // below.
+ template <class T>
+ void deleteUntracked(T* p) {
+ if (p) {
+ p->~T();
+ js_free(p);
+ }
+ }
+
+ // Delete a C++ object that was associated with a GC thing and update the
+ // memory accounting. The size is determined by the type T.
+ //
+ // The memory should have been associated with the GC thing using
+ // js::InitReservedSlot or js::InitObjectPrivate, or possibly
+ // js::AddCellMemory.
+ template <class T>
+ void delete_(Cell* cell, T* p, MemoryUse use) {
+ delete_(cell, p, sizeof(T), use);
+ }
+
+ // Delete a C++ object that was associated with a GC thing and update the
+ // memory accounting.
+ //
+ // The memory should have been associated with the GC thing using
+ // js::InitReservedSlot or js::InitObjectPrivate, or possibly
+ // js::AddCellMemory.
+ template <class T>
+ void delete_(Cell* cell, T* p, size_t nbytes, MemoryUse use) {
+ if (p) {
+ p->~T();
+ free_(cell, p, nbytes, use);
+ }
+ }
+
+ // Release a RefCounted object that was associated with a GC thing and update
+ // the memory accounting.
+ //
+ // The memory should have been associated with the GC thing using
+ // js::InitReservedSlot or js::InitObjectPrivate, or possibly
+ // js::AddCellMemory.
+ //
+ // This counts the memory once per association with a GC thing. It's not
+ // expected that the same object is associated with more than one GC thing in
+ // each zone. If this is the case then some other form of accounting would be
+ // more appropriate.
+ template <class T>
+ void release(Cell* cell, T* p, MemoryUse use) {
+ release(cell, p, sizeof(T), use);
+ }
+
+ // Release a RefCounted object and that was associated with a GC thing and
+ // update the memory accounting.
+ //
+ // The memory should have been associated with the GC thing using
+ // js::InitReservedSlot or js::InitObjectPrivate, or possibly
+ // js::AddCellMemory.
+ template <class T>
+ void release(Cell* cell, T* p, size_t nbytes, MemoryUse use);
+
+ // Update the memory accounting for a GC for memory freed by some other
+ // method.
+ void removeCellMemory(Cell* cell, size_t nbytes, MemoryUse use);
+};
+
+} // namespace JS
+
+namespace js {
+
+/* Thread Local Storage for storing the GCContext for a thread. */
+extern MOZ_THREAD_LOCAL(JS::GCContext*) TlsGCContext;
+
+inline JS::GCContext* MaybeGetGCContext() {
+ if (!TlsGCContext.init()) {
+ return nullptr;
+ }
+ return TlsGCContext.get();
+}
+
+class MOZ_RAII AutoTouchingGrayThings {
+ public:
+#ifdef DEBUG
+ AutoTouchingGrayThings() { TlsGCContext.get()->isTouchingGrayThings_++; }
+ ~AutoTouchingGrayThings() {
+ JS::GCContext* gcx = TlsGCContext.get();
+ MOZ_ASSERT(gcx->isTouchingGrayThings_);
+ gcx->isTouchingGrayThings_--;
+ }
+#else
+ AutoTouchingGrayThings() {}
+#endif
+};
+
+#ifdef DEBUG
+
+inline bool CurrentThreadIsGCMarking() {
+ JS::GCContext* gcx = MaybeGetGCContext();
+ return gcx && gcx->gcUse() == gc::GCUse::Marking;
+}
+
+inline bool CurrentThreadIsGCSweeping() {
+ JS::GCContext* gcx = MaybeGetGCContext();
+ return gcx && gcx->gcUse() == gc::GCUse::Sweeping;
+}
+
+inline bool CurrentThreadIsGCFinalizing() {
+ JS::GCContext* gcx = MaybeGetGCContext();
+ return gcx && gcx->gcUse() == gc::GCUse::Finalizing;
+}
+
+inline bool CurrentThreadIsTouchingGrayThings() {
+ JS::GCContext* gcx = MaybeGetGCContext();
+ return gcx && gcx->isTouchingGrayThings();
+}
+
+inline bool CurrentThreadIsPerformingGC() {
+ JS::GCContext* gcx = MaybeGetGCContext();
+ return gcx && gcx->isCollecting();
+}
+
+#endif
+
+} // namespace js
+
+#endif // gc_GCContext_h
diff --git a/js/src/gc/GCEnum.h b/js/src/gc/GCEnum.h
new file mode 100644
index 0000000000..7434e34b1c
--- /dev/null
+++ b/js/src/gc/GCEnum.h
@@ -0,0 +1,160 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal enum definitions.
+ */
+
+#ifndef gc_GCEnum_h
+#define gc_GCEnum_h
+
+#include <stdint.h>
+
+#include "js/MemoryFunctions.h" // JS_FOR_EACH_PUBLIC_MEMORY_USE
+
+namespace js {
+namespace gc {
+
+// The phases of an incremental GC.
+#define GCSTATES(D) \
+ D(NotActive) \
+ D(Prepare) \
+ D(MarkRoots) \
+ D(Mark) \
+ D(Sweep) \
+ D(Finalize) \
+ D(Compact) \
+ D(Decommit) \
+ D(Finish)
+enum class State {
+#define MAKE_STATE(name) name,
+ GCSTATES(MAKE_STATE)
+#undef MAKE_STATE
+};
+
+#define JS_FOR_EACH_ZEAL_MODE(D) \
+ D(RootsChange, 1) \
+ D(Alloc, 2) \
+ D(VerifierPre, 4) \
+ D(YieldBeforeRootMarking, 6) \
+ D(GenerationalGC, 7) \
+ D(YieldBeforeMarking, 8) \
+ D(YieldBeforeSweeping, 9) \
+ D(IncrementalMultipleSlices, 10) \
+ D(IncrementalMarkingValidator, 11) \
+ D(ElementsBarrier, 12) \
+ D(CheckHashTablesOnMinorGC, 13) \
+ D(Compact, 14) \
+ D(CheckHeapAfterGC, 15) \
+ D(YieldBeforeSweepingAtoms, 17) \
+ D(CheckGrayMarking, 18) \
+ D(YieldBeforeSweepingCaches, 19) \
+ D(YieldBeforeSweepingObjects, 21) \
+ D(YieldBeforeSweepingNonObjects, 22) \
+ D(YieldBeforeSweepingPropMapTrees, 23) \
+ D(CheckWeakMapMarking, 24) \
+ D(YieldWhileGrayMarking, 25)
+
+enum class ZealMode {
+#define ZEAL_MODE(name, value) name = value,
+ JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE)
+#undef ZEAL_MODE
+ Count,
+ Limit = Count - 1
+};
+
+} /* namespace gc */
+
+// Reasons we reset an ongoing incremental GC or perform a non-incremental GC.
+#define GC_ABORT_REASONS(D) \
+ D(None, 0) \
+ D(NonIncrementalRequested, 1) \
+ D(AbortRequested, 2) \
+ D(Unused1, 3) \
+ D(IncrementalDisabled, 4) \
+ D(ModeChange, 5) \
+ D(MallocBytesTrigger, 6) \
+ D(GCBytesTrigger, 7) \
+ D(ZoneChange, 8) \
+ D(CompartmentRevived, 9) \
+ D(GrayRootBufferingFailed, 10) \
+ D(JitCodeBytesTrigger, 11)
+enum class GCAbortReason {
+#define MAKE_REASON(name, num) name = num,
+ GC_ABORT_REASONS(MAKE_REASON)
+#undef MAKE_REASON
+};
+
+#define JS_FOR_EACH_INTERNAL_MEMORY_USE(_) \
+ _(ArrayBufferContents) \
+ _(StringContents) \
+ _(ObjectElements) \
+ _(ObjectSlots) \
+ _(ScriptPrivateData) \
+ _(MapObjectTable) \
+ _(BigIntDigits) \
+ _(ScopeData) \
+ _(WeakMapObject) \
+ _(ShapeSetForAdd) \
+ _(PropMapChildren) \
+ _(PropMapTable) \
+ _(ModuleBindingMap) \
+ _(ModuleCyclicFields) \
+ _(ModuleExports) \
+ _(BaselineScript) \
+ _(IonScript) \
+ _(ArgumentsData) \
+ _(RareArgumentsData) \
+ _(RegExpSharedBytecode) \
+ _(RegExpSharedNamedCaptureData) \
+ _(TypedArrayElements) \
+ _(NativeIterator) \
+ _(JitScript) \
+ _(ScriptDebugScript) \
+ _(BreakpointSite) \
+ _(Breakpoint) \
+ _(ForOfPIC) \
+ _(ForOfPICStub) \
+ _(WasmInstanceExports) \
+ _(WasmInstanceScopes) \
+ _(WasmInstanceGlobals) \
+ _(WasmInstanceInstance) \
+ _(WasmMemoryObservers) \
+ _(WasmGlobalCell) \
+ _(WasmResolveResponseClosure) \
+ _(WasmModule) \
+ _(WasmTableTable) \
+ _(WasmExceptionData) \
+ _(WasmTagType) \
+ _(FileObjectFile) \
+ _(Debugger) \
+ _(DebuggerFrameGeneratorInfo) \
+ _(DebuggerFrameIterData) \
+ _(DebuggerOnStepHandler) \
+ _(DebuggerOnPopHandler) \
+ _(ICUObject) \
+ _(FinalizationRegistryRecordVector) \
+ _(FinalizationRegistryRegistrations) \
+ _(FinalizationRecordVector) \
+ _(TrackedAllocPolicy) \
+ _(SharedArrayRawBuffer) \
+ _(XDRBufferElements) \
+ _(GlobalObjectData) \
+ _(ProxyExternalValueArray)
+
+#define JS_FOR_EACH_MEMORY_USE(_) \
+ JS_FOR_EACH_PUBLIC_MEMORY_USE(_) \
+ JS_FOR_EACH_INTERNAL_MEMORY_USE(_)
+
+enum class MemoryUse : uint8_t {
+#define DEFINE_MEMORY_USE(Name) Name,
+ JS_FOR_EACH_MEMORY_USE(DEFINE_MEMORY_USE)
+#undef DEFINE_MEMORY_USE
+};
+
+} /* namespace js */
+
+#endif /* gc_GCEnum_h */
diff --git a/js/src/gc/GCInternals.h b/js/src/gc/GCInternals.h
new file mode 100644
index 0000000000..c234ad4b2b
--- /dev/null
+++ b/js/src/gc/GCInternals.h
@@ -0,0 +1,344 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definitions.
+ */
+
+#ifndef gc_GCInternals_h
+#define gc_GCInternals_h
+
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/Cell.h"
+#include "gc/GC.h"
+#include "gc/GCContext.h"
+#include "vm/GeckoProfiler.h"
+#include "vm/HelperThreads.h"
+#include "vm/JSContext.h"
+
+namespace js {
+namespace gc {
+
+/*
+ * There are a couple of classes here that serve mostly as "tokens" indicating
+ * that a precondition holds. Some functions force the caller to possess such a
+ * token because they require the precondition to hold, and it is better to make
+ * the precondition explicit at the API entry point than to crash in an
+ * assertion later on when it is relied upon.
+ */
+
+struct MOZ_RAII AutoAssertNoNurseryAlloc {
+#ifdef DEBUG
+ AutoAssertNoNurseryAlloc();
+ ~AutoAssertNoNurseryAlloc();
+#else
+ AutoAssertNoNurseryAlloc() {}
+#endif
+};
+
+/*
+ * A class that serves as a token that the nursery in the current thread's zone
+ * group is empty.
+ */
+class MOZ_RAII AutoAssertEmptyNursery {
+ protected:
+ JSContext* cx;
+
+ mozilla::Maybe<AutoAssertNoNurseryAlloc> noAlloc;
+
+ // Check that the nursery is empty.
+ void checkCondition(JSContext* cx);
+
+ // For subclasses that need to empty the nursery in their constructors.
+ AutoAssertEmptyNursery() : cx(nullptr) {}
+
+ public:
+ explicit AutoAssertEmptyNursery(JSContext* cx) : cx(nullptr) {
+ checkCondition(cx);
+ }
+
+ AutoAssertEmptyNursery(const AutoAssertEmptyNursery& other)
+ : AutoAssertEmptyNursery(other.cx) {}
+};
+
+/*
+ * Evict the nursery upon construction. Serves as a token indicating that the
+ * nursery is empty. (See AutoAssertEmptyNursery, above.)
+ */
+class MOZ_RAII AutoEmptyNursery : public AutoAssertEmptyNursery {
+ public:
+ explicit AutoEmptyNursery(JSContext* cx);
+};
+
+// Abstract base class for exclusive heap access for tracing or GC.
+class MOZ_RAII AutoHeapSession {
+ public:
+ ~AutoHeapSession();
+
+ protected:
+ AutoHeapSession(GCRuntime* gc, JS::HeapState state);
+
+ private:
+ AutoHeapSession(const AutoHeapSession&) = delete;
+ void operator=(const AutoHeapSession&) = delete;
+
+ GCRuntime* gc;
+ JS::HeapState prevState;
+ mozilla::Maybe<AutoGeckoProfilerEntry> profilingStackFrame;
+};
+
+class MOZ_RAII AutoGCSession : public AutoHeapSession {
+ public:
+ explicit AutoGCSession(GCRuntime* gc, JS::HeapState state)
+ : AutoHeapSession(gc, state) {}
+};
+
+class MOZ_RAII AutoMajorGCProfilerEntry : public AutoGeckoProfilerEntry {
+ public:
+ explicit AutoMajorGCProfilerEntry(GCRuntime* gc);
+};
+
+class MOZ_RAII AutoTraceSession : public AutoHeapSession {
+ public:
+ explicit AutoTraceSession(JSRuntime* rt)
+ : AutoHeapSession(&rt->gc, JS::HeapState::Tracing) {}
+};
+
+struct MOZ_RAII AutoFinishGC {
+ explicit AutoFinishGC(JSContext* cx, JS::GCReason reason) {
+ FinishGC(cx, reason);
+ }
+};
+
+// This class should be used by any code that needs exclusive access to the heap
+// in order to trace through it.
+class MOZ_RAII AutoPrepareForTracing : private AutoFinishGC,
+ public AutoTraceSession {
+ public:
+ explicit AutoPrepareForTracing(JSContext* cx)
+ : AutoFinishGC(cx, JS::GCReason::PREPARE_FOR_TRACING),
+ AutoTraceSession(cx->runtime()) {}
+};
+
+// This class should be used by any code that needs exclusive access to the heap
+// in order to trace through it.
+//
+// This version also empties the nursery after finishing any ongoing GC.
+class MOZ_RAII AutoEmptyNurseryAndPrepareForTracing : private AutoFinishGC,
+ public AutoEmptyNursery,
+ public AutoTraceSession {
+ public:
+ explicit AutoEmptyNurseryAndPrepareForTracing(JSContext* cx)
+ : AutoFinishGC(cx, JS::GCReason::PREPARE_FOR_TRACING),
+ AutoEmptyNursery(cx),
+ AutoTraceSession(cx->runtime()) {}
+};
+
+/*
+ * Temporarily disable incremental barriers.
+ */
+class AutoDisableBarriers {
+ public:
+ explicit AutoDisableBarriers(GCRuntime* gc);
+ ~AutoDisableBarriers();
+
+ private:
+ GCRuntime* gc;
+};
+
+// Set compartments' maybeAlive flags if anything is marked while this class is
+// live. This is used while marking roots.
+class AutoUpdateLiveCompartments {
+ GCRuntime* gc;
+
+ public:
+ explicit AutoUpdateLiveCompartments(GCRuntime* gc);
+ ~AutoUpdateLiveCompartments();
+};
+
+class MOZ_RAII AutoRunParallelTask : public GCParallelTask {
+ // This class takes a pointer to a member function of GCRuntime.
+ using TaskFunc = JS_MEMBER_FN_PTR_TYPE(GCRuntime, void);
+
+ TaskFunc func_;
+ AutoLockHelperThreadState& lock_;
+
+ public:
+ AutoRunParallelTask(GCRuntime* gc, TaskFunc func, gcstats::PhaseKind phase,
+ GCUse use, AutoLockHelperThreadState& lock)
+ : GCParallelTask(gc, phase, use), func_(func), lock_(lock) {
+ gc->startTask(*this, lock_);
+ }
+
+ ~AutoRunParallelTask() { gc->joinTask(*this, lock_); }
+
+ void run(AutoLockHelperThreadState& lock) override {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ // The hazard analysis can't tell what the call to func_ will do but it's
+ // not allowed to GC.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ // Call pointer to member function on |gc|.
+ JS_CALL_MEMBER_FN_PTR(gc, func_);
+ }
+};
+
+GCAbortReason IsIncrementalGCUnsafe(JSRuntime* rt);
+
+#ifdef JS_GC_ZEAL
+
+class MOZ_RAII AutoStopVerifyingBarriers {
+ GCRuntime* gc;
+ bool restartPreVerifier;
+
+ public:
+ AutoStopVerifyingBarriers(JSRuntime* rt, bool isShutdown) : gc(&rt->gc) {
+ if (gc->isVerifyPreBarriersEnabled()) {
+ gc->endVerifyPreBarriers();
+ restartPreVerifier = !isShutdown;
+ } else {
+ restartPreVerifier = false;
+ }
+ }
+
+ ~AutoStopVerifyingBarriers() {
+ // Nasty special case: verification runs a minor GC, which *may* nest
+ // inside of an outer minor GC. This is not allowed by the
+ // gc::Statistics phase tree. So we pause the "real" GC, if in fact one
+ // is in progress.
+ gcstats::PhaseKind outer = gc->stats().currentPhaseKind();
+ if (outer != gcstats::PhaseKind::NONE) {
+ gc->stats().endPhase(outer);
+ }
+ MOZ_ASSERT(gc->stats().currentPhaseKind() == gcstats::PhaseKind::NONE);
+
+ if (restartPreVerifier) {
+ gc->startVerifyPreBarriers();
+ }
+
+ if (outer != gcstats::PhaseKind::NONE) {
+ gc->stats().beginPhase(outer);
+ }
+ }
+};
+#else
+struct MOZ_RAII AutoStopVerifyingBarriers {
+ AutoStopVerifyingBarriers(JSRuntime*, bool) {}
+};
+#endif /* JS_GC_ZEAL */
+
+class MOZ_RAII AutoPoisonFreedJitCode {
+ JS::GCContext* const gcx;
+
+ public:
+ explicit AutoPoisonFreedJitCode(JS::GCContext* gcx) : gcx(gcx) {}
+ ~AutoPoisonFreedJitCode() { gcx->poisonJitCode(); }
+};
+
+// Set/restore the GCContext GC use flag for the current thread.
+
+class MOZ_RAII AutoSetThreadGCUse {
+ public:
+ AutoSetThreadGCUse(JS::GCContext* gcx, GCUse use)
+ : gcx(gcx), prevUse(gcx->gcUse_) {
+ gcx->gcUse_ = use;
+ }
+ explicit AutoSetThreadGCUse(GCUse use)
+ : AutoSetThreadGCUse(TlsGCContext.get(), use) {}
+
+ ~AutoSetThreadGCUse() { gcx->gcUse_ = prevUse; }
+
+ protected:
+ JS::GCContext* gcx;
+ GCUse prevUse;
+};
+
+template <GCUse Use>
+class AutoSetThreadGCUseT : public AutoSetThreadGCUse {
+ public:
+ explicit AutoSetThreadGCUseT(JS::GCContext* gcx)
+ : AutoSetThreadGCUse(gcx, Use) {}
+ AutoSetThreadGCUseT() : AutoSetThreadGCUseT(TlsGCContext.get()) {}
+};
+
+using AutoSetThreadIsPerformingGC = AutoSetThreadGCUseT<GCUse::Unspecified>;
+using AutoSetThreadIsMarking = AutoSetThreadGCUseT<GCUse::Marking>;
+using AutoSetThreadIsFinalizing = AutoSetThreadGCUseT<GCUse::Finalizing>;
+
+class AutoSetThreadIsSweeping : public AutoSetThreadGCUseT<GCUse::Sweeping> {
+ public:
+ explicit AutoSetThreadIsSweeping(JS::GCContext* gcx,
+ JS::Zone* sweepZone = nullptr)
+ : AutoSetThreadGCUseT(gcx) {
+#ifdef DEBUG
+ prevZone = gcx->gcSweepZone_;
+ gcx->gcSweepZone_ = sweepZone;
+#endif
+ }
+ explicit AutoSetThreadIsSweeping(JS::Zone* sweepZone = nullptr)
+ : AutoSetThreadIsSweeping(TlsGCContext.get(), sweepZone) {}
+
+ ~AutoSetThreadIsSweeping() {
+#ifdef DEBUG
+ MOZ_ASSERT_IF(prevUse == GCUse::None, !prevZone);
+ gcx->gcSweepZone_ = prevZone;
+#endif
+ }
+
+ private:
+#ifdef DEBUG
+ JS::Zone* prevZone;
+#endif
+};
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void CheckHashTablesAfterMovingGC(JSRuntime* rt);
+void CheckHeapAfterGC(JSRuntime* rt);
+#endif
+
+struct MovingTracer final : public GenericTracerImpl<MovingTracer> {
+ explicit MovingTracer(JSRuntime* rt);
+
+ private:
+ template <typename T>
+ void onEdge(T** thingp, const char* name);
+ friend class GenericTracerImpl<MovingTracer>;
+};
+
+struct MinorSweepingTracer final
+ : public GenericTracerImpl<MinorSweepingTracer> {
+ explicit MinorSweepingTracer(JSRuntime* rt);
+
+ private:
+ template <typename T>
+ void onEdge(T** thingp, const char* name);
+ friend class GenericTracerImpl<MinorSweepingTracer>;
+};
+
+extern void DelayCrossCompartmentGrayMarking(GCMarker* maybeMarker,
+ JSObject* src);
+
+inline bool IsOOMReason(JS::GCReason reason) {
+ return reason == JS::GCReason::LAST_DITCH ||
+ reason == JS::GCReason::MEM_PRESSURE;
+}
+
+void* AllocateCellInGC(JS::Zone* zone, AllocKind thingKind);
+
+void ReadProfileEnv(const char* envName, const char* helpText, bool* enableOut,
+ bool* workersOut, mozilla::TimeDuration* thresholdOut);
+
+bool ShouldPrintProfile(JSRuntime* runtime, bool enable, bool workers,
+ mozilla::TimeDuration threshold,
+ mozilla::TimeDuration duration);
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_GCInternals_h */
diff --git a/js/src/gc/GCLock.h b/js/src/gc/GCLock.h
new file mode 100644
index 0000000000..64c28ac544
--- /dev/null
+++ b/js/src/gc/GCLock.h
@@ -0,0 +1,110 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal classes for acquiring and releasing the GC lock.
+ */
+
+#ifndef gc_GCLock_h
+#define gc_GCLock_h
+
+#include "vm/Runtime.h"
+
+namespace js {
+
+class AutoUnlockGC;
+
+/*
+ * RAII class that takes the GC lock while it is live.
+ *
+ * Usually functions will pass const references of this class. However
+ * non-const references can be used to either temporarily release the lock by
+ * use of AutoUnlockGC or to start background allocation when the lock is
+ * released.
+ */
+class MOZ_RAII AutoLockGC {
+ public:
+ explicit AutoLockGC(gc::GCRuntime* gc) : gc(gc) { lock(); }
+ explicit AutoLockGC(JSRuntime* rt) : AutoLockGC(&rt->gc) {}
+
+ ~AutoLockGC() { lockGuard_.reset(); }
+
+ js::LockGuard<js::Mutex>& guard() { return lockGuard_.ref(); }
+
+ protected:
+ void lock() {
+ MOZ_ASSERT(lockGuard_.isNothing());
+ lockGuard_.emplace(gc->lock);
+ }
+
+ void unlock() {
+ MOZ_ASSERT(lockGuard_.isSome());
+ lockGuard_.reset();
+ }
+
+ gc::GCRuntime* const gc;
+
+ private:
+ mozilla::Maybe<js::LockGuard<js::Mutex>> lockGuard_;
+
+ AutoLockGC(const AutoLockGC&) = delete;
+ AutoLockGC& operator=(const AutoLockGC&) = delete;
+
+ friend class AutoUnlockGC; // For lock/unlock.
+};
+
+/*
+ * Same as AutoLockGC except it can optionally start a background chunk
+ * allocation task when the lock is released.
+ */
+class MOZ_RAII AutoLockGCBgAlloc : public AutoLockGC {
+ public:
+ explicit AutoLockGCBgAlloc(gc::GCRuntime* gc) : AutoLockGC(gc) {}
+ explicit AutoLockGCBgAlloc(JSRuntime* rt) : AutoLockGCBgAlloc(&rt->gc) {}
+
+ ~AutoLockGCBgAlloc() {
+ unlock();
+
+ /*
+ * We have to do this after releasing the lock because it may acquire
+ * the helper lock which could cause lock inversion if we still held
+ * the GC lock.
+ */
+ if (startBgAlloc) {
+ gc->startBackgroundAllocTaskIfIdle();
+ }
+ }
+
+ /*
+ * This can be used to start a background allocation task (if one isn't
+ * already running) that allocates chunks and makes them available in the
+ * free chunks list. This happens after the lock is released in order to
+ * avoid lock inversion.
+ */
+ void tryToStartBackgroundAllocation() { startBgAlloc = true; }
+
+ private:
+ // true if we should start a background chunk allocation task after the
+ // lock is released.
+ bool startBgAlloc = false;
+};
+
+class MOZ_RAII AutoUnlockGC {
+ public:
+ explicit AutoUnlockGC(AutoLockGC& lock) : lock(lock) { lock.unlock(); }
+
+ ~AutoUnlockGC() { lock.lock(); }
+
+ private:
+ AutoLockGC& lock;
+
+ AutoUnlockGC(const AutoUnlockGC&) = delete;
+ AutoUnlockGC& operator=(const AutoUnlockGC&) = delete;
+};
+
+} // namespace js
+
+#endif /* gc_GCLock_h */
diff --git a/js/src/gc/GCMarker.h b/js/src/gc/GCMarker.h
new file mode 100644
index 0000000000..053ba90e18
--- /dev/null
+++ b/js/src/gc/GCMarker.h
@@ -0,0 +1,598 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCMarker_h
+#define gc_GCMarker_h
+
+#include "mozilla/Maybe.h"
+#include "mozilla/Variant.h"
+
+#include "ds/OrderedHashTable.h"
+#include "gc/Barrier.h"
+#include "js/TracingAPI.h"
+#include "js/TypeDecls.h"
+#include "threading/ProtectedData.h"
+
+class JSRope;
+
+namespace js {
+
+class GCMarker;
+class SliceBudget;
+class WeakMapBase;
+
+static const size_t MARK_STACK_BASE_CAPACITY = 4096;
+
+enum class SlotsOrElementsKind {
+ Unused = 0, // Must match SlotsOrElementsRangeTag
+ Elements,
+ FixedSlots,
+ DynamicSlots
+};
+
+namespace gc {
+
+enum IncrementalProgress { NotFinished = 0, Finished };
+
+class AutoSetMarkColor;
+struct Cell;
+class ParallelMarker;
+class UnmarkGrayTracer;
+
+struct EphemeronEdgeTableHashPolicy {
+ using Lookup = Cell*;
+ static HashNumber hash(const Lookup& v,
+ const mozilla::HashCodeScrambler& hcs) {
+ return hcs.scramble(mozilla::HashGeneric(v));
+ }
+ static bool match(Cell* const& k, const Lookup& l) { return k == l; }
+ static bool isEmpty(Cell* const& v) { return !v; }
+ static void makeEmpty(Cell** vp) { *vp = nullptr; }
+};
+
+// Ephemeron edges have two source nodes and one target, and mark the target
+// with the minimum (least-marked) color of the sources. Currently, one of
+// those sources will always be a WeakMapBase, so this will refer to its color
+// at the time the edge is traced through. The other source's color will be
+// given by the current mark color of the GCMarker.
+struct EphemeronEdge {
+ CellColor color;
+ Cell* target;
+
+ EphemeronEdge(CellColor color_, Cell* cell) : color(color_), target(cell) {}
+};
+
+using EphemeronEdgeVector = Vector<EphemeronEdge, 2, js::SystemAllocPolicy>;
+
+using EphemeronEdgeTable =
+ OrderedHashMap<Cell*, EphemeronEdgeVector, EphemeronEdgeTableHashPolicy,
+ js::SystemAllocPolicy>;
+
+/*
+ * The mark stack. Pointers in this stack are "gray" in the GC sense, but
+ * their references may be marked either black or gray (in the CC sense).
+ *
+ * When the mark stack is full, the GC does not call js::TraceChildren to mark
+ * the reachable "children" of the thing. Rather the thing is put aside and
+ * js::TraceChildren is called later when the mark stack is empty.
+ *
+ * To implement such delayed marking of the children with minimal overhead for
+ * the normal case of sufficient stack, we link arenas into a list using
+ * Arena::setNextDelayedMarkingArena(). The head of the list is stored in
+ * GCMarker::delayedMarkingList. GCMarker::delayMarkingChildren() adds arenas
+ * to the list as necessary while markAllDelayedChildren() pops the arenas from
+ * the stack until it is empty.
+ */
+class MarkStack {
+ public:
+ /*
+ * We use a common mark stack to mark GC things of different types and use
+ * the explicit tags to distinguish them when it cannot be deduced from
+ * the context of push or pop operation.
+ */
+ enum Tag {
+ SlotsOrElementsRangeTag = 0, // Must match SlotsOrElementsKind::Unused.
+ ObjectTag,
+ JitCodeTag,
+ ScriptTag,
+ TempRopeTag,
+
+ LastTag = TempRopeTag
+ };
+
+ static const uintptr_t TagMask = 7;
+ static_assert(TagMask >= uintptr_t(LastTag),
+ "The tag mask must subsume the tags.");
+ static_assert(TagMask <= gc::CellAlignMask,
+ "The tag mask must be embeddable in a Cell*.");
+
+ class TaggedPtr {
+ uintptr_t bits;
+
+ Cell* ptr() const;
+
+ public:
+ TaggedPtr() = default;
+ TaggedPtr(Tag tag, Cell* ptr);
+ Tag tag() const;
+ uintptr_t tagUnchecked() const;
+ template <typename T>
+ T* as() const;
+
+ JSObject* asRangeObject() const;
+ JSRope* asTempRope() const;
+
+ void assertValid() const;
+ };
+
+ struct SlotsOrElementsRange {
+ SlotsOrElementsRange(SlotsOrElementsKind kind, JSObject* obj, size_t start);
+ void assertValid() const;
+
+ SlotsOrElementsKind kind() const;
+ size_t start() const;
+ TaggedPtr ptr() const;
+
+ static constexpr size_t StartShift = 2;
+ static constexpr size_t KindMask = (1 << StartShift) - 1;
+
+ private:
+ uintptr_t startAndKind_;
+ TaggedPtr ptr_;
+ };
+
+ MarkStack();
+ ~MarkStack();
+
+ explicit MarkStack(const MarkStack& other);
+ MarkStack& operator=(const MarkStack& other);
+
+ MarkStack(MarkStack&& other);
+ MarkStack& operator=(MarkStack&& other);
+
+ // The unit for MarkStack::capacity() is mark stack words.
+ size_t capacity() { return stack().length(); }
+
+ size_t position() const { return topIndex_; }
+
+ [[nodiscard]] bool init();
+ [[nodiscard]] bool resetStackCapacity();
+
+#ifdef JS_GC_ZEAL
+ void setMaxCapacity(size_t maxCapacity);
+#endif
+
+ template <typename T>
+ [[nodiscard]] bool push(T* ptr);
+
+ [[nodiscard]] bool push(JSObject* obj, SlotsOrElementsKind kind,
+ size_t start);
+ [[nodiscard]] bool push(const TaggedPtr& ptr);
+ [[nodiscard]] bool push(const SlotsOrElementsRange& array);
+ void infalliblePush(const TaggedPtr& ptr);
+ void infalliblePush(const SlotsOrElementsRange& array);
+
+ // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary
+ // storage to hold rope pointers.
+ [[nodiscard]] bool pushTempRope(JSRope* ptr);
+
+ bool isEmpty() const { return position() == 0; }
+ bool hasEntries() const { return !isEmpty(); }
+
+ Tag peekTag() const;
+ TaggedPtr popPtr();
+ SlotsOrElementsRange popSlotsOrElementsRange();
+
+ void clearAndResetCapacity();
+ void clearAndFreeStack();
+
+ void poisonUnused();
+
+ [[nodiscard]] bool ensureSpace(size_t count);
+
+ static void moveWork(MarkStack& dst, MarkStack& src);
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+
+ private:
+ using StackVector = Vector<TaggedPtr, 0, SystemAllocPolicy>;
+ const StackVector& stack() const { return stack_.ref(); }
+ StackVector& stack() { return stack_.ref(); }
+
+ /* Grow the stack, ensuring there is space for at least count elements. */
+ [[nodiscard]] bool enlarge(size_t count);
+
+ [[nodiscard]] bool resize(size_t newCapacity);
+
+ TaggedPtr* topPtr();
+
+ const TaggedPtr& peekPtr() const;
+ [[nodiscard]] bool pushTaggedPtr(Tag tag, Cell* ptr);
+
+ bool indexIsEntryBase(size_t index) const;
+
+ // Vector containing allocated stack memory. Unused beyond topIndex_.
+ MainThreadOrGCTaskData<StackVector> stack_;
+
+ // Index of the top of the stack.
+ MainThreadOrGCTaskData<size_t> topIndex_;
+
+#ifdef JS_GC_ZEAL
+ // The maximum stack capacity to grow to.
+ MainThreadOrGCTaskData<size_t> maxCapacity_{SIZE_MAX};
+#endif
+};
+
+static_assert(unsigned(SlotsOrElementsKind::Unused) ==
+ unsigned(MarkStack::SlotsOrElementsRangeTag),
+ "To split the mark stack we depend on being able to tell the "
+ "difference between SlotsOrElementsRange::startAndKind_ and a "
+ "tagged SlotsOrElementsRange");
+
+// Bitmask of options to parameterize MarkingTracerT.
+namespace MarkingOptions {
+enum : uint32_t {
+ // Set the compartment's hasMarkedCells flag for roots.
+ MarkRootCompartments = 1,
+
+ // The marking tracer is operating in parallel. Use appropriate atomic
+ // accesses to update the mark bits correctly.
+ ParallelMarking = 2,
+
+ // Mark any implicit edges if we are in weak marking mode.
+ MarkImplicitEdges = 4,
+};
+} // namespace MarkingOptions
+
+constexpr uint32_t NormalMarkingOptions = MarkingOptions::MarkImplicitEdges;
+
+template <uint32_t markingOptions>
+class MarkingTracerT
+ : public GenericTracerImpl<MarkingTracerT<markingOptions>> {
+ public:
+ MarkingTracerT(JSRuntime* runtime, GCMarker* marker);
+ virtual ~MarkingTracerT() = default;
+
+ template <typename T>
+ void onEdge(T** thingp, const char* name);
+ friend class GenericTracerImpl<MarkingTracerT<markingOptions>>;
+
+ GCMarker* getMarker();
+};
+
+using MarkingTracer = MarkingTracerT<NormalMarkingOptions>;
+using RootMarkingTracer = MarkingTracerT<MarkingOptions::MarkRootCompartments>;
+using ParallelMarkingTracer = MarkingTracerT<MarkingOptions::ParallelMarking>;
+
+enum ShouldReportMarkTime : bool {
+ ReportMarkTime = true,
+ DontReportMarkTime = false
+};
+
+} /* namespace gc */
+
+class GCMarker {
+ enum MarkingState : uint8_t {
+ // Have not yet started marking.
+ NotActive,
+
+ // Root marking mode. This sets the hasMarkedCells flag on compartments
+ // containing objects and scripts, which is used to make sure we clean up
+ // dead compartments.
+ RootMarking,
+
+ // Main marking mode. Weakmap marking will be populating the
+ // gcEphemeronEdges tables but not consulting them. The state will
+ // transition to WeakMarking until it is done, then back to RegularMarking.
+ RegularMarking,
+
+ // Like RegularMarking but with multiple threads running in parallel.
+ ParallelMarking,
+
+ // Same as RegularMarking except now every marked obj/script is immediately
+ // looked up in the gcEphemeronEdges table to find edges generated by
+ // weakmap keys, and traversing them to their values. Transitions back to
+ // RegularMarking when done.
+ WeakMarking,
+ };
+
+ public:
+ explicit GCMarker(JSRuntime* rt);
+ [[nodiscard]] bool init();
+
+ JSRuntime* runtime() { return runtime_; }
+ JSTracer* tracer() {
+ return tracer_.match([](auto& t) -> JSTracer* { return &t; });
+ }
+
+#ifdef JS_GC_ZEAL
+ void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); }
+#endif
+
+ bool isActive() const { return state != NotActive; }
+ bool isRegularMarking() const { return state == RegularMarking; }
+ bool isParallelMarking() const { return state == ParallelMarking; }
+ bool isWeakMarking() const { return state == WeakMarking; }
+
+ gc::MarkColor markColor() const { return markColor_; }
+
+ bool isDrained() const { return stack.isEmpty() && otherStack.isEmpty(); }
+
+ bool hasEntriesForCurrentColor() { return stack.hasEntries(); }
+ bool hasBlackEntries() const { return hasEntries(gc::MarkColor::Black); }
+ bool hasGrayEntries() const { return hasEntries(gc::MarkColor::Gray); }
+ bool hasEntries(gc::MarkColor color) const;
+
+ bool canDonateWork() const;
+
+ void start();
+ void stop();
+ void reset();
+
+ [[nodiscard]] bool markUntilBudgetExhausted(
+ SliceBudget& budget,
+ gc::ShouldReportMarkTime reportTime = gc::ReportMarkTime);
+
+ void setRootMarkingMode(bool newState);
+
+ bool enterWeakMarkingMode();
+ void leaveWeakMarkingMode();
+
+ void enterParallelMarkingMode(gc::ParallelMarker* pm);
+ void leaveParallelMarkingMode();
+
+ // Do not use linear-time weak marking for the rest of this collection.
+ // Currently, this will only be triggered by an OOM when updating needed data
+ // structures.
+ void abortLinearWeakMarking();
+
+ // 'delegate' is no longer the delegate of 'key'.
+ void severWeakDelegate(JSObject* key, JSObject* delegate);
+
+ // 'delegate' is now the delegate of 'key'. Update weakmap marking state.
+ void restoreWeakDelegate(JSObject* key, JSObject* delegate);
+
+#ifdef DEBUG
+ // We can't check atom marking if the helper thread lock is already held by
+ // the current thread. This allows us to disable the check.
+ void setCheckAtomMarking(bool check);
+
+ bool shouldCheckCompartments() { return strictCompartmentChecking; }
+#endif
+
+ bool markCurrentColorInParallel(SliceBudget& budget);
+
+ template <uint32_t markingOptions, gc::MarkColor>
+ bool markOneColor(SliceBudget& budget);
+
+ static void moveWork(GCMarker* dst, GCMarker* src);
+
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+
+ static GCMarker* fromTracer(JSTracer* trc) {
+ MOZ_ASSERT(trc->isMarkingTracer());
+ auto* marker = reinterpret_cast<GCMarker*>(uintptr_t(trc) -
+ offsetof(GCMarker, tracer_));
+ MOZ_ASSERT(marker->tracer() == trc);
+ return marker;
+ }
+
+ // Internal public methods, for ease of use by the rest of the GC:
+
+ // If |thing| is unmarked, mark it and then traverse its children.
+ template <uint32_t, typename T>
+ void markAndTraverse(T* thing);
+
+ template <typename T>
+ void markImplicitEdges(T* oldThing);
+
+ private:
+ /*
+ * Care must be taken changing the mark color from gray to black. The cycle
+ * collector depends on the invariant that there are no black to gray edges
+ * in the GC heap. This invariant lets the CC not trace through black
+ * objects. If this invariant is violated, the cycle collector may free
+ * objects that are still reachable.
+ */
+ void setMarkColor(gc::MarkColor newColor);
+ friend class js::gc::AutoSetMarkColor;
+
+ template <typename Tracer>
+ void setMarkingStateAndTracer(MarkingState prev, MarkingState next);
+
+ template <uint32_t markingOptions>
+ bool processMarkStackTop(SliceBudget& budget);
+ friend class gc::GCRuntime;
+
+ // Helper methods that coerce their second argument to the base pointer
+ // type.
+ template <uint32_t markingOptions, typename S>
+ void markAndTraverseObjectEdge(S source, JSObject* target) {
+ markAndTraverseEdge<markingOptions>(source, target);
+ }
+ template <uint32_t markingOptions, typename S>
+ void markAndTraverseStringEdge(S source, JSString* target) {
+ markAndTraverseEdge<markingOptions>(source, target);
+ }
+
+ template <uint32_t markingOptions, typename S, typename T>
+ void markAndTraverseEdge(S source, T* target);
+ template <uint32_t markingOptions, typename S, typename T>
+ void markAndTraverseEdge(S source, const T& target);
+
+ template <typename S, typename T>
+ void checkTraversedEdge(S source, T* target);
+
+ // Mark the given GC thing, but do not trace its children. Return true
+ // if the thing became marked.
+ template <uint32_t markingOptions, typename T>
+ [[nodiscard]] bool mark(T* thing);
+
+ // Traverse a GC thing's children, using a strategy depending on the type.
+ // This can either processing them immediately or push them onto the mark
+ // stack for later.
+#define DEFINE_TRAVERSE_METHOD(_1, Type, _2, _3) \
+ template <uint32_t> \
+ void traverse(Type* thing);
+ JS_FOR_EACH_TRACEKIND(DEFINE_TRAVERSE_METHOD)
+#undef DEFINE_TRAVERSE_METHOD
+
+ // Process a marked thing's children by calling T::traceChildren().
+ template <uint32_t markingOptions, typename T>
+ void traceChildren(T* thing);
+
+ // Process a marked thing's children recursively using an iterative loop and
+ // manual dispatch, for kinds where this is possible.
+ template <uint32_t markingOptions, typename T>
+ void scanChildren(T* thing);
+
+ // Push a marked thing onto the mark stack. Its children will be marked later.
+ template <uint32_t markingOptions, typename T>
+ void pushThing(T* thing);
+
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(JSLinearString* str);
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(JSRope* rope);
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(JSString* str);
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(Shape* shape);
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(PropMap* map);
+ template <uint32_t markingOptions>
+ void eagerlyMarkChildren(Scope* scope);
+
+ template <typename T>
+ inline void pushTaggedPtr(T* ptr);
+
+ inline void pushValueRange(JSObject* obj, SlotsOrElementsKind kind,
+ size_t start, size_t end);
+
+ // Push an object onto the stack for later tracing and assert that it has
+ // already been marked.
+ inline void repush(JSObject* obj);
+
+ template <typename T>
+ void markImplicitEdgesHelper(T oldThing);
+
+ // Mark through edges whose target color depends on the colors of two source
+ // entities (eg a WeakMap and one of its keys), and push the target onto the
+ // mark stack.
+ void markEphemeronEdges(gc::EphemeronEdgeVector& edges,
+ gc::CellColor srcColor);
+ friend class JS::Zone;
+
+#ifdef DEBUG
+ void checkZone(void* p);
+#else
+ void checkZone(void* p) {}
+#endif
+
+ template <uint32_t markingOptions>
+ bool doMarking(SliceBudget& budget, gc::ShouldReportMarkTime reportTime);
+
+ void delayMarkingChildrenOnOOM(gc::Cell* cell);
+
+ /*
+ * The JSTracer used for marking. This can change depending on the current
+ * state.
+ */
+ mozilla::Variant<gc::MarkingTracer, gc::RootMarkingTracer,
+ gc::ParallelMarkingTracer>
+ tracer_;
+
+ JSRuntime* const runtime_;
+
+ // The main mark stack, holding entries of color |markColor_|.
+ gc::MarkStack stack;
+
+ // The auxiliary mark stack, which may contain entries of the other color.
+ gc::MarkStack otherStack;
+
+ // Track whether we're using the main or auxiliary stack.
+ MainThreadOrGCTaskData<bool> haveSwappedStacks;
+
+ // The current mark stack color.
+ MainThreadOrGCTaskData<gc::MarkColor> markColor_;
+
+ MainThreadOrGCTaskData<gc::ParallelMarker*> parallelMarker_;
+
+ Vector<JS::GCCellPtr, 0, SystemAllocPolicy> unmarkGrayStack;
+ friend class gc::UnmarkGrayTracer;
+
+ /* Track the state of marking. */
+ MainThreadOrGCTaskData<MarkingState> state;
+
+ /* Whether we successfully added all edges to the implicit edges table. */
+ MainThreadOrGCTaskData<bool> haveAllImplicitEdges;
+
+ public:
+ /*
+ * Whether weakmaps can be marked incrementally.
+ *
+ * JSGC_INCREMENTAL_WEAKMAP_ENABLED
+ * pref: javascript.options.mem.incremental_weakmap
+ */
+ MainThreadOrGCTaskData<bool> incrementalWeakMapMarkingEnabled;
+
+#ifdef DEBUG
+ private:
+ /* Assert that start and stop are called with correct ordering. */
+ MainThreadOrGCTaskData<bool> started;
+
+ /*
+ * Whether to check that atoms traversed are present in atom marking
+ * bitmap.
+ */
+ MainThreadOrGCTaskData<bool> checkAtomMarking;
+
+ /*
+ * If this is true, all marked objects must belong to a compartment being
+ * GCed. This is used to look for compartment bugs.
+ */
+ MainThreadOrGCTaskData<bool> strictCompartmentChecking;
+
+ public:
+ /*
+ * The compartment and zone of the object whose trace hook is currently being
+ * called, if any. Used to catch cross-compartment edges traced without use of
+ * TraceCrossCompartmentEdge.
+ */
+ MainThreadOrGCTaskData<Compartment*> tracingCompartment;
+ MainThreadOrGCTaskData<Zone*> tracingZone;
+#endif // DEBUG
+};
+
+namespace gc {
+
+/*
+ * Temporarily change the mark color while this class is on the stack.
+ *
+ * During incremental sweeping this also transitions zones in the
+ * current sweep group into the Mark or MarkGray state as appropriate.
+ */
+class MOZ_RAII AutoSetMarkColor {
+ GCMarker& marker_;
+ MarkColor initialColor_;
+
+ public:
+ AutoSetMarkColor(GCMarker& marker, MarkColor newColor)
+ : marker_(marker), initialColor_(marker.markColor()) {
+ marker_.setMarkColor(newColor);
+ }
+
+ AutoSetMarkColor(GCMarker& marker, CellColor newColor)
+ : AutoSetMarkColor(marker, newColor.asMarkColor()) {}
+
+ ~AutoSetMarkColor() { marker_.setMarkColor(initialColor_); }
+};
+
+} /* namespace gc */
+
+} /* namespace js */
+
+#endif /* gc_GCMarker_h */
diff --git a/js/src/gc/GCParallelTask.cpp b/js/src/gc/GCParallelTask.cpp
new file mode 100644
index 0000000000..029315b119
--- /dev/null
+++ b/js/src/gc/GCParallelTask.cpp
@@ -0,0 +1,231 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/GCParallelTask.h"
+
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/GCContext.h"
+#include "gc/GCInternals.h"
+#include "gc/ParallelWork.h"
+#include "vm/HelperThreadState.h"
+#include "vm/Runtime.h"
+#include "vm/Time.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::Maybe;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+js::GCParallelTask::~GCParallelTask() {
+ // The LinkedListElement destructor will remove us from any list we are part
+ // of without synchronization, so ensure that doesn't happen.
+ MOZ_DIAGNOSTIC_ASSERT(!isInList());
+
+ // Only most-derived classes' destructors may do the join: base class
+ // destructors run after those for derived classes' members, so a join in a
+ // base class can't ensure that the task is done using the members. All we
+ // can do now is check that someone has previously stopped the task.
+ assertIdle();
+}
+
+static bool ShouldMeasureTaskStartDelay() {
+ // We use many tasks during GC so randomly sample a small fraction for the
+ // purposes of recording telemetry.
+ return (rand() % 100) == 0;
+}
+
+void js::GCParallelTask::startWithLockHeld(AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(CanUseExtraThreads());
+ MOZ_ASSERT(HelperThreadState().isInitialized(lock));
+ assertIdle();
+
+ maybeQueueTime_ = TimeStamp();
+ if (ShouldMeasureTaskStartDelay()) {
+ maybeQueueTime_ = TimeStamp::Now();
+ }
+
+ setDispatched(lock);
+ HelperThreadState().submitTask(this, lock);
+}
+
+void js::GCParallelTask::start() {
+ if (!CanUseExtraThreads()) {
+ runFromMainThread();
+ return;
+ }
+
+ AutoLockHelperThreadState lock;
+ startWithLockHeld(lock);
+}
+
+void js::GCParallelTask::startOrRunIfIdle(AutoLockHelperThreadState& lock) {
+ if (wasStarted(lock)) {
+ return;
+ }
+
+ // Join the previous invocation of the task. This will return immediately
+ // if the thread has never been started.
+ joinWithLockHeld(lock);
+
+ if (!CanUseExtraThreads()) {
+ runFromMainThread(lock);
+ return;
+ }
+
+ startWithLockHeld(lock);
+}
+
+void js::GCParallelTask::cancelAndWait() {
+ MOZ_ASSERT(!isCancelled());
+ cancel_ = true;
+ join();
+ cancel_ = false;
+}
+
+void js::GCParallelTask::join(Maybe<TimeStamp> deadline) {
+ AutoLockHelperThreadState lock;
+ joinWithLockHeld(lock, deadline);
+}
+
+void js::GCParallelTask::joinWithLockHeld(AutoLockHelperThreadState& lock,
+ Maybe<TimeStamp> deadline) {
+ // Task has not been started; there's nothing to do.
+ if (isIdle(lock)) {
+ return;
+ }
+
+ if (isDispatched(lock) && deadline.isNothing()) {
+ // If the task was dispatched but has not yet started then cancel the task
+ // and run it from the main thread. This stops us from blocking here when
+ // the helper threads are busy with other tasks.
+ cancelDispatchedTask(lock);
+ runFromMainThread(lock);
+ } else {
+ // Otherwise wait for the task to complete.
+ joinNonIdleTask(deadline, lock);
+ }
+
+ if (isIdle(lock)) {
+ recordDuration();
+ }
+}
+
+void GCParallelTask::recordDuration() {
+ if (phaseKind != gcstats::PhaseKind::NONE) {
+ gc->stats().recordParallelPhase(phaseKind, duration_);
+ }
+}
+
+void js::GCParallelTask::joinNonIdleTask(Maybe<TimeStamp> deadline,
+ AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(!isIdle(lock));
+
+ while (!isFinished(lock)) {
+ TimeDuration timeout = TimeDuration::Forever();
+ if (deadline) {
+ TimeStamp now = TimeStamp::Now();
+ if (*deadline <= now) {
+ break;
+ }
+ timeout = *deadline - now;
+ }
+
+ HelperThreadState().wait(lock, timeout);
+ }
+
+ if (isFinished(lock)) {
+ setIdle(lock);
+ }
+}
+
+void js::GCParallelTask::cancelDispatchedTask(AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isDispatched(lock));
+ MOZ_ASSERT(isInList());
+ remove();
+ setIdle(lock);
+}
+
+void js::GCParallelTask::runFromMainThread(AutoLockHelperThreadState& lock) {
+ assertIdle();
+ MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(gc->rt));
+ state_ = State::Running;
+ runTask(gc->rt->gcContext(), lock);
+ state_ = State::Idle;
+}
+
+void js::GCParallelTask::runFromMainThread() {
+ AutoLockHelperThreadState lock;
+ runFromMainThread(lock);
+}
+
+class MOZ_RAII AutoGCContext {
+ JS::GCContext context;
+
+ public:
+ explicit AutoGCContext(JSRuntime* runtime) : context(runtime) {
+ MOZ_RELEASE_ASSERT(TlsGCContext.init(),
+ "Failed to initialize TLS for GC context");
+
+ MOZ_ASSERT(!TlsGCContext.get());
+ TlsGCContext.set(&context);
+ }
+
+ ~AutoGCContext() {
+ MOZ_ASSERT(TlsGCContext.get() == &context);
+ TlsGCContext.set(nullptr);
+ }
+
+ JS::GCContext* get() { return &context; }
+};
+
+void js::GCParallelTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
+ setRunning(lock);
+
+ AutoGCContext gcContext(gc->rt);
+
+ runTask(gcContext.get(), lock);
+
+ setFinished(lock);
+}
+
+void GCParallelTask::runTask(JS::GCContext* gcx,
+ AutoLockHelperThreadState& lock) {
+ // Run the task from either the main thread or a helper thread.
+
+ AutoSetThreadGCUse setUse(gcx, use);
+
+ // The hazard analysis can't tell what the call to func_ will do but it's not
+ // allowed to GC.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ TimeStamp timeStart = TimeStamp::Now();
+ run(lock);
+ duration_ = TimeSince(timeStart);
+
+ if (maybeQueueTime_) {
+ TimeDuration delay = timeStart - maybeQueueTime_;
+ gc->rt->metrics().GC_TASK_START_DELAY_US(delay);
+ }
+}
+
+bool js::GCParallelTask::isIdle() const {
+ AutoLockHelperThreadState lock;
+ return isIdle(lock);
+}
+
+bool js::GCParallelTask::wasStarted() const {
+ AutoLockHelperThreadState lock;
+ return wasStarted(lock);
+}
+
+/* static */
+size_t js::gc::GCRuntime::parallelWorkerCount() const {
+ return std::min(helperThreadCount.ref(), MaxParallelWorkers);
+}
diff --git a/js/src/gc/GCParallelTask.h b/js/src/gc/GCParallelTask.h
new file mode 100644
index 0000000000..4784382ab5
--- /dev/null
+++ b/js/src/gc/GCParallelTask.h
@@ -0,0 +1,246 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCParallelTask_h
+#define gc_GCParallelTask_h
+
+#include "mozilla/LinkedList.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include <utility>
+
+#include "gc/GCContext.h"
+#include "js/Utility.h"
+#include "threading/ProtectedData.h"
+#include "vm/HelperThreads.h"
+#include "vm/HelperThreadTask.h"
+
+#define JS_MEMBER_FN_PTR_TYPE(ClassT, ReturnT, /* ArgTs */...) \
+ ReturnT (ClassT::*)(__VA_ARGS__)
+
+#define JS_CALL_MEMBER_FN_PTR(Receiver, Ptr, /* Args */...) \
+ ((Receiver)->*(Ptr))(__VA_ARGS__)
+
+namespace js {
+
+namespace gcstats {
+enum class PhaseKind : uint8_t;
+}
+
+namespace gc {
+
+class GCRuntime;
+
+static inline mozilla::TimeDuration TimeSince(mozilla::TimeStamp prev) {
+ mozilla::TimeStamp now = mozilla::TimeStamp::Now();
+ // Sadly this happens sometimes.
+ MOZ_ASSERT(now >= prev);
+ if (now < prev) {
+ now = prev;
+ }
+ return now - prev;
+}
+
+} // namespace gc
+
+class AutoLockHelperThreadState;
+class GCParallelTask;
+class HelperThread;
+
+// A wrapper around a linked list to enforce synchronization.
+class GCParallelTaskList {
+ mozilla::LinkedList<GCParallelTask> tasks;
+
+ public:
+ bool isEmpty(const AutoLockHelperThreadState& lock) {
+ gHelperThreadLock.assertOwnedByCurrentThread();
+ return tasks.isEmpty();
+ }
+
+ void insertBack(GCParallelTask* task, const AutoLockHelperThreadState& lock) {
+ gHelperThreadLock.assertOwnedByCurrentThread();
+ tasks.insertBack(task);
+ }
+
+ GCParallelTask* popFirst(const AutoLockHelperThreadState& lock) {
+ gHelperThreadLock.assertOwnedByCurrentThread();
+ return tasks.popFirst();
+ }
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf,
+ const AutoLockHelperThreadState& lock) const {
+ gHelperThreadLock.assertOwnedByCurrentThread();
+ return tasks.sizeOfExcludingThis(aMallocSizeOf);
+ }
+};
+
+// A generic task used to dispatch work to the helper thread system.
+// Users override the pure-virtual run() method.
+class GCParallelTask : private mozilla::LinkedListElement<GCParallelTask>,
+ public HelperThreadTask {
+ friend class mozilla::LinkedList<GCParallelTask>;
+ friend class mozilla::LinkedListElement<GCParallelTask>;
+
+ public:
+ gc::GCRuntime* const gc;
+
+ // This can be PhaseKind::NONE for tasks that take place outside a GC.
+ const gcstats::PhaseKind phaseKind;
+
+ gc::GCUse use;
+
+ private:
+ // The state of the parallel computation.
+ enum class State {
+ // The task is idle. Either start() has not been called or join() has
+ // returned.
+ Idle,
+
+ // The task has been started but has not yet begun running on a helper
+ // thread.
+ Dispatched,
+
+ // The task is currently running on a helper thread.
+ Running,
+
+ // The task has finished running but has not yet been joined by the main
+ // thread.
+ Finished
+ };
+
+ UnprotectedData<State> state_;
+
+ // May be set to the time this task was queued to collect telemetry.
+ mozilla::TimeStamp maybeQueueTime_;
+
+ // Amount of time this task took to execute.
+ MainThreadOrGCTaskData<mozilla::TimeDuration> duration_;
+
+ explicit GCParallelTask(const GCParallelTask&) = delete;
+
+ protected:
+ // A flag to signal a request for early completion of the off-thread task.
+ mozilla::Atomic<bool, mozilla::MemoryOrdering::ReleaseAcquire> cancel_;
+
+ public:
+ explicit GCParallelTask(gc::GCRuntime* gc, gcstats::PhaseKind phaseKind,
+ gc::GCUse use = gc::GCUse::Unspecified)
+ : gc(gc),
+ phaseKind(phaseKind),
+ use(use),
+ state_(State::Idle),
+ cancel_(false) {}
+ GCParallelTask(GCParallelTask&& other)
+ : gc(other.gc),
+ phaseKind(other.phaseKind),
+ use(other.use),
+ state_(other.state_),
+ cancel_(false) {}
+
+ // Derived classes must override this to ensure that join() gets called
+ // before members get destructed.
+ virtual ~GCParallelTask();
+
+ // Time spent in the most recent invocation of this task.
+ mozilla::TimeDuration duration() const { return duration_; }
+
+ // The simple interface to a parallel task works exactly like pthreads.
+ void start();
+ void join(mozilla::Maybe<mozilla::TimeStamp> deadline = mozilla::Nothing());
+
+ // If multiple tasks are to be started or joined at once, it is more
+ // efficient to take the helper thread lock once and use these methods.
+ void startWithLockHeld(AutoLockHelperThreadState& lock);
+ void joinWithLockHeld(
+ AutoLockHelperThreadState& lock,
+ mozilla::Maybe<mozilla::TimeStamp> deadline = mozilla::Nothing());
+ void joinNonIdleTask(mozilla::Maybe<mozilla::TimeStamp> deadline,
+ AutoLockHelperThreadState& lock);
+
+ // Instead of dispatching to a helper, run the task on the current thread.
+ void runFromMainThread();
+ void runFromMainThread(AutoLockHelperThreadState& lock);
+
+ // If the task is not already running, either start it or run it on the main
+ // thread if that fails.
+ void startOrRunIfIdle(AutoLockHelperThreadState& lock);
+
+ // Cancel a dispatched task before it started executing.
+ void cancelDispatchedTask(AutoLockHelperThreadState& lock);
+
+ // Set the cancel flag and wait for the task to finish.
+ void cancelAndWait();
+
+ // Report whether the task is idle. This means either before start() has been
+ // called or after join() has been called.
+ bool isIdle() const;
+ bool isIdle(const AutoLockHelperThreadState& lock) const {
+ return state_ == State::Idle;
+ }
+
+ // Report whether the task has been started. This means after start() has been
+ // called but before the task has run to completion. The task may not yet have
+ // started running.
+ bool wasStarted() const;
+ bool wasStarted(const AutoLockHelperThreadState& lock) const {
+ return isDispatched(lock) || isRunning(lock);
+ }
+
+ bool isDispatched(const AutoLockHelperThreadState& lock) const {
+ return state_ == State::Dispatched;
+ }
+
+ protected:
+ // Override this method to provide the task's functionality.
+ virtual void run(AutoLockHelperThreadState& lock) = 0;
+
+ virtual void recordDuration();
+
+ bool isCancelled() const { return cancel_; }
+
+ private:
+ void assertIdle() const {
+ // Don't lock here because that adds extra synchronization in debug
+ // builds that may hide bugs. There's no race if the assertion passes.
+ MOZ_ASSERT(state_ == State::Idle);
+ }
+
+ bool isRunning(const AutoLockHelperThreadState& lock) const {
+ return state_ == State::Running;
+ }
+ bool isFinished(const AutoLockHelperThreadState& lock) const {
+ return state_ == State::Finished;
+ }
+
+ void setDispatched(const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isIdle(lock));
+ state_ = State::Dispatched;
+ }
+ void setRunning(const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isDispatched(lock));
+ state_ = State::Running;
+ }
+ void setFinished(const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isRunning(lock));
+ state_ = State::Finished;
+ }
+ void setIdle(const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isDispatched(lock) || isFinished(lock));
+ state_ = State::Idle;
+ }
+
+ void runTask(JS::GCContext* gcx, AutoLockHelperThreadState& lock);
+
+ // Implement the HelperThreadTask interface.
+ ThreadType threadType() override {
+ return ThreadType::THREAD_TYPE_GCPARALLEL;
+ }
+ void runHelperThreadTask(AutoLockHelperThreadState& locked) override;
+};
+
+} /* namespace js */
+#endif /* gc_GCParallelTask_h */
diff --git a/js/src/gc/GCProbes.h b/js/src/gc/GCProbes.h
new file mode 100644
index 0000000000..6942610493
--- /dev/null
+++ b/js/src/gc/GCProbes.h
@@ -0,0 +1,49 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCProbes_h
+#define gc_GCProbes_h
+
+/*
+ * This interface can be used to insert probes for GC related events.
+ *
+ * The code must be built with JS_GC_PROBES for these probes to be called
+ * from JIT code.
+ */
+
+#include <stddef.h>
+
+#include "gc/AllocKind.h"
+#include "js/TraceKind.h"
+
+class JSObject;
+
+namespace js {
+namespace gc {
+
+class GCRuntime;
+struct Cell;
+
+namespace gcprobes {
+
+inline void Init(gc::GCRuntime* gc) {}
+inline void Finish(gc::GCRuntime* gc) {}
+inline void NurseryAlloc(void* ptr, JS::TraceKind kind) {}
+inline void TenuredAlloc(void* ptr, gc::AllocKind kind) {}
+inline void CreateObject(JSObject* object) {}
+inline void MinorGCStart() {}
+inline void PromoteToTenured(gc::Cell* src, gc::Cell* dst) {}
+inline void MinorGCEnd() {}
+inline void MajorGCStart() {}
+inline void TenuredFinalize(gc::Cell* thing) {
+} // May be called off main thread.
+inline void MajorGCEnd() {}
+
+} // namespace gcprobes
+} // namespace gc
+} // namespace js
+
+#endif // gc_GCProbes_h
diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h
new file mode 100644
index 0000000000..5336114020
--- /dev/null
+++ b/js/src/gc/GCRuntime.h
@@ -0,0 +1,1444 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCRuntime_h
+#define gc_GCRuntime_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/DoublyLinkedList.h"
+#include "mozilla/EnumSet.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/ArenaList.h"
+#include "gc/AtomMarking.h"
+#include "gc/GCContext.h"
+#include "gc/GCMarker.h"
+#include "gc/GCParallelTask.h"
+#include "gc/IteratorUtils.h"
+#include "gc/Nursery.h"
+#include "gc/Scheduling.h"
+#include "gc/Statistics.h"
+#include "gc/StoreBuffer.h"
+#include "js/friend/PerformanceHint.h"
+#include "js/GCAnnotations.h"
+#include "js/UniquePtr.h"
+#include "vm/AtomsTable.h"
+
+namespace js {
+
+class AutoLockGC;
+class AutoLockGCBgAlloc;
+class AutoLockHelperThreadState;
+class FinalizationRegistryObject;
+class FinalizationRecordObject;
+class FinalizationQueueObject;
+class GlobalObject;
+class VerifyPreTracer;
+class WeakRefObject;
+
+namespace gc {
+
+using BlackGrayEdgeVector = Vector<TenuredCell*, 0, SystemAllocPolicy>;
+using ZoneVector = Vector<JS::Zone*, 4, SystemAllocPolicy>;
+
+class AutoCallGCCallbacks;
+class AutoGCSession;
+class AutoHeapSession;
+class AutoTraceSession;
+struct FinalizePhase;
+class MarkingValidator;
+struct MovingTracer;
+class ParallelMarkTask;
+enum class ShouldCheckThresholds;
+class SweepGroupsIter;
+
+// Interface to a sweep action.
+struct SweepAction {
+ // The arguments passed to each action.
+ struct Args {
+ GCRuntime* gc;
+ JS::GCContext* gcx;
+ SliceBudget& budget;
+ };
+
+ virtual ~SweepAction() = default;
+ virtual IncrementalProgress run(Args& state) = 0;
+ virtual void assertFinished() const = 0;
+ virtual bool shouldSkip() { return false; }
+};
+
+class ChunkPool {
+ TenuredChunk* head_;
+ size_t count_;
+
+ public:
+ ChunkPool() : head_(nullptr), count_(0) {}
+ ChunkPool(const ChunkPool& other) = delete;
+ ChunkPool(ChunkPool&& other) { *this = std::move(other); }
+
+ ~ChunkPool() {
+ MOZ_ASSERT(!head_);
+ MOZ_ASSERT(count_ == 0);
+ }
+
+ ChunkPool& operator=(const ChunkPool& other) = delete;
+ ChunkPool& operator=(ChunkPool&& other) {
+ head_ = other.head_;
+ other.head_ = nullptr;
+ count_ = other.count_;
+ other.count_ = 0;
+ return *this;
+ }
+
+ bool empty() const { return !head_; }
+ size_t count() const { return count_; }
+
+ TenuredChunk* head() {
+ MOZ_ASSERT(head_);
+ return head_;
+ }
+ TenuredChunk* pop();
+ void push(TenuredChunk* chunk);
+ TenuredChunk* remove(TenuredChunk* chunk);
+
+ void sort();
+
+ private:
+ TenuredChunk* mergeSort(TenuredChunk* list, size_t count);
+ bool isSorted() const;
+
+#ifdef DEBUG
+ public:
+ bool contains(TenuredChunk* chunk) const;
+ bool verify() const;
+ void verifyChunks() const;
+#endif
+
+ public:
+ // Pool mutation does not invalidate an Iter unless the mutation
+ // is of the TenuredChunk currently being visited by the Iter.
+ class Iter {
+ public:
+ explicit Iter(ChunkPool& pool) : current_(pool.head_) {}
+ bool done() const { return !current_; }
+ void next();
+ TenuredChunk* get() const { return current_; }
+ operator TenuredChunk*() const { return get(); }
+ TenuredChunk* operator->() const { return get(); }
+
+ private:
+ TenuredChunk* current_;
+ };
+};
+
+class BackgroundMarkTask : public GCParallelTask {
+ public:
+ explicit BackgroundMarkTask(GCRuntime* gc);
+ void setBudget(const SliceBudget& budget) { this->budget = budget; }
+ void run(AutoLockHelperThreadState& lock) override;
+
+ private:
+ SliceBudget budget;
+};
+
+class BackgroundUnmarkTask : public GCParallelTask {
+ public:
+ explicit BackgroundUnmarkTask(GCRuntime* gc);
+ void initZones();
+ void run(AutoLockHelperThreadState& lock) override;
+
+ ZoneVector zones;
+};
+
+class BackgroundSweepTask : public GCParallelTask {
+ public:
+ explicit BackgroundSweepTask(GCRuntime* gc);
+ void run(AutoLockHelperThreadState& lock) override;
+};
+
+class BackgroundFreeTask : public GCParallelTask {
+ public:
+ explicit BackgroundFreeTask(GCRuntime* gc);
+ void run(AutoLockHelperThreadState& lock) override;
+};
+
+// Performs extra allocation off thread so that when memory is required on the
+// main thread it will already be available and waiting.
+class BackgroundAllocTask : public GCParallelTask {
+ // Guarded by the GC lock.
+ GCLockData<ChunkPool&> chunkPool_;
+
+ const bool enabled_;
+
+ public:
+ BackgroundAllocTask(GCRuntime* gc, ChunkPool& pool);
+ bool enabled() const { return enabled_; }
+
+ void run(AutoLockHelperThreadState& lock) override;
+};
+
+// Search the provided chunks for free arenas and decommit them.
+class BackgroundDecommitTask : public GCParallelTask {
+ public:
+ explicit BackgroundDecommitTask(GCRuntime* gc);
+ void run(AutoLockHelperThreadState& lock) override;
+};
+
+template <typename F>
+struct Callback {
+ F op;
+ void* data;
+
+ Callback() : op(nullptr), data(nullptr) {}
+ Callback(F op, void* data) : op(op), data(data) {}
+};
+
+template <typename F>
+using CallbackVector = Vector<Callback<F>, 4, SystemAllocPolicy>;
+
+typedef HashMap<Value*, const char*, DefaultHasher<Value*>, SystemAllocPolicy>
+ RootedValueMap;
+
+using AllocKinds = mozilla::EnumSet<AllocKind, uint64_t>;
+
+// A singly linked list of zones.
+class ZoneList {
+ static Zone* const End;
+
+ Zone* head;
+ Zone* tail;
+
+ public:
+ ZoneList();
+ ~ZoneList();
+
+ bool isEmpty() const;
+ Zone* front() const;
+
+ void prepend(Zone* zone);
+ void append(Zone* zone);
+ void prependList(ZoneList&& other);
+ void appendList(ZoneList&& other);
+ Zone* removeFront();
+ void clear();
+
+ private:
+ explicit ZoneList(Zone* singleZone);
+ void check() const;
+
+ ZoneList(const ZoneList& other) = delete;
+ ZoneList& operator=(const ZoneList& other) = delete;
+};
+
+struct WeakCacheToSweep {
+ JS::detail::WeakCacheBase* cache;
+ JS::Zone* zone;
+};
+
+class WeakCacheSweepIterator {
+ using WeakCacheBase = JS::detail::WeakCacheBase;
+
+ JS::Zone* sweepZone;
+ WeakCacheBase* sweepCache;
+
+ public:
+ explicit WeakCacheSweepIterator(JS::Zone* sweepGroup);
+
+ bool done() const;
+ WeakCacheToSweep get() const;
+ void next();
+
+ private:
+ void settle();
+};
+
+struct SweepingTracer final : public GenericTracerImpl<SweepingTracer> {
+ explicit SweepingTracer(JSRuntime* rt);
+
+ private:
+ template <typename T>
+ void onEdge(T** thingp, const char* name);
+ friend class GenericTracerImpl<SweepingTracer>;
+};
+
+class GCRuntime {
+ public:
+ explicit GCRuntime(JSRuntime* rt);
+ [[nodiscard]] bool init(uint32_t maxbytes);
+ bool wasInitialized() const { return initialized; }
+ void finishRoots();
+ void finish();
+
+ Zone* atomsZone() {
+ Zone* zone = zones()[0];
+ MOZ_ASSERT(JS::shadow::Zone::from(zone)->isAtomsZone());
+ return zone;
+ }
+ Zone* maybeSharedAtomsZone() { return sharedAtomsZone_; }
+
+ [[nodiscard]] bool freezeSharedAtomsZone();
+ void restoreSharedAtomsZone();
+
+ JS::HeapState heapState() const { return heapState_; }
+
+ inline bool hasZealMode(ZealMode mode);
+ inline void clearZealMode(ZealMode mode);
+ inline bool needZealousGC();
+ inline bool hasIncrementalTwoSliceZealMode();
+
+ [[nodiscard]] bool addRoot(Value* vp, const char* name);
+ void removeRoot(Value* vp);
+
+ [[nodiscard]] bool setParameter(JSContext* cx, JSGCParamKey key,
+ uint32_t value);
+ void resetParameter(JSContext* cx, JSGCParamKey key);
+ uint32_t getParameter(JSGCParamKey key);
+
+ void setPerformanceHint(PerformanceHint hint);
+ bool isInPageLoad() const { return inPageLoadCount != 0; }
+
+ [[nodiscard]] bool triggerGC(JS::GCReason reason);
+ // Check whether to trigger a zone GC after allocating GC cells.
+ void maybeTriggerGCAfterAlloc(Zone* zone);
+ // Check whether to trigger a zone GC after malloc memory.
+ void maybeTriggerGCAfterMalloc(Zone* zone);
+ bool maybeTriggerGCAfterMalloc(Zone* zone, const HeapSize& heap,
+ const HeapThreshold& threshold,
+ JS::GCReason reason);
+ // The return value indicates if we were able to do the GC.
+ bool triggerZoneGC(Zone* zone, JS::GCReason reason, size_t usedBytes,
+ size_t thresholdBytes);
+
+ void maybeGC();
+
+ // Return whether we want to run a major GC. If eagerOk is true, include eager
+ // triggers (eg EAGER_ALLOC_TRIGGER) in this determination, and schedule all
+ // zones that exceed the eager thresholds.
+ JS::GCReason wantMajorGC(bool eagerOk);
+ bool checkEagerAllocTrigger(const HeapSize& size,
+ const HeapThreshold& threshold);
+
+ // Do a minor GC if requested, followed by a major GC if requested. The return
+ // value indicates whether a major GC was performed.
+ bool gcIfRequested() { return gcIfRequestedImpl(false); }
+
+ // Internal function to do a GC if previously requested. But if not and
+ // eagerOk, do an eager GC for all Zones that have exceeded the eager
+ // thresholds.
+ //
+ // Return whether a major GC was performed or started.
+ bool gcIfRequestedImpl(bool eagerOk);
+
+ void gc(JS::GCOptions options, JS::GCReason reason);
+ void startGC(JS::GCOptions options, JS::GCReason reason,
+ const SliceBudget& budget);
+ void gcSlice(JS::GCReason reason, const SliceBudget& budget);
+ void finishGC(JS::GCReason reason);
+ void abortGC();
+ void startDebugGC(JS::GCOptions options, const SliceBudget& budget);
+ void debugGCSlice(const SliceBudget& budget);
+
+ void runDebugGC();
+ void notifyRootsRemoved();
+
+ enum TraceOrMarkRuntime { TraceRuntime, MarkRuntime };
+ void traceRuntime(JSTracer* trc, AutoTraceSession& session);
+ void traceRuntimeForMinorGC(JSTracer* trc, AutoGCSession& session);
+
+ void purgeRuntimeForMinorGC();
+
+ void shrinkBuffers();
+ void onOutOfMallocMemory();
+ void onOutOfMallocMemory(const AutoLockGC& lock);
+
+ Nursery& nursery() { return nursery_.ref(); }
+ gc::StoreBuffer& storeBuffer() { return storeBuffer_.ref(); }
+
+ void minorGC(JS::GCReason reason,
+ gcstats::PhaseKind phase = gcstats::PhaseKind::MINOR_GC)
+ JS_HAZ_GC_CALL;
+ void evictNursery(JS::GCReason reason = JS::GCReason::EVICT_NURSERY) {
+ minorGC(reason, gcstats::PhaseKind::EVICT_NURSERY);
+ }
+
+ void* addressOfNurseryPosition() {
+ return nursery_.refNoCheck().addressOfPosition();
+ }
+
+ const void* addressOfLastBufferedWholeCell() {
+ return storeBuffer_.refNoCheck().addressOfLastBufferedWholeCell();
+ }
+
+#ifdef JS_GC_ZEAL
+ const uint32_t* addressOfZealModeBits() { return &zealModeBits.refNoCheck(); }
+ void getZealBits(uint32_t* zealBits, uint32_t* frequency,
+ uint32_t* nextScheduled);
+ void setZeal(uint8_t zeal, uint32_t frequency);
+ void unsetZeal(uint8_t zeal);
+ bool parseAndSetZeal(const char* str);
+ void setNextScheduled(uint32_t count);
+ void verifyPreBarriers();
+ void maybeVerifyPreBarriers(bool always);
+ bool selectForMarking(JSObject* object);
+ void clearSelectedForMarking();
+ void setDeterministic(bool enable);
+ void setMarkStackLimit(size_t limit, AutoLockGC& lock);
+#endif
+
+ uint64_t nextCellUniqueId() {
+ MOZ_ASSERT(nextCellUniqueId_ > 0);
+ uint64_t uid = ++nextCellUniqueId_;
+ return uid;
+ }
+
+ void setLowMemoryState(bool newState) { lowMemoryState = newState; }
+ bool systemHasLowMemory() const { return lowMemoryState; }
+
+ public:
+ // Internal public interface
+ ZoneVector& zones() { return zones_.ref(); }
+ gcstats::Statistics& stats() { return stats_.ref(); }
+ const gcstats::Statistics& stats() const { return stats_.ref(); }
+ State state() const { return incrementalState; }
+ bool isHeapCompacting() const { return state() == State::Compact; }
+ bool isForegroundSweeping() const { return state() == State::Sweep; }
+ bool isBackgroundSweeping() const { return sweepTask.wasStarted(); }
+ bool isBackgroundMarking() const { return markTask.wasStarted(); }
+ void waitBackgroundSweepEnd();
+ void waitBackgroundAllocEnd() { allocTask.cancelAndWait(); }
+ void waitBackgroundFreeEnd();
+ void waitForBackgroundTasks();
+ bool isWaitingOnBackgroundTask() const;
+
+ void lockGC() { lock.lock(); }
+ bool tryLockGC() { return lock.tryLock(); }
+ void unlockGC() { lock.unlock(); }
+
+#ifdef DEBUG
+ void assertCurrentThreadHasLockedGC() const {
+ lock.assertOwnedByCurrentThread();
+ }
+#endif // DEBUG
+
+ void setAlwaysPreserveCode() { alwaysPreserveCode = true; }
+
+ bool isIncrementalGCAllowed() const { return incrementalAllowed; }
+ void disallowIncrementalGC() { incrementalAllowed = false; }
+
+ void setIncrementalGCEnabled(bool enabled);
+
+ bool isIncrementalGCEnabled() const { return incrementalGCEnabled; }
+ bool isPerZoneGCEnabled() const { return perZoneGCEnabled; }
+ bool isCompactingGCEnabled() const;
+ bool isParallelMarkingEnabled() const { return parallelMarkingEnabled; }
+
+ bool isIncrementalGCInProgress() const {
+ return state() != State::NotActive && !isVerifyPreBarriersEnabled();
+ }
+
+ bool hasForegroundWork() const;
+
+ bool isShrinkingGC() const { return gcOptions() == JS::GCOptions::Shrink; }
+
+ bool isShutdownGC() const { return gcOptions() == JS::GCOptions::Shutdown; }
+
+#ifdef DEBUG
+ bool isShuttingDown() const { return hadShutdownGC; }
+#endif
+
+ bool initSweepActions();
+
+ void setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data);
+ [[nodiscard]] bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data);
+ void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data);
+ void clearBlackAndGrayRootTracers();
+
+ void setGCCallback(JSGCCallback callback, void* data);
+ void callGCCallback(JSGCStatus status, JS::GCReason reason) const;
+ void setObjectsTenuredCallback(JSObjectsTenuredCallback callback, void* data);
+ void callObjectsTenuredCallback();
+ [[nodiscard]] bool addFinalizeCallback(JSFinalizeCallback callback,
+ void* data);
+ void removeFinalizeCallback(JSFinalizeCallback func);
+ void setHostCleanupFinalizationRegistryCallback(
+ JSHostCleanupFinalizationRegistryCallback callback, void* data);
+ void callHostCleanupFinalizationRegistryCallback(
+ JSFunction* doCleanup, GlobalObject* incumbentGlobal);
+ [[nodiscard]] bool addWeakPointerZonesCallback(
+ JSWeakPointerZonesCallback callback, void* data);
+ void removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback);
+ [[nodiscard]] bool addWeakPointerCompartmentCallback(
+ JSWeakPointerCompartmentCallback callback, void* data);
+ void removeWeakPointerCompartmentCallback(
+ JSWeakPointerCompartmentCallback callback);
+ JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback);
+ JS::GCNurseryCollectionCallback setNurseryCollectionCallback(
+ JS::GCNurseryCollectionCallback callback);
+ JS::DoCycleCollectionCallback setDoCycleCollectionCallback(
+ JS::DoCycleCollectionCallback callback);
+
+ bool addFinalizationRegistry(JSContext* cx,
+ Handle<FinalizationRegistryObject*> registry);
+ bool registerWithFinalizationRegistry(JSContext* cx, HandleObject target,
+ HandleObject record);
+ void queueFinalizationRegistryForCleanup(FinalizationQueueObject* queue);
+
+ void nukeFinalizationRecordWrapper(JSObject* wrapper,
+ FinalizationRecordObject* record);
+ void nukeWeakRefWrapper(JSObject* wrapper, WeakRefObject* record);
+
+ void setFullCompartmentChecks(bool enable);
+
+ // Get the main marking tracer.
+ GCMarker& marker() { return *markers[0]; }
+
+ JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; }
+ unsigned getCurrentSweepGroupIndex() {
+ MOZ_ASSERT_IF(unsigned(state()) < unsigned(State::Sweep),
+ sweepGroupIndex == 0);
+ return sweepGroupIndex;
+ }
+
+ uint64_t gcNumber() const { return number; }
+ void incGcNumber() { ++number; }
+
+ uint64_t minorGCCount() const { return minorGCNumber; }
+ void incMinorGcNumber() { ++minorGCNumber; }
+
+ uint64_t majorGCCount() const { return majorGCNumber; }
+ void incMajorGcNumber() { ++majorGCNumber; }
+
+ uint64_t gcSliceCount() const { return sliceNumber; }
+ void incGcSliceNumber() { ++sliceNumber; }
+
+ int64_t defaultSliceBudgetMS() const { return defaultTimeBudgetMS_; }
+
+ bool isIncrementalGc() const { return isIncremental; }
+ bool isFullGc() const { return isFull; }
+ bool isCompactingGc() const { return isCompacting; }
+ bool didCompactZones() const { return isCompacting && zonesCompacted; }
+
+ bool areGrayBitsValid() const { return grayBitsValid; }
+ void setGrayBitsInvalid() { grayBitsValid = false; }
+
+ mozilla::TimeStamp lastGCStartTime() const { return lastGCStartTime_; }
+ mozilla::TimeStamp lastGCEndTime() const { return lastGCEndTime_; }
+
+ bool majorGCRequested() const {
+ return majorGCTriggerReason != JS::GCReason::NO_REASON;
+ }
+
+ double computeHeapGrowthFactor(size_t lastBytes);
+ size_t computeTriggerBytes(double growthFactor, size_t lastBytes);
+
+ inline void updateOnFreeArenaAlloc(const TenuredChunkInfo& info);
+ void updateOnArenaFree() { ++numArenasFreeCommitted; }
+
+ ChunkPool& fullChunks(const AutoLockGC& lock) { return fullChunks_.ref(); }
+ ChunkPool& availableChunks(const AutoLockGC& lock) {
+ return availableChunks_.ref();
+ }
+ ChunkPool& emptyChunks(const AutoLockGC& lock) { return emptyChunks_.ref(); }
+ const ChunkPool& fullChunks(const AutoLockGC& lock) const {
+ return fullChunks_.ref();
+ }
+ const ChunkPool& availableChunks(const AutoLockGC& lock) const {
+ return availableChunks_.ref();
+ }
+ const ChunkPool& emptyChunks(const AutoLockGC& lock) const {
+ return emptyChunks_.ref();
+ }
+ using NonEmptyChunksIter = ChainedIterator<ChunkPool::Iter, 2>;
+ NonEmptyChunksIter allNonEmptyChunks(const AutoLockGC& lock) {
+ return NonEmptyChunksIter(availableChunks(lock), fullChunks(lock));
+ }
+ uint32_t minEmptyChunkCount(const AutoLockGC& lock) const {
+ return minEmptyChunkCount_;
+ }
+ uint32_t maxEmptyChunkCount(const AutoLockGC& lock) const {
+ return maxEmptyChunkCount_;
+ }
+#ifdef DEBUG
+ void verifyAllChunks();
+#endif
+
+ TenuredChunk* getOrAllocChunk(AutoLockGCBgAlloc& lock);
+ void recycleChunk(TenuredChunk* chunk, const AutoLockGC& lock);
+
+#ifdef JS_GC_ZEAL
+ void startVerifyPreBarriers();
+ void endVerifyPreBarriers();
+ void finishVerifier();
+ bool isVerifyPreBarriersEnabled() const { return verifyPreData.refNoCheck(); }
+ bool shouldYieldForZeal(ZealMode mode);
+#else
+ bool isVerifyPreBarriersEnabled() const { return false; }
+#endif
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+ void checkHashTablesAfterMovingGC();
+#endif
+
+#ifdef DEBUG
+ // Crawl the heap to check whether an arbitary pointer is within a cell of
+ // the given kind.
+ bool isPointerWithinTenuredCell(void* ptr, JS::TraceKind traceKind);
+
+ bool hasZone(Zone* target);
+#endif
+
+ // Queue memory memory to be freed on a background thread if possible.
+ void queueUnusedLifoBlocksForFree(LifoAlloc* lifo);
+ void queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo);
+ void queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers);
+
+ // Public here for ReleaseArenaLists and FinalizeTypedArenas.
+ void releaseArena(Arena* arena, const AutoLockGC& lock);
+
+ // Allocator
+ template <AllowGC allowGC>
+ [[nodiscard]] bool checkAllocatorState(JSContext* cx, AllocKind kind);
+ template <JS::TraceKind kind, AllowGC allowGC>
+ void* tryNewNurseryCell(JSContext* cx, size_t thingSize, AllocSite* site);
+ template <AllowGC allowGC>
+ static void* tryNewTenuredThing(JSContext* cx, AllocKind kind,
+ size_t thingSize);
+ static void* refillFreeListInGC(Zone* zone, AllocKind thingKind);
+
+ // Delayed marking.
+ void delayMarkingChildren(gc::Cell* cell, MarkColor color);
+ bool hasDelayedMarking() const;
+ void markAllDelayedChildren(ShouldReportMarkTime reportTime);
+
+ /*
+ * Concurrent sweep infrastructure.
+ */
+ void startTask(GCParallelTask& task, AutoLockHelperThreadState& lock);
+ void joinTask(GCParallelTask& task, AutoLockHelperThreadState& lock);
+ void updateHelperThreadCount();
+ bool updateMarkersVector();
+ size_t parallelWorkerCount() const;
+ size_t markingWorkerCount() const;
+
+ // WeakRefs
+ bool registerWeakRef(HandleObject target, HandleObject weakRef);
+ void traceKeptObjects(JSTracer* trc);
+
+ JS::GCReason lastStartReason() const { return initialReason; }
+
+ void updateAllocationRates();
+
+#ifdef DEBUG
+ const GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>& getTestMarkQueue()
+ const;
+ [[nodiscard]] bool appendTestMarkQueue(const JS::Value& value);
+ void clearTestMarkQueue();
+ size_t testMarkQueuePos() const;
+#endif
+
+ private:
+ enum IncrementalResult { ResetIncremental = 0, Ok };
+
+ [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value,
+ AutoLockGC& lock);
+ void resetParameter(JSGCParamKey key, AutoLockGC& lock);
+ uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock);
+ bool setThreadParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock);
+ void resetThreadParameter(JSGCParamKey key, AutoLockGC& lock);
+ void updateThreadDataStructures(AutoLockGC& lock);
+
+ JS::GCOptions gcOptions() const { return maybeGcOptions.ref().ref(); }
+
+ TriggerResult checkHeapThreshold(Zone* zone, const HeapSize& heapSize,
+ const HeapThreshold& heapThreshold);
+
+ void updateSchedulingStateOnGCStart();
+ void updateSchedulingStateAfterCollection(mozilla::TimeStamp currentTime);
+ void updateAllGCStartThresholds();
+
+ // For ArenaLists::allocateFromArena()
+ friend class ArenaLists;
+ TenuredChunk* pickChunk(AutoLockGCBgAlloc& lock);
+ Arena* allocateArena(TenuredChunk* chunk, Zone* zone, AllocKind kind,
+ ShouldCheckThresholds checkThresholds,
+ const AutoLockGC& lock);
+
+ // Allocator internals
+ void gcIfNeededAtAllocation(JSContext* cx);
+ static void* refillFreeList(JSContext* cx, AllocKind thingKind);
+ void attemptLastDitchGC(JSContext* cx);
+#ifdef DEBUG
+ static void checkIncrementalZoneState(JSContext* cx, void* ptr);
+#endif
+
+ /*
+ * Return the list of chunks that can be released outside the GC lock.
+ * Must be called either during the GC or with the GC lock taken.
+ */
+ friend class BackgroundDecommitTask;
+ bool tooManyEmptyChunks(const AutoLockGC& lock);
+ ChunkPool expireEmptyChunkPool(const AutoLockGC& lock);
+ void freeEmptyChunks(const AutoLockGC& lock);
+ void prepareToFreeChunk(TenuredChunkInfo& info);
+ void setMinEmptyChunkCount(uint32_t value, const AutoLockGC& lock);
+ void setMaxEmptyChunkCount(uint32_t value, const AutoLockGC& lock);
+
+ friend class BackgroundAllocTask;
+ bool wantBackgroundAllocation(const AutoLockGC& lock) const;
+ void startBackgroundAllocTaskIfIdle();
+
+ void requestMajorGC(JS::GCReason reason);
+ SliceBudget defaultBudget(JS::GCReason reason, int64_t millis);
+ bool maybeIncreaseSliceBudget(SliceBudget& budget);
+ bool maybeIncreaseSliceBudgetForLongCollections(SliceBudget& budget);
+ bool maybeIncreaseSliceBudgetForUrgentCollections(SliceBudget& budget);
+ IncrementalResult budgetIncrementalGC(bool nonincrementalByAPI,
+ JS::GCReason reason,
+ SliceBudget& budget);
+ void checkZoneIsScheduled(Zone* zone, JS::GCReason reason,
+ const char* trigger);
+ IncrementalResult resetIncrementalGC(GCAbortReason reason);
+
+ // Assert if the system state is such that we should never
+ // receive a request to do GC work.
+ void checkCanCallAPI();
+
+ // Check if the system state is such that GC has been supressed
+ // or otherwise delayed.
+ [[nodiscard]] bool checkIfGCAllowedInCurrentState(JS::GCReason reason);
+
+ gcstats::ZoneGCStats scanZonesBeforeGC();
+
+ void setGCOptions(JS::GCOptions options);
+
+ void collect(bool nonincrementalByAPI, const SliceBudget& budget,
+ JS::GCReason reason) JS_HAZ_GC_CALL;
+
+ /*
+ * Run one GC "cycle" (either a slice of incremental GC or an entire
+ * non-incremental GC).
+ *
+ * Returns:
+ * * ResetIncremental if we "reset" an existing incremental GC, which would
+ * force us to run another cycle or
+ * * Ok otherwise.
+ */
+ [[nodiscard]] IncrementalResult gcCycle(bool nonincrementalByAPI,
+ const SliceBudget& budgetArg,
+ JS::GCReason reason);
+ bool shouldRepeatForDeadZone(JS::GCReason reason);
+
+ void incrementalSlice(SliceBudget& budget, JS::GCReason reason,
+ bool budgetWasIncreased);
+
+ bool mightSweepInThisSlice(bool nonIncremental);
+ void collectNurseryFromMajorGC(JS::GCReason reason);
+ void collectNursery(JS::GCOptions options, JS::GCReason reason,
+ gcstats::PhaseKind phase);
+
+ friend class AutoCallGCCallbacks;
+ void maybeCallGCCallback(JSGCStatus status, JS::GCReason reason);
+
+ void startCollection(JS::GCReason reason);
+
+ void purgeRuntime();
+ [[nodiscard]] bool beginPreparePhase(JS::GCReason reason,
+ AutoGCSession& session);
+ bool prepareZonesForCollection(JS::GCReason reason, bool* isFullOut);
+ void unmarkWeakMaps();
+ void endPreparePhase(JS::GCReason reason);
+ void beginMarkPhase(AutoGCSession& session);
+ bool shouldPreserveJITCode(JS::Realm* realm,
+ const mozilla::TimeStamp& currentTime,
+ JS::GCReason reason, bool canAllocateMoreCode,
+ bool isActiveCompartment);
+ void discardJITCodeForGC();
+ void startBackgroundFreeAfterMinorGC();
+ void relazifyFunctionsForShrinkingGC();
+ void purgePropMapTablesForShrinkingGC();
+ void purgeSourceURLsForShrinkingGC();
+ void traceRuntimeForMajorGC(JSTracer* trc, AutoGCSession& session);
+ void traceRuntimeAtoms(JSTracer* trc);
+ void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark);
+ void traceEmbeddingBlackRoots(JSTracer* trc);
+ void traceEmbeddingGrayRoots(JSTracer* trc);
+ IncrementalProgress traceEmbeddingGrayRoots(JSTracer* trc,
+ SliceBudget& budget);
+ void checkNoRuntimeRoots(AutoGCSession& session);
+ void maybeDoCycleCollection();
+ void findDeadCompartments();
+
+ friend class BackgroundMarkTask;
+ enum ParallelMarking : bool {
+ SingleThreadedMarking = false,
+ AllowParallelMarking = true
+ };
+ IncrementalProgress markUntilBudgetExhausted(
+ SliceBudget& sliceBudget,
+ ParallelMarking allowParallelMarking = SingleThreadedMarking,
+ ShouldReportMarkTime reportTime = ReportMarkTime);
+ bool canMarkInParallel() const;
+
+ bool hasMarkingWork(MarkColor color) const;
+
+ void drainMarkStack();
+
+#ifdef DEBUG
+ void assertNoMarkingWork() const;
+#else
+ void assertNoMarkingWork() const {}
+#endif
+
+ void markDelayedChildren(gc::Arena* arena, MarkColor color);
+ void processDelayedMarkingList(gc::MarkColor color);
+ void rebuildDelayedMarkingList();
+ void appendToDelayedMarkingList(gc::Arena** listTail, gc::Arena* arena);
+ void resetDelayedMarking();
+ template <typename F>
+ void forEachDelayedMarkingArena(F&& f);
+
+ template <class ZoneIterT>
+ IncrementalProgress markWeakReferences(SliceBudget& budget);
+ IncrementalProgress markWeakReferencesInCurrentGroup(SliceBudget& budget);
+ template <class ZoneIterT>
+ IncrementalProgress markGrayRoots(SliceBudget& budget,
+ gcstats::PhaseKind phase);
+ void markBufferedGrayRoots(JS::Zone* zone);
+ IncrementalProgress markAllWeakReferences();
+ void markAllGrayReferences(gcstats::PhaseKind phase);
+
+ // The mark queue is a testing-only feature for controlling mark ordering and
+ // yield timing.
+ enum MarkQueueProgress {
+ QueueYielded, // End this incremental GC slice, if possible
+ QueueComplete, // Done with the queue
+ QueueSuspended // Continue the GC without ending the slice
+ };
+ MarkQueueProgress processTestMarkQueue();
+
+ // GC Sweeping. Implemented in Sweeping.cpp.
+ void beginSweepPhase(JS::GCReason reason, AutoGCSession& session);
+ void dropStringWrappers();
+ void groupZonesForSweeping(JS::GCReason reason);
+ [[nodiscard]] bool findSweepGroupEdges();
+ [[nodiscard]] bool addEdgesForMarkQueue();
+ void getNextSweepGroup();
+ void resetGrayList(Compartment* comp);
+ IncrementalProgress beginMarkingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget);
+ IncrementalProgress markGrayRootsInCurrentGroup(JS::GCContext* gcx,
+ SliceBudget& budget);
+ IncrementalProgress markGray(JS::GCContext* gcx, SliceBudget& budget);
+ IncrementalProgress endMarkingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget);
+ void markIncomingGrayCrossCompartmentPointers();
+ IncrementalProgress beginSweepingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget);
+ void initBackgroundSweep(Zone* zone, JS::GCContext* gcx,
+ const FinalizePhase& phase);
+ IncrementalProgress markDuringSweeping(JS::GCContext* gcx,
+ SliceBudget& budget);
+ void updateAtomsBitmap();
+ void sweepCCWrappers();
+ void sweepRealmGlobals();
+ void sweepEmbeddingWeakPointers(JS::GCContext* gcx);
+ void sweepMisc();
+ void sweepCompressionTasks();
+ void sweepWeakMaps();
+ void sweepUniqueIds();
+ void sweepDebuggerOnMainThread(JS::GCContext* gcx);
+ void sweepJitDataOnMainThread(JS::GCContext* gcx);
+ void sweepFinalizationObserversOnMainThread();
+ void traceWeakFinalizationObserverEdges(JSTracer* trc, Zone* zone);
+ void sweepWeakRefs();
+ IncrementalProgress endSweepingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget);
+ IncrementalProgress performSweepActions(SliceBudget& sliceBudget);
+ void startSweepingAtomsTable();
+ IncrementalProgress sweepAtomsTable(JS::GCContext* gcx, SliceBudget& budget);
+ IncrementalProgress sweepWeakCaches(JS::GCContext* gcx, SliceBudget& budget);
+ IncrementalProgress finalizeAllocKind(JS::GCContext* gcx,
+ SliceBudget& budget);
+ bool foregroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind thingKind,
+ js::SliceBudget& sliceBudget,
+ SortedArenaList& sweepList);
+ IncrementalProgress sweepPropMapTree(JS::GCContext* gcx, SliceBudget& budget);
+ void endSweepPhase(bool lastGC);
+ void queueZonesAndStartBackgroundSweep(ZoneList&& zones);
+ void sweepFromBackgroundThread(AutoLockHelperThreadState& lock);
+ void startBackgroundFree();
+ void freeFromBackgroundThread(AutoLockHelperThreadState& lock);
+ void sweepBackgroundThings(ZoneList& zones);
+ void backgroundFinalize(JS::GCContext* gcx, Zone* zone, AllocKind kind,
+ Arena** empty);
+ void assertBackgroundSweepingFinished();
+
+ bool allCCVisibleZonesWereCollected();
+ void sweepZones(JS::GCContext* gcx, bool destroyingRuntime);
+ bool shouldDecommit() const;
+ void startDecommit();
+ void decommitEmptyChunks(const bool& cancel, AutoLockGC& lock);
+ void decommitFreeArenas(const bool& cancel, AutoLockGC& lock);
+ void decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock);
+
+ // Compacting GC. Implemented in Compacting.cpp.
+ bool shouldCompact();
+ void beginCompactPhase();
+ IncrementalProgress compactPhase(JS::GCReason reason,
+ SliceBudget& sliceBudget,
+ AutoGCSession& session);
+ void endCompactPhase();
+ void sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone);
+ bool canRelocateZone(Zone* zone) const;
+ [[nodiscard]] bool relocateArenas(Zone* zone, JS::GCReason reason,
+ Arena*& relocatedListOut,
+ SliceBudget& sliceBudget);
+ void updateCellPointers(Zone* zone, AllocKinds kinds);
+ void updateAllCellPointers(MovingTracer* trc, Zone* zone);
+ void updateZonePointersToRelocatedCells(Zone* zone);
+ void updateRuntimePointersToRelocatedCells(AutoGCSession& session);
+ void clearRelocatedArenas(Arena* arenaList, JS::GCReason reason);
+ void clearRelocatedArenasWithoutUnlocking(Arena* arenaList,
+ JS::GCReason reason,
+ const AutoLockGC& lock);
+ void releaseRelocatedArenas(Arena* arenaList);
+ void releaseRelocatedArenasWithoutUnlocking(Arena* arenaList,
+ const AutoLockGC& lock);
+#ifdef DEBUG
+ void protectOrReleaseRelocatedArenas(Arena* arenaList, JS::GCReason reason);
+ void protectAndHoldArenas(Arena* arenaList);
+ void unprotectHeldRelocatedArenas(const AutoLockGC& lock);
+ void releaseHeldRelocatedArenas();
+ void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock);
+#endif
+
+ /*
+ * Whether to immediately trigger a slice after a background task
+ * finishes. This may not happen at a convenient time, so the consideration is
+ * whether the slice will run quickly or may take a long time.
+ */
+ enum ShouldTriggerSliceWhenFinished : bool {
+ DontTriggerSliceWhenFinished = false,
+ TriggerSliceWhenFinished = true
+ };
+
+ IncrementalProgress waitForBackgroundTask(
+ GCParallelTask& task, const SliceBudget& budget, bool shouldPauseMutator,
+ ShouldTriggerSliceWhenFinished triggerSlice);
+
+ void maybeRequestGCAfterBackgroundTask(const AutoLockHelperThreadState& lock);
+ void cancelRequestedGCAfterBackgroundTask();
+ void finishCollection(JS::GCReason reason);
+ void maybeStopPretenuring();
+ void checkGCStateNotInUse();
+ IncrementalProgress joinBackgroundMarkTask();
+
+#ifdef JS_GC_ZEAL
+ void computeNonIncrementalMarkingForValidation(AutoGCSession& session);
+ void validateIncrementalMarking();
+ void finishMarkingValidation();
+#endif
+
+#ifdef DEBUG
+ void checkForCompartmentMismatches();
+#endif
+
+ void callFinalizeCallbacks(JS::GCContext* gcx, JSFinalizeStatus status) const;
+ void callWeakPointerZonesCallbacks(JSTracer* trc) const;
+ void callWeakPointerCompartmentCallbacks(JSTracer* trc,
+ JS::Compartment* comp) const;
+ void callDoCycleCollectionCallback(JSContext* cx);
+
+ public:
+ JSRuntime* const rt;
+
+ // Embedders can use this zone however they wish.
+ MainThreadData<JS::Zone*> systemZone;
+
+ MainThreadData<JS::GCContext> mainThreadContext;
+
+ private:
+ // For parent runtimes, a zone containing atoms that is shared by child
+ // runtimes.
+ MainThreadData<Zone*> sharedAtomsZone_;
+
+ // All zones in the runtime. The first element is always the atoms zone.
+ MainThreadOrGCTaskData<ZoneVector> zones_;
+
+ // Any activity affecting the heap.
+ MainThreadOrGCTaskData<JS::HeapState> heapState_;
+ friend class AutoHeapSession;
+ friend class JS::AutoEnterCycleCollection;
+
+ UnprotectedData<gcstats::Statistics> stats_;
+
+ public:
+ js::StringStats stringStats;
+
+ Vector<UniquePtr<GCMarker>, 1, SystemAllocPolicy> markers;
+
+ // Delayed marking support in case we OOM pushing work onto the mark stack.
+ MainThreadOrGCTaskData<js::gc::Arena*> delayedMarkingList;
+ MainThreadOrGCTaskData<bool> delayedMarkingWorkAdded;
+#ifdef DEBUG
+ /* Count of arenas that are currently in the stack. */
+ MainThreadOrGCTaskData<size_t> markLaterArenas;
+#endif
+
+ SweepingTracer sweepingTracer;
+
+ /* Track total GC heap size for this runtime. */
+ HeapSize heapSize;
+
+ /* GC scheduling state and parameters. */
+ GCSchedulingTunables tunables;
+ GCSchedulingState schedulingState;
+ MainThreadData<bool> fullGCRequested;
+
+ // Helper thread configuration.
+ MainThreadData<double> helperThreadRatio;
+ MainThreadData<size_t> maxHelperThreads;
+ MainThreadOrGCTaskData<size_t> helperThreadCount;
+ MainThreadData<size_t> markingThreadCount;
+
+ // State used for managing atom mark bitmaps in each zone.
+ AtomMarkingRuntime atomMarking;
+
+ /*
+ * Pointer to a callback that, if set, will be used to create a
+ * budget for internally-triggered GCs.
+ */
+ MainThreadData<JS::CreateSliceBudgetCallback> createBudgetCallback;
+
+ private:
+ // Arenas used for permanent things created at startup and shared by child
+ // runtimes.
+ MainThreadData<ArenaList> permanentAtoms;
+ MainThreadData<ArenaList> permanentFatInlineAtoms;
+ MainThreadData<ArenaList> permanentWellKnownSymbols;
+
+ // When chunks are empty, they reside in the emptyChunks pool and are
+ // re-used as needed or eventually expired if not re-used. The emptyChunks
+ // pool gets refilled from the background allocation task heuristically so
+ // that empty chunks should always be available for immediate allocation
+ // without syscalls.
+ GCLockData<ChunkPool> emptyChunks_;
+
+ // Chunks which have had some, but not all, of their arenas allocated live
+ // in the available chunk lists. When all available arenas in a chunk have
+ // been allocated, the chunk is removed from the available list and moved
+ // to the fullChunks pool. During a GC, if all arenas are free, the chunk
+ // is moved back to the emptyChunks pool and scheduled for eventual
+ // release.
+ GCLockData<ChunkPool> availableChunks_;
+
+ // When all arenas in a chunk are used, it is moved to the fullChunks pool
+ // so as to reduce the cost of operations on the available lists.
+ GCLockData<ChunkPool> fullChunks_;
+
+ /*
+ * JSGC_MIN_EMPTY_CHUNK_COUNT
+ * JSGC_MAX_EMPTY_CHUNK_COUNT
+ *
+ * Controls the number of empty chunks reserved for future allocation.
+ *
+ * They can be read off main thread by the background allocation task and the
+ * background decommit task.
+ */
+ GCLockData<uint32_t> minEmptyChunkCount_;
+ GCLockData<uint32_t> maxEmptyChunkCount_;
+
+ MainThreadData<RootedValueMap> rootsHash;
+
+ // An incrementing id used to assign unique ids to cells that require one.
+ MainThreadData<uint64_t> nextCellUniqueId_;
+
+ /*
+ * Number of the committed arenas in all GC chunks including empty chunks.
+ */
+ mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire> numArenasFreeCommitted;
+ MainThreadData<VerifyPreTracer*> verifyPreData;
+
+ MainThreadData<mozilla::TimeStamp> lastGCStartTime_;
+ MainThreadData<mozilla::TimeStamp> lastGCEndTime_;
+
+ WriteOnceData<bool> initialized;
+ MainThreadData<bool> incrementalGCEnabled;
+ MainThreadData<bool> perZoneGCEnabled;
+
+ mozilla::Atomic<size_t, mozilla::ReleaseAcquire> numActiveZoneIters;
+
+ /* During shutdown, the GC needs to clean up every possible object. */
+ MainThreadData<bool> cleanUpEverything;
+
+ /*
+ * The gray bits can become invalid if UnmarkGray overflows the stack. A
+ * full GC will reset this bit, since it fills in all the gray bits.
+ */
+ UnprotectedData<bool> grayBitsValid;
+
+ mozilla::Atomic<JS::GCReason, mozilla::ReleaseAcquire> majorGCTriggerReason;
+
+ /* Incremented at the start of every minor GC. */
+ MainThreadData<uint64_t> minorGCNumber;
+
+ /* Incremented at the start of every major GC. */
+ MainThreadData<uint64_t> majorGCNumber;
+
+ /* Incremented on every GC slice or minor collection. */
+ MainThreadData<uint64_t> number;
+
+ /* Incremented on every GC slice. */
+ MainThreadData<uint64_t> sliceNumber;
+
+ /* Whether the currently running GC can finish in multiple slices. */
+ MainThreadOrGCTaskData<bool> isIncremental;
+
+ /* Whether all zones are being collected in first GC slice. */
+ MainThreadData<bool> isFull;
+
+ /* Whether the heap will be compacted at the end of GC. */
+ MainThreadData<bool> isCompacting;
+
+ /* The invocation kind of the current GC, set at the start of collection. */
+ MainThreadOrGCTaskData<mozilla::Maybe<JS::GCOptions>> maybeGcOptions;
+
+ /* The initial GC reason, taken from the first slice. */
+ MainThreadData<JS::GCReason> initialReason;
+
+ /*
+ * The current incremental GC phase. This is also used internally in
+ * non-incremental GC.
+ */
+ MainThreadOrGCTaskData<State> incrementalState;
+
+ /* The incremental state at the start of this slice. */
+ MainThreadOrGCTaskData<State> initialState;
+
+ /* Whether to pay attention the zeal settings in this incremental slice. */
+#ifdef JS_GC_ZEAL
+ MainThreadData<bool> useZeal;
+#else
+ const bool useZeal;
+#endif
+
+ /* Indicates that the last incremental slice exhausted the mark stack. */
+ MainThreadData<bool> lastMarkSlice;
+
+ // Whether it's currently safe to yield to the mutator in an incremental GC.
+ MainThreadData<bool> safeToYield;
+
+ // Whether to do any marking caused by barriers on a background thread during
+ // incremental sweeping, while also sweeping zones which have finished
+ // marking.
+ MainThreadData<bool> markOnBackgroundThreadDuringSweeping;
+
+ // Whether any sweeping and decommitting will run on a separate GC helper
+ // thread.
+ MainThreadData<bool> useBackgroundThreads;
+
+#ifdef DEBUG
+ /* Shutdown has started. Further collections must be shutdown collections. */
+ MainThreadData<bool> hadShutdownGC;
+#endif
+
+ /* Singly linked list of zones to be swept in the background. */
+ HelperThreadLockData<ZoneList> backgroundSweepZones;
+
+ /*
+ * Whether to trigger a GC slice after a background task is complete, so that
+ * the collector can continue or finsish collecting. This is only used for the
+ * tasks that run concurrently with the mutator, which are background
+ * finalization and background decommit.
+ */
+ HelperThreadLockData<bool> requestSliceAfterBackgroundTask;
+
+ /*
+ * Free LIFO blocks are transferred to these allocators before being freed on
+ * a background thread.
+ */
+ HelperThreadLockData<LifoAlloc> lifoBlocksToFree;
+ MainThreadData<LifoAlloc> lifoBlocksToFreeAfterMinorGC;
+ HelperThreadLockData<Nursery::BufferSet> buffersToFreeAfterMinorGC;
+
+ /* Index of current sweep group (for stats). */
+ MainThreadData<unsigned> sweepGroupIndex;
+
+ /*
+ * Incremental sweep state.
+ */
+ MainThreadData<JS::Zone*> sweepGroups;
+ MainThreadOrGCTaskData<JS::Zone*> currentSweepGroup;
+ MainThreadData<UniquePtr<SweepAction>> sweepActions;
+ MainThreadOrGCTaskData<JS::Zone*> sweepZone;
+ MainThreadOrGCTaskData<AllocKind> sweepAllocKind;
+ MainThreadData<mozilla::Maybe<AtomsTable::SweepIterator>> maybeAtomsToSweep;
+ MainThreadOrGCTaskData<mozilla::Maybe<WeakCacheSweepIterator>>
+ weakCachesToSweep;
+ MainThreadData<bool> abortSweepAfterCurrentGroup;
+ MainThreadOrGCTaskData<IncrementalProgress> sweepMarkResult;
+
+#ifdef DEBUG
+ /*
+ * List of objects to mark at the beginning of a GC for testing purposes. May
+ * also contain string directives to change mark color or wait until different
+ * phases of the GC.
+ *
+ * This is a WeakCache because not everything in this list is guaranteed to
+ * end up marked (eg if you insert an object from an already-processed sweep
+ * group in the middle of an incremental GC). Also, the mark queue is not
+ * used during shutdown GCs. In either case, unmarked objects may need to be
+ * discarded.
+ */
+ JS::WeakCache<GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>>
+ testMarkQueue;
+
+ /* Position within the test mark queue. */
+ size_t queuePos;
+
+ /* The test marking queue might want to be marking a particular color. */
+ mozilla::Maybe<js::gc::MarkColor> queueMarkColor;
+
+ // During gray marking, delay AssertCellIsNotGray checks by
+ // recording the cell pointers here and checking after marking has
+ // finished.
+ MainThreadData<Vector<const Cell*, 0, SystemAllocPolicy>>
+ cellsToAssertNotGray;
+ friend void js::gc::detail::AssertCellIsNotGray(const Cell*);
+#endif
+
+ friend class SweepGroupsIter;
+
+ /*
+ * Incremental compacting state.
+ */
+ MainThreadData<bool> startedCompacting;
+ MainThreadData<ZoneList> zonesToMaybeCompact;
+ MainThreadData<size_t> zonesCompacted;
+#ifdef DEBUG
+ GCLockData<Arena*> relocatedArenasToRelease;
+#endif
+
+#ifdef JS_GC_ZEAL
+ MainThreadData<MarkingValidator*> markingValidator;
+#endif
+
+ /*
+ * Default budget for incremental GC slice. See js/SliceBudget.h.
+ *
+ * JSGC_SLICE_TIME_BUDGET_MS
+ * pref: javascript.options.mem.gc_incremental_slice_ms,
+ */
+ MainThreadData<int64_t> defaultTimeBudgetMS_;
+
+ /*
+ * We disable incremental GC if we encounter a Class with a trace hook
+ * that does not implement write barriers.
+ */
+ MainThreadData<bool> incrementalAllowed;
+
+ /*
+ * Whether compacting GC can is enabled globally.
+ *
+ * JSGC_COMPACTING_ENABLED
+ * pref: javascript.options.mem.gc_compacting
+ */
+ MainThreadData<bool> compactingEnabled;
+
+ /*
+ * Whether parallel marking is enabled globally.
+ *
+ * JSGC_PARALLEL_MARKING_ENABLED
+ * pref: javascript.options.mem.gc_parallel_marking
+ */
+ MainThreadData<bool> parallelMarkingEnabled;
+
+ MainThreadData<bool> rootsRemoved;
+
+ /*
+ * These options control the zealousness of the GC. At every allocation,
+ * nextScheduled is decremented. When it reaches zero we do a full GC.
+ *
+ * At this point, if zeal_ is one of the types that trigger periodic
+ * collection, then nextScheduled is reset to the value of zealFrequency.
+ * Otherwise, no additional GCs take place.
+ *
+ * You can control these values in several ways:
+ * - Set the JS_GC_ZEAL environment variable
+ * - Call gczeal() or schedulegc() from inside shell-executed JS code
+ * (see the help for details)
+ *
+ * If gcZeal_ == 1 then we perform GCs in select places (during MaybeGC and
+ * whenever we are notified that GC roots have been removed). This option is
+ * mainly useful to embedders.
+ *
+ * We use zeal_ == 4 to enable write barrier verification. See the comment
+ * in gc/Verifier.cpp for more information about this.
+ *
+ * zeal_ values from 8 to 10 periodically run different types of
+ * incremental GC.
+ *
+ * zeal_ value 14 performs periodic shrinking collections.
+ */
+#ifdef JS_GC_ZEAL
+ static_assert(size_t(ZealMode::Count) <= 32,
+ "Too many zeal modes to store in a uint32_t");
+ MainThreadData<uint32_t> zealModeBits;
+ MainThreadData<int> zealFrequency;
+ MainThreadData<int> nextScheduled;
+ MainThreadData<bool> deterministicOnly;
+ MainThreadData<int> zealSliceBudget;
+ MainThreadData<size_t> maybeMarkStackLimit;
+
+ MainThreadData<PersistentRooted<GCVector<JSObject*, 0, SystemAllocPolicy>>>
+ selectedForMarking;
+#endif
+
+ MainThreadData<bool> fullCompartmentChecks;
+
+ MainThreadData<uint32_t> gcCallbackDepth;
+
+ MainThreadData<Callback<JSGCCallback>> gcCallback;
+ MainThreadData<Callback<JS::DoCycleCollectionCallback>>
+ gcDoCycleCollectionCallback;
+ MainThreadData<Callback<JSObjectsTenuredCallback>> tenuredCallback;
+ MainThreadData<CallbackVector<JSFinalizeCallback>> finalizeCallbacks;
+ MainThreadOrGCTaskData<Callback<JSHostCleanupFinalizationRegistryCallback>>
+ hostCleanupFinalizationRegistryCallback;
+ MainThreadData<CallbackVector<JSWeakPointerZonesCallback>>
+ updateWeakPointerZonesCallbacks;
+ MainThreadData<CallbackVector<JSWeakPointerCompartmentCallback>>
+ updateWeakPointerCompartmentCallbacks;
+
+ /*
+ * The trace operations to trace embedding-specific GC roots. One is for
+ * tracing through black roots and the other is for tracing through gray
+ * roots. The black/gray distinction is only relevant to the cycle
+ * collector.
+ */
+ MainThreadData<CallbackVector<JSTraceDataOp>> blackRootTracers;
+ MainThreadOrGCTaskData<Callback<JSGrayRootsTracer>> grayRootTracer;
+
+ /* Always preserve JIT code during GCs, for testing. */
+ MainThreadData<bool> alwaysPreserveCode;
+
+ /* Count of the number of zones that are currently in page load. */
+ MainThreadData<size_t> inPageLoadCount;
+
+ MainThreadData<bool> lowMemoryState;
+
+ /*
+ * General purpose GC lock, used for synchronising operations on
+ * arenas and during parallel marking.
+ */
+ friend class js::AutoLockGC;
+ friend class js::AutoLockGCBgAlloc;
+ js::Mutex lock MOZ_UNANNOTATED;
+
+ /* Lock used to synchronise access to delayed marking state. */
+ js::Mutex delayedMarkingLock MOZ_UNANNOTATED;
+
+ friend class BackgroundSweepTask;
+ friend class BackgroundFreeTask;
+
+ BackgroundAllocTask allocTask;
+ BackgroundUnmarkTask unmarkTask;
+ BackgroundMarkTask markTask;
+ BackgroundSweepTask sweepTask;
+ BackgroundFreeTask freeTask;
+ BackgroundDecommitTask decommitTask;
+
+ /*
+ * During incremental sweeping, this field temporarily holds the arenas of
+ * the current AllocKind being swept in order of increasing free space.
+ */
+ MainThreadData<SortedArenaList> incrementalSweepList;
+
+ MainThreadData<Nursery> nursery_;
+
+ // The store buffer used to track tenured to nursery edges for generational
+ // GC. This is accessed off main thread when sweeping WeakCaches.
+ MainThreadOrGCTaskData<gc::StoreBuffer> storeBuffer_;
+
+ mozilla::TimeStamp lastLastDitchTime;
+
+ // The last time per-zone allocation rates were updated.
+ MainThreadData<mozilla::TimeStamp> lastAllocRateUpdateTime;
+
+ // Total collector time since per-zone allocation rates were last updated.
+ MainThreadData<mozilla::TimeDuration> collectorTimeSinceAllocRateUpdate;
+
+ friend class MarkingValidator;
+ friend class AutoEnterIteration;
+};
+
+/* Prevent compartments and zones from being collected during iteration. */
+class MOZ_RAII AutoEnterIteration {
+ GCRuntime* gc;
+
+ public:
+ explicit AutoEnterIteration(GCRuntime* gc_) : gc(gc_) {
+ ++gc->numActiveZoneIters;
+ }
+
+ ~AutoEnterIteration() {
+ MOZ_ASSERT(gc->numActiveZoneIters);
+ --gc->numActiveZoneIters;
+ }
+};
+
+#ifdef JS_GC_ZEAL
+
+inline bool GCRuntime::hasZealMode(ZealMode mode) {
+ static_assert(size_t(ZealMode::Limit) < sizeof(zealModeBits) * 8,
+ "Zeal modes must fit in zealModeBits");
+ return zealModeBits & (1 << uint32_t(mode));
+}
+
+inline void GCRuntime::clearZealMode(ZealMode mode) {
+ zealModeBits &= ~(1 << uint32_t(mode));
+ MOZ_ASSERT(!hasZealMode(mode));
+}
+
+inline bool GCRuntime::needZealousGC() {
+ if (nextScheduled > 0 && --nextScheduled == 0) {
+ if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::GenerationalGC) ||
+ hasZealMode(ZealMode::IncrementalMultipleSlices) ||
+ hasZealMode(ZealMode::Compact) || hasIncrementalTwoSliceZealMode()) {
+ nextScheduled = zealFrequency;
+ }
+ return true;
+ }
+ return false;
+}
+
+inline bool GCRuntime::hasIncrementalTwoSliceZealMode() {
+ return hasZealMode(ZealMode::YieldBeforeRootMarking) ||
+ hasZealMode(ZealMode::YieldBeforeMarking) ||
+ hasZealMode(ZealMode::YieldBeforeSweeping) ||
+ hasZealMode(ZealMode::YieldBeforeSweepingAtoms) ||
+ hasZealMode(ZealMode::YieldBeforeSweepingCaches) ||
+ hasZealMode(ZealMode::YieldBeforeSweepingObjects) ||
+ hasZealMode(ZealMode::YieldBeforeSweepingNonObjects) ||
+ hasZealMode(ZealMode::YieldBeforeSweepingPropMapTrees) ||
+ hasZealMode(ZealMode::YieldWhileGrayMarking);
+}
+
+#else
+inline bool GCRuntime::hasZealMode(ZealMode mode) { return false; }
+inline void GCRuntime::clearZealMode(ZealMode mode) {}
+inline bool GCRuntime::needZealousGC() { return false; }
+inline bool GCRuntime::hasIncrementalTwoSliceZealMode() { return false; }
+#endif
+
+bool IsCurrentlyAnimating(const mozilla::TimeStamp& lastAnimationTime,
+ const mozilla::TimeStamp& currentTime);
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif
diff --git a/js/src/gc/GenerateStatsPhases.py b/js/src/gc/GenerateStatsPhases.py
new file mode 100644
index 0000000000..355abaf09d
--- /dev/null
+++ b/js/src/gc/GenerateStatsPhases.py
@@ -0,0 +1,404 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# flake8: noqa: F821
+
+# Generate graph structures for GC statistics recording.
+#
+# Stats phases are nested and form a directed acyclic graph starting
+# from a set of root phases. Importantly, a phase may appear under more
+# than one parent phase.
+#
+# For example, the following arrangement is possible:
+#
+# +---+
+# | A |
+# +---+
+# |
+# +-------+-------+
+# | | |
+# v v v
+# +---+ +---+ +---+
+# | B | | C | | D |
+# +---+ +---+ +---+
+# | |
+# +---+---+
+# |
+# v
+# +---+
+# | E |
+# +---+
+#
+# This graph is expanded into a tree (or really a forest) and phases
+# with multiple parents are duplicated.
+#
+# For example, the input example above would be expanded to:
+#
+# +---+
+# | A |
+# +---+
+# |
+# +-------+-------+
+# | | |
+# v v v
+# +---+ +---+ +---+
+# | B | | C | | D |
+# +---+ +---+ +---+
+# | |
+# v v
+# +---+ +---+
+# | E | | E'|
+# +---+ +---+
+
+# NOTE: If you add new phases here the current next phase kind number can be
+# found at the end of js/src/gc/StatsPhasesGenerated.inc
+
+import collections
+import re
+
+
+class PhaseKind:
+ def __init__(self, name, descr, bucket, children=[]):
+ self.name = name
+ self.descr = descr
+ # For telemetry
+ self.bucket = bucket
+ self.children = children
+
+
+AllPhaseKinds = []
+PhaseKindsByName = dict()
+
+
+def addPhaseKind(name, descr, bucket, children=[]):
+ assert name not in PhaseKindsByName
+ phaseKind = PhaseKind(name, descr, bucket, children)
+ AllPhaseKinds.append(phaseKind)
+ PhaseKindsByName[name] = phaseKind
+ return phaseKind
+
+
+def getPhaseKind(name):
+ return PhaseKindsByName[name]
+
+
+PhaseKindGraphRoots = [
+ addPhaseKind("MUTATOR", "Mutator Running", 0),
+ addPhaseKind("GC_BEGIN", "Begin Callback", 1),
+ addPhaseKind(
+ "EVICT_NURSERY_FOR_MAJOR_GC",
+ "Evict Nursery For Major GC",
+ 70,
+ [
+ addPhaseKind(
+ "MARK_ROOTS",
+ "Mark Roots",
+ 48,
+ [
+ addPhaseKind("MARK_CCWS", "Mark Cross Compartment Wrappers", 50),
+ addPhaseKind("MARK_STACK", "Mark C and JS stacks", 51),
+ addPhaseKind("MARK_RUNTIME_DATA", "Mark Runtime-wide Data", 52),
+ addPhaseKind("MARK_EMBEDDING", "Mark Embedding", 53),
+ ],
+ )
+ ],
+ ),
+ addPhaseKind("WAIT_BACKGROUND_THREAD", "Wait Background Thread", 2),
+ addPhaseKind(
+ "PREPARE",
+ "Prepare For Collection",
+ 69,
+ [
+ addPhaseKind("UNMARK", "Unmark", 7),
+ addPhaseKind("UNMARK_WEAKMAPS", "Unmark WeakMaps", 76),
+ addPhaseKind("MARK_DISCARD_CODE", "Mark Discard Code", 3),
+ addPhaseKind("RELAZIFY_FUNCTIONS", "Relazify Functions", 4),
+ addPhaseKind("PURGE", "Purge", 5),
+ addPhaseKind("PURGE_PROP_MAP_TABLES", "Purge PropMapTables", 60),
+ addPhaseKind("PURGE_SOURCE_URLS", "Purge Source URLs", 73),
+ addPhaseKind("JOIN_PARALLEL_TASKS", "Join Parallel Tasks", 67),
+ ],
+ ),
+ addPhaseKind(
+ "MARK",
+ "Mark",
+ 6,
+ [
+ getPhaseKind("MARK_ROOTS"),
+ addPhaseKind("MARK_DELAYED", "Mark Delayed", 8),
+ addPhaseKind(
+ "MARK_WEAK",
+ "Mark Weak",
+ 13,
+ [
+ getPhaseKind("MARK_DELAYED"),
+ addPhaseKind("MARK_GRAY_WEAK", "Mark Gray and Weak", 16),
+ ],
+ ),
+ addPhaseKind("MARK_INCOMING_GRAY", "Mark Incoming Gray Pointers", 14),
+ addPhaseKind("MARK_GRAY", "Mark Gray", 15),
+ addPhaseKind(
+ "PARALLEL_MARK",
+ "Parallel marking",
+ 78,
+ [
+ getPhaseKind("JOIN_PARALLEL_TASKS"),
+ # The following are only used for parallel phase times:
+ addPhaseKind("PARALLEL_MARK_MARK", "Parallel marking work", 79),
+ addPhaseKind("PARALLEL_MARK_WAIT", "Waiting for work", 80),
+ ],
+ ),
+ ],
+ ),
+ addPhaseKind(
+ "SWEEP",
+ "Sweep",
+ 9,
+ [
+ getPhaseKind("MARK"),
+ addPhaseKind(
+ "FINALIZE_START",
+ "Finalize Start Callbacks",
+ 17,
+ [
+ addPhaseKind("WEAK_ZONES_CALLBACK", "Per-Slice Weak Callback", 57),
+ addPhaseKind(
+ "WEAK_COMPARTMENT_CALLBACK", "Per-Compartment Weak Callback", 58
+ ),
+ ],
+ ),
+ addPhaseKind("UPDATE_ATOMS_BITMAP", "Sweep Atoms Bitmap", 68),
+ addPhaseKind("SWEEP_ATOMS_TABLE", "Sweep Atoms Table", 18),
+ addPhaseKind(
+ "SWEEP_COMPARTMENTS",
+ "Sweep Compartments",
+ 20,
+ [
+ addPhaseKind("SWEEP_DISCARD_CODE", "Sweep Discard Code", 21),
+ addPhaseKind("SWEEP_INNER_VIEWS", "Sweep Inner Views", 22),
+ addPhaseKind(
+ "SWEEP_CC_WRAPPER", "Sweep Cross Compartment Wrappers", 23
+ ),
+ addPhaseKind("SWEEP_BASE_SHAPE", "Sweep Base Shapes", 24),
+ addPhaseKind("SWEEP_INITIAL_SHAPE", "Sweep Initial Shapes", 25),
+ addPhaseKind("SWEEP_REGEXP", "Sweep Regexps", 28),
+ addPhaseKind("SWEEP_COMPRESSION", "Sweep Compression Tasks", 62),
+ addPhaseKind("SWEEP_WEAKMAPS", "Sweep WeakMaps", 63),
+ addPhaseKind("SWEEP_UNIQUEIDS", "Sweep Unique IDs", 64),
+ addPhaseKind(
+ "SWEEP_FINALIZATION_OBSERVERS",
+ "Sweep FinalizationRegistries and WeakRefs",
+ 74,
+ ),
+ addPhaseKind("SWEEP_JIT_DATA", "Sweep JIT Data", 65),
+ addPhaseKind("SWEEP_WEAK_CACHES", "Sweep Weak Caches", 66),
+ addPhaseKind("SWEEP_MISC", "Sweep Miscellaneous", 29),
+ getPhaseKind("JOIN_PARALLEL_TASKS"),
+ ],
+ ),
+ addPhaseKind("FINALIZE_OBJECT", "Finalize Objects", 33),
+ addPhaseKind("FINALIZE_NON_OBJECT", "Finalize Non-objects", 34),
+ addPhaseKind("SWEEP_PROP_MAP", "Sweep PropMap Tree", 77),
+ addPhaseKind("FINALIZE_END", "Finalize End Callback", 38),
+ addPhaseKind("DESTROY", "Deallocate", 39),
+ getPhaseKind("JOIN_PARALLEL_TASKS"),
+ addPhaseKind("FIND_DEAD_COMPARTMENTS", "Find Dead Compartments", 54),
+ ],
+ ),
+ addPhaseKind(
+ "COMPACT",
+ "Compact",
+ 40,
+ [
+ addPhaseKind("COMPACT_MOVE", "Compact Move", 41),
+ addPhaseKind(
+ "COMPACT_UPDATE",
+ "Compact Update",
+ 42,
+ [
+ getPhaseKind("MARK_ROOTS"),
+ addPhaseKind("COMPACT_UPDATE_CELLS", "Compact Update Cells", 43),
+ getPhaseKind("JOIN_PARALLEL_TASKS"),
+ ],
+ ),
+ ],
+ ),
+ addPhaseKind("DECOMMIT", "Decommit", 72),
+ addPhaseKind("GC_END", "End Callback", 44),
+ addPhaseKind(
+ "MINOR_GC",
+ "All Minor GCs",
+ 45,
+ [
+ getPhaseKind("MARK_ROOTS"),
+ ],
+ ),
+ addPhaseKind(
+ "EVICT_NURSERY",
+ "Minor GCs to Evict Nursery",
+ 46,
+ [
+ getPhaseKind("MARK_ROOTS"),
+ ],
+ ),
+ addPhaseKind(
+ "TRACE_HEAP",
+ "Trace Heap",
+ 47,
+ [
+ getPhaseKind("MARK_ROOTS"),
+ ],
+ ),
+]
+
+
+class Phase:
+ # Expand the DAG into a tree, duplicating phases which have more than
+ # one parent.
+ def __init__(self, phaseKind, parent):
+ self.phaseKind = phaseKind
+ self.parent = parent
+ self.depth = parent.depth + 1 if parent else 0
+ self.children = []
+ self.nextSibling = None
+ self.nextInPhaseKind = None
+
+ self.path = re.sub(r"\W+", "_", phaseKind.name.lower())
+ if parent is not None:
+ self.path = parent.path + "." + self.path
+
+
+def expandPhases():
+ phases = []
+ phasesForKind = collections.defaultdict(list)
+
+ def traverse(phaseKind, parent):
+ ep = Phase(phaseKind, parent)
+ phases.append(ep)
+
+ # Update list of expanded phases for this phase kind.
+ if phasesForKind[phaseKind]:
+ phasesForKind[phaseKind][-1].nextInPhaseKind = ep
+ phasesForKind[phaseKind].append(ep)
+
+ # Recurse over children.
+ for child in phaseKind.children:
+ child_ep = traverse(child, ep)
+ if ep.children:
+ ep.children[-1].nextSibling = child_ep
+ ep.children.append(child_ep)
+ return ep
+
+ for phaseKind in PhaseKindGraphRoots:
+ traverse(phaseKind, None)
+
+ return phases, phasesForKind
+
+
+AllPhases, PhasesForPhaseKind = expandPhases()
+
+# Name phases based on phase kind name and index if there are multiple phases
+# corresponding to a single phase kind.
+
+for phaseKind in AllPhaseKinds:
+ phases = PhasesForPhaseKind[phaseKind]
+ if len(phases) == 1:
+ phases[0].name = "%s" % phaseKind.name
+ else:
+ for index, phase in enumerate(phases):
+ phase.name = "%s_%d" % (phaseKind.name, index + 1)
+
+# Find the maximum phase nesting.
+MaxPhaseNesting = max(phase.depth for phase in AllPhases) + 1
+
+# And the maximum bucket number.
+MaxBucket = max(kind.bucket for kind in AllPhaseKinds)
+
+# Generate code.
+
+
+def writeList(out, items):
+ if items:
+ out.write(",\n".join(" " + item for item in items) + "\n")
+
+
+def writeEnumClass(out, name, type, items, extraItems):
+ items = ["FIRST"] + list(items) + ["LIMIT"] + list(extraItems)
+ items[1] += " = " + items[0]
+ out.write("enum class %s : %s {\n" % (name, type))
+ writeList(out, items)
+ out.write("};\n")
+
+
+def generateHeader(out):
+ #
+ # Generate PhaseKind enum.
+ #
+ phaseKindNames = map(lambda phaseKind: phaseKind.name, AllPhaseKinds)
+ extraPhaseKinds = [
+ "NONE = LIMIT",
+ "EXPLICIT_SUSPENSION = LIMIT",
+ "IMPLICIT_SUSPENSION",
+ ]
+ writeEnumClass(out, "PhaseKind", "uint8_t", phaseKindNames, extraPhaseKinds)
+ out.write("\n")
+
+ #
+ # Generate Phase enum.
+ #
+ phaseNames = map(lambda phase: phase.name, AllPhases)
+ extraPhases = ["NONE = LIMIT", "EXPLICIT_SUSPENSION = LIMIT", "IMPLICIT_SUSPENSION"]
+ writeEnumClass(out, "Phase", "uint8_t", phaseNames, extraPhases)
+ out.write("\n")
+
+ #
+ # Generate MAX_PHASE_NESTING constant.
+ #
+ out.write("static const size_t MAX_PHASE_NESTING = %d;\n" % MaxPhaseNesting)
+
+
+def generateCpp(out):
+ #
+ # Generate the PhaseKindInfo table.
+ #
+ out.write("static constexpr PhaseKindTable phaseKinds = {\n")
+ for phaseKind in AllPhaseKinds:
+ phase = PhasesForPhaseKind[phaseKind][0]
+ out.write(
+ ' /* PhaseKind::%s */ PhaseKindInfo { Phase::%s, %d, "%s" },\n'
+ % (phaseKind.name, phase.name, phaseKind.bucket, phaseKind.name)
+ )
+ out.write("};\n")
+ out.write("\n")
+
+ #
+ # Generate the PhaseInfo tree.
+ #
+ def name(phase):
+ return "Phase::" + phase.name if phase else "Phase::NONE"
+
+ out.write("static constexpr PhaseTable phases = {\n")
+ for phase in AllPhases:
+ firstChild = phase.children[0] if phase.children else None
+ phaseKind = phase.phaseKind
+ out.write(
+ ' /* %s */ PhaseInfo { %s, %s, %s, %s, PhaseKind::%s, %d, "%s", "%s" },\n'
+ % ( # NOQA: E501
+ name(phase),
+ name(phase.parent),
+ name(firstChild),
+ name(phase.nextSibling),
+ name(phase.nextInPhaseKind),
+ phaseKind.name,
+ phase.depth,
+ phaseKind.descr,
+ phase.path,
+ )
+ )
+ out.write("};\n")
+
+ #
+ # Print in a comment the next available phase kind number.
+ #
+ out.write("// The next available phase kind number is: %d\n" % (MaxBucket + 1))
diff --git a/js/src/gc/HashUtil.h b/js/src/gc/HashUtil.h
new file mode 100644
index 0000000000..52a3058f8a
--- /dev/null
+++ b/js/src/gc/HashUtil.h
@@ -0,0 +1,84 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_HashUtil_h
+#define gc_HashUtil_h
+
+#include <type_traits>
+
+#include "vm/JSContext.h"
+
+namespace js {
+
+/*
+ * Used to add entries to a js::HashMap or HashSet where the key depends on a GC
+ * thing that may be moved by generational or compacting GC between the call to
+ * lookupForAdd() and relookupOrAdd().
+ */
+template <class T>
+struct DependentAddPtr {
+ typedef typename T::AddPtr AddPtr;
+ typedef typename T::Entry Entry;
+
+ template <class Lookup>
+ DependentAddPtr(const JSContext* cx, T& table, const Lookup& lookup)
+ : addPtr(table.lookupForAdd(lookup)),
+ originalGcNumber(cx->runtime()->gc.gcNumber()) {}
+
+ DependentAddPtr(DependentAddPtr&& other)
+ : addPtr(other.addPtr), originalGcNumber(other.originalGcNumber) {}
+
+ template <class KeyInput, class ValueInput>
+ bool add(JSContext* cx, T& table, const KeyInput& key,
+ const ValueInput& value) {
+ refreshAddPtr(cx, table, key);
+ if (!table.relookupOrAdd(addPtr, key, value)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ return true;
+ }
+
+ template <class KeyInput>
+ void remove(JSContext* cx, T& table, const KeyInput& key) {
+ refreshAddPtr(cx, table, key);
+ if (addPtr) {
+ table.remove(addPtr);
+ }
+ }
+
+ bool found() const { return addPtr.found(); }
+ explicit operator bool() const { return found(); }
+ const Entry& operator*() const { return *addPtr; }
+ const Entry* operator->() const { return &*addPtr; }
+
+ private:
+ AddPtr addPtr;
+ const uint64_t originalGcNumber;
+
+ template <class KeyInput>
+ void refreshAddPtr(JSContext* cx, T& table, const KeyInput& key) {
+ bool gcHappened = originalGcNumber != cx->runtime()->gc.gcNumber();
+ if (gcHappened) {
+ addPtr = table.lookupForAdd(key);
+ }
+ }
+
+ DependentAddPtr() = delete;
+ DependentAddPtr(const DependentAddPtr&) = delete;
+ DependentAddPtr& operator=(const DependentAddPtr&) = delete;
+};
+
+template <typename T, typename Lookup>
+inline auto MakeDependentAddPtr(const JSContext* cx, T& table,
+ const Lookup& lookup) {
+ using Ptr = DependentAddPtr<std::remove_reference_t<decltype(table)>>;
+ return Ptr(cx, table, lookup);
+}
+
+} // namespace js
+
+#endif
diff --git a/js/src/gc/Heap-inl.h b/js/src/gc/Heap-inl.h
new file mode 100644
index 0000000000..87dcdc900c
--- /dev/null
+++ b/js/src/gc/Heap-inl.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Heap_inl_h
+#define gc_Heap_inl_h
+
+#include "gc/Heap.h"
+
+#include "gc/StoreBuffer.h"
+#include "gc/Zone.h"
+#include "util/Poison.h"
+#include "vm/Runtime.h"
+
+inline void js::gc::Arena::init(JS::Zone* zoneArg, AllocKind kind,
+ const AutoLockGC& lock) {
+#ifdef DEBUG
+ MOZ_MAKE_MEM_DEFINED(&zone, sizeof(zone));
+ MOZ_ASSERT((uintptr_t(zone) & 0xff) == JS_FREED_ARENA_PATTERN);
+#endif
+
+ MOZ_ASSERT(firstFreeSpan.isEmpty());
+ MOZ_ASSERT(!allocated());
+ MOZ_ASSERT(!onDelayedMarkingList_);
+ MOZ_ASSERT(!hasDelayedBlackMarking_);
+ MOZ_ASSERT(!hasDelayedGrayMarking_);
+ MOZ_ASSERT(!nextDelayedMarkingArena_);
+
+ MOZ_MAKE_MEM_UNDEFINED(this, ArenaSize);
+
+ zone = zoneArg;
+ allocKind = kind;
+ isNewlyCreated_ = 1;
+ onDelayedMarkingList_ = 0;
+ hasDelayedBlackMarking_ = 0;
+ hasDelayedGrayMarking_ = 0;
+ nextDelayedMarkingArena_ = 0;
+ if (zone->isAtomsZone()) {
+ zone->runtimeFromAnyThread()->gc.atomMarking.registerArena(this, lock);
+ } else {
+ bufferedCells() = &ArenaCellSet::Empty;
+ }
+
+ setAsFullyUnused();
+}
+
+inline void js::gc::Arena::release(const AutoLockGC& lock) {
+ if (zone->isAtomsZone()) {
+ zone->runtimeFromAnyThread()->gc.atomMarking.unregisterArena(this, lock);
+ }
+ setAsNotAllocated();
+}
+
+inline js::gc::ArenaCellSet*& js::gc::Arena::bufferedCells() {
+ MOZ_ASSERT(zone && !zone->isAtomsZone());
+ return bufferedCells_;
+}
+
+inline size_t& js::gc::Arena::atomBitmapStart() {
+ MOZ_ASSERT(zone && zone->isAtomsZone());
+ return atomBitmapStart_;
+}
+
+inline js::gc::NurseryCellHeader::NurseryCellHeader(AllocSite* site,
+ JS::TraceKind kind)
+ : allocSiteAndTraceKind(MakeValue(site, kind)) {}
+
+#endif
diff --git a/js/src/gc/Heap.cpp b/js/src/gc/Heap.cpp
new file mode 100644
index 0000000000..fdf24fbb7c
--- /dev/null
+++ b/js/src/gc/Heap.cpp
@@ -0,0 +1,635 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Tenured heap management.
+ *
+ * This file contains method definitions for the following classes for code that
+ * is not specific to a particular phase of GC:
+ *
+ * - Arena
+ * - ArenaList
+ * - FreeLists
+ * - ArenaLists
+ * - TenuredChunk
+ * - ChunkPool
+ */
+
+#include "gc/Heap-inl.h"
+
+#include "gc/GCLock.h"
+#include "gc/Memory.h"
+#include "jit/Assembler.h"
+#include "vm/BigIntType.h"
+#include "vm/RegExpShared.h"
+#include "vm/Scope.h"
+
+#include "gc/ArenaList-inl.h"
+#include "gc/PrivateIterators-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+// Check that reserved bits of a Cell are compatible with our typical allocators
+// since most derived classes will store a pointer in the first word.
+static const size_t MinFirstWordAlignment = 1u << CellFlagBitsReservedForGC;
+static_assert(js::detail::LIFO_ALLOC_ALIGN >= MinFirstWordAlignment,
+ "CellFlagBitsReservedForGC should support LifoAlloc");
+static_assert(CellAlignBytes >= MinFirstWordAlignment,
+ "CellFlagBitsReservedForGC should support gc::Cell");
+static_assert(js::jit::CodeAlignment >= MinFirstWordAlignment,
+ "CellFlagBitsReservedForGC should support JIT code");
+static_assert(js::gc::JSClassAlignBytes >= MinFirstWordAlignment,
+ "CellFlagBitsReservedForGC should support JSClass pointers");
+static_assert(js::ScopeDataAlignBytes >= MinFirstWordAlignment,
+ "CellFlagBitsReservedForGC should support scope data pointers");
+
+#define CHECK_THING_SIZE(allocKind, traceKind, type, sizedType, bgFinal, \
+ nursery, compact) \
+ static_assert(sizeof(sizedType) >= SortedArenaList::MinThingSize, \
+ #sizedType " is smaller than SortedArenaList::MinThingSize!"); \
+ static_assert(sizeof(sizedType) >= sizeof(FreeSpan), \
+ #sizedType " is smaller than FreeSpan"); \
+ static_assert(sizeof(sizedType) % CellAlignBytes == 0, \
+ "Size of " #sizedType " is not a multiple of CellAlignBytes"); \
+ static_assert(sizeof(sizedType) >= MinCellSize, \
+ "Size of " #sizedType " is smaller than the minimum size");
+FOR_EACH_ALLOCKIND(CHECK_THING_SIZE);
+#undef CHECK_THING_SIZE
+
+FreeSpan FreeLists::emptySentinel;
+
+template <typename T>
+struct ArenaLayout {
+ static constexpr size_t thingSize() { return sizeof(T); }
+ static constexpr size_t thingsPerArena() {
+ return (ArenaSize - ArenaHeaderSize) / thingSize();
+ }
+ static constexpr size_t firstThingOffset() {
+ return ArenaSize - thingSize() * thingsPerArena();
+ }
+};
+
+const uint8_t Arena::ThingSizes[] = {
+#define EXPAND_THING_SIZE(_1, _2, _3, sizedType, _4, _5, _6) \
+ ArenaLayout<sizedType>::thingSize(),
+ FOR_EACH_ALLOCKIND(EXPAND_THING_SIZE)
+#undef EXPAND_THING_SIZE
+};
+
+const uint8_t Arena::FirstThingOffsets[] = {
+#define EXPAND_FIRST_THING_OFFSET(_1, _2, _3, sizedType, _4, _5, _6) \
+ ArenaLayout<sizedType>::firstThingOffset(),
+ FOR_EACH_ALLOCKIND(EXPAND_FIRST_THING_OFFSET)
+#undef EXPAND_FIRST_THING_OFFSET
+};
+
+const uint8_t Arena::ThingsPerArena[] = {
+#define EXPAND_THINGS_PER_ARENA(_1, _2, _3, sizedType, _4, _5, _6) \
+ ArenaLayout<sizedType>::thingsPerArena(),
+ FOR_EACH_ALLOCKIND(EXPAND_THINGS_PER_ARENA)
+#undef EXPAND_THINGS_PER_ARENA
+};
+
+void Arena::unmarkAll() {
+ MarkBitmapWord* arenaBits = chunk()->markBits.arenaBits(this);
+ for (size_t i = 0; i < ArenaBitmapWords; i++) {
+ arenaBits[i] = 0;
+ }
+}
+
+void Arena::unmarkPreMarkedFreeCells() {
+ for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) {
+ MOZ_ASSERT(cell->isMarkedBlack());
+ cell->unmark();
+ }
+}
+
+#ifdef DEBUG
+
+void Arena::checkNoMarkedFreeCells() {
+ for (ArenaFreeCellIter cell(this); !cell.done(); cell.next()) {
+ MOZ_ASSERT(!cell->isMarkedAny());
+ }
+}
+
+void Arena::checkAllCellsMarkedBlack() {
+ for (ArenaCellIter cell(this); !cell.done(); cell.next()) {
+ MOZ_ASSERT(cell->isMarkedBlack());
+ }
+}
+
+#endif
+
+#if defined(DEBUG) || defined(JS_GC_ZEAL)
+void Arena::checkNoMarkedCells() {
+ for (ArenaCellIter cell(this); !cell.done(); cell.next()) {
+ MOZ_ASSERT(!cell->isMarkedAny());
+ }
+}
+#endif
+
+/* static */
+void Arena::staticAsserts() {
+ static_assert(size_t(AllocKind::LIMIT) <= 255,
+ "All AllocKinds and AllocKind::LIMIT must fit in a uint8_t.");
+ static_assert(std::size(ThingSizes) == AllocKindCount,
+ "We haven't defined all thing sizes.");
+ static_assert(std::size(FirstThingOffsets) == AllocKindCount,
+ "We haven't defined all offsets.");
+ static_assert(std::size(ThingsPerArena) == AllocKindCount,
+ "We haven't defined all counts.");
+}
+
+/* static */
+void Arena::checkLookupTables() {
+#ifdef DEBUG
+ for (size_t i = 0; i < AllocKindCount; i++) {
+ MOZ_ASSERT(
+ FirstThingOffsets[i] + ThingsPerArena[i] * ThingSizes[i] == ArenaSize,
+ "Inconsistent arena lookup table data");
+ }
+#endif
+}
+
+#ifdef DEBUG
+void js::gc::ArenaList::dump() {
+ fprintf(stderr, "ArenaList %p:", this);
+ if (cursorp_ == &head_) {
+ fprintf(stderr, " *");
+ }
+ for (Arena* arena = head(); arena; arena = arena->next) {
+ fprintf(stderr, " %p", arena);
+ if (cursorp_ == &arena->next) {
+ fprintf(stderr, " *");
+ }
+ }
+ fprintf(stderr, "\n");
+}
+#endif
+
+Arena* ArenaList::removeRemainingArenas(Arena** arenap) {
+ // This is only ever called to remove arenas that are after the cursor, so
+ // we don't need to update it.
+#ifdef DEBUG
+ for (Arena* arena = *arenap; arena; arena = arena->next) {
+ MOZ_ASSERT(cursorp_ != &arena->next);
+ }
+#endif
+ Arena* remainingArenas = *arenap;
+ *arenap = nullptr;
+ check();
+ return remainingArenas;
+}
+
+FreeLists::FreeLists() {
+ for (auto i : AllAllocKinds()) {
+ freeLists_[i] = &emptySentinel;
+ }
+}
+
+ArenaLists::ArenaLists(Zone* zone)
+ : zone_(zone),
+ incrementalSweptArenaKind(AllocKind::LIMIT),
+ gcCompactPropMapArenasToUpdate(nullptr),
+ gcNormalPropMapArenasToUpdate(nullptr),
+ savedEmptyArenas(nullptr) {
+ for (auto i : AllAllocKinds()) {
+ concurrentUse(i) = ConcurrentUse::None;
+ }
+}
+
+void ReleaseArenas(JSRuntime* rt, Arena* arena, const AutoLockGC& lock) {
+ Arena* next;
+ for (; arena; arena = next) {
+ next = arena->next;
+ rt->gc.releaseArena(arena, lock);
+ }
+}
+
+void ReleaseArenaList(JSRuntime* rt, ArenaList& arenaList,
+ const AutoLockGC& lock) {
+ ReleaseArenas(rt, arenaList.head(), lock);
+ arenaList.clear();
+}
+
+ArenaLists::~ArenaLists() {
+ AutoLockGC lock(runtime());
+
+ for (auto i : AllAllocKinds()) {
+ /*
+ * We can only call this during the shutdown after the last GC when
+ * the background finalization is disabled.
+ */
+ MOZ_ASSERT(concurrentUse(i) == ConcurrentUse::None);
+ ReleaseArenaList(runtime(), arenaList(i), lock);
+ }
+ ReleaseArenaList(runtime(), incrementalSweptArenas.ref(), lock);
+
+ ReleaseArenas(runtime(), savedEmptyArenas, lock);
+}
+
+void ArenaLists::moveArenasToCollectingLists() {
+ checkEmptyFreeLists();
+ for (AllocKind kind : AllAllocKinds()) {
+ MOZ_ASSERT(collectingArenaList(kind).isEmpty());
+ collectingArenaList(kind) = std::move(arenaList(kind));
+ MOZ_ASSERT(arenaList(kind).isEmpty());
+ }
+}
+
+void ArenaLists::mergeArenasFromCollectingLists() {
+ for (AllocKind kind : AllAllocKinds()) {
+ collectingArenaList(kind).insertListWithCursorAtEnd(arenaList(kind));
+ arenaList(kind) = std::move(collectingArenaList(kind));
+ MOZ_ASSERT(collectingArenaList(kind).isEmpty());
+ }
+}
+
+Arena* ArenaLists::takeSweptEmptyArenas() {
+ Arena* arenas = savedEmptyArenas;
+ savedEmptyArenas = nullptr;
+ return arenas;
+}
+
+void ArenaLists::setIncrementalSweptArenas(AllocKind kind,
+ SortedArenaList& arenas) {
+ incrementalSweptArenaKind = kind;
+ incrementalSweptArenas.ref().clear();
+ incrementalSweptArenas = arenas.toArenaList();
+}
+
+void ArenaLists::clearIncrementalSweptArenas() {
+ incrementalSweptArenaKind = AllocKind::LIMIT;
+ incrementalSweptArenas.ref().clear();
+}
+
+void ArenaLists::checkGCStateNotInUse() {
+ // Called before and after collection to check the state is as expected.
+#ifdef DEBUG
+ checkSweepStateNotInUse();
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT(collectingArenaList(i).isEmpty());
+ }
+#endif
+}
+
+void ArenaLists::checkSweepStateNotInUse() {
+#ifdef DEBUG
+ checkNoArenasToUpdate();
+ MOZ_ASSERT(incrementalSweptArenaKind == AllocKind::LIMIT);
+ MOZ_ASSERT(incrementalSweptArenas.ref().isEmpty());
+ MOZ_ASSERT(!savedEmptyArenas);
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT(concurrentUse(i) == ConcurrentUse::None);
+ }
+#endif
+}
+
+void ArenaLists::checkNoArenasToUpdate() {
+ MOZ_ASSERT(!gcCompactPropMapArenasToUpdate);
+ MOZ_ASSERT(!gcNormalPropMapArenasToUpdate);
+}
+
+void ArenaLists::checkNoArenasToUpdateForKind(AllocKind kind) {
+#ifdef DEBUG
+ switch (kind) {
+ case AllocKind::COMPACT_PROP_MAP:
+ MOZ_ASSERT(!gcCompactPropMapArenasToUpdate);
+ break;
+ case AllocKind::NORMAL_PROP_MAP:
+ MOZ_ASSERT(!gcNormalPropMapArenasToUpdate);
+ break;
+ default:
+ break;
+ }
+#endif
+}
+
+inline bool TenuredChunk::canDecommitPage(size_t pageIndex) const {
+ if (decommittedPages[pageIndex]) {
+ return false;
+ }
+
+ size_t arenaIndex = pageIndex * ArenasPerPage;
+ for (size_t i = 0; i < ArenasPerPage; i++) {
+ if (!freeCommittedArenas[arenaIndex + i]) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void TenuredChunk::decommitFreeArenas(GCRuntime* gc, const bool& cancel,
+ AutoLockGC& lock) {
+ MOZ_ASSERT(DecommitEnabled());
+
+ for (size_t i = 0; i < PagesPerChunk; i++) {
+ if (cancel) {
+ break;
+ }
+
+ if (canDecommitPage(i) && !decommitOneFreePage(gc, i, lock)) {
+ break;
+ }
+ }
+}
+
+void TenuredChunk::recycleArena(Arena* arena, SortedArenaList& dest,
+ size_t thingsPerArena) {
+ arena->setAsFullyUnused();
+ dest.insertAt(arena, thingsPerArena);
+}
+
+void TenuredChunk::releaseArena(GCRuntime* gc, Arena* arena,
+ const AutoLockGC& lock) {
+ MOZ_ASSERT(!arena->allocated());
+ MOZ_ASSERT(!freeCommittedArenas[arenaIndex(arena)]);
+
+ freeCommittedArenas[arenaIndex(arena)] = true;
+ ++info.numArenasFreeCommitted;
+ ++info.numArenasFree;
+ gc->updateOnArenaFree();
+
+ verify();
+
+ updateChunkListAfterFree(gc, 1, lock);
+}
+
+bool TenuredChunk::decommitOneFreePage(GCRuntime* gc, size_t pageIndex,
+ AutoLockGC& lock) {
+ MOZ_ASSERT(DecommitEnabled());
+ MOZ_ASSERT(canDecommitPage(pageIndex));
+ MOZ_ASSERT(info.numArenasFreeCommitted >= ArenasPerPage);
+
+ // Temporarily mark the page as allocated while we decommit.
+ for (size_t i = 0; i < ArenasPerPage; i++) {
+ size_t arenaIndex = pageIndex * ArenasPerPage + i;
+ MOZ_ASSERT(freeCommittedArenas[arenaIndex]);
+ freeCommittedArenas[arenaIndex] = false;
+ }
+ info.numArenasFreeCommitted -= ArenasPerPage;
+ info.numArenasFree -= ArenasPerPage;
+ updateChunkListAfterAlloc(gc, lock);
+
+ verify();
+
+ bool ok;
+ {
+ AutoUnlockGC unlock(lock);
+ ok = !oom::ShouldFailWithOOM() &&
+ MarkPagesUnusedSoft(pageAddress(pageIndex), PageSize);
+ }
+
+ // Mark the page as decommited if successful or restore the original free
+ // state.
+ if (ok) {
+ decommittedPages[pageIndex] = true;
+ } else {
+ for (size_t i = 0; i < ArenasPerPage; i++) {
+ size_t arenaIndex = pageIndex * ArenasPerPage + i;
+ MOZ_ASSERT(!freeCommittedArenas[arenaIndex]);
+ freeCommittedArenas[arenaIndex] = true;
+ }
+ info.numArenasFreeCommitted += ArenasPerPage;
+ }
+
+ info.numArenasFree += ArenasPerPage;
+ updateChunkListAfterFree(gc, ArenasPerPage, lock);
+
+ verify();
+
+ return ok;
+}
+
+void TenuredChunk::decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock) {
+ MOZ_ASSERT(DecommitEnabled());
+
+ for (size_t i = 0; i < PagesPerChunk; i++) {
+ if (!canDecommitPage(i)) {
+ continue;
+ }
+
+ MOZ_ASSERT(!decommittedPages[i]);
+ MOZ_ASSERT(info.numArenasFreeCommitted >= ArenasPerPage);
+
+ if (js::oom::ShouldFailWithOOM() ||
+ !MarkPagesUnusedSoft(pageAddress(i), SystemPageSize())) {
+ break;
+ }
+
+ decommittedPages[i] = true;
+ for (size_t j = 0; j < ArenasPerPage; ++j) {
+ size_t arenaIndex = i * ArenasPerPage + j;
+ MOZ_ASSERT(freeCommittedArenas[arenaIndex]);
+ freeCommittedArenas[arenaIndex] = false;
+ }
+ info.numArenasFreeCommitted -= ArenasPerPage;
+ }
+
+ verify();
+}
+
+void TenuredChunk::updateChunkListAfterAlloc(GCRuntime* gc,
+ const AutoLockGC& lock) {
+ if (MOZ_UNLIKELY(!hasAvailableArenas())) {
+ gc->availableChunks(lock).remove(this);
+ gc->fullChunks(lock).push(this);
+ }
+}
+
+void TenuredChunk::updateChunkListAfterFree(GCRuntime* gc, size_t numArenasFree,
+ const AutoLockGC& lock) {
+ if (info.numArenasFree == numArenasFree) {
+ gc->fullChunks(lock).remove(this);
+ gc->availableChunks(lock).push(this);
+ } else if (!unused()) {
+ MOZ_ASSERT(gc->availableChunks(lock).contains(this));
+ } else {
+ MOZ_ASSERT(unused());
+ gc->availableChunks(lock).remove(this);
+ gc->recycleChunk(this, lock);
+ }
+}
+
+TenuredChunk* ChunkPool::pop() {
+ MOZ_ASSERT(bool(head_) == bool(count_));
+ if (!count_) {
+ return nullptr;
+ }
+ return remove(head_);
+}
+
+void ChunkPool::push(TenuredChunk* chunk) {
+ MOZ_ASSERT(!chunk->info.next);
+ MOZ_ASSERT(!chunk->info.prev);
+
+ chunk->info.next = head_;
+ if (head_) {
+ head_->info.prev = chunk;
+ }
+ head_ = chunk;
+ ++count_;
+}
+
+TenuredChunk* ChunkPool::remove(TenuredChunk* chunk) {
+ MOZ_ASSERT(count_ > 0);
+ MOZ_ASSERT(contains(chunk));
+
+ if (head_ == chunk) {
+ head_ = chunk->info.next;
+ }
+ if (chunk->info.prev) {
+ chunk->info.prev->info.next = chunk->info.next;
+ }
+ if (chunk->info.next) {
+ chunk->info.next->info.prev = chunk->info.prev;
+ }
+ chunk->info.next = chunk->info.prev = nullptr;
+ --count_;
+
+ return chunk;
+}
+
+// We could keep the chunk pool sorted, but that's likely to be more expensive.
+// This sort is nlogn, but keeping it sorted is likely to be m*n, with m being
+// the number of operations (likely higher than n).
+void ChunkPool::sort() {
+ // Only sort if the list isn't already sorted.
+ if (!isSorted()) {
+ head_ = mergeSort(head(), count());
+
+ // Fixup prev pointers.
+ TenuredChunk* prev = nullptr;
+ for (TenuredChunk* cur = head_; cur; cur = cur->info.next) {
+ cur->info.prev = prev;
+ prev = cur;
+ }
+ }
+
+ MOZ_ASSERT(verify());
+ MOZ_ASSERT(isSorted());
+}
+
+TenuredChunk* ChunkPool::mergeSort(TenuredChunk* list, size_t count) {
+ MOZ_ASSERT(bool(list) == bool(count));
+
+ if (count < 2) {
+ return list;
+ }
+
+ size_t half = count / 2;
+
+ // Split;
+ TenuredChunk* front = list;
+ TenuredChunk* back;
+ {
+ TenuredChunk* cur = list;
+ for (size_t i = 0; i < half - 1; i++) {
+ MOZ_ASSERT(cur);
+ cur = cur->info.next;
+ }
+ back = cur->info.next;
+ cur->info.next = nullptr;
+ }
+
+ front = mergeSort(front, half);
+ back = mergeSort(back, count - half);
+
+ // Merge
+ list = nullptr;
+ TenuredChunk** cur = &list;
+ while (front || back) {
+ if (!front) {
+ *cur = back;
+ break;
+ }
+ if (!back) {
+ *cur = front;
+ break;
+ }
+
+ // Note that the sort is stable due to the <= here. Nothing depends on
+ // this but it could.
+ if (front->info.numArenasFree <= back->info.numArenasFree) {
+ *cur = front;
+ front = front->info.next;
+ cur = &(*cur)->info.next;
+ } else {
+ *cur = back;
+ back = back->info.next;
+ cur = &(*cur)->info.next;
+ }
+ }
+
+ return list;
+}
+
+bool ChunkPool::isSorted() const {
+ uint32_t last = 1;
+ for (TenuredChunk* cursor = head_; cursor; cursor = cursor->info.next) {
+ if (cursor->info.numArenasFree < last) {
+ return false;
+ }
+ last = cursor->info.numArenasFree;
+ }
+ return true;
+}
+
+#ifdef DEBUG
+
+bool ChunkPool::contains(TenuredChunk* chunk) const {
+ verify();
+ for (TenuredChunk* cursor = head_; cursor; cursor = cursor->info.next) {
+ if (cursor == chunk) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool ChunkPool::verify() const {
+ MOZ_ASSERT(bool(head_) == bool(count_));
+ uint32_t count = 0;
+ for (TenuredChunk* cursor = head_; cursor;
+ cursor = cursor->info.next, ++count) {
+ MOZ_ASSERT_IF(cursor->info.prev, cursor->info.prev->info.next == cursor);
+ MOZ_ASSERT_IF(cursor->info.next, cursor->info.next->info.prev == cursor);
+ }
+ MOZ_ASSERT(count_ == count);
+ return true;
+}
+
+void ChunkPool::verifyChunks() const {
+ for (TenuredChunk* chunk = head_; chunk; chunk = chunk->info.next) {
+ chunk->verify();
+ }
+}
+
+void TenuredChunk::verify() const {
+ MOZ_ASSERT(info.numArenasFree <= ArenasPerChunk);
+ MOZ_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree);
+
+ size_t decommittedCount = decommittedPages.Count() * ArenasPerPage;
+ size_t freeCommittedCount = freeCommittedArenas.Count();
+ size_t freeCount = freeCommittedCount + decommittedCount;
+
+ MOZ_ASSERT(freeCount == info.numArenasFree);
+ MOZ_ASSERT(freeCommittedCount == info.numArenasFreeCommitted);
+
+ for (size_t i = 0; i < ArenasPerChunk; ++i) {
+ MOZ_ASSERT(!(decommittedPages[pageIndex(i)] && freeCommittedArenas[i]));
+ MOZ_ASSERT_IF(freeCommittedArenas[i], !arenas[i].allocated());
+ }
+}
+
+#endif
+
+void ChunkPool::Iter::next() {
+ MOZ_ASSERT(!done());
+ current_ = current_->info.next;
+}
diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h
new file mode 100644
index 0000000000..9a1b25d508
--- /dev/null
+++ b/js/src/gc/Heap.h
@@ -0,0 +1,846 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Heap_h
+#define gc_Heap_h
+
+#include "mozilla/DebugOnly.h"
+
+#include "gc/AllocKind.h"
+#include "gc/Pretenuring.h"
+#include "js/HeapAPI.h"
+#include "js/TypeDecls.h"
+#include "util/Poison.h"
+
+namespace js {
+
+class AutoLockGC;
+class AutoLockGCBgAlloc;
+class Nursery;
+
+// To prevent false sharing, some data structures are aligned to a typical cache
+// line size.
+static constexpr size_t TypicalCacheLineSize = 64;
+
+namespace gc {
+
+class Arena;
+class ArenaCellSet;
+class ArenaList;
+class GCRuntime;
+class MarkingValidator;
+class SortedArenaList;
+class TenuredCell;
+
+// Cells are aligned to CellAlignShift, so the largest tagged null pointer is:
+const uintptr_t LargestTaggedNullCellPointer = (1 << CellAlignShift) - 1;
+
+/*
+ * The minimum cell size ends up as twice the cell alignment because the mark
+ * bitmap contains one bit per CellBytesPerMarkBit bytes (which is equal to
+ * CellAlignBytes) and we need two mark bits per cell.
+ */
+const size_t MinCellSize = CellBytesPerMarkBit * MarkBitsPerCell;
+
+static_assert(ArenaSize % CellAlignBytes == 0,
+ "Arena size must be a multiple of cell alignment");
+
+/*
+ * A FreeSpan represents a contiguous sequence of free cells in an Arena. It
+ * can take two forms.
+ *
+ * - In an empty span, |first| and |last| are both zero.
+ *
+ * - In a non-empty span, |first| is the address of the first free thing in the
+ * span, and |last| is the address of the last free thing in the span.
+ * Furthermore, the memory pointed to by |last| holds a FreeSpan structure
+ * that points to the next span (which may be empty); this works because
+ * sizeof(FreeSpan) is less than the smallest thingSize.
+ */
+class FreeSpan {
+ friend class Arena;
+ friend class ArenaCellIter;
+ friend class ArenaFreeCellIter;
+
+ uint16_t first;
+ uint16_t last;
+
+ public:
+ // This inits just |first| and |last|; if the span is non-empty it doesn't
+ // do anything with the next span stored at |last|.
+ void initBounds(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) {
+ checkRange(firstArg, lastArg, arena);
+ first = firstArg;
+ last = lastArg;
+ }
+
+ void initAsEmpty() {
+ first = 0;
+ last = 0;
+ }
+
+ // This sets |first| and |last|, and also sets the next span stored at
+ // |last| as empty. (As a result, |firstArg| and |lastArg| cannot represent
+ // an empty span.)
+ void initFinal(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) {
+ initBounds(firstArg, lastArg, arena);
+ FreeSpan* last = nextSpanUnchecked(arena);
+ last->initAsEmpty();
+ checkSpan(arena);
+ }
+
+ bool isEmpty() const { return !first; }
+
+ Arena* getArenaUnchecked() { return reinterpret_cast<Arena*>(this); }
+ inline Arena* getArena();
+
+ static size_t offsetOfFirst() { return offsetof(FreeSpan, first); }
+
+ static size_t offsetOfLast() { return offsetof(FreeSpan, last); }
+
+ // Like nextSpan(), but no checking of the following span is done.
+ FreeSpan* nextSpanUnchecked(const Arena* arena) const {
+ MOZ_ASSERT(arena && !isEmpty());
+ return reinterpret_cast<FreeSpan*>(uintptr_t(arena) + last);
+ }
+
+ const FreeSpan* nextSpan(const Arena* arena) const {
+ checkSpan(arena);
+ return nextSpanUnchecked(arena);
+ }
+
+ MOZ_ALWAYS_INLINE TenuredCell* allocate(size_t thingSize) {
+ // Eschew the usual checks, because this might be the placeholder span.
+ // If this is somehow an invalid, non-empty span, checkSpan() will catch it.
+ Arena* arena = getArenaUnchecked();
+ checkSpan(arena);
+ uintptr_t thing = uintptr_t(arena) + first;
+ if (first < last) {
+ // We have space for at least two more things, so do a simple
+ // bump-allocate.
+ first += thingSize;
+ } else if (MOZ_LIKELY(first)) {
+ // The last space points to the next free span (which may be empty).
+ const FreeSpan* next = nextSpan(arena);
+ first = next->first;
+ last = next->last;
+ } else {
+ return nullptr; // The span is empty.
+ }
+ checkSpan(arena);
+ DebugOnlyPoison(reinterpret_cast<void*>(thing),
+ JS_ALLOCATED_TENURED_PATTERN, thingSize,
+ MemCheckKind::MakeUndefined);
+ return reinterpret_cast<TenuredCell*>(thing);
+ }
+
+ inline void checkSpan(const Arena* arena) const;
+ inline void checkRange(uintptr_t first, uintptr_t last,
+ const Arena* arena) const;
+};
+
+/*
+ * Arenas are the allocation units of the tenured heap in the GC. An arena
+ * is 4kiB in size and 4kiB-aligned. It starts with several header fields
+ * followed by some bytes of padding. The remainder of the arena is filled
+ * with GC things of a particular AllocKind. The padding ensures that the
+ * GC thing array ends exactly at the end of the arena:
+ *
+ * <----------------------------------------------> = ArenaSize bytes
+ * +---------------+---------+----+----+-----+----+
+ * | header fields | padding | T0 | T1 | ... | Tn |
+ * +---------------+---------+----+----+-----+----+
+ * <-------------------------> = first thing offset
+ */
+class alignas(ArenaSize) Arena {
+ static JS_PUBLIC_DATA const uint8_t ThingSizes[];
+ static JS_PUBLIC_DATA const uint8_t FirstThingOffsets[];
+ static JS_PUBLIC_DATA const uint8_t ThingsPerArena[];
+ /*
+ * The first span of free things in the arena. Most of these spans are
+ * stored as offsets in free regions of the data array, and most operations
+ * on FreeSpans take an Arena pointer for safety. However, the FreeSpans
+ * used for allocation are stored here, at the start of an Arena, and use
+ * their own address to grab the next span within the same Arena.
+ */
+ FreeSpan firstFreeSpan;
+
+ public:
+ /*
+ * One of the AllocKind constants or AllocKind::LIMIT when the arena does
+ * not contain any GC things and is on the list of empty arenas in the GC
+ * chunk.
+ */
+ AllocKind allocKind;
+
+ /*
+ * The zone that this Arena is contained within, when allocated. The offset
+ * of this field must match the ArenaZoneOffset stored in js/HeapAPI.h,
+ * as is statically asserted below.
+ */
+ JS::Zone* zone;
+
+ /*
+ * Arena::next has two purposes: when unallocated, it points to the next
+ * available Arena. When allocated, it points to the next Arena in the same
+ * zone and with the same alloc kind.
+ */
+ Arena* next;
+
+ private:
+ static const size_t ARENA_FLAG_BITS = 4;
+ static const size_t DELAYED_MARKING_ARENA_BITS =
+ JS_BITS_PER_WORD - ArenaShift;
+ static_assert(
+ ARENA_FLAG_BITS + DELAYED_MARKING_ARENA_BITS <= JS_BITS_PER_WORD,
+ "Not enough space to pack flags and nextDelayedMarkingArena_ pointer "
+ "into a single word.");
+
+ /*
+ * True until the arena is swept for the first time.
+ */
+ size_t isNewlyCreated_ : 1;
+
+ /*
+ * When recursive marking uses too much stack we delay marking of arenas and
+ * link them into a list for later processing. This uses the following fields.
+ */
+ size_t onDelayedMarkingList_ : 1;
+ size_t hasDelayedBlackMarking_ : 1;
+ size_t hasDelayedGrayMarking_ : 1;
+ size_t nextDelayedMarkingArena_ : DELAYED_MARKING_ARENA_BITS;
+
+ union {
+ /*
+ * For arenas in zones other than the atoms zone, if non-null, points
+ * to an ArenaCellSet that represents the set of cells in this arena
+ * that are in the nursery's store buffer.
+ */
+ ArenaCellSet* bufferedCells_;
+
+ /*
+ * For arenas in the atoms zone, the starting index into zone atom
+ * marking bitmaps (see AtomMarking.h) of the things in this zone.
+ * Atoms never refer to nursery things, so no store buffer index is
+ * needed.
+ */
+ size_t atomBitmapStart_;
+ };
+
+ public:
+ /*
+ * The size of data should be |ArenaSize - offsetof(data)|, but the offset
+ * is not yet known to the compiler, so we do it by hand. |firstFreeSpan|
+ * takes up 8 bytes on 64-bit due to alignment requirements; the rest are
+ * obvious. This constant is stored in js/HeapAPI.h.
+ */
+ uint8_t data[ArenaSize - ArenaHeaderSize];
+
+ void init(JS::Zone* zoneArg, AllocKind kind, const AutoLockGC& lock);
+
+ // Sets |firstFreeSpan| to the Arena's entire valid range, and
+ // also sets the next span stored at |firstFreeSpan.last| as empty.
+ void setAsFullyUnused() {
+ AllocKind kind = getAllocKind();
+ firstFreeSpan.first = firstThingOffset(kind);
+ firstFreeSpan.last = lastThingOffset(kind);
+ FreeSpan* last = firstFreeSpan.nextSpanUnchecked(this);
+ last->initAsEmpty();
+ }
+
+ // Initialize an arena to its unallocated state. For arenas that were
+ // previously allocated for some zone, use release() instead.
+ void setAsNotAllocated() {
+ firstFreeSpan.initAsEmpty();
+
+ // Poison zone pointer to highlight UAF on released arenas in crash data.
+ AlwaysPoison(&zone, JS_FREED_ARENA_PATTERN, sizeof(zone),
+ MemCheckKind::MakeNoAccess);
+
+ allocKind = AllocKind::LIMIT;
+ onDelayedMarkingList_ = 0;
+ hasDelayedBlackMarking_ = 0;
+ hasDelayedGrayMarking_ = 0;
+ nextDelayedMarkingArena_ = 0;
+ bufferedCells_ = nullptr;
+
+ MOZ_ASSERT(!allocated());
+ }
+
+ // Return an allocated arena to its unallocated state.
+ inline void release(const AutoLockGC& lock);
+
+ uintptr_t address() const {
+ checkAddress();
+ return uintptr_t(this);
+ }
+
+ inline void checkAddress() const;
+
+ inline TenuredChunk* chunk() const;
+
+ bool allocated() const {
+ MOZ_ASSERT(IsAllocKind(AllocKind(allocKind)));
+ return IsValidAllocKind(AllocKind(allocKind));
+ }
+
+ AllocKind getAllocKind() const {
+ MOZ_ASSERT(allocated());
+ return allocKind;
+ }
+
+ FreeSpan* getFirstFreeSpan() { return &firstFreeSpan; }
+
+ static size_t thingSize(AllocKind kind) { return ThingSizes[size_t(kind)]; }
+ static size_t thingsPerArena(AllocKind kind) {
+ return ThingsPerArena[size_t(kind)];
+ }
+ static size_t thingsSpan(AllocKind kind) {
+ return thingsPerArena(kind) * thingSize(kind);
+ }
+
+ static size_t firstThingOffset(AllocKind kind) {
+ return FirstThingOffsets[size_t(kind)];
+ }
+ static size_t lastThingOffset(AllocKind kind) {
+ return ArenaSize - thingSize(kind);
+ }
+
+ size_t getThingSize() const { return thingSize(getAllocKind()); }
+ size_t getThingsPerArena() const { return thingsPerArena(getAllocKind()); }
+ size_t getThingsSpan() const { return getThingsPerArena() * getThingSize(); }
+ size_t getFirstThingOffset() const {
+ return firstThingOffset(getAllocKind());
+ }
+
+ uintptr_t thingsStart() const { return address() + getFirstThingOffset(); }
+ uintptr_t thingsEnd() const { return address() + ArenaSize; }
+
+ bool isEmpty() const {
+ // Arena is empty if its first span covers the whole arena.
+ firstFreeSpan.checkSpan(this);
+ AllocKind kind = getAllocKind();
+ return firstFreeSpan.first == firstThingOffset(kind) &&
+ firstFreeSpan.last == lastThingOffset(kind);
+ }
+
+ bool hasFreeThings() const { return !firstFreeSpan.isEmpty(); }
+
+ size_t numFreeThings(size_t thingSize) const {
+ firstFreeSpan.checkSpan(this);
+ size_t numFree = 0;
+ const FreeSpan* span = &firstFreeSpan;
+ for (; !span->isEmpty(); span = span->nextSpan(this)) {
+ numFree += (span->last - span->first) / thingSize + 1;
+ }
+ return numFree;
+ }
+
+ size_t countFreeCells() { return numFreeThings(getThingSize()); }
+ size_t countUsedCells() { return getThingsPerArena() - countFreeCells(); }
+
+#ifdef DEBUG
+ bool inFreeList(uintptr_t thing) {
+ uintptr_t base = address();
+ const FreeSpan* span = &firstFreeSpan;
+ for (; !span->isEmpty(); span = span->nextSpan(this)) {
+ // If the thing comes before the current span, it's not free.
+ if (thing < base + span->first) {
+ return false;
+ }
+
+ // If we find it before the end of the span, it's free.
+ if (thing <= base + span->last) {
+ return true;
+ }
+ }
+ return false;
+ }
+#endif
+
+ static bool isAligned(uintptr_t thing, size_t thingSize) {
+ /* Things ends at the arena end. */
+ uintptr_t tailOffset = ArenaSize - (thing & ArenaMask);
+ return tailOffset % thingSize == 0;
+ }
+
+ bool isNewlyCreated() const { return isNewlyCreated_; }
+
+ bool onDelayedMarkingList() const { return onDelayedMarkingList_; }
+
+ Arena* getNextDelayedMarking() const {
+ MOZ_ASSERT(onDelayedMarkingList_);
+ return reinterpret_cast<Arena*>(nextDelayedMarkingArena_ << ArenaShift);
+ }
+
+ void setNextDelayedMarkingArena(Arena* arena) {
+ MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask));
+ MOZ_ASSERT(!onDelayedMarkingList_);
+ MOZ_ASSERT(!hasDelayedBlackMarking_);
+ MOZ_ASSERT(!hasDelayedGrayMarking_);
+ MOZ_ASSERT(!nextDelayedMarkingArena_);
+ onDelayedMarkingList_ = 1;
+ if (arena) {
+ nextDelayedMarkingArena_ = arena->address() >> ArenaShift;
+ }
+ }
+
+ void updateNextDelayedMarkingArena(Arena* arena) {
+ MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask));
+ MOZ_ASSERT(onDelayedMarkingList_);
+ nextDelayedMarkingArena_ = arena ? arena->address() >> ArenaShift : 0;
+ }
+
+ bool hasDelayedMarking(MarkColor color) const {
+ MOZ_ASSERT(onDelayedMarkingList_);
+ return color == MarkColor::Black ? hasDelayedBlackMarking_
+ : hasDelayedGrayMarking_;
+ }
+
+ bool hasAnyDelayedMarking() const {
+ MOZ_ASSERT(onDelayedMarkingList_);
+ return hasDelayedBlackMarking_ || hasDelayedGrayMarking_;
+ }
+
+ void setHasDelayedMarking(MarkColor color, bool value) {
+ MOZ_ASSERT(onDelayedMarkingList_);
+ if (color == MarkColor::Black) {
+ hasDelayedBlackMarking_ = value;
+ } else {
+ hasDelayedGrayMarking_ = value;
+ }
+ }
+
+ void clearDelayedMarkingState() {
+ MOZ_ASSERT(onDelayedMarkingList_);
+ onDelayedMarkingList_ = 0;
+ hasDelayedBlackMarking_ = 0;
+ hasDelayedGrayMarking_ = 0;
+ nextDelayedMarkingArena_ = 0;
+ }
+
+ inline ArenaCellSet*& bufferedCells();
+ inline size_t& atomBitmapStart();
+
+ template <typename T>
+ size_t finalize(JS::GCContext* gcx, AllocKind thingKind, size_t thingSize);
+
+ static void staticAsserts();
+ static void checkLookupTables();
+
+ void unmarkAll();
+ void unmarkPreMarkedFreeCells();
+
+ void arenaAllocatedDuringGC();
+
+#ifdef DEBUG
+ void checkNoMarkedFreeCells();
+ void checkAllCellsMarkedBlack();
+#endif
+
+#if defined(DEBUG) || defined(JS_GC_ZEAL)
+ void checkNoMarkedCells();
+#endif
+};
+
+static_assert(ArenaZoneOffset == offsetof(Arena, zone),
+ "The hardcoded API zone offset must match the actual offset.");
+
+static_assert(sizeof(Arena) == ArenaSize,
+ "ArenaSize must match the actual size of the Arena structure.");
+
+static_assert(
+ offsetof(Arena, data) == ArenaHeaderSize,
+ "ArenaHeaderSize must match the actual size of the header fields.");
+
+inline Arena* FreeSpan::getArena() {
+ Arena* arena = getArenaUnchecked();
+ arena->checkAddress();
+ return arena;
+}
+
+inline void FreeSpan::checkSpan(const Arena* arena) const {
+#ifdef DEBUG
+ if (!first) {
+ MOZ_ASSERT(!first && !last);
+ return;
+ }
+
+ arena->checkAddress();
+ checkRange(first, last, arena);
+
+ // If there's a following span, it must have a higher address,
+ // and the gap must be at least 2 * thingSize.
+ const FreeSpan* next = nextSpanUnchecked(arena);
+ if (next->first) {
+ checkRange(next->first, next->last, arena);
+ size_t thingSize = arena->getThingSize();
+ MOZ_ASSERT(last + 2 * thingSize <= next->first);
+ }
+#endif
+}
+
+inline void FreeSpan::checkRange(uintptr_t first, uintptr_t last,
+ const Arena* arena) const {
+#ifdef DEBUG
+ MOZ_ASSERT(arena);
+ MOZ_ASSERT(first <= last);
+ AllocKind thingKind = arena->getAllocKind();
+ MOZ_ASSERT(first >= Arena::firstThingOffset(thingKind));
+ MOZ_ASSERT(last <= Arena::lastThingOffset(thingKind));
+ MOZ_ASSERT((last - first) % Arena::thingSize(thingKind) == 0);
+#endif
+}
+
+// Mark bitmap API:
+
+MOZ_ALWAYS_INLINE bool MarkBitmap::markBit(const TenuredCell* cell,
+ ColorBit colorBit) {
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ getMarkWordAndMask(cell, colorBit, &word, &mask);
+ return *word & mask;
+}
+
+MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedAny(const TenuredCell* cell) {
+ return markBit(cell, ColorBit::BlackBit) ||
+ markBit(cell, ColorBit::GrayOrBlackBit);
+}
+
+MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedBlack(const TenuredCell* cell) {
+ return markBit(cell, ColorBit::BlackBit);
+}
+
+MOZ_ALWAYS_INLINE bool MarkBitmap::isMarkedGray(const TenuredCell* cell) {
+ return !markBit(cell, ColorBit::BlackBit) &&
+ markBit(cell, ColorBit::GrayOrBlackBit);
+}
+
+// The following methods that update the mark bits are not thread safe and must
+// not be called in parallel with each other.
+//
+// They use separate read and write operations to avoid an unnecessarily strict
+// atomic update on the marking bitmap.
+//
+// They may be called in parallel with read operations on the mark bitmap where
+// there is no required ordering between the operations. This happens when gray
+// unmarking occurs in parallel with background sweeping.
+
+// The return value indicates if the cell went from unmarked to marked.
+MOZ_ALWAYS_INLINE bool MarkBitmap::markIfUnmarked(const TenuredCell* cell,
+ MarkColor color) {
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
+ if (*word & mask) {
+ return false;
+ }
+ if (color == MarkColor::Black) {
+ uintptr_t bits = *word;
+ *word = bits | mask;
+ } else {
+ // We use getMarkWordAndMask to recalculate both mask and word as doing just
+ // mask << color may overflow the mask.
+ getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask);
+ if (*word & mask) {
+ return false;
+ }
+ uintptr_t bits = *word;
+ *word = bits | mask;
+ }
+ return true;
+}
+
+MOZ_ALWAYS_INLINE bool MarkBitmap::markIfUnmarkedAtomic(const TenuredCell* cell,
+ MarkColor color) {
+ // This version of the method is safe in the face of concurrent writes to the
+ // mark bitmap but may return false positives. The extra synchronisation
+ // necessary to avoid this resulted in worse performance overall.
+
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
+ if (*word & mask) {
+ return false;
+ }
+ if (color == MarkColor::Black) {
+ *word |= mask;
+ } else {
+ // We use getMarkWordAndMask to recalculate both mask and word as doing just
+ // mask << color may overflow the mask.
+ getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask);
+ if (*word & mask) {
+ return false;
+ }
+ *word |= mask;
+ }
+ return true;
+}
+
+MOZ_ALWAYS_INLINE void MarkBitmap::markBlack(const TenuredCell* cell) {
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
+ uintptr_t bits = *word;
+ *word = bits | mask;
+}
+
+MOZ_ALWAYS_INLINE void MarkBitmap::markBlackAtomic(const TenuredCell* cell) {
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
+ *word |= mask;
+}
+
+MOZ_ALWAYS_INLINE void MarkBitmap::copyMarkBit(TenuredCell* dst,
+ const TenuredCell* src,
+ ColorBit colorBit) {
+ TenuredChunkBase* srcChunk = detail::GetCellChunkBase(src);
+ MarkBitmapWord* srcWord;
+ uintptr_t srcMask;
+ srcChunk->markBits.getMarkWordAndMask(src, colorBit, &srcWord, &srcMask);
+
+ MarkBitmapWord* dstWord;
+ uintptr_t dstMask;
+ getMarkWordAndMask(dst, colorBit, &dstWord, &dstMask);
+
+ uintptr_t bits = *dstWord;
+ bits &= ~dstMask;
+ if (*srcWord & srcMask) {
+ bits |= dstMask;
+ }
+ *dstWord = bits;
+}
+
+MOZ_ALWAYS_INLINE void MarkBitmap::unmark(const TenuredCell* cell) {
+ MarkBitmapWord* word;
+ uintptr_t mask;
+ uintptr_t bits;
+ getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
+ bits = *word;
+ *word = bits & ~mask;
+ getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask);
+ bits = *word;
+ *word = bits & ~mask;
+}
+
+inline MarkBitmapWord* MarkBitmap::arenaBits(Arena* arena) {
+ static_assert(
+ ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD,
+ "We assume that the part of the bitmap corresponding to the arena "
+ "has the exact number of words so we do not need to deal with a word "
+ "that covers bits from two arenas.");
+
+ MarkBitmapWord* word;
+ uintptr_t unused;
+ getMarkWordAndMask(reinterpret_cast<TenuredCell*>(arena->address()),
+ ColorBit::BlackBit, &word, &unused);
+ return word;
+}
+
+/*
+ * A chunk in the tenured heap. TenuredChunks contain arenas and associated data
+ * structures (mark bitmap, delayed marking state).
+ */
+class TenuredChunk : public TenuredChunkBase {
+ Arena arenas[ArenasPerChunk];
+
+ friend class GCRuntime;
+ friend class MarkingValidator;
+
+ public:
+ static TenuredChunk* fromAddress(uintptr_t addr) {
+ addr &= ~ChunkMask;
+ return reinterpret_cast<TenuredChunk*>(addr);
+ }
+
+ static bool withinValidRange(uintptr_t addr) {
+ uintptr_t offset = addr & ChunkMask;
+ if (TenuredChunk::fromAddress(addr)->isNurseryChunk()) {
+ return offset >= sizeof(ChunkBase) && offset < ChunkSize;
+ }
+ return offset >= offsetof(TenuredChunk, arenas) && offset < ChunkSize;
+ }
+
+ static size_t arenaIndex(const Arena* arena) {
+ uintptr_t addr = arena->address();
+ MOZ_ASSERT(!TenuredChunk::fromAddress(addr)->isNurseryChunk());
+ MOZ_ASSERT(withinValidRange(addr));
+ uintptr_t offset = addr & ChunkMask;
+ return (offset - offsetof(TenuredChunk, arenas)) >> ArenaShift;
+ }
+
+ explicit TenuredChunk(JSRuntime* runtime) : TenuredChunkBase(runtime) {}
+
+ uintptr_t address() const {
+ uintptr_t addr = reinterpret_cast<uintptr_t>(this);
+ MOZ_ASSERT(!(addr & ChunkMask));
+ return addr;
+ }
+
+ bool unused() const { return info.numArenasFree == ArenasPerChunk; }
+
+ bool hasAvailableArenas() const { return info.numArenasFree != 0; }
+
+ bool isNurseryChunk() const { return storeBuffer; }
+
+ Arena* allocateArena(GCRuntime* gc, JS::Zone* zone, AllocKind kind,
+ const AutoLockGC& lock);
+
+ void releaseArena(GCRuntime* gc, Arena* arena, const AutoLockGC& lock);
+ void recycleArena(Arena* arena, SortedArenaList& dest, size_t thingsPerArena);
+
+ void decommitFreeArenas(GCRuntime* gc, const bool& cancel, AutoLockGC& lock);
+ [[nodiscard]] bool decommitOneFreePage(GCRuntime* gc, size_t pageIndex,
+ AutoLockGC& lock);
+ void decommitAllArenas();
+
+ // This will decommit each unused not-already decommitted arena. It performs a
+ // system call for each arena but is only used during OOM.
+ void decommitFreeArenasWithoutUnlocking(const AutoLockGC& lock);
+
+ static void* allocate(GCRuntime* gc);
+ static TenuredChunk* emplace(void* ptr, GCRuntime* gc,
+ bool allMemoryCommitted);
+
+ /* Unlink and return the freeArenasHead. */
+ Arena* fetchNextFreeArena(GCRuntime* gc);
+
+#ifdef DEBUG
+ void verify() const;
+#else
+ void verify() const {}
+#endif
+
+ private:
+ void commitOnePage(GCRuntime* gc);
+
+ void updateChunkListAfterAlloc(GCRuntime* gc, const AutoLockGC& lock);
+ void updateChunkListAfterFree(GCRuntime* gc, size_t numArenasFree,
+ const AutoLockGC& lock);
+
+ // Check if all arenas in a page are free.
+ bool canDecommitPage(size_t pageIndex) const;
+
+ // Check the arena from freeArenasList is located in a free page.
+ // Unlike the isPageFree(size_t) version, this isPageFree(Arena*) will see the
+ // following arenas from the freeArenasHead are also located in the same page,
+ // to prevent not to access the arenas mprotect'ed during compaction in debug
+ // build.
+ bool isPageFree(const Arena* arena) const;
+
+ // Get the page index of the arena.
+ size_t pageIndex(const Arena* arena) const {
+ return pageIndex(arenaIndex(arena));
+ }
+ size_t pageIndex(size_t arenaIndex) const {
+ return arenaIndex / ArenasPerPage;
+ }
+
+ Arena* pageAddress(size_t pageIndex) {
+ return &arenas[pageIndex * ArenasPerPage];
+ }
+};
+
+inline void Arena::checkAddress() const {
+ mozilla::DebugOnly<uintptr_t> addr = uintptr_t(this);
+ MOZ_ASSERT(addr);
+ MOZ_ASSERT(!(addr & ArenaMask));
+ MOZ_ASSERT(TenuredChunk::withinValidRange(addr));
+}
+
+inline TenuredChunk* Arena::chunk() const {
+ return TenuredChunk::fromAddress(address());
+}
+
+// Cell header stored before all nursery cells.
+struct alignas(gc::CellAlignBytes) NurseryCellHeader {
+ // Store zone pointer with the trace kind in the lowest three bits.
+ const uintptr_t allocSiteAndTraceKind;
+
+ // We only need to store a subset of trace kinds so this doesn't cover the
+ // full range.
+ static const uintptr_t TraceKindMask = 3;
+
+ static uintptr_t MakeValue(AllocSite* const site, JS::TraceKind kind) {
+ MOZ_ASSERT(uintptr_t(kind) < TraceKindMask);
+ MOZ_ASSERT((uintptr_t(site) & TraceKindMask) == 0);
+ return uintptr_t(site) | uintptr_t(kind);
+ }
+
+ inline NurseryCellHeader(AllocSite* site, JS::TraceKind kind);
+
+ AllocSite* allocSite() const {
+ return reinterpret_cast<AllocSite*>(allocSiteAndTraceKind & ~TraceKindMask);
+ }
+
+ JS::Zone* zone() const { return allocSite()->zone(); }
+
+ JS::TraceKind traceKind() const {
+ return JS::TraceKind(allocSiteAndTraceKind & TraceKindMask);
+ }
+
+ static const NurseryCellHeader* from(const Cell* cell) {
+ MOZ_ASSERT(IsInsideNursery(cell));
+ return reinterpret_cast<const NurseryCellHeader*>(
+ uintptr_t(cell) - sizeof(NurseryCellHeader));
+ }
+};
+
+static_assert(uintptr_t(JS::TraceKind::Object) <=
+ NurseryCellHeader::TraceKindMask);
+static_assert(uintptr_t(JS::TraceKind::String) <=
+ NurseryCellHeader::TraceKindMask);
+static_assert(uintptr_t(JS::TraceKind::BigInt) <=
+ NurseryCellHeader::TraceKindMask);
+
+} /* namespace gc */
+
+namespace debug {
+
+// Utility functions meant to be called from an interactive debugger.
+enum class MarkInfo : int {
+ BLACK = 0,
+ GRAY = 1,
+ UNMARKED = -1,
+ NURSERY = -2,
+};
+
+// Get the mark color for a cell, in a way easily usable from a debugger.
+MOZ_NEVER_INLINE MarkInfo GetMarkInfo(js::gc::Cell* cell);
+
+// Sample usage from gdb:
+//
+// (gdb) p $word = js::debug::GetMarkWordAddress(obj)
+// $1 = (uintptr_t *) 0x7fa56d5fe360
+// (gdb) p/x $mask = js::debug::GetMarkMask(obj, js::gc::GRAY)
+// $2 = 0x200000000
+// (gdb) watch *$word
+// Hardware watchpoint 7: *$word
+// (gdb) cond 7 *$word & $mask
+// (gdb) cont
+//
+// Note that this is *not* a watchpoint on a single bit. It is a watchpoint on
+// the whole word, which will trigger whenever the word changes and the
+// selected bit is set after the change.
+//
+// So if the bit changing is the desired one, this is exactly what you want.
+// But if a different bit changes (either set or cleared), you may still stop
+// execution if the $mask bit happened to already be set. gdb does not expose
+// enough information to restrict the watchpoint to just a single bit.
+
+// Return the address of the word containing the mark bits for the given cell,
+// or nullptr if the cell is in the nursery.
+MOZ_NEVER_INLINE uintptr_t* GetMarkWordAddress(js::gc::Cell* cell);
+
+// Return the mask for the given cell and color bit, or 0 if the cell is in the
+// nursery.
+MOZ_NEVER_INLINE uintptr_t GetMarkMask(js::gc::Cell* cell, uint32_t colorBit);
+
+} /* namespace debug */
+} /* namespace js */
+
+#endif /* gc_Heap_h */
diff --git a/js/src/gc/IteratorUtils.h b/js/src/gc/IteratorUtils.h
new file mode 100644
index 0000000000..614fd12100
--- /dev/null
+++ b/js/src/gc/IteratorUtils.h
@@ -0,0 +1,121 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_IteratorUtils_h
+#define gc_IteratorUtils_h
+
+#include "mozilla/Array.h"
+#include "mozilla/Maybe.h"
+
+#include <initializer_list>
+
+namespace js {
+
+/*
+ * Create an iterator that yields the values from IteratorB(a) for all a in
+ * IteratorA(). Equivalent to nested for loops over IteratorA and IteratorB
+ * where IteratorB is constructed with a value from IteratorA.
+ */
+template <typename IteratorA, typename IteratorB>
+class NestedIterator {
+ using T = decltype(std::declval<IteratorB>().get());
+
+ IteratorA a;
+ mozilla::Maybe<IteratorB> b;
+
+ public:
+ template <typename... Args>
+ explicit NestedIterator(Args&&... args) : a(std::forward<Args>(args)...) {
+ settle();
+ }
+
+ bool done() const { return b.isNothing(); }
+
+ T get() const {
+ MOZ_ASSERT(!done());
+ return b.ref().get();
+ }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ b->next();
+ if (b->done()) {
+ b.reset();
+ a.next();
+ settle();
+ }
+ }
+
+ const IteratorB& ref() const { return *b; }
+
+ operator T() const { return get(); }
+
+ T operator->() const { return get(); }
+
+ private:
+ void settle() {
+ MOZ_ASSERT(b.isNothing());
+ while (!a.done()) {
+ b.emplace(a.get());
+ if (!b->done()) {
+ break;
+ }
+ b.reset();
+ a.next();
+ }
+ }
+};
+
+/*
+ * An iterator the yields values from each of N of instances of Iterator in
+ * sequence.
+ */
+template <typename Iterator, size_t N>
+class ChainedIterator {
+ using T = decltype(std::declval<Iterator>().get());
+
+ mozilla::Array<Iterator, N> iterators;
+ size_t index = 0;
+
+ public:
+ template <typename... Args>
+ MOZ_IMPLICIT ChainedIterator(Args&&... args)
+ : iterators(Iterator(std::forward<Args>(args))...) {
+ static_assert(N > 1);
+ settle();
+ }
+
+ bool done() const { return index == N; }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ iterators[index].next();
+ settle();
+ }
+
+ T get() const {
+ MOZ_ASSERT(!done());
+ return iterators[index].get();
+ }
+
+ operator T() const { return get(); }
+ T operator->() const { return get(); }
+
+ private:
+ void settle() {
+ MOZ_ASSERT(!done());
+ while (iterators[index].done()) {
+ index++;
+ if (done()) {
+ break;
+ }
+ }
+ }
+};
+
+} /* namespace js */
+
+#endif // gc_IteratorUtils_h
diff --git a/js/src/gc/MallocedBlockCache.cpp b/js/src/gc/MallocedBlockCache.cpp
new file mode 100644
index 0000000000..602d1a80b0
--- /dev/null
+++ b/js/src/gc/MallocedBlockCache.cpp
@@ -0,0 +1,144 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sw=2 et tw=80:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/MallocedBlockCache.h"
+#include "mozilla/MemoryChecking.h"
+
+using js::PointerAndUint7;
+using js::gc::MallocedBlockCache;
+
+MallocedBlockCache::~MallocedBlockCache() { clear(); }
+
+PointerAndUint7 MallocedBlockCache::alloc(size_t size) {
+ // Figure out which free list can give us a block of size `size`, after it
+ // has been rounded up to a multiple of `step`.
+ //
+ // Example mapping for STEP = 16 and NUM_LISTS = 8, after rounding up:
+ // 0 never holds any blocks (denotes "too large")
+ // 1 holds blocks of size 16
+ // 2 holds blocks of size 32
+ // 3 holds blocks of size 48
+ // 4 holds blocks of size 64
+ // 5 holds blocks of size 80
+ // 6 holds blocks of size 96
+ // 7 holds blocks of size 112
+ //
+ // For a request of size n:
+ // * if n == 0, fail
+ // * else
+ // round n up to a multiple of STEP
+ // let i = n / STEP
+ // if i >= NUM_LISTS
+ // alloc direct from js_malloc, and return listID = 0
+ // if lists[i] is nonempty, use lists[i] and return listID = i.
+ // else
+ // let p = js_malloc(n)
+ // return p and listID = i.
+
+ // We're never expected to handle zero-sized blocks.
+ MOZ_ASSERT(size > 0);
+
+ size = js::RoundUp(size, STEP);
+ size_t i = size / STEP;
+
+ // Too large to cache; go straight to js_malloc.
+ if (MOZ_UNLIKELY(i >= NUM_LISTS)) {
+ void* p = js_malloc(size);
+ // If p is nullptr, that fact is carried into the PointerAndUint7, and the
+ // caller is expected to check that.
+ return PointerAndUint7(p, OVERSIZE_BLOCK_LIST_ID);
+ }
+
+ // The case we hope is common. First, see if we can pull a block from the
+ // relevant list.
+ MOZ_ASSERT(i >= 1 && i < NUM_LISTS);
+ // Check that i is the right list
+ MOZ_ASSERT(i * STEP == size);
+ if (MOZ_LIKELY(!lists[i].empty())) {
+ void* block = lists[i].popCopy();
+ return PointerAndUint7(block, i);
+ }
+
+ // No luck.
+ void* p = js_malloc(size);
+ if (MOZ_UNLIKELY(!p)) {
+ return PointerAndUint7(nullptr, 0); // OOM
+ }
+ return PointerAndUint7(p, i);
+}
+
+void MallocedBlockCache::free(PointerAndUint7 blockAndListID) {
+ // This is a whole lot simpler than the ::alloc case, since we are given the
+ // listId and don't have to compute it (not that we have any way to).
+ void* block = blockAndListID.pointer();
+ uint32_t listID = blockAndListID.uint7();
+ MOZ_ASSERT(block);
+ MOZ_ASSERT(listID < NUM_LISTS);
+ if (MOZ_UNLIKELY(listID == OVERSIZE_BLOCK_LIST_ID)) {
+ // It was too large for recycling; go straight to js_free.
+ js_free(block);
+ return;
+ }
+
+ // Put it back on list `listId`, first poisoning it for safety.
+ memset(block, JS_NOTINUSE_TRAILER_PATTERN, listID * STEP);
+ MOZ_MAKE_MEM_UNDEFINED(block, listID * STEP);
+ if (MOZ_UNLIKELY(!lists[listID].append(block))) {
+ // OOM'd while doing admin. Hand it off to js_free and forget about the
+ // OOM.
+ js_free(block);
+ }
+}
+
+void MallocedBlockCache::preen(float percentOfBlocksToDiscard) {
+ MOZ_ASSERT(percentOfBlocksToDiscard >= 0.0 &&
+ percentOfBlocksToDiscard <= 100.0);
+ MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty());
+ for (size_t listID = 1; listID < NUM_LISTS; listID++) {
+ MallocedBlockVector& list = lists[listID];
+ size_t numToFree =
+ size_t(float(list.length()) * (percentOfBlocksToDiscard / 100.0));
+ MOZ_RELEASE_ASSERT(numToFree <= list.length());
+ while (numToFree > 0) {
+ void* block = list.popCopy();
+ MOZ_ASSERT(block);
+ js_free(block);
+ numToFree--;
+ }
+ }
+}
+
+void MallocedBlockCache::clear() {
+ MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty());
+ for (size_t i = 1; i < NUM_LISTS; i++) {
+ MallocedBlockVector& list = lists[i];
+ for (size_t j = 0; j < list.length(); j++) {
+ MOZ_ASSERT(list[j]);
+ js_free(list[j]);
+ list[j] = nullptr; // for safety
+ }
+ list.clear();
+ }
+}
+
+size_t MallocedBlockCache::sizeOfExcludingThis(
+ mozilla::MallocSizeOf mallocSizeOf) const {
+ MOZ_ASSERT(lists[OVERSIZE_BLOCK_LIST_ID].empty());
+ size_t nBytes = 0;
+ for (size_t listID = 0; listID < NUM_LISTS; listID++) {
+ const MallocedBlockVector& list = lists[listID];
+ nBytes += list.sizeOfExcludingThis(mallocSizeOf);
+ // The payload size of each block in `list` is the same. Hence, we could
+ // possibly do better here (measure once and multiply by the length) if we
+ // believe that the metadata size for each block is also the same.
+ for (size_t i = 0; i < list.length(); i++) {
+ MOZ_ASSERT(list[i]);
+ nBytes += mallocSizeOf(list[i]);
+ }
+ }
+ return nBytes;
+}
diff --git a/js/src/gc/MallocedBlockCache.h b/js/src/gc/MallocedBlockCache.h
new file mode 100644
index 0000000000..546d7e50af
--- /dev/null
+++ b/js/src/gc/MallocedBlockCache.h
@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sw=2 et tw=80:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_MallocedBlockCache_h
+#define gc_MallocedBlockCache_h
+
+#include "ds/PointerAndUint7.h"
+#include "js/AllocPolicy.h"
+#include "js/Vector.h"
+#include "util/Poison.h"
+
+namespace js {
+namespace gc {
+
+// MallocedBlockCache implements a lightweight wrapper around js_malloc/js_free.
+//
+// Blocks are requested by ::alloc and must be returned with ::free, at which
+// point the cache may decide to hold on to the block rather than hand it back
+// to js_free. Subsequent ::alloc calls may be satisfied from the cached
+// blocks rather than calling js_malloc. The mechanism is designed to be much
+// cheaper than calling js_malloc/js_free directly. One consequence is that
+// there is no locking; it is essential therefore to use each cache only from
+// a single thread.
+//
+// The intended use is for lightweight management of OOL (malloc'd) storage
+// associated with WasmStructObject and WasmArrayObject. The mechanism is
+// general and potentially has other uses. Blocks of size STEP * NUM_LISTS
+// and larger are never cached, though.
+//
+// Request sizes are rounded up to a multiple of STEP. There are NUM_LISTS-1
+// free lists, with a "List ID" indicating the multiple of STEP stored on the
+// list. So for example, blocks of size 3 * STEP (after rounding up) are
+// stored on the list with ID 3. List ID 0 indicates blocks which are too
+// large to live on any freelist. With the default settings, this gives
+// separate freelists for blocks of size 16, 32, 48, .. 496. Blocks of size
+// zero are not supported, and `lists[0]` will always be empty.
+//
+// Allocation of a block produces not only the block's address but also its
+// List ID. When freeing, both values must be presented, because there is
+// otherwise no way for ::free to know the size of the original allocation,
+// and hence which freelist it should go on. For this reason, the ::alloc and
+// ::free methods produce and take a `PointerAndUint7`, not a `void*`.
+//
+// Resizing of blocks is not supported.
+
+class MallocedBlockCache {
+ public:
+ static const size_t STEP = 16;
+
+ static const size_t NUM_LISTS = 32;
+ // This limitation exists because allocation returns a PointerAndUint7, and
+ // a List-ID value (viz, 0 .. NUM_LISTS-1) is stored in the Uint7 part.
+ static_assert(NUM_LISTS <= (1 << 7));
+
+ // list[0] must always remain empty. List ID 0 indicates a block which
+ // cannot participate in the freelist machinery because it is too large.
+ //
+ // list[i], for 1 <= i < NUM_LISTS, holds blocks of size i * STEP only.
+ // All requests are rounded up to multiple of SIZE.
+ //
+ // We do not expect to be required to issue or accept blocks of size zero.
+ static const size_t OVERSIZE_BLOCK_LIST_ID = 0;
+ using MallocedBlockVector = Vector<void*, 0, SystemAllocPolicy>;
+
+ MallocedBlockVector lists[NUM_LISTS];
+
+ ~MallocedBlockCache();
+
+ // Allocation and freeing.
+ [[nodiscard]] PointerAndUint7 alloc(size_t size);
+ void free(PointerAndUint7 blockAndListID);
+
+ // Allows users to gradually hand blocks back to js_free, so as to avoid
+ // space leaks in long-running scenarios. The specified percentage of
+ // blocks in each list is discarded.
+ void preen(float percentOfBlocksToDiscard);
+
+ // Return all blocks in the cache to js_free.
+ void clear();
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_MallocedBlockCache_h
diff --git a/js/src/gc/Marking-inl.h b/js/src/gc/Marking-inl.h
new file mode 100644
index 0000000000..07cfffb8da
--- /dev/null
+++ b/js/src/gc/Marking-inl.h
@@ -0,0 +1,196 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Marking_inl_h
+#define gc_Marking_inl_h
+
+#include "gc/Marking.h"
+
+#include <type_traits>
+
+#include "gc/RelocationOverlay.h"
+#include "js/Id.h"
+#include "js/Value.h"
+#include "vm/StringType.h"
+#include "vm/TaggedProto.h"
+
+#include "gc/Nursery-inl.h"
+
+namespace js {
+namespace gc {
+
+// An abstraction to re-wrap any kind of typed pointer back to the tagged
+// pointer it came from with |TaggedPtr<TargetType>::wrap(sourcePtr)|.
+template <typename T>
+struct TaggedPtr {};
+
+template <>
+struct TaggedPtr<JS::Value> {
+ static JS::Value wrap(JSObject* obj) {
+ if (!obj) {
+ return JS::NullValue();
+ }
+#ifdef ENABLE_RECORD_TUPLE
+ if (MaybeForwardedIsExtendedPrimitive(*obj)) {
+ return JS::ExtendedPrimitiveValue(*obj);
+ }
+#endif
+ return JS::ObjectValue(*obj);
+ }
+ static JS::Value wrap(JSString* str) { return JS::StringValue(str); }
+ static JS::Value wrap(JS::Symbol* sym) { return JS::SymbolValue(sym); }
+ static JS::Value wrap(JS::BigInt* bi) { return JS::BigIntValue(bi); }
+ template <typename T>
+ static JS::Value wrap(T* priv) {
+ static_assert(std::is_base_of_v<Cell, T>,
+ "Type must be a GC thing derived from js::gc::Cell");
+ return JS::PrivateGCThingValue(priv);
+ }
+ static JS::Value empty() { return JS::UndefinedValue(); }
+};
+
+template <>
+struct TaggedPtr<jsid> {
+ static jsid wrap(JSString* str) { return JS::PropertyKey::NonIntAtom(str); }
+ static jsid wrap(JS::Symbol* sym) { return PropertyKey::Symbol(sym); }
+ static jsid empty() { return JS::PropertyKey::Void(); }
+};
+
+template <>
+struct TaggedPtr<TaggedProto> {
+ static TaggedProto wrap(JSObject* obj) { return TaggedProto(obj); }
+ static TaggedProto empty() { return TaggedProto(); }
+};
+
+template <typename T>
+struct MightBeForwarded {
+ static_assert(std::is_base_of_v<Cell, T>);
+ static_assert(!std::is_same_v<Cell, T> && !std::is_same_v<TenuredCell, T>);
+
+#define CAN_FORWARD_KIND_OR(_1, _2, Type, _3, _4, _5, canCompact) \
+ std::is_base_of_v<Type, T> ? canCompact:
+
+ // FOR_EACH_ALLOCKIND doesn't cover every possible type: make sure
+ // to default to `true` for unknown types.
+ static constexpr bool value = FOR_EACH_ALLOCKIND(CAN_FORWARD_KIND_OR) true;
+#undef CAN_FORWARD_KIND_OR
+};
+
+template <typename T>
+inline bool IsForwarded(const T* t) {
+ if (!MightBeForwarded<T>::value) {
+ MOZ_ASSERT(!t->isForwarded());
+ return false;
+ }
+
+ return t->isForwarded();
+}
+
+template <typename T>
+inline T* Forwarded(const T* t) {
+ const RelocationOverlay* overlay = RelocationOverlay::fromCell(t);
+ MOZ_ASSERT(overlay->isForwarded());
+ return reinterpret_cast<T*>(overlay->forwardingAddress());
+}
+
+template <typename T>
+inline T MaybeForwarded(T t) {
+ if (IsForwarded(t)) {
+ t = Forwarded(t);
+ }
+ MOZ_ASSERT(!IsForwarded(t));
+ return t;
+}
+
+inline const JSClass* MaybeForwardedObjectClass(const JSObject* obj) {
+ Shape* shape = MaybeForwarded(obj->shapeMaybeForwarded());
+ BaseShape* baseShape = MaybeForwarded(shape->base());
+ return baseShape->clasp();
+}
+
+template <typename T>
+inline bool MaybeForwardedObjectIs(const JSObject* obj) {
+ MOZ_ASSERT(!obj->isForwarded());
+ return MaybeForwardedObjectClass(obj) == &T::class_;
+}
+
+template <typename T>
+inline T& MaybeForwardedObjectAs(JSObject* obj) {
+ MOZ_ASSERT(MaybeForwardedObjectIs<T>(obj));
+ return *static_cast<T*>(obj);
+}
+
+inline RelocationOverlay::RelocationOverlay(Cell* dst) {
+ MOZ_ASSERT(dst->flags() == 0);
+ uintptr_t ptr = uintptr_t(dst);
+ header_.setForwardingAddress(ptr);
+}
+
+/* static */
+inline RelocationOverlay* RelocationOverlay::forwardCell(Cell* src, Cell* dst) {
+ MOZ_ASSERT(!src->isForwarded());
+ MOZ_ASSERT(!dst->isForwarded());
+ return new (src) RelocationOverlay(dst);
+}
+
+inline bool IsAboutToBeFinalizedDuringMinorSweep(Cell** cellp) {
+ MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+
+ if ((*cellp)->isTenured()) {
+ return false;
+ }
+
+ return !Nursery::getForwardedPointer(cellp);
+}
+
+// Special case pre-write barrier for strings used during rope flattening. This
+// avoids eager marking of ropes which does not immediately mark the cells if we
+// hit OOM. This does not traverse ropes and is instead called on every node in
+// a rope during flattening.
+inline void PreWriteBarrierDuringFlattening(JSString* str) {
+ MOZ_ASSERT(str);
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+
+ if (IsInsideNursery(str)) {
+ return;
+ }
+
+ auto* cell = reinterpret_cast<TenuredCell*>(str);
+ JS::shadow::Zone* zone = cell->shadowZoneFromAnyThread();
+ if (!zone->needsIncrementalBarrier()) {
+ return;
+ }
+
+ MOZ_ASSERT(!str->isPermanentAndMayBeShared());
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(zone->runtimeFromAnyThread()));
+ PerformIncrementalBarrierDuringFlattening(str);
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+template <typename T>
+inline bool IsGCThingValidAfterMovingGC(T* t) {
+ return !IsInsideNursery(t) && !t->isForwarded();
+}
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(T* t) {
+ if (t) {
+ MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t));
+ }
+}
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(const WeakHeapPtr<T*>& t) {
+ CheckGCThingAfterMovingGC(t.unbarrieredGet());
+}
+
+#endif // JSGC_HASH_TABLE_CHECKS
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif // gc_Marking_inl_h
diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp
new file mode 100644
index 0000000000..b92cd5f3ac
--- /dev/null
+++ b/js/src/gc/Marking.cpp
@@ -0,0 +1,2774 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Marking-inl.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/IntegerRange.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/ScopeExit.h"
+
+#include <algorithm>
+#include <type_traits>
+
+#include "gc/GCInternals.h"
+#include "gc/ParallelMarking.h"
+#include "gc/TraceKind.h"
+#include "jit/JitCode.h"
+#include "js/GCTypeMacros.h" // JS_FOR_EACH_PUBLIC_{,TAGGED_}GC_POINTER_TYPE
+#include "js/SliceBudget.h"
+#include "util/Poison.h"
+#include "vm/GeneratorObject.h"
+
+#include "gc/GC-inl.h"
+#include "gc/PrivateIterators-inl.h"
+#include "gc/TraceMethods-inl.h"
+#include "gc/WeakMap-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using JS::MapTypeToTraceKind;
+
+using mozilla::DebugOnly;
+using mozilla::IntegerRange;
+using mozilla::PodCopy;
+
+// [SMDOC] GC Tracing
+//
+// Tracing Overview
+// ================
+//
+// Tracing, in this context, refers to an abstract visitation of some or all of
+// the GC-controlled heap. The effect of tracing an edge of the graph depends
+// on the subclass of the JSTracer on whose behalf we are tracing.
+//
+// Marking
+// -------
+//
+// The primary JSTracer is the GCMarker. The marking tracer causes the target
+// of each traversed edge to be marked black and the target edge's children to
+// be marked either gray (in the gc algorithm sense) or immediately black.
+//
+// Callback
+// --------
+//
+// The secondary JSTracer is the CallbackTracer. This simply invokes a callback
+// on each edge in a child.
+//
+// The following is a rough outline of the general struture of the tracing
+// internals.
+//
+/* clang-format off */
+//
+// +-------------------+ ......................
+// | | : :
+// | v v +---+---+
+// | TraceRoot TraceEdge TraceRange GCMarker:: | |
+// | | | | processMarkStackTop | Mark |
+// | +-----------------------+ | | Stack |
+// | | | | |
+// | v | +---+---+
+// | TraceEdgeInternal | ^
+// | | +<-------------+ :
+// | | | | :
+// | v v | :
+// | CallbackTracer:: markAndTraverseEdge | :
+// | onSomeEdge | | :
+// | | | | :
+// | | | | :
+// | +-------------+---------------+ | :
+// | | | :
+// | v | :
+// | markAndTraverse | :
+// | | | :
+// | | | :
+// | traverse | :
+// | | | :
+// | +--------------------------------------+ | :
+// | | | | | :
+// | v v v | :
+// | markAndTraceChildren markAndPush eagerlyMarkChildren | :
+// | | : | | :
+// | v : +-----------+ :
+// | T::traceChildren : :
+// | | : :
+// +-------------+ ......................................
+//
+// Legend:
+// ------- Direct calls
+// ....... Data flow
+//
+/* clang-format on */
+
+/*** Tracing Invariants *****************************************************/
+
+template <typename T>
+static inline bool IsOwnedByOtherRuntime(JSRuntime* rt, T thing) {
+ bool other = thing->runtimeFromAnyThread() != rt;
+ MOZ_ASSERT_IF(other, thing->isPermanentAndMayBeShared());
+ return other;
+}
+
+#ifdef DEBUG
+
+static inline bool IsInFreeList(TenuredCell* cell) {
+ Arena* arena = cell->arena();
+ uintptr_t addr = reinterpret_cast<uintptr_t>(cell);
+ MOZ_ASSERT(Arena::isAligned(addr, arena->getThingSize()));
+ return arena->inFreeList(addr);
+}
+
+template <typename T>
+void js::CheckTracedThing(JSTracer* trc, T* thing) {
+ MOZ_ASSERT(trc);
+ MOZ_ASSERT(thing);
+
+ if (IsForwarded(thing)) {
+ JS::TracerKind kind = trc->kind();
+ MOZ_ASSERT(kind == JS::TracerKind::Tenuring ||
+ kind == JS::TracerKind::MinorSweeping ||
+ kind == JS::TracerKind::Moving);
+ thing = Forwarded(thing);
+ }
+
+ /* This function uses data that's not available in the nursery. */
+ if (IsInsideNursery(thing)) {
+ return;
+ }
+
+ /*
+ * Permanent shared things that are not associated with this runtime will be
+ * ignored during marking.
+ */
+ Zone* zone = thing->zoneFromAnyThread();
+ if (IsOwnedByOtherRuntime(trc->runtime(), thing)) {
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(thing->isMarkedBlack());
+ return;
+ }
+
+ JSRuntime* rt = trc->runtime();
+ MOZ_ASSERT(zone->runtimeFromAnyThread() == rt);
+
+ bool isGcMarkingTracer = trc->isMarkingTracer();
+ bool isUnmarkGrayTracer = IsTracerKind(trc, JS::TracerKind::UnmarkGray);
+ bool isClearEdgesTracer = IsTracerKind(trc, JS::TracerKind::ClearEdges);
+
+ if (TlsContext.get()) {
+ // If we're on the main thread we must have access to the runtime and zone.
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(CurrentThreadCanAccessZone(zone));
+ } else {
+ MOZ_ASSERT(isGcMarkingTracer || isUnmarkGrayTracer || isClearEdgesTracer ||
+ IsTracerKind(trc, JS::TracerKind::Moving) ||
+ IsTracerKind(trc, JS::TracerKind::Sweeping));
+ MOZ_ASSERT_IF(!isClearEdgesTracer, CurrentThreadIsPerformingGC());
+ }
+
+ MOZ_ASSERT(thing->isAligned());
+ MOZ_ASSERT(MapTypeToTraceKind<std::remove_pointer_t<T>>::kind ==
+ thing->getTraceKind());
+
+ /*
+ * Check that we only mark allocated cells.
+ *
+ * This check is restricted to marking for two reasons: Firstly, if background
+ * sweeping is running and concurrently modifying the free list then it is not
+ * safe. Secondly, it was thought to be slow so this is a compromise so as to
+ * not affect test times too much.
+ */
+ MOZ_ASSERT_IF(zone->isGCMarking(), !IsInFreeList(&thing->asTenured()));
+}
+
+template <typename T>
+void js::CheckTracedThing(JSTracer* trc, const T& thing) {
+ ApplyGCThingTyped(thing, [trc](auto t) { CheckTracedThing(trc, t); });
+}
+
+template <typename T>
+static void CheckMarkedThing(GCMarker* gcMarker, T* thing) {
+ Zone* zone = thing->zoneFromAnyThread();
+
+ MOZ_ASSERT(zone->shouldMarkInZone(gcMarker->markColor()));
+
+ MOZ_ASSERT_IF(gcMarker->shouldCheckCompartments(),
+ zone->isCollectingFromAnyThread() || zone->isAtomsZone());
+
+ MOZ_ASSERT_IF(gcMarker->markColor() == MarkColor::Gray,
+ !zone->isGCMarkingBlackOnly() || zone->isAtomsZone());
+
+ MOZ_ASSERT(!(zone->isGCSweeping() || zone->isGCFinished() ||
+ zone->isGCCompacting()));
+
+ // Check that we don't stray from the current compartment and zone without
+ // using TraceCrossCompartmentEdge.
+ Compartment* comp = thing->maybeCompartment();
+ MOZ_ASSERT_IF(gcMarker->tracingCompartment && comp,
+ gcMarker->tracingCompartment == comp);
+ MOZ_ASSERT_IF(gcMarker->tracingZone,
+ gcMarker->tracingZone == zone || zone->isAtomsZone());
+}
+
+namespace js {
+
+# define IMPL_CHECK_TRACED_THING(_, type, _1, _2) \
+ template void CheckTracedThing<type>(JSTracer*, type*);
+JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING);
+# undef IMPL_CHECK_TRACED_THING
+
+template void CheckTracedThing<Value>(JSTracer*, const Value&);
+
+} // namespace js
+
+#endif
+
+static inline bool ShouldMarkCrossCompartment(GCMarker* marker, JSObject* src,
+ Cell* dstCell) {
+ MarkColor color = marker->markColor();
+
+ if (!dstCell->isTenured()) {
+#ifdef DEBUG
+ // Bug 1743098: This shouldn't be possible but it does seem to happen. Log
+ // some useful information in debug builds.
+ if (color != MarkColor::Black) {
+ fprintf(stderr,
+ "ShouldMarkCrossCompartment: cross compartment edge from gray "
+ "object to nursery thing\n");
+ fprintf(stderr, "src: ");
+ src->dump();
+ fprintf(stderr, "dst: ");
+ dstCell->dump();
+ }
+#endif
+ MOZ_ASSERT(color == MarkColor::Black);
+ return false;
+ }
+ TenuredCell& dst = dstCell->asTenured();
+
+ JS::Zone* dstZone = dst.zone();
+ if (!src->zone()->isGCMarking() && !dstZone->isGCMarking()) {
+ return false;
+ }
+
+ if (color == MarkColor::Black) {
+ // Check our sweep groups are correct: we should never have to
+ // mark something in a zone that we have started sweeping.
+ MOZ_ASSERT_IF(!dst.isMarkedBlack(), !dstZone->isGCSweeping());
+
+ /*
+ * Having black->gray edges violates our promise to the cycle collector so
+ * we ensure that gray things we encounter when marking black end up getting
+ * marked black.
+ *
+ * This can happen for two reasons:
+ *
+ * 1) If we're collecting a compartment and it has an edge to an uncollected
+ * compartment it's possible that the source and destination of the
+ * cross-compartment edge should be gray, but the source was marked black by
+ * the write barrier.
+ *
+ * 2) If we yield during gray marking and the write barrier marks a gray
+ * thing black.
+ *
+ * We handle the first case before returning whereas the second case happens
+ * as part of normal marking.
+ */
+ if (dst.isMarkedGray() && !dstZone->isGCMarking()) {
+ UnmarkGrayGCThingUnchecked(marker,
+ JS::GCCellPtr(&dst, dst.getTraceKind()));
+ return false;
+ }
+
+ return dstZone->isGCMarking();
+ } else {
+ // Check our sweep groups are correct as above.
+ MOZ_ASSERT_IF(!dst.isMarkedAny(), !dstZone->isGCSweeping());
+
+ if (dstZone->isGCMarkingBlackOnly()) {
+ /*
+ * The destination compartment is being not being marked gray now,
+ * but it will be later, so record the cell so it can be marked gray
+ * at the appropriate time.
+ */
+ if (!dst.isMarkedAny()) {
+ DelayCrossCompartmentGrayMarking(marker, src);
+ }
+ return false;
+ }
+
+ return dstZone->isGCMarkingBlackAndGray();
+ }
+}
+
+static bool ShouldTraceCrossCompartment(JSTracer* trc, JSObject* src,
+ Cell* dstCell) {
+ if (!trc->isMarkingTracer()) {
+ return true;
+ }
+
+ return ShouldMarkCrossCompartment(GCMarker::fromTracer(trc), src, dstCell);
+}
+
+static bool ShouldTraceCrossCompartment(JSTracer* trc, JSObject* src,
+ const Value& val) {
+ return val.isGCThing() &&
+ ShouldTraceCrossCompartment(trc, src, val.toGCThing());
+}
+
+#ifdef DEBUG
+
+inline void js::gc::AssertShouldMarkInZone(GCMarker* marker, Cell* thing) {
+ if (!thing->isMarkedBlack()) {
+ Zone* zone = thing->zone();
+ MOZ_ASSERT(zone->isAtomsZone() ||
+ zone->shouldMarkInZone(marker->markColor()));
+ }
+}
+
+void js::gc::AssertRootMarkingPhase(JSTracer* trc) {
+ MOZ_ASSERT_IF(trc->isMarkingTracer(),
+ trc->runtime()->gc.state() == State::NotActive ||
+ trc->runtime()->gc.state() == State::MarkRoots);
+}
+
+#endif // DEBUG
+
+/*** Tracing Interface ******************************************************/
+
+template <typename T>
+static void TraceExternalEdgeHelper(JSTracer* trc, T* thingp,
+ const char* name) {
+ MOZ_ASSERT(InternalBarrierMethods<T>::isMarkable(*thingp));
+ TraceEdgeInternal(trc, ConvertToBase(thingp), name);
+}
+
+JS_PUBLIC_API void js::UnsafeTraceManuallyBarrieredEdge(JSTracer* trc,
+ JSObject** thingp,
+ const char* name) {
+ TraceEdgeInternal(trc, ConvertToBase(thingp), name);
+}
+
+template <typename T>
+static void TraceRootHelper(JSTracer* trc, T* thingp, const char* name) {
+ MOZ_ASSERT(thingp);
+ js::TraceNullableRoot(trc, thingp, name);
+}
+
+namespace js {
+class AbstractGeneratorObject;
+class SavedFrame;
+} // namespace js
+
+#define DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION(type) \
+ JS_PUBLIC_API void js::gc::TraceExternalEdge(JSTracer* trc, type* thingp, \
+ const char* name) { \
+ TraceExternalEdgeHelper(trc, thingp, name); \
+ }
+
+// Define TraceExternalEdge for each public GC pointer type.
+JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION)
+JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION)
+
+#undef DEFINE_TRACE_EXTERNAL_EDGE_FUNCTION
+
+#define DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(type) \
+ JS_PUBLIC_API void JS::TraceRoot(JSTracer* trc, type* thingp, \
+ const char* name) { \
+ TraceRootHelper(trc, thingp, name); \
+ }
+
+// Define TraceRoot for each public GC pointer type.
+JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(DEFINE_UNSAFE_TRACE_ROOT_FUNCTION)
+JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(DEFINE_UNSAFE_TRACE_ROOT_FUNCTION)
+
+// Also, for the moment, define TraceRoot for internal GC pointer types.
+DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(AbstractGeneratorObject*)
+DEFINE_UNSAFE_TRACE_ROOT_FUNCTION(SavedFrame*)
+
+#undef DEFINE_UNSAFE_TRACE_ROOT_FUNCTION
+
+namespace js {
+namespace gc {
+
+#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type) \
+ template void TraceRangeInternal<type>(JSTracer*, size_t len, type*, \
+ const char*);
+
+#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND(_1, type, _2, _3) \
+ INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type*)
+
+JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND)
+JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS)
+INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(TaggedProto)
+
+#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND
+#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS
+
+} // namespace gc
+} // namespace js
+
+// In debug builds, makes a note of the current compartment before calling a
+// trace hook or traceChildren() method on a GC thing.
+class MOZ_RAII AutoSetTracingSource {
+#ifndef DEBUG
+ public:
+ template <typename T>
+ AutoSetTracingSource(JSTracer* trc, T* thing) {}
+ ~AutoSetTracingSource() {}
+#else
+ GCMarker* marker = nullptr;
+
+ public:
+ template <typename T>
+ AutoSetTracingSource(JSTracer* trc, T* thing) {
+ if (trc->isMarkingTracer() && thing) {
+ marker = GCMarker::fromTracer(trc);
+ MOZ_ASSERT(!marker->tracingZone);
+ marker->tracingZone = thing->asTenured().zone();
+ MOZ_ASSERT(!marker->tracingCompartment);
+ marker->tracingCompartment = thing->maybeCompartment();
+ }
+ }
+
+ ~AutoSetTracingSource() {
+ if (marker) {
+ marker->tracingZone = nullptr;
+ marker->tracingCompartment = nullptr;
+ }
+ }
+#endif
+};
+
+// In debug builds, clear the trace hook compartment. This happens after the
+// trace hook has called back into one of our trace APIs and we've checked the
+// traced thing.
+class MOZ_RAII AutoClearTracingSource {
+#ifndef DEBUG
+ public:
+ explicit AutoClearTracingSource(GCMarker* marker) {}
+ explicit AutoClearTracingSource(JSTracer* trc) {}
+ ~AutoClearTracingSource() {}
+#else
+ GCMarker* marker = nullptr;
+ JS::Zone* prevZone = nullptr;
+ Compartment* prevCompartment = nullptr;
+
+ public:
+ explicit AutoClearTracingSource(JSTracer* trc) {
+ if (trc->isMarkingTracer()) {
+ marker = GCMarker::fromTracer(trc);
+ prevZone = marker->tracingZone;
+ marker->tracingZone = nullptr;
+ prevCompartment = marker->tracingCompartment;
+ marker->tracingCompartment = nullptr;
+ }
+ }
+ ~AutoClearTracingSource() {
+ if (marker) {
+ marker->tracingZone = prevZone;
+ marker->tracingCompartment = prevCompartment;
+ }
+ }
+#endif
+};
+
+template <typename T>
+void js::TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc,
+ JSObject* src, T* dst,
+ const char* name) {
+ // Clear expected compartment for cross-compartment edge.
+ AutoClearTracingSource acts(trc);
+
+ if (ShouldTraceCrossCompartment(trc, src, *dst)) {
+ TraceEdgeInternal(trc, dst, name);
+ }
+}
+template void js::TraceManuallyBarrieredCrossCompartmentEdge<Value>(
+ JSTracer*, JSObject*, Value*, const char*);
+template void js::TraceManuallyBarrieredCrossCompartmentEdge<JSObject*>(
+ JSTracer*, JSObject*, JSObject**, const char*);
+template void js::TraceManuallyBarrieredCrossCompartmentEdge<BaseScript*>(
+ JSTracer*, JSObject*, BaseScript**, const char*);
+
+template <typename T>
+void js::TraceSameZoneCrossCompartmentEdge(JSTracer* trc,
+ const WriteBarriered<T>* dst,
+ const char* name) {
+#ifdef DEBUG
+ if (trc->isMarkingTracer()) {
+ MOZ_ASSERT((*dst)->maybeCompartment(),
+ "Use TraceEdge for GC things without a compartment");
+
+ GCMarker* gcMarker = GCMarker::fromTracer(trc);
+ MOZ_ASSERT_IF(gcMarker->tracingZone,
+ (*dst)->zone() == gcMarker->tracingZone);
+ }
+
+ // Skip compartment checks for this edge.
+ if (trc->kind() == JS::TracerKind::CompartmentCheck) {
+ return;
+ }
+#endif
+
+ // Clear expected compartment for cross-compartment edge.
+ AutoClearTracingSource acts(trc);
+ TraceEdgeInternal(trc, ConvertToBase(dst->unbarrieredAddress()), name);
+}
+template void js::TraceSameZoneCrossCompartmentEdge(
+ JSTracer*, const WriteBarriered<Shape*>*, const char*);
+
+template <typename T>
+void js::TraceWeakMapKeyEdgeInternal(JSTracer* trc, Zone* weakMapZone,
+ T** thingp, const char* name) {
+ // We can't use ShouldTraceCrossCompartment here because that assumes the
+ // source of the edge is a CCW object which could be used to delay gray
+ // marking. Instead, assert that the weak map zone is in the same marking
+ // state as the target thing's zone and therefore we can go ahead and mark it.
+#ifdef DEBUG
+ auto thing = *thingp;
+ if (trc->isMarkingTracer()) {
+ MOZ_ASSERT(weakMapZone->isGCMarking());
+ MOZ_ASSERT(weakMapZone->gcState() == thing->zone()->gcState());
+ }
+#endif
+
+ // Clear expected compartment for cross-compartment edge.
+ AutoClearTracingSource acts(trc);
+
+ TraceEdgeInternal(trc, thingp, name);
+}
+
+template void js::TraceWeakMapKeyEdgeInternal<JSObject>(JSTracer*, Zone*,
+ JSObject**,
+ const char*);
+template void js::TraceWeakMapKeyEdgeInternal<BaseScript>(JSTracer*, Zone*,
+ BaseScript**,
+ const char*);
+
+static Cell* TraceGenericPointerRootAndType(JSTracer* trc, Cell* thing,
+ JS::TraceKind kind,
+ const char* name) {
+ return MapGCThingTyped(thing, kind, [trc, name](auto t) -> Cell* {
+ TraceRoot(trc, &t, name);
+ return t;
+ });
+}
+
+void js::TraceGenericPointerRoot(JSTracer* trc, Cell** thingp,
+ const char* name) {
+ MOZ_ASSERT(thingp);
+ Cell* thing = *thingp;
+ if (!thing) {
+ return;
+ }
+
+ Cell* traced =
+ TraceGenericPointerRootAndType(trc, thing, thing->getTraceKind(), name);
+ if (traced != thing) {
+ *thingp = traced;
+ }
+}
+
+void js::TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, Cell** thingp,
+ const char* name) {
+ MOZ_ASSERT(thingp);
+ Cell* thing = *thingp;
+ if (!*thingp) {
+ return;
+ }
+
+ auto traced = MapGCThingTyped(thing, thing->getTraceKind(),
+ [trc, name](auto t) -> Cell* {
+ TraceManuallyBarrieredEdge(trc, &t, name);
+ return t;
+ });
+ if (traced != thing) {
+ *thingp = traced;
+ }
+}
+
+void js::TraceGCCellPtrRoot(JSTracer* trc, JS::GCCellPtr* thingp,
+ const char* name) {
+ Cell* thing = thingp->asCell();
+ if (!thing) {
+ return;
+ }
+
+ Cell* traced =
+ TraceGenericPointerRootAndType(trc, thing, thingp->kind(), name);
+
+ if (!traced) {
+ *thingp = JS::GCCellPtr();
+ } else if (traced != thingp->asCell()) {
+ *thingp = JS::GCCellPtr(traced, thingp->kind());
+ }
+}
+
+void js::TraceManuallyBarrieredGCCellPtr(JSTracer* trc, JS::GCCellPtr* thingp,
+ const char* name) {
+ Cell* thing = thingp->asCell();
+ if (!thing) {
+ return;
+ }
+
+ Cell* traced = MapGCThingTyped(thing, thing->getTraceKind(),
+ [trc, name](auto t) -> Cell* {
+ TraceManuallyBarrieredEdge(trc, &t, name);
+ return t;
+ });
+
+ if (!traced) {
+ // If we are clearing edges, also erase the type. This happens when using
+ // ClearEdgesTracer.
+ *thingp = JS::GCCellPtr();
+ } else if (traced != thingp->asCell()) {
+ *thingp = JS::GCCellPtr(traced, thingp->kind());
+ }
+}
+
+template <typename T>
+inline bool TraceTaggedPtrEdge(JSTracer* trc, T* thingp, const char* name) {
+ // Return true by default. For some types the lambda below won't be called.
+ bool ret = true;
+ auto thing = MapGCThingTyped(*thingp, [&](auto thing) {
+ if (!TraceEdgeInternal(trc, &thing, name)) {
+ ret = false;
+ return TaggedPtr<T>::empty();
+ }
+
+ return TaggedPtr<T>::wrap(thing);
+ });
+
+ // Only update *thingp if the value changed, to avoid TSan false positives for
+ // template objects when using DumpHeapTracer or UbiNode tracers while Ion
+ // compiling off-thread.
+ if (thing.isSome() && thing.value() != *thingp) {
+ *thingp = thing.value();
+ }
+
+ return ret;
+}
+
+bool js::gc::TraceEdgeInternal(JSTracer* trc, Value* thingp, const char* name) {
+ return TraceTaggedPtrEdge(trc, thingp, name);
+}
+bool js::gc::TraceEdgeInternal(JSTracer* trc, jsid* thingp, const char* name) {
+ return TraceTaggedPtrEdge(trc, thingp, name);
+}
+bool js::gc::TraceEdgeInternal(JSTracer* trc, TaggedProto* thingp,
+ const char* name) {
+ return TraceTaggedPtrEdge(trc, thingp, name);
+}
+
+template <typename T>
+void js::gc::TraceRangeInternal(JSTracer* trc, size_t len, T* vec,
+ const char* name) {
+ JS::AutoTracingIndex index(trc);
+ for (auto i : IntegerRange(len)) {
+ if (InternalBarrierMethods<T>::isMarkable(vec[i])) {
+ TraceEdgeInternal(trc, &vec[i], name);
+ }
+ ++index;
+ }
+}
+
+/*** GC Marking Interface ***************************************************/
+
+namespace js {
+
+using HasNoImplicitEdgesType = bool;
+
+template <typename T>
+struct ImplicitEdgeHolderType {
+ using Type = HasNoImplicitEdgesType;
+};
+
+// For now, we only handle JSObject* and BaseScript* keys, but the linear time
+// algorithm can be easily extended by adding in more types here, then making
+// GCMarker::traverse<T> call markImplicitEdges.
+template <>
+struct ImplicitEdgeHolderType<JSObject*> {
+ using Type = JSObject*;
+};
+
+template <>
+struct ImplicitEdgeHolderType<BaseScript*> {
+ using Type = BaseScript*;
+};
+
+void GCMarker::markEphemeronEdges(EphemeronEdgeVector& edges,
+ gc::CellColor srcColor) {
+ // This is called as part of GC weak marking or by barriers outside of GC.
+ MOZ_ASSERT_IF(CurrentThreadIsPerformingGC(),
+ state == MarkingState::WeakMarking);
+
+ DebugOnly<size_t> initialLength = edges.length();
+
+ for (auto& edge : edges) {
+ CellColor targetColor = std::min(srcColor, edge.color);
+ MOZ_ASSERT(CellColor(markColor()) >= targetColor);
+ if (targetColor == markColor()) {
+ ApplyGCThingTyped(
+ edge.target, edge.target->getTraceKind(),
+ [this](auto t) { markAndTraverse<NormalMarkingOptions>(t); });
+ }
+ }
+
+ // The above marking always goes through markAndPush, which will not cause
+ // 'edges' to be appended to while iterating.
+ MOZ_ASSERT(edges.length() == initialLength);
+
+ // This is not just an optimization. When nuking a CCW, we conservatively
+ // mark through the related edges and then lose the CCW->target connection
+ // that induces a sweep group edge. As a result, it is possible for the
+ // delegate zone to get marked later, look up an edge in this table, and
+ // then try to mark something in a Zone that is no longer marking.
+ if (srcColor == CellColor::Black && markColor() == MarkColor::Black) {
+ edges.eraseIf([](auto& edge) { return edge.color == MarkColor::Black; });
+ }
+}
+
+// 'delegate' is no longer the delegate of 'key'.
+void GCMarker::severWeakDelegate(JSObject* key, JSObject* delegate) {
+ MOZ_ASSERT(CurrentThreadIsMainThread());
+
+ JS::Zone* zone = delegate->zone();
+ if (!zone->needsIncrementalBarrier()) {
+ MOZ_ASSERT(
+ !zone->gcEphemeronEdges(delegate).get(delegate),
+ "non-collecting zone should not have populated gcEphemeronEdges");
+ return;
+ }
+ auto* p = zone->gcEphemeronEdges(delegate).get(delegate);
+ if (!p) {
+ return;
+ }
+
+ // We are losing 3 edges here: key -> delegate, delegate -> key, and
+ // <delegate, map> -> value. Maintain snapshot-at-beginning (hereafter,
+ // S-A-B) by conservatively assuming the delegate will end up black and
+ // marking through the latter 2 edges.
+ //
+ // Note that this does not fully give S-A-B:
+ //
+ // 1. If the map is gray, then the value will only be marked gray here even
+ // though the map could later be discovered to be black.
+ //
+ // 2. If the map has not yet been marked, we won't have any entries to mark
+ // here in the first place.
+ //
+ // 3. We're not marking the delegate, since that would cause eg nukeAllCCWs
+ // to keep everything alive for another collection.
+ //
+ // We can't even assume that the delegate passed in here is live, because we
+ // could have gotten here from nukeAllCCWs, which iterates over all CCWs
+ // including dead ones.
+ //
+ // This is ok because S-A-B is only needed to prevent the case where an
+ // unmarked object is removed from the graph and then re-inserted where it is
+ // reachable only by things that have already been marked. None of the 3
+ // target objects will be re-inserted anywhere as a result of this action.
+
+ EphemeronEdgeVector& edges = p->value;
+ MOZ_ASSERT(markColor() == MarkColor::Black);
+ markEphemeronEdges(edges, MarkColor::Black);
+}
+
+// 'delegate' is now the delegate of 'key'. Update weakmap marking state.
+void GCMarker::restoreWeakDelegate(JSObject* key, JSObject* delegate) {
+ MOZ_ASSERT(CurrentThreadIsMainThread());
+
+ if (!key->zone()->needsIncrementalBarrier()) {
+ // Temporary diagnostic printouts for when this would have asserted.
+ if (key->zone()->gcEphemeronEdges(key).has(key)) {
+ fprintf(stderr, "key zone: %d\n", int(key->zone()->gcState()));
+#ifdef DEBUG
+ key->dump();
+#endif
+ fprintf(stderr, "delegate zone: %d\n", int(delegate->zone()->gcState()));
+#ifdef DEBUG
+ delegate->dump();
+#endif
+ }
+ MOZ_ASSERT(
+ !key->zone()->gcEphemeronEdges(key).has(key),
+ "non-collecting zone should not have populated gcEphemeronEdges");
+ return;
+ }
+ if (!delegate->zone()->needsIncrementalBarrier()) {
+ // Normally we should not have added the key -> value edge if the delegate
+ // zone is not marking (because the delegate would have been seen as black,
+ // so we would mark the key immediately instead). But if there wasn't a
+ // delegate (the key was nuked), then we won't have consulted it. So we
+ // can't do the same assertion as above.
+ //
+ // Specifically, the sequence would be:
+ // 1. Nuke the key.
+ // 2. Start the incremental GC.
+ // 3. Mark the WeakMap. Insert a key->value edge with a DeadObjectProxy key.
+ // 4. Un-nuke the key with a delegate in a nonmarking Zone.
+ //
+ // The result is an ephemeron edge (from <map,key> to value, but stored
+ // as key to value) involving a key with a delegate in a nonmarking Zone,
+ // something that ordinarily would not happen.
+ return;
+ }
+ auto* p = key->zone()->gcEphemeronEdges(key).get(key);
+ if (!p) {
+ return;
+ }
+
+ // Similar to severWeakDelegate above, mark through the key -> value edge.
+ EphemeronEdgeVector& edges = p->value;
+ MOZ_ASSERT(markColor() == MarkColor::Black);
+ markEphemeronEdges(edges, MarkColor::Black);
+}
+
+template <typename T>
+void GCMarker::markImplicitEdgesHelper(T markedThing) {
+ if (!isWeakMarking()) {
+ return;
+ }
+
+ Zone* zone = markedThing->asTenured().zone();
+ MOZ_ASSERT(zone->isGCMarking());
+ MOZ_ASSERT(!zone->isGCSweeping());
+
+ auto p = zone->gcEphemeronEdges().get(markedThing);
+ if (!p) {
+ return;
+ }
+ EphemeronEdgeVector& edges = p->value;
+
+ // markedThing might be a key in a debugger weakmap, which can end up marking
+ // values that are in a different compartment.
+ AutoClearTracingSource acts(tracer());
+
+ CellColor thingColor = gc::detail::GetEffectiveColor(this, markedThing);
+ markEphemeronEdges(edges, thingColor);
+}
+
+template <>
+void GCMarker::markImplicitEdgesHelper(HasNoImplicitEdgesType) {}
+
+template <typename T>
+void GCMarker::markImplicitEdges(T* thing) {
+ markImplicitEdgesHelper<typename ImplicitEdgeHolderType<T*>::Type>(thing);
+}
+
+template void GCMarker::markImplicitEdges(JSObject*);
+template void GCMarker::markImplicitEdges(BaseScript*);
+
+} // namespace js
+
+template <typename T>
+static inline bool ShouldMark(GCMarker* gcmarker, T* thing) {
+ // We may encounter nursery things during normal marking since we don't
+ // collect the nursery at the start of every GC slice.
+ if (!thing->isTenured()) {
+ return false;
+ }
+
+ // Don't mark things outside a zone if we are in a per-zone GC. Don't mark
+ // permanent shared things owned by other runtimes (we will never observe
+ // their zone being collected).
+ Zone* zone = thing->asTenured().zoneFromAnyThread();
+ return zone->shouldMarkInZone(gcmarker->markColor());
+}
+
+template <uint32_t opts>
+MarkingTracerT<opts>::MarkingTracerT(JSRuntime* runtime, GCMarker* marker)
+ : GenericTracerImpl<MarkingTracerT<opts>>(
+ runtime, JS::TracerKind::Marking,
+ JS::TraceOptions(JS::WeakMapTraceAction::Expand,
+ JS::WeakEdgeTraceAction::Skip)) {
+ // Marking tracers are owned by (and part of) a GCMarker.
+ MOZ_ASSERT(this == marker->tracer());
+ MOZ_ASSERT(getMarker() == marker);
+}
+
+template <uint32_t opts>
+MOZ_ALWAYS_INLINE GCMarker* MarkingTracerT<opts>::getMarker() {
+ return GCMarker::fromTracer(this);
+}
+
+template <uint32_t opts>
+template <typename T>
+void MarkingTracerT<opts>::onEdge(T** thingp, const char* name) {
+ T* thing = *thingp;
+
+ // Do per-type marking precondition checks.
+ GCMarker* marker = getMarker();
+ if (!ShouldMark(marker, thing)) {
+ MOZ_ASSERT(gc::detail::GetEffectiveColor(marker, thing) ==
+ js::gc::CellColor::Black);
+ return;
+ }
+
+ MOZ_ASSERT(!IsOwnedByOtherRuntime(this->runtime(), thing));
+
+#ifdef DEBUG
+ CheckMarkedThing(marker, thing);
+#endif
+
+ AutoClearTracingSource acts(this);
+ marker->markAndTraverse<opts>(thing);
+}
+
+#define INSTANTIATE_ONEDGE_METHOD(name, type, _1, _2) \
+ template void MarkingTracerT<NormalMarkingOptions>::onEdge<type>( \
+ type * *thingp, const char* name); \
+ template void \
+ MarkingTracerT<MarkingOptions::MarkRootCompartments>::onEdge<type>( \
+ type * *thingp, const char* name);
+JS_FOR_EACH_TRACEKIND(INSTANTIATE_ONEDGE_METHOD)
+#undef INSTANTIATE_ONEDGE_METHOD
+
+static void TraceEdgeForBarrier(GCMarker* gcmarker, TenuredCell* thing,
+ JS::TraceKind kind) {
+ // Dispatch to markAndTraverse without checking ShouldMark.
+ ApplyGCThingTyped(thing, kind, [gcmarker](auto thing) {
+ MOZ_ASSERT(ShouldMark(gcmarker, thing));
+ CheckTracedThing(gcmarker->tracer(), thing);
+ AutoClearTracingSource acts(gcmarker->tracer());
+ gcmarker->markAndTraverse<NormalMarkingOptions>(thing);
+ });
+}
+
+JS_PUBLIC_API void js::gc::PerformIncrementalReadBarrier(JS::GCCellPtr thing) {
+ // Optimized marking for read barriers. This is called from
+ // ExposeGCThingToActiveJS which has already checked the prerequisites for
+ // performing a read barrier. This means we can skip a bunch of checks and
+ // call into the tracer directly.
+
+ MOZ_ASSERT(thing);
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+
+ TenuredCell* cell = &thing.asCell()->asTenured();
+ MOZ_ASSERT(!cell->isMarkedBlack());
+
+ Zone* zone = cell->zone();
+ MOZ_ASSERT(zone->needsIncrementalBarrier());
+
+ // Skip dispatching on known tracer type.
+ GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer());
+ TraceEdgeForBarrier(gcmarker, cell, thing.kind());
+}
+
+void js::gc::PerformIncrementalReadBarrier(TenuredCell* cell) {
+ // Internal version of previous function.
+
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+
+ if (cell->isMarkedBlack()) {
+ return;
+ }
+
+ Zone* zone = cell->zone();
+ MOZ_ASSERT(zone->needsIncrementalBarrier());
+
+ // Skip dispatching on known tracer type.
+ GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer());
+ TraceEdgeForBarrier(gcmarker, cell, cell->getTraceKind());
+}
+
+void js::gc::PerformIncrementalPreWriteBarrier(TenuredCell* cell) {
+ // The same as PerformIncrementalReadBarrier except for an extra check on the
+ // runtime for cells in atoms zone.
+
+ Zone* zone = cell->zoneFromAnyThread();
+ MOZ_ASSERT(zone->needsIncrementalBarrier());
+
+ MOZ_ASSERT(cell);
+ if (cell->isMarkedBlack()) {
+ return;
+ }
+
+ // Barriers can be triggered off the main thread by background finalization of
+ // HeapPtrs to the atoms zone. We don't want to trigger the barrier in this
+ // case.
+ bool checkThread = zone->isAtomsZone();
+ JSRuntime* runtime = cell->runtimeFromAnyThread();
+ if (checkThread && !CurrentThreadCanAccessRuntime(runtime)) {
+ MOZ_ASSERT(CurrentThreadIsGCFinalizing());
+ return;
+ }
+
+ MOZ_ASSERT(CurrentThreadIsMainThread());
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+
+ // Skip dispatching on known tracer type.
+ GCMarker* gcmarker = GCMarker::fromTracer(zone->barrierTracer());
+ TraceEdgeForBarrier(gcmarker, cell, cell->getTraceKind());
+}
+
+void js::gc::PerformIncrementalBarrierDuringFlattening(JSString* str) {
+ TenuredCell* cell = &str->asTenured();
+
+ // Skip eager marking of ropes during flattening. Their children will also be
+ // barriered by flattening process so we don't need to traverse them.
+ if (str->isRope()) {
+ cell->markBlack();
+ return;
+ }
+
+ PerformIncrementalPreWriteBarrier(cell);
+}
+
+template <uint32_t opts, typename T>
+void js::GCMarker::markAndTraverse(T* thing) {
+ if (mark<opts>(thing)) {
+ // We only mark permanent things during initialization.
+ MOZ_ASSERT_IF(thing->isPermanentAndMayBeShared(),
+ !runtime()->permanentAtomsPopulated());
+
+ // We don't need to pass MarkRootCompartments options on to children.
+ constexpr uint32_t traverseOpts =
+ opts & ~MarkingOptions::MarkRootCompartments;
+
+ traverse<traverseOpts>(thing);
+
+ if constexpr (bool(opts & MarkingOptions::MarkRootCompartments)) {
+ // Mark the compartment as live.
+ SetCompartmentHasMarkedCells(thing);
+ }
+ }
+}
+
+// The |traverse| method overloads select the traversal strategy for each kind.
+//
+// There are three possible strategies:
+//
+// 1. traceChildren
+//
+// The simplest traversal calls out to the fully generic traceChildren
+// function to visit the child edges. In the absence of other traversal
+// mechanisms, this function will rapidly grow the stack past its bounds and
+// crash the process. Thus, this generic tracing should only be used in cases
+// where subsequent tracing will not recurse.
+//
+// 2. scanChildren
+//
+// Strings, Shapes, and Scopes are extremely common, but have simple patterns
+// of recursion. We traverse trees of these edges immediately, with
+// aggressive, manual inlining, implemented by eagerlyTraceChildren.
+//
+// 3. pushThing
+//
+// Objects are extremely common and can contain arbitrarily nested graphs, so
+// are not trivially inlined. In this case we use the mark stack to control
+// recursion. JitCode shares none of these properties, but is included for
+// historical reasons. JSScript normally cannot recurse, but may be used as a
+// weakmap key and thereby recurse into weakmapped values.
+
+template <uint32_t opts>
+void GCMarker::traverse(BaseShape* thing) {
+ traceChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(GetterSetter* thing) {
+ traceChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(JS::Symbol* thing) {
+ traceChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(JS::BigInt* thing) {
+ traceChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(RegExpShared* thing) {
+ traceChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(JSString* thing) {
+ scanChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(Shape* thing) {
+ scanChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(PropMap* thing) {
+ scanChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(js::Scope* thing) {
+ scanChildren<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(JSObject* thing) {
+ pushThing<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(jit::JitCode* thing) {
+ pushThing<opts>(thing);
+}
+template <uint32_t opts>
+void GCMarker::traverse(BaseScript* thing) {
+ pushThing<opts>(thing);
+}
+
+template <uint32_t opts, typename T>
+void js::GCMarker::traceChildren(T* thing) {
+ MOZ_ASSERT(!thing->isPermanentAndMayBeShared());
+ MOZ_ASSERT(thing->isMarkedAny());
+ AutoSetTracingSource asts(tracer(), thing);
+ thing->traceChildren(tracer());
+}
+
+template <uint32_t opts, typename T>
+void js::GCMarker::scanChildren(T* thing) {
+ MOZ_ASSERT(!thing->isPermanentAndMayBeShared());
+ MOZ_ASSERT(thing->isMarkedAny());
+ eagerlyMarkChildren<opts>(thing);
+}
+
+template <uint32_t opts, typename T>
+void js::GCMarker::pushThing(T* thing) {
+ MOZ_ASSERT(!thing->isPermanentAndMayBeShared());
+ MOZ_ASSERT(thing->isMarkedAny());
+ pushTaggedPtr(thing);
+}
+
+template void js::GCMarker::markAndTraverse<NormalMarkingOptions, JSObject>(
+ JSObject* thing);
+template void js::GCMarker::markAndTraverse<
+ MarkingOptions::MarkRootCompartments, JSObject>(JSObject* thing);
+
+#ifdef DEBUG
+void GCMarker::setCheckAtomMarking(bool check) {
+ MOZ_ASSERT(check != checkAtomMarking);
+ checkAtomMarking = check;
+}
+#endif
+
+template <typename S, typename T>
+inline void GCMarker::checkTraversedEdge(S source, T* target) {
+#ifdef DEBUG
+ // Atoms and Symbols do not have or mark their internal pointers,
+ // respectively.
+ MOZ_ASSERT(!source->isPermanentAndMayBeShared());
+
+ // Shared things are already black so we will not mark them.
+ if (target->isPermanentAndMayBeShared()) {
+ Zone* zone = target->zoneFromAnyThread();
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ MOZ_ASSERT(target->isMarkedBlack());
+ MOZ_ASSERT(!target->maybeCompartment());
+ return;
+ }
+
+ Zone* sourceZone = source->zone();
+ Zone* targetZone = target->zone();
+
+ // Atoms and Symbols do not have access to a compartment pointer, or we'd need
+ // to adjust the subsequent check to catch that case.
+ MOZ_ASSERT_IF(targetZone->isAtomsZone(), !target->maybeCompartment());
+
+ // The Zones must match, unless the target is an atom.
+ MOZ_ASSERT(targetZone == sourceZone || targetZone->isAtomsZone());
+
+ // If we are marking an atom, that atom must be marked in the source zone's
+ // atom bitmap.
+ if (checkAtomMarking && !sourceZone->isAtomsZone() &&
+ targetZone->isAtomsZone()) {
+ MOZ_ASSERT(target->runtimeFromAnyThread()->gc.atomMarking.atomIsMarked(
+ sourceZone, reinterpret_cast<TenuredCell*>(target)));
+ }
+
+ // If we have access to a compartment pointer for both things, they must
+ // match.
+ MOZ_ASSERT_IF(source->maybeCompartment() && target->maybeCompartment(),
+ source->maybeCompartment() == target->maybeCompartment());
+#endif
+}
+
+template <uint32_t opts, typename S, typename T>
+void js::GCMarker::markAndTraverseEdge(S source, T* target) {
+ checkTraversedEdge(source, target);
+ markAndTraverse<opts>(target);
+}
+
+template <uint32_t opts, typename S, typename T>
+void js::GCMarker::markAndTraverseEdge(S source, const T& thing) {
+ ApplyGCThingTyped(thing, [this, source](auto t) {
+ this->markAndTraverseEdge<opts>(source, t);
+ });
+}
+
+template <uint32_t opts, typename T>
+bool js::GCMarker::mark(T* thing) {
+ if (!thing->isTenured()) {
+ return false;
+ }
+
+ AssertShouldMarkInZone(this, thing);
+
+ MarkColor color =
+ TraceKindCanBeGray<T>::value ? markColor() : MarkColor::Black;
+
+ if constexpr (bool(opts & MarkingOptions::ParallelMarking)) {
+ return thing->asTenured().markIfUnmarkedAtomic(color);
+ }
+
+ return thing->asTenured().markIfUnmarked(color);
+}
+
+/*** Mark-stack Marking *****************************************************/
+
+// Call the trace hook set on the object, if present.
+static inline void CallTraceHook(JSTracer* trc, JSObject* obj) {
+ const JSClass* clasp = obj->getClass();
+ MOZ_ASSERT(clasp);
+
+ if (clasp->hasTrace()) {
+ AutoSetTracingSource asts(trc, obj);
+ clasp->doTrace(trc, obj);
+ }
+}
+
+static gcstats::PhaseKind GrayMarkingPhaseForCurrentPhase(
+ const gcstats::Statistics& stats) {
+ using namespace gcstats;
+ switch (stats.currentPhaseKind()) {
+ case PhaseKind::MARK:
+ return PhaseKind::MARK_GRAY;
+ case PhaseKind::MARK_WEAK:
+ return PhaseKind::MARK_GRAY_WEAK;
+ default:
+ MOZ_CRASH("Unexpected current phase");
+ }
+}
+
+void GCMarker::moveWork(GCMarker* dst, GCMarker* src) {
+ MOZ_ASSERT(dst->stack.isEmpty());
+ MOZ_ASSERT(src->canDonateWork());
+
+ MarkStack::moveWork(dst->stack, src->stack);
+}
+
+bool GCMarker::markUntilBudgetExhausted(SliceBudget& budget,
+ ShouldReportMarkTime reportTime) {
+#ifdef DEBUG
+ MOZ_ASSERT(!strictCompartmentChecking);
+ strictCompartmentChecking = true;
+ auto acc = mozilla::MakeScopeExit([&] { strictCompartmentChecking = false; });
+#endif
+
+ if (budget.isOverBudget()) {
+ return false;
+ }
+
+ return doMarking<NormalMarkingOptions>(budget, reportTime);
+}
+
+template <uint32_t opts>
+bool GCMarker::doMarking(SliceBudget& budget, ShouldReportMarkTime reportTime) {
+ GCRuntime& gc = runtime()->gc;
+
+ // This method leaves the mark color as it found it.
+
+ if (hasBlackEntries() && !markOneColor<opts, MarkColor::Black>(budget)) {
+ return false;
+ }
+
+ if (hasGrayEntries()) {
+ mozilla::Maybe<gcstats::AutoPhase> ap;
+ if (reportTime) {
+ auto& stats = runtime()->gc.stats();
+ ap.emplace(stats, GrayMarkingPhaseForCurrentPhase(stats));
+ }
+
+ if (!markOneColor<opts, MarkColor::Gray>(budget)) {
+ return false;
+ }
+ }
+
+ // Mark children of things that caused too deep recursion during the above
+ // tracing. All normal marking happens before any delayed marking.
+ if (gc.hasDelayedMarking()) {
+ gc.markAllDelayedChildren(reportTime);
+ }
+
+ MOZ_ASSERT(!gc.hasDelayedMarking());
+ MOZ_ASSERT(isDrained());
+
+ return true;
+}
+
+template <uint32_t opts, MarkColor color>
+bool GCMarker::markOneColor(SliceBudget& budget) {
+ AutoSetMarkColor setColor(*this, color);
+
+ while (processMarkStackTop<opts>(budget)) {
+ if (stack.isEmpty()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool GCMarker::markCurrentColorInParallel(SliceBudget& budget) {
+ ParallelMarker::AtomicCount& waitingTaskCount =
+ parallelMarker_->waitingTaskCountRef();
+
+ while (processMarkStackTop<MarkingOptions::ParallelMarking>(budget)) {
+ if (stack.isEmpty()) {
+ return true;
+ }
+
+ // TODO: It might be better to only check this occasionally, possibly
+ // combined with the slice budget check. Experiments with giving this its
+ // own counter resulted in worse performance.
+ if (waitingTaskCount && canDonateWork()) {
+ parallelMarker_->donateWorkFrom(this);
+ }
+ }
+
+ return false;
+}
+
+static inline void CheckForCompartmentMismatch(JSObject* obj, JSObject* obj2) {
+#ifdef DEBUG
+ if (MOZ_UNLIKELY(obj->compartment() != obj2->compartment())) {
+ fprintf(
+ stderr,
+ "Compartment mismatch in pointer from %s object slot to %s object\n",
+ obj->getClass()->name, obj2->getClass()->name);
+ MOZ_CRASH("Compartment mismatch");
+ }
+#endif
+}
+
+static inline size_t NumUsedFixedSlots(NativeObject* obj) {
+ return std::min(obj->numFixedSlots(), obj->slotSpan());
+}
+
+static inline size_t NumUsedDynamicSlots(NativeObject* obj) {
+ size_t nfixed = obj->numFixedSlots();
+ size_t nslots = obj->slotSpan();
+ if (nslots < nfixed) {
+ return 0;
+ }
+
+ return nslots - nfixed;
+}
+
+template <uint32_t opts>
+inline bool GCMarker::processMarkStackTop(SliceBudget& budget) {
+ /*
+ * This function uses explicit goto and scans objects directly. This allows us
+ * to eliminate tail recursion and significantly improve the marking
+ * performance, see bug 641025.
+ *
+ * Note that the mutator can change the size and layout of objects between
+ * marking slices, so we must check slots and element ranges read from the
+ * stack.
+ */
+
+ MOZ_ASSERT(!stack.isEmpty());
+ MOZ_ASSERT_IF(markColor() == MarkColor::Gray, !hasBlackEntries());
+
+ JSObject* obj; // The object being scanned.
+ SlotsOrElementsKind kind; // The kind of slot range being scanned, if any.
+ HeapSlot* base; // Slot range base pointer.
+ size_t index; // Index of the next slot to mark.
+ size_t end; // End of slot range to mark.
+
+ if (stack.peekTag() == MarkStack::SlotsOrElementsRangeTag) {
+ auto range = stack.popSlotsOrElementsRange();
+ obj = range.ptr().asRangeObject();
+ NativeObject* nobj = &obj->as<NativeObject>();
+ kind = range.kind();
+ index = range.start();
+
+ switch (kind) {
+ case SlotsOrElementsKind::FixedSlots: {
+ base = nobj->fixedSlots();
+ end = NumUsedFixedSlots(nobj);
+ break;
+ }
+
+ case SlotsOrElementsKind::DynamicSlots: {
+ base = nobj->slots_;
+ end = NumUsedDynamicSlots(nobj);
+ break;
+ }
+
+ case SlotsOrElementsKind::Elements: {
+ base = nobj->getDenseElements();
+
+ // Account for shifted elements.
+ size_t numShifted = nobj->getElementsHeader()->numShiftedElements();
+ size_t initlen = nobj->getDenseInitializedLength();
+ index = std::max(index, numShifted) - numShifted;
+ end = initlen;
+ break;
+ }
+
+ case SlotsOrElementsKind::Unused: {
+ MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("Unused SlotsOrElementsKind");
+ }
+ }
+
+ goto scan_value_range;
+ }
+
+ budget.step();
+ if (budget.isOverBudget()) {
+ return false;
+ }
+
+ {
+ MarkStack::TaggedPtr ptr = stack.popPtr();
+ switch (ptr.tag()) {
+ case MarkStack::ObjectTag: {
+ obj = ptr.as<JSObject>();
+ AssertShouldMarkInZone(this, obj);
+ goto scan_obj;
+ }
+
+ case MarkStack::JitCodeTag: {
+ auto* code = ptr.as<jit::JitCode>();
+ AutoSetTracingSource asts(tracer(), code);
+ code->traceChildren(tracer());
+ return true;
+ }
+
+ case MarkStack::ScriptTag: {
+ auto* script = ptr.as<BaseScript>();
+ if constexpr (bool(opts & MarkingOptions::MarkImplicitEdges)) {
+ markImplicitEdges(script);
+ }
+ AutoSetTracingSource asts(tracer(), script);
+ script->traceChildren(tracer());
+ return true;
+ }
+
+ default:
+ MOZ_CRASH("Invalid tag in mark stack");
+ }
+ }
+
+ return true;
+
+scan_value_range:
+ while (index < end) {
+ budget.step();
+ if (budget.isOverBudget()) {
+ pushValueRange(obj, kind, index, end);
+ return false;
+ }
+
+ const Value& v = base[index];
+ index++;
+
+ if (v.isString()) {
+ markAndTraverseEdge<opts>(obj, v.toString());
+ } else if (v.hasObjectPayload()) {
+ JSObject* obj2 = &v.getObjectPayload();
+#ifdef DEBUG
+ if (!obj2) {
+ fprintf(stderr,
+ "processMarkStackTop found ObjectValue(nullptr) "
+ "at %zu Values from end of range in object:\n",
+ size_t(end - (index - 1)));
+ obj->dump();
+ }
+#endif
+ CheckForCompartmentMismatch(obj, obj2);
+ if (mark<opts>(obj2)) {
+ // Save the rest of this value range for later and start scanning obj2's
+ // children.
+ pushValueRange(obj, kind, index, end);
+ obj = obj2;
+ goto scan_obj;
+ }
+ } else if (v.isSymbol()) {
+ markAndTraverseEdge<opts>(obj, v.toSymbol());
+ } else if (v.isBigInt()) {
+ markAndTraverseEdge<opts>(obj, v.toBigInt());
+ } else if (v.isPrivateGCThing()) {
+ // v.toGCCellPtr cannot be inlined, so construct one manually.
+ Cell* cell = v.toGCThing();
+ markAndTraverseEdge<opts>(obj, JS::GCCellPtr(cell, cell->getTraceKind()));
+ }
+ }
+
+ return true;
+
+scan_obj : {
+ AssertShouldMarkInZone(this, obj);
+
+ if constexpr (bool(opts & MarkingOptions::MarkImplicitEdges)) {
+ markImplicitEdges(obj);
+ }
+ markAndTraverseEdge<opts>(obj, obj->shape());
+
+ CallTraceHook(tracer(), obj);
+
+ if (!obj->is<NativeObject>()) {
+ return true;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+
+ unsigned nslots = nobj->slotSpan();
+
+ do {
+ if (nobj->hasEmptyElements()) {
+ break;
+ }
+
+ base = nobj->getDenseElements();
+ kind = SlotsOrElementsKind::Elements;
+ index = 0;
+ end = nobj->getDenseInitializedLength();
+
+ if (!nslots) {
+ goto scan_value_range;
+ }
+ pushValueRange(nobj, kind, index, end);
+ } while (false);
+
+ unsigned nfixed = nobj->numFixedSlots();
+
+ base = nobj->fixedSlots();
+ kind = SlotsOrElementsKind::FixedSlots;
+ index = 0;
+
+ if (nslots > nfixed) {
+ pushValueRange(nobj, kind, index, nfixed);
+ kind = SlotsOrElementsKind::DynamicSlots;
+ base = nobj->slots_;
+ end = nslots - nfixed;
+ goto scan_value_range;
+ }
+
+ MOZ_ASSERT(nslots <= nobj->numFixedSlots());
+ end = nslots;
+ goto scan_value_range;
+}
+}
+
+/*** Mark Stack *************************************************************/
+
+static_assert(sizeof(MarkStack::TaggedPtr) == sizeof(uintptr_t),
+ "A TaggedPtr should be the same size as a pointer");
+static_assert((sizeof(MarkStack::SlotsOrElementsRange) % sizeof(uintptr_t)) ==
+ 0,
+ "SlotsOrElementsRange size should be a multiple of "
+ "the pointer size");
+
+static const size_t ValueRangeWords =
+ sizeof(MarkStack::SlotsOrElementsRange) / sizeof(uintptr_t);
+
+template <typename T>
+struct MapTypeToMarkStackTag {};
+template <>
+struct MapTypeToMarkStackTag<JSObject*> {
+ static const auto value = MarkStack::ObjectTag;
+};
+template <>
+struct MapTypeToMarkStackTag<jit::JitCode*> {
+ static const auto value = MarkStack::JitCodeTag;
+};
+template <>
+struct MapTypeToMarkStackTag<BaseScript*> {
+ static const auto value = MarkStack::ScriptTag;
+};
+
+#ifdef DEBUG
+static inline bool TagIsRangeTag(MarkStack::Tag tag) {
+ return tag == MarkStack::SlotsOrElementsRangeTag;
+}
+#endif
+
+inline MarkStack::TaggedPtr::TaggedPtr(Tag tag, Cell* ptr)
+ : bits(tag | uintptr_t(ptr)) {
+ assertValid();
+}
+
+inline uintptr_t MarkStack::TaggedPtr::tagUnchecked() const {
+ return bits & TagMask;
+}
+
+inline MarkStack::Tag MarkStack::TaggedPtr::tag() const {
+ auto tag = Tag(bits & TagMask);
+ MOZ_ASSERT(tag <= LastTag);
+ return tag;
+}
+
+inline Cell* MarkStack::TaggedPtr::ptr() const {
+ return reinterpret_cast<Cell*>(bits & ~TagMask);
+}
+
+inline void MarkStack::TaggedPtr::assertValid() const {
+ (void)tag();
+ MOZ_ASSERT(IsCellPointerValid(ptr()));
+}
+
+template <typename T>
+inline T* MarkStack::TaggedPtr::as() const {
+ MOZ_ASSERT(tag() == MapTypeToMarkStackTag<T*>::value);
+ MOZ_ASSERT(ptr()->isTenured());
+ MOZ_ASSERT(ptr()->is<T>());
+ return static_cast<T*>(ptr());
+}
+
+inline JSObject* MarkStack::TaggedPtr::asRangeObject() const {
+ MOZ_ASSERT(TagIsRangeTag(tag()));
+ MOZ_ASSERT(ptr()->isTenured());
+ return ptr()->as<JSObject>();
+}
+
+inline JSRope* MarkStack::TaggedPtr::asTempRope() const {
+ MOZ_ASSERT(tag() == TempRopeTag);
+ return &ptr()->as<JSString>()->asRope();
+}
+
+inline MarkStack::SlotsOrElementsRange::SlotsOrElementsRange(
+ SlotsOrElementsKind kindArg, JSObject* obj, size_t startArg)
+ : startAndKind_((startArg << StartShift) | size_t(kindArg)),
+ ptr_(SlotsOrElementsRangeTag, obj) {
+ assertValid();
+ MOZ_ASSERT(kind() == kindArg);
+ MOZ_ASSERT(start() == startArg);
+}
+
+inline void MarkStack::SlotsOrElementsRange::assertValid() const {
+ ptr_.assertValid();
+ MOZ_ASSERT(TagIsRangeTag(ptr_.tag()));
+}
+
+inline SlotsOrElementsKind MarkStack::SlotsOrElementsRange::kind() const {
+ return SlotsOrElementsKind(startAndKind_ & KindMask);
+}
+
+inline size_t MarkStack::SlotsOrElementsRange::start() const {
+ return startAndKind_ >> StartShift;
+}
+
+inline MarkStack::TaggedPtr MarkStack::SlotsOrElementsRange::ptr() const {
+ return ptr_;
+}
+
+MarkStack::MarkStack() { MOZ_ASSERT(isEmpty()); }
+
+MarkStack::~MarkStack() { MOZ_ASSERT(isEmpty()); }
+
+MarkStack::MarkStack(const MarkStack& other) {
+ MOZ_CRASH("Compiler requires this but doesn't call it");
+}
+
+MarkStack& MarkStack::operator=(const MarkStack& other) {
+ new (this) MarkStack(other);
+ return *this;
+}
+
+MarkStack::MarkStack(MarkStack&& other)
+ : stack_(std::move(other.stack_.ref())),
+ topIndex_(other.topIndex_.ref())
+#ifdef JS_GC_ZEAL
+ ,
+ maxCapacity_(other.maxCapacity_)
+#endif
+{
+ other.topIndex_ = 0;
+}
+
+MarkStack& MarkStack::operator=(MarkStack&& other) {
+ new (this) MarkStack(std::move(other));
+ return *this;
+}
+
+bool MarkStack::init() { return resetStackCapacity(); }
+
+bool MarkStack::resetStackCapacity() {
+ MOZ_ASSERT(isEmpty());
+
+ size_t capacity = MARK_STACK_BASE_CAPACITY;
+
+#ifdef JS_GC_ZEAL
+ capacity = std::min(capacity, maxCapacity_.ref());
+#endif
+
+ return resize(capacity);
+}
+
+#ifdef JS_GC_ZEAL
+void MarkStack::setMaxCapacity(size_t maxCapacity) {
+ MOZ_ASSERT(maxCapacity != 0);
+ MOZ_ASSERT(isEmpty());
+
+ maxCapacity_ = maxCapacity;
+ if (capacity() > maxCapacity_) {
+ // If the realloc fails, just keep using the existing stack; it's
+ // not ideal but better than failing.
+ (void)resize(maxCapacity_);
+ }
+}
+#endif
+
+MOZ_ALWAYS_INLINE bool MarkStack::indexIsEntryBase(size_t index) const {
+ // The mark stack holds both TaggedPtr and SlotsOrElementsRange entries, which
+ // are one or two words long respectively. Determine whether |index| points to
+ // the base of an entry (i.e. the lowest word in memory).
+ //
+ // The possible cases are that |index| points to:
+ // 1. a single word TaggedPtr entry => true
+ // 2. the startAndKind_ word of SlotsOrElementsRange => true
+ // (startAndKind_ is a uintptr_t tagged with SlotsOrElementsKind)
+ // 3. the ptr_ word of SlotsOrElementsRange (itself a TaggedPtr) => false
+ //
+ // To check for case 3, interpret the word as a TaggedPtr: if it is tagged as
+ // a SlotsOrElementsRange tagged pointer then we are inside such a range and
+ // |index| does not point to the base of an entry. This requires that no
+ // startAndKind_ word can be interpreted as such, which is arranged by making
+ // SlotsOrElementsRangeTag zero and all SlotsOrElementsKind tags non-zero.
+
+ MOZ_ASSERT(index < position());
+ return stack()[index].tagUnchecked() != SlotsOrElementsRangeTag;
+}
+
+/* static */
+void MarkStack::moveWork(MarkStack& dst, MarkStack& src) {
+ // Move some work from |src| to |dst|. Assumes |dst| is empty.
+ //
+ // When this method runs during parallel marking, we are on the thread that
+ // owns |src|, and the thread that owns |dst| is blocked waiting on the
+ // ParallelMarkTask::resumed condition variable.
+
+ // Limit the size of moves to stop threads with work spending too much time
+ // donating.
+ static const size_t MaxWordsToMove = 4096;
+
+ size_t totalWords = src.position();
+ size_t wordsToMove = std::min(totalWords / 2, MaxWordsToMove);
+
+ size_t targetPos = src.position() - wordsToMove;
+
+ // Adjust the target position in case it points to the middle of a two word
+ // entry.
+ if (!src.indexIsEntryBase(targetPos)) {
+ targetPos--;
+ wordsToMove++;
+ }
+ MOZ_ASSERT(src.indexIsEntryBase(targetPos));
+ MOZ_ASSERT(targetPos < src.position());
+ MOZ_ASSERT(targetPos > 0);
+ MOZ_ASSERT(wordsToMove == src.position() - targetPos);
+
+ if (!dst.ensureSpace(wordsToMove)) {
+ return;
+ }
+
+ // TODO: This doesn't have good cache behaviour when moving work between
+ // threads. It might be better if the original thread ended up with the top
+ // part of the stack, in src words if this method stole from the bottom of
+ // the stack rather than the top.
+
+ mozilla::PodCopy(dst.topPtr(), src.stack().begin() + targetPos, wordsToMove);
+ dst.topIndex_ += wordsToMove;
+ dst.peekPtr().assertValid();
+
+ src.topIndex_ = targetPos;
+#ifdef DEBUG
+ src.poisonUnused();
+#endif
+ src.peekPtr().assertValid();
+}
+
+void MarkStack::clearAndResetCapacity() {
+ // Fall back to the smaller initial capacity so we don't hold on to excess
+ // memory between GCs.
+ stack().clear();
+ topIndex_ = 0;
+ (void)resetStackCapacity();
+}
+
+void MarkStack::clearAndFreeStack() {
+ // Free all stack memory so we don't hold on to excess memory between GCs.
+ stack().clearAndFree();
+ topIndex_ = 0;
+}
+
+inline MarkStack::TaggedPtr* MarkStack::topPtr() { return &stack()[topIndex_]; }
+
+template <typename T>
+inline bool MarkStack::push(T* ptr) {
+ return push(TaggedPtr(MapTypeToMarkStackTag<T*>::value, ptr));
+}
+
+inline bool MarkStack::pushTempRope(JSRope* rope) {
+ return push(TaggedPtr(TempRopeTag, rope));
+}
+
+inline bool MarkStack::push(const TaggedPtr& ptr) {
+ if (!ensureSpace(1)) {
+ return false;
+ }
+
+ infalliblePush(ptr);
+ return true;
+}
+
+inline void MarkStack::infalliblePush(const TaggedPtr& ptr) {
+ *topPtr() = ptr;
+ topIndex_++;
+ MOZ_ASSERT(position() <= capacity());
+}
+
+inline bool MarkStack::push(JSObject* obj, SlotsOrElementsKind kind,
+ size_t start) {
+ return push(SlotsOrElementsRange(kind, obj, start));
+}
+
+inline bool MarkStack::push(const SlotsOrElementsRange& array) {
+ array.assertValid();
+
+ if (!ensureSpace(ValueRangeWords)) {
+ return false;
+ }
+
+ infalliblePush(array);
+ return true;
+}
+
+inline void MarkStack::infalliblePush(const SlotsOrElementsRange& array) {
+ *reinterpret_cast<SlotsOrElementsRange*>(topPtr()) = array;
+ topIndex_ += ValueRangeWords;
+ MOZ_ASSERT(position() <= capacity());
+ MOZ_ASSERT(TagIsRangeTag(peekTag()));
+}
+
+inline const MarkStack::TaggedPtr& MarkStack::peekPtr() const {
+ MOZ_ASSERT(!isEmpty());
+ return stack()[topIndex_ - 1];
+}
+
+inline MarkStack::Tag MarkStack::peekTag() const {
+ MOZ_ASSERT(!isEmpty());
+ return peekPtr().tag();
+}
+
+inline MarkStack::TaggedPtr MarkStack::popPtr() {
+ MOZ_ASSERT(!isEmpty());
+ MOZ_ASSERT(!TagIsRangeTag(peekTag()));
+ peekPtr().assertValid();
+ topIndex_--;
+ return *topPtr();
+}
+
+inline MarkStack::SlotsOrElementsRange MarkStack::popSlotsOrElementsRange() {
+ MOZ_ASSERT(!isEmpty());
+ MOZ_ASSERT(TagIsRangeTag(peekTag()));
+ MOZ_ASSERT(position() >= ValueRangeWords);
+
+ topIndex_ -= ValueRangeWords;
+ const auto& array = *reinterpret_cast<SlotsOrElementsRange*>(topPtr());
+ array.assertValid();
+ return array;
+}
+
+inline bool MarkStack::ensureSpace(size_t count) {
+ if (MOZ_LIKELY((topIndex_ + count) <= capacity())) {
+ return !js::oom::ShouldFailWithOOM();
+ }
+
+ return enlarge(count);
+}
+
+MOZ_NEVER_INLINE bool MarkStack::enlarge(size_t count) {
+ size_t required = capacity() + count;
+ size_t newCapacity = mozilla::RoundUpPow2(required);
+
+#ifdef JS_GC_ZEAL
+ newCapacity = std::min(newCapacity, maxCapacity_.ref());
+ if (newCapacity < required) {
+ return false;
+ }
+#endif
+
+ return resize(newCapacity);
+}
+
+bool MarkStack::resize(size_t newCapacity) {
+ MOZ_ASSERT(newCapacity != 0);
+ MOZ_ASSERT(newCapacity >= position());
+
+ if (!stack().resize(newCapacity)) {
+ return false;
+ }
+
+ poisonUnused();
+ return true;
+}
+
+inline void MarkStack::poisonUnused() {
+ static_assert((JS_FRESH_MARK_STACK_PATTERN & TagMask) > LastTag,
+ "The mark stack poison pattern must not look like a valid "
+ "tagged pointer");
+
+ AlwaysPoison(stack().begin() + topIndex_, JS_FRESH_MARK_STACK_PATTERN,
+ stack().capacity() - topIndex_, MemCheckKind::MakeUndefined);
+}
+
+size_t MarkStack::sizeOfExcludingThis(
+ mozilla::MallocSizeOf mallocSizeOf) const {
+ return stack().sizeOfExcludingThis(mallocSizeOf);
+}
+
+/*** GCMarker ***************************************************************/
+
+/*
+ * WeakMapTraceAction::Expand: the GC is recomputing the liveness of WeakMap
+ * entries by expanding each live WeakMap into its constituent key->value edges,
+ * a table of which will be consulted in a later phase whenever marking a
+ * potential key.
+ */
+GCMarker::GCMarker(JSRuntime* rt)
+ : tracer_(mozilla::VariantType<MarkingTracer>(), rt, this),
+ runtime_(rt),
+ haveSwappedStacks(false),
+ markColor_(MarkColor::Black),
+ state(NotActive),
+ incrementalWeakMapMarkingEnabled(
+ TuningDefaults::IncrementalWeakMapMarkingEnabled)
+#ifdef DEBUG
+ ,
+ checkAtomMarking(true),
+ strictCompartmentChecking(false)
+#endif
+{
+}
+
+bool GCMarker::init() { return stack.init(); }
+
+void GCMarker::start() {
+ MOZ_ASSERT(state == NotActive);
+ MOZ_ASSERT(stack.isEmpty());
+ state = RegularMarking;
+ haveAllImplicitEdges = true;
+ setMarkColor(MarkColor::Black);
+}
+
+static void ClearEphemeronEdges(JSRuntime* rt) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing weak keys in GCMarker::stop()");
+ }
+ if (!zone->gcNurseryEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing (nursery) weak keys in GCMarker::stop()");
+ }
+ }
+}
+
+void GCMarker::stop() {
+ MOZ_ASSERT(isDrained());
+ MOZ_ASSERT(markColor() == MarkColor::Black);
+ MOZ_ASSERT(!haveSwappedStacks);
+
+ if (state == NotActive) {
+ return;
+ }
+ state = NotActive;
+
+ (void)stack.resetStackCapacity();
+ otherStack.clearAndFreeStack();
+ ClearEphemeronEdges(runtime());
+ unmarkGrayStack.clearAndFree();
+}
+
+void GCMarker::reset() {
+ stack.clearAndResetCapacity();
+ otherStack.clearAndFreeStack();
+ ClearEphemeronEdges(runtime());
+ MOZ_ASSERT(isDrained());
+
+ setMarkColor(MarkColor::Black);
+ MOZ_ASSERT(!haveSwappedStacks);
+
+ unmarkGrayStack.clearAndFree();
+}
+
+void GCMarker::setMarkColor(gc::MarkColor newColor) {
+ if (markColor_ == newColor) {
+ return;
+ }
+
+ // We don't support gray marking while there is black marking work to do.
+ MOZ_ASSERT(!hasBlackEntries());
+
+ markColor_ = newColor;
+
+ // Switch stacks. We only need to do this if there are any stack entries (as
+ // empty stacks are interchangeable) or to swtich back to the original stack.
+ if (!isDrained() || haveSwappedStacks) {
+ std::swap(stack, otherStack);
+ haveSwappedStacks = !haveSwappedStacks;
+ }
+}
+
+bool GCMarker::hasEntries(MarkColor color) const {
+ const MarkStack& stackForColor = color == markColor() ? stack : otherStack;
+ return stackForColor.hasEntries();
+}
+
+template <typename T>
+inline void GCMarker::pushTaggedPtr(T* ptr) {
+ checkZone(ptr);
+ if (!stack.push(ptr)) {
+ delayMarkingChildrenOnOOM(ptr);
+ }
+}
+
+inline void GCMarker::pushValueRange(JSObject* obj, SlotsOrElementsKind kind,
+ size_t start, size_t end) {
+ checkZone(obj);
+ MOZ_ASSERT(obj->is<NativeObject>());
+ MOZ_ASSERT(start <= end);
+
+ if (start == end) {
+ return;
+ }
+
+ if (MOZ_UNLIKELY(!stack.push(obj, kind, start))) {
+ delayMarkingChildrenOnOOM(obj);
+ }
+}
+
+void GCMarker::repush(JSObject* obj) {
+ MOZ_ASSERT(obj->asTenured().isMarkedAtLeast(markColor()));
+ pushTaggedPtr(obj);
+}
+
+void GCMarker::setRootMarkingMode(bool newState) {
+ if (newState) {
+ setMarkingStateAndTracer<RootMarkingTracer>(RegularMarking, RootMarking);
+ } else {
+ setMarkingStateAndTracer<MarkingTracer>(RootMarking, RegularMarking);
+ }
+}
+
+void GCMarker::enterParallelMarkingMode(ParallelMarker* pm) {
+ MOZ_ASSERT(pm);
+ MOZ_ASSERT(!parallelMarker_);
+ setMarkingStateAndTracer<ParallelMarkingTracer>(RegularMarking,
+ ParallelMarking);
+ parallelMarker_ = pm;
+}
+
+void GCMarker::leaveParallelMarkingMode() {
+ MOZ_ASSERT(parallelMarker_);
+ setMarkingStateAndTracer<MarkingTracer>(ParallelMarking, RegularMarking);
+ parallelMarker_ = nullptr;
+}
+
+bool GCMarker::canDonateWork() const {
+ // It's not worth the overhead of donating very few entries. For some
+ // (non-parallelizable) workloads this can lead to constantly interrupting
+ // marking work and makes parallel marking slower than single threaded.
+ constexpr size_t MinWordCount = 12;
+
+ static_assert(MinWordCount >= ValueRangeWords,
+ "We must always leave at least one stack entry.");
+
+ return stack.position() > MinWordCount;
+}
+
+template <typename Tracer>
+void GCMarker::setMarkingStateAndTracer(MarkingState prev, MarkingState next) {
+ MOZ_ASSERT(state == prev);
+ state = next;
+ tracer_.emplace<Tracer>(runtime(), this);
+}
+
+bool GCMarker::enterWeakMarkingMode() {
+ MOZ_ASSERT(tracer()->weakMapAction() == JS::WeakMapTraceAction::Expand);
+ MOZ_ASSERT(state == RegularMarking);
+ if (!haveAllImplicitEdges) {
+ return false;
+ }
+
+ // During weak marking mode, we maintain a table mapping weak keys to
+ // entries in known-live weakmaps. Initialize it with the keys of marked
+ // weakmaps -- or more precisely, the keys of marked weakmaps that are
+ // mapped to not yet live values. (Once bug 1167452 implements incremental
+ // weakmap marking, this initialization step will become unnecessary, as
+ // the table will already hold all such keys.)
+
+ // Set state before doing anything else, so any new key that is marked
+ // during the following gcEphemeronEdges scan will itself be looked up in
+ // gcEphemeronEdges and marked according to ephemeron rules.
+ state = WeakMarking;
+
+ return true;
+}
+
+IncrementalProgress JS::Zone::enterWeakMarkingMode(GCMarker* marker,
+ SliceBudget& budget) {
+ MOZ_ASSERT(marker->isWeakMarking());
+
+ if (!marker->incrementalWeakMapMarkingEnabled) {
+ for (WeakMapBase* m : gcWeakMapList()) {
+ if (m->mapColor) {
+ (void)m->markEntries(marker);
+ }
+ }
+ return IncrementalProgress::Finished;
+ }
+
+ // gcEphemeronEdges contains the keys from all weakmaps marked so far, or at
+ // least the keys that might still need to be marked through. Scan through
+ // gcEphemeronEdges and mark all values whose keys are marked. This marking
+ // may recursively mark through other weakmap entries (immediately since we
+ // are now in WeakMarking mode). The end result is a consistent state where
+ // all values are marked if both their map and key are marked -- though note
+ // that we may later leave weak marking mode, do some more marking, and then
+ // enter back in.
+ if (!isGCMarking()) {
+ return IncrementalProgress::Finished;
+ }
+
+ MOZ_ASSERT(gcNurseryEphemeronEdges().count() == 0);
+
+ // An OrderedHashMap::MutableRange stays valid even when the underlying table
+ // (zone->gcEphemeronEdges) is mutated, which is useful here since we may add
+ // additional entries while iterating over the Range.
+ EphemeronEdgeTable::MutableRange r = gcEphemeronEdges().mutableAll();
+ while (!r.empty()) {
+ Cell* src = r.front().key;
+ CellColor srcColor = gc::detail::GetEffectiveColor(marker, src);
+ auto& edges = r.front().value;
+ r.popFront(); // Pop before any mutations happen.
+
+ if (edges.length() > 0) {
+ uint32_t steps = edges.length();
+ marker->markEphemeronEdges(edges, srcColor);
+ budget.step(steps);
+ if (budget.isOverBudget()) {
+ return NotFinished;
+ }
+ }
+ }
+
+ return IncrementalProgress::Finished;
+}
+
+void GCMarker::leaveWeakMarkingMode() {
+ MOZ_ASSERT(state == WeakMarking || state == RegularMarking);
+ state = RegularMarking;
+
+ // The gcEphemeronEdges table is still populated and may be used during a
+ // future weak marking mode within this GC.
+}
+
+void GCMarker::abortLinearWeakMarking() {
+ haveAllImplicitEdges = false;
+ if (state == WeakMarking) {
+ leaveWeakMarkingMode();
+ }
+}
+
+MOZ_NEVER_INLINE void GCMarker::delayMarkingChildrenOnOOM(Cell* cell) {
+ runtime()->gc.delayMarkingChildren(cell, markColor());
+}
+
+bool GCRuntime::hasDelayedMarking() const {
+ bool result = delayedMarkingList;
+ MOZ_ASSERT(result == (markLaterArenas != 0));
+ return result;
+}
+
+void GCRuntime::delayMarkingChildren(Cell* cell, MarkColor color) {
+ // Synchronize access to delayed marking state during parallel marking.
+ LockGuard<Mutex> lock(delayedMarkingLock);
+
+ Arena* arena = cell->asTenured().arena();
+ if (!arena->onDelayedMarkingList()) {
+ arena->setNextDelayedMarkingArena(delayedMarkingList);
+ delayedMarkingList = arena;
+#ifdef DEBUG
+ markLaterArenas++;
+#endif
+ }
+
+ if (!arena->hasDelayedMarking(color)) {
+ arena->setHasDelayedMarking(color, true);
+ delayedMarkingWorkAdded = true;
+ }
+}
+
+void GCRuntime::markDelayedChildren(Arena* arena, MarkColor color) {
+ JSTracer* trc = marker().tracer();
+ JS::TraceKind kind = MapAllocToTraceKind(arena->getAllocKind());
+ MarkColor colorToCheck =
+ TraceKindCanBeMarkedGray(kind) ? color : MarkColor::Black;
+
+ for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+ if (cell->isMarked(colorToCheck)) {
+ ApplyGCThingTyped(cell, kind, [trc, this](auto t) {
+ t->traceChildren(trc);
+ marker().markImplicitEdges(t);
+ });
+ }
+ }
+}
+
+/*
+ * Process arenas from |delayedMarkingList| by marking the unmarked children of
+ * marked cells of color |color|.
+ *
+ * This is called twice, first to mark gray children and then to mark black
+ * children.
+ */
+void GCRuntime::processDelayedMarkingList(MarkColor color) {
+ // Marking delayed children may add more arenas to the list, including arenas
+ // we are currently processing or have previously processed. Handle this by
+ // clearing a flag on each arena before marking its children. This flag will
+ // be set again if the arena is re-added. Iterate the list until no new arenas
+ // were added.
+
+ AutoSetMarkColor setColor(marker(), color);
+
+ do {
+ delayedMarkingWorkAdded = false;
+ for (Arena* arena = delayedMarkingList; arena;
+ arena = arena->getNextDelayedMarking()) {
+ if (arena->hasDelayedMarking(color)) {
+ arena->setHasDelayedMarking(color, false);
+ markDelayedChildren(arena, color);
+ }
+ }
+ while (marker().hasEntriesForCurrentColor()) {
+ SliceBudget budget = SliceBudget::unlimited();
+ MOZ_ALWAYS_TRUE(
+ marker().processMarkStackTop<NormalMarkingOptions>(budget));
+ }
+ } while (delayedMarkingWorkAdded);
+
+ MOZ_ASSERT(marker().isDrained());
+}
+
+void GCRuntime::markAllDelayedChildren(ShouldReportMarkTime reportTime) {
+ MOZ_ASSERT(CurrentThreadIsMainThread() || CurrentThreadIsPerformingGC());
+ MOZ_ASSERT(marker().isDrained());
+ MOZ_ASSERT(hasDelayedMarking());
+
+ mozilla::Maybe<gcstats::AutoPhase> ap;
+ if (reportTime) {
+ ap.emplace(stats(), gcstats::PhaseKind::MARK_DELAYED);
+ }
+
+ // We have a list of arenas containing marked cells with unmarked children
+ // where we ran out of stack space during marking. Both black and gray cells
+ // in these arenas may have unmarked children. Mark black children first.
+
+ const MarkColor colors[] = {MarkColor::Black, MarkColor::Gray};
+ for (MarkColor color : colors) {
+ processDelayedMarkingList(color);
+ rebuildDelayedMarkingList();
+ }
+
+ MOZ_ASSERT(!hasDelayedMarking());
+}
+
+void GCRuntime::rebuildDelayedMarkingList() {
+ // Rebuild the delayed marking list, removing arenas which do not need further
+ // marking.
+
+ Arena* listTail = nullptr;
+ forEachDelayedMarkingArena([&](Arena* arena) {
+ if (!arena->hasAnyDelayedMarking()) {
+ arena->clearDelayedMarkingState();
+#ifdef DEBUG
+ MOZ_ASSERT(markLaterArenas);
+ markLaterArenas--;
+#endif
+ return;
+ }
+
+ appendToDelayedMarkingList(&listTail, arena);
+ });
+ appendToDelayedMarkingList(&listTail, nullptr);
+}
+
+void GCRuntime::resetDelayedMarking() {
+ MOZ_ASSERT(CurrentThreadIsMainThread());
+
+ forEachDelayedMarkingArena([&](Arena* arena) {
+ MOZ_ASSERT(arena->onDelayedMarkingList());
+ arena->clearDelayedMarkingState();
+#ifdef DEBUG
+ MOZ_ASSERT(markLaterArenas);
+ markLaterArenas--;
+#endif
+ });
+ delayedMarkingList = nullptr;
+ MOZ_ASSERT(!markLaterArenas);
+}
+
+inline void GCRuntime::appendToDelayedMarkingList(Arena** listTail,
+ Arena* arena) {
+ if (*listTail) {
+ (*listTail)->updateNextDelayedMarkingArena(arena);
+ } else {
+ delayedMarkingList = arena;
+ }
+ *listTail = arena;
+}
+
+template <typename F>
+inline void GCRuntime::forEachDelayedMarkingArena(F&& f) {
+ Arena* arena = delayedMarkingList;
+ Arena* next;
+ while (arena) {
+ next = arena->getNextDelayedMarking();
+ f(arena);
+ arena = next;
+ }
+}
+
+#ifdef DEBUG
+void GCMarker::checkZone(void* p) {
+ MOZ_ASSERT(state != NotActive);
+ DebugOnly<Cell*> cell = static_cast<Cell*>(p);
+ MOZ_ASSERT_IF(cell->isTenured(),
+ cell->asTenured().zone()->isCollectingFromAnyThread());
+}
+#endif
+
+size_t GCMarker::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return mallocSizeOf(this) + stack.sizeOfExcludingThis(mallocSizeOf) +
+ otherStack.sizeOfExcludingThis(mallocSizeOf);
+}
+
+/*** IsMarked / IsAboutToBeFinalized ****************************************/
+
+template <typename T>
+static inline void CheckIsMarkedThing(T* thing) {
+#define IS_SAME_TYPE_OR(name, type, _, _1) std::is_same_v<type, T> ||
+ static_assert(JS_FOR_EACH_TRACEKIND(IS_SAME_TYPE_OR) false,
+ "Only the base cell layout types are allowed into "
+ "marking/tracing internals");
+#undef IS_SAME_TYPE_OR
+
+#ifdef DEBUG
+ MOZ_ASSERT(thing);
+
+ // Allow any thread access to uncollected things.
+ Zone* zone = thing->zoneFromAnyThread();
+ if (thing->isPermanentAndMayBeShared()) {
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ MOZ_ASSERT(thing->isMarkedBlack());
+ return;
+ }
+
+ // Allow the current thread access if it is sweeping or in sweep-marking, but
+ // try to check the zone. Some threads have access to all zones when sweeping.
+ JS::GCContext* gcx = TlsGCContext.get();
+ MOZ_ASSERT(gcx->gcUse() != GCUse::Finalizing);
+ if (gcx->gcUse() == GCUse::Sweeping || gcx->gcUse() == GCUse::Marking) {
+ MOZ_ASSERT_IF(gcx->gcSweepZone(),
+ gcx->gcSweepZone() == zone || zone->isAtomsZone());
+ return;
+ }
+
+ // Otherwise only allow access from the main thread or this zone's associated
+ // thread.
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread()) ||
+ CurrentThreadCanAccessZone(thing->zoneFromAnyThread()));
+#endif
+}
+
+template <typename T>
+bool js::gc::IsMarkedInternal(JSRuntime* rt, T* thing) {
+ // Don't depend on the mark state of other cells during finalization.
+ MOZ_ASSERT(!CurrentThreadIsGCFinalizing());
+ MOZ_ASSERT(rt->heapState() != JS::HeapState::MinorCollecting);
+ MOZ_ASSERT(thing);
+ CheckIsMarkedThing(thing);
+
+ // This is not used during minor sweeping nor used to update moved GC things.
+ MOZ_ASSERT(!IsForwarded(thing));
+
+ // Permanent things are never marked by non-owning runtimes.
+ TenuredCell* cell = &thing->asTenured();
+ Zone* zone = cell->zoneFromAnyThread();
+#ifdef DEBUG
+ if (IsOwnedByOtherRuntime(rt, thing)) {
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(thing->isMarkedBlack());
+ }
+#endif
+
+ return !zone->isGCMarking() || TenuredThingIsMarkedAny(thing);
+}
+
+template <typename T>
+bool js::gc::IsAboutToBeFinalizedInternal(T* thing) {
+ // Don't depend on the mark state of other cells during finalization.
+ MOZ_ASSERT(!CurrentThreadIsGCFinalizing());
+ MOZ_ASSERT(thing);
+ CheckIsMarkedThing(thing);
+
+ // This is not used during minor sweeping nor used to update moved GC things.
+ MOZ_ASSERT(!IsForwarded(thing));
+
+ if (!thing->isTenured()) {
+ return false;
+ }
+
+ // Permanent things are never finalized by non-owning runtimes.
+ TenuredCell* cell = &thing->asTenured();
+ Zone* zone = cell->zoneFromAnyThread();
+#ifdef DEBUG
+ JSRuntime* rt = TlsGCContext.get()->runtimeFromAnyThread();
+ if (IsOwnedByOtherRuntime(rt, thing)) {
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(thing->isMarkedBlack());
+ }
+#endif
+
+ return zone->isGCSweeping() && !TenuredThingIsMarkedAny(thing);
+}
+
+template <typename T>
+bool js::gc::IsAboutToBeFinalizedInternal(const T& thing) {
+ bool dying = false;
+ ApplyGCThingTyped(
+ thing, [&dying](auto t) { dying = IsAboutToBeFinalizedInternal(t); });
+ return dying;
+}
+
+SweepingTracer::SweepingTracer(JSRuntime* rt)
+ : GenericTracerImpl(rt, JS::TracerKind::Sweeping,
+ JS::WeakMapTraceAction::TraceKeysAndValues) {}
+
+template <typename T>
+inline void SweepingTracer::onEdge(T** thingp, const char* name) {
+ T* thing = *thingp;
+ CheckIsMarkedThing(thing);
+
+ if (!thing->isTenured()) {
+ return;
+ }
+
+ // Permanent things are never finalized by non-owning runtimes.
+ TenuredCell* cell = &thing->asTenured();
+ Zone* zone = cell->zoneFromAnyThread();
+#ifdef DEBUG
+ if (IsOwnedByOtherRuntime(runtime(), thing)) {
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(thing->isMarkedBlack());
+ }
+#endif
+
+ // It would be nice if we could assert that the zone of the tenured cell is in
+ // the Sweeping state, but that isn't always true for:
+ // - atoms
+ // - the jitcode map
+ // - the mark queue
+ if (zone->isGCSweeping() && !cell->isMarkedAny()) {
+ *thingp = nullptr;
+ }
+}
+
+namespace js {
+namespace gc {
+
+template <typename T>
+JS_PUBLIC_API bool TraceWeakEdge(JSTracer* trc, JS::Heap<T>* thingp) {
+ return TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unsafeGet()),
+ "JS::Heap edge");
+}
+
+template <typename T>
+JS_PUBLIC_API bool EdgeNeedsSweepUnbarrieredSlow(T* thingp) {
+ return IsAboutToBeFinalizedInternal(*ConvertToBase(thingp));
+}
+
+// Instantiate a copy of the Tracing templates for each public GC type.
+#define INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS(type) \
+ template JS_PUBLIC_API bool TraceWeakEdge<type>(JSTracer * trc, \
+ JS::Heap<type>*); \
+ template JS_PUBLIC_API bool EdgeNeedsSweepUnbarrieredSlow<type>(type*);
+JS_FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS)
+JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(
+ INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS)
+
+#define INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION(type) \
+ template bool IsMarkedInternal(JSRuntime* rt, type thing);
+
+#define INSTANTIATE_INTERNAL_IATBF_FUNCTION(type) \
+ template bool IsAboutToBeFinalizedInternal(type thingp);
+
+#define INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND(_1, type, _2, \
+ _3) \
+ INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION(type*) \
+ INSTANTIATE_INTERNAL_IATBF_FUNCTION(type*)
+
+JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND)
+
+#define INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER(type) \
+ INSTANTIATE_INTERNAL_IATBF_FUNCTION(const type&)
+
+JS_FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(
+ INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER)
+
+#undef INSTANTIATE_INTERNAL_IS_MARKED_FUNCTION
+#undef INSTANTIATE_INTERNAL_IATBF_FUNCTION
+#undef INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND
+#undef INSTANTIATE_IATBF_FUNCTION_FOR_TAGGED_POINTER
+
+} /* namespace gc */
+} /* namespace js */
+
+/*** Cycle Collector Barrier Implementation *********************************/
+
+/*
+ * The GC and CC are run independently. Consequently, the following sequence of
+ * events can occur:
+ * 1. GC runs and marks an object gray.
+ * 2. The mutator runs (specifically, some C++ code with access to gray
+ * objects) and creates a pointer from a JS root or other black object to
+ * the gray object. If we re-ran a GC at this point, the object would now be
+ * black.
+ * 3. Now we run the CC. It may think it can collect the gray object, even
+ * though it's reachable from the JS heap.
+ *
+ * To prevent this badness, we unmark the gray bit of an object when it is
+ * accessed by callers outside XPConnect. This would cause the object to go
+ * black in step 2 above. This must be done on everything reachable from the
+ * object being returned. The following code takes care of the recursive
+ * re-coloring.
+ *
+ * There is an additional complication for certain kinds of edges that are not
+ * contained explicitly in the source object itself, such as from a weakmap key
+ * to its value. These "implicit edges" are represented in some other
+ * container object, such as the weakmap itself. In these
+ * cases, calling unmark gray on an object won't find all of its children.
+ *
+ * Handling these implicit edges has two parts:
+ * - A special pass enumerating all of the containers that know about the
+ * implicit edges to fix any black-gray edges that have been created. This
+ * is implemented in nsXPConnect::FixWeakMappingGrayBits.
+ * - To prevent any incorrectly gray objects from escaping to live JS outside
+ * of the containers, we must add unmark-graying read barriers to these
+ * containers.
+ */
+
+#ifdef DEBUG
+struct AssertNonGrayTracer final : public JS::CallbackTracer {
+ // This is used by the UnmarkGray tracer only, and needs to report itself as
+ // the non-gray tracer to not trigger assertions. Do not use it in another
+ // context without making this more generic.
+ explicit AssertNonGrayTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt, JS::TracerKind::UnmarkGray) {}
+ void onChild(JS::GCCellPtr thing, const char* name) override {
+ MOZ_ASSERT(!thing.asCell()->isMarkedGray());
+ }
+};
+#endif
+
+class js::gc::UnmarkGrayTracer final : public JS::CallbackTracer {
+ public:
+ // We set weakMapAction to WeakMapTraceAction::Skip because the cycle
+ // collector will fix up any color mismatches involving weakmaps when it runs.
+ explicit UnmarkGrayTracer(GCMarker* marker)
+ : JS::CallbackTracer(marker->runtime(), JS::TracerKind::UnmarkGray,
+ JS::WeakMapTraceAction::Skip),
+ unmarkedAny(false),
+ oom(false),
+ marker(marker),
+ stack(marker->unmarkGrayStack) {}
+
+ void unmark(JS::GCCellPtr cell);
+
+ // Whether we unmarked anything.
+ bool unmarkedAny;
+
+ // Whether we ran out of memory.
+ bool oom;
+
+ private:
+ // Marker to use if we need to unmark in zones that are currently being
+ // marked.
+ GCMarker* marker;
+
+ // Stack of cells to traverse.
+ Vector<JS::GCCellPtr, 0, SystemAllocPolicy>& stack;
+
+ void onChild(JS::GCCellPtr thing, const char* name) override;
+};
+
+void UnmarkGrayTracer::onChild(JS::GCCellPtr thing, const char* name) {
+ Cell* cell = thing.asCell();
+
+ // Cells in the nursery cannot be gray, and nor can certain kinds of tenured
+ // cells. These must necessarily point only to black edges.
+ if (!cell->isTenured() || !TraceKindCanBeMarkedGray(thing.kind())) {
+#ifdef DEBUG
+ MOZ_ASSERT(!cell->isMarkedGray());
+ AssertNonGrayTracer nongray(runtime());
+ JS::TraceChildren(&nongray, thing);
+#endif
+ return;
+ }
+
+ TenuredCell& tenured = cell->asTenured();
+ Zone* zone = tenured.zone();
+
+ // If the cell is in a zone whose mark bits are being cleared, then it will
+ // end up white.
+ if (zone->isGCPreparing()) {
+ return;
+ }
+
+ // If the cell is in a zone that we're currently marking, then it's possible
+ // that it is currently white but will end up gray. To handle this case,
+ // trigger the barrier for any cells in zones that are currently being
+ // marked. This will ensure they will eventually get marked black.
+ if (zone->isGCMarking()) {
+ if (!cell->isMarkedBlack()) {
+ TraceEdgeForBarrier(marker, &tenured, thing.kind());
+ unmarkedAny = true;
+ }
+ return;
+ }
+
+ if (!tenured.isMarkedGray()) {
+ return;
+ }
+
+ // TODO: It may be a small improvement to only use the atomic version during
+ // parallel marking.
+ tenured.markBlackAtomic();
+ unmarkedAny = true;
+
+ if (!stack.append(thing)) {
+ oom = true;
+ }
+}
+
+void UnmarkGrayTracer::unmark(JS::GCCellPtr cell) {
+ MOZ_ASSERT(stack.empty());
+
+ onChild(cell, "unmarking root");
+
+ while (!stack.empty() && !oom) {
+ TraceChildren(this, stack.popCopy());
+ }
+
+ if (oom) {
+ // If we run out of memory, we take a drastic measure: require that we
+ // GC again before the next CC.
+ stack.clear();
+ runtime()->gc.setGrayBitsInvalid();
+ return;
+ }
+}
+
+bool js::gc::UnmarkGrayGCThingUnchecked(GCMarker* marker, JS::GCCellPtr thing) {
+ MOZ_ASSERT(thing);
+ MOZ_ASSERT(thing.asCell()->isMarkedGray());
+
+ mozilla::Maybe<AutoGeckoProfilerEntry> profilingStackFrame;
+ if (JSContext* cx = TlsContext.get()) {
+ profilingStackFrame.emplace(cx, "UnmarkGrayGCThing",
+ JS::ProfilingCategoryPair::GCCC_UnmarkGray);
+ }
+
+ UnmarkGrayTracer unmarker(marker);
+ unmarker.unmark(thing);
+ return unmarker.unmarkedAny;
+}
+
+JS_PUBLIC_API bool JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+ MOZ_ASSERT(!JS::RuntimeHeapIsCycleCollecting());
+
+ JSRuntime* rt = thing.asCell()->runtimeFromMainThread();
+ if (thing.asCell()->zone()->isGCPreparing()) {
+ // Mark bits are being cleared in preparation for GC.
+ return false;
+ }
+
+ return UnmarkGrayGCThingUnchecked(&rt->gc.marker(), thing);
+}
+
+void js::gc::UnmarkGrayGCThingRecursively(TenuredCell* cell) {
+ JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(cell, cell->getTraceKind()));
+}
+
+bool js::UnmarkGrayShapeRecursively(Shape* shape) {
+ return JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(shape));
+}
+
+#ifdef DEBUG
+Cell* js::gc::UninlinedForwarded(const Cell* cell) { return Forwarded(cell); }
+#endif
+
+namespace js {
+namespace debug {
+
+MarkInfo GetMarkInfo(Cell* rawCell) {
+ if (!rawCell->isTenured()) {
+ return MarkInfo::NURSERY;
+ }
+
+ TenuredCell* cell = &rawCell->asTenured();
+ if (cell->isMarkedGray()) {
+ return MarkInfo::GRAY;
+ }
+ if (cell->isMarkedBlack()) {
+ return MarkInfo::BLACK;
+ }
+ return MarkInfo::UNMARKED;
+}
+
+uintptr_t* GetMarkWordAddress(Cell* cell) {
+ if (!cell->isTenured()) {
+ return nullptr;
+ }
+
+ MarkBitmapWord* wordp;
+ uintptr_t mask;
+ TenuredChunkBase* chunk = gc::detail::GetCellChunkBase(&cell->asTenured());
+ chunk->markBits.getMarkWordAndMask(&cell->asTenured(), ColorBit::BlackBit,
+ &wordp, &mask);
+ return reinterpret_cast<uintptr_t*>(wordp);
+}
+
+uintptr_t GetMarkMask(Cell* cell, uint32_t colorBit) {
+ MOZ_ASSERT(colorBit == 0 || colorBit == 1);
+
+ if (!cell->isTenured()) {
+ return 0;
+ }
+
+ ColorBit bit = colorBit == 0 ? ColorBit::BlackBit : ColorBit::GrayOrBlackBit;
+ MarkBitmapWord* wordp;
+ uintptr_t mask;
+ TenuredChunkBase* chunk = gc::detail::GetCellChunkBase(&cell->asTenured());
+ chunk->markBits.getMarkWordAndMask(&cell->asTenured(), bit, &wordp, &mask);
+ return mask;
+}
+
+} // namespace debug
+} // namespace js
diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h
new file mode 100644
index 0000000000..7043eccc53
--- /dev/null
+++ b/js/src/gc/Marking.h
@@ -0,0 +1,150 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Marking and sweeping APIs for use by implementations of different GC cell
+ * kinds.
+ */
+
+#ifndef gc_Marking_h
+#define gc_Marking_h
+
+#include "gc/Barrier.h"
+#include "js/TypeDecls.h"
+
+class JSTracer;
+struct JSClass;
+
+namespace js {
+class GCMarker;
+class Shape;
+class WeakMapBase;
+
+namespace gc {
+
+struct Cell;
+
+/*** Liveness ***/
+
+// The IsMarkedInternal and IsAboutToBeFinalizedInternal function templates are
+// used to implement the IsMarked and IsAboutToBeFinalized set of functions.
+// These internal functions are instantiated for the base GC types and should
+// not be called directly.
+//
+// Note that there are two function templates declared for each, not one
+// template and a specialization. This is necessary so that pointer arguments
+// (e.g. JSObject**) and tagged value arguments (e.g. JS::Value*) are routed to
+// separate implementations.
+
+template <typename T>
+bool IsMarkedInternal(JSRuntime* rt, T* thing);
+
+template <typename T>
+bool IsAboutToBeFinalizedInternal(T* thing);
+template <typename T>
+bool IsAboutToBeFinalizedInternal(const T& thing);
+
+// Report whether a GC thing has been marked with any color. Things which are in
+// zones that are not currently being collected or are owned by another runtime
+// are always reported as being marked.
+template <typename T>
+inline bool IsMarked(JSRuntime* rt, const BarrieredBase<T>& thing) {
+ return IsMarkedInternal(rt, *ConvertToBase(thing.unbarrieredAddress()));
+}
+template <typename T>
+inline bool IsMarkedUnbarriered(JSRuntime* rt, T thing) {
+ return IsMarkedInternal(rt, *ConvertToBase(&thing));
+}
+
+// Report whether a GC thing is dead and will be finalized in the current sweep
+// group. This is mainly used in read barriers for incremental sweeping.
+//
+// This no longer updates pointers moved by the GC (tracing should be used for
+// this instead).
+template <typename T>
+inline bool IsAboutToBeFinalized(const BarrieredBase<T>& thing) {
+ return IsAboutToBeFinalizedInternal(
+ *ConvertToBase(thing.unbarrieredAddress()));
+}
+template <typename T>
+inline bool IsAboutToBeFinalizedUnbarriered(T thing) {
+ return IsAboutToBeFinalizedInternal(*ConvertToBase(&thing));
+}
+
+inline bool IsAboutToBeFinalizedDuringMinorSweep(Cell* cell);
+
+inline Cell* ToMarkable(const Value& v) {
+ if (v.isGCThing()) {
+ return (Cell*)v.toGCThing();
+ }
+ return nullptr;
+}
+
+inline Cell* ToMarkable(Cell* cell) { return cell; }
+
+bool UnmarkGrayGCThingUnchecked(GCMarker* marker, JS::GCCellPtr thing);
+
+} /* namespace gc */
+
+// The return value indicates if anything was unmarked.
+bool UnmarkGrayShapeRecursively(Shape* shape);
+
+namespace gc {
+
+// Functions for checking and updating GC thing pointers that might have been
+// moved by compacting GC. Overloads are also provided that work with Values.
+//
+// IsForwarded - check whether a pointer refers to an GC thing that has been
+// moved.
+//
+// Forwarded - return a pointer to the new location of a GC thing given a
+// pointer to old location.
+//
+// MaybeForwarded - used before dereferencing a pointer that may refer to a
+// moved GC thing without updating it. For JSObjects this will
+// also update the object's shape pointer if it has been moved
+// to allow slots to be accessed.
+
+template <typename T>
+inline bool IsForwarded(const T* t);
+
+template <typename T>
+inline T* Forwarded(const T* t);
+
+inline Value Forwarded(const JS::Value& value);
+
+template <typename T>
+inline T MaybeForwarded(T t);
+
+// Helper functions for use in situations where the object's group might be
+// forwarded, for example while marking.
+
+inline const JSClass* MaybeForwardedObjectClass(const JSObject* obj);
+
+template <typename T>
+inline bool MaybeForwardedObjectIs(const JSObject* obj);
+
+template <typename T>
+inline T& MaybeForwardedObjectAs(JSObject* obj);
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+template <typename T>
+inline bool IsGCThingValidAfterMovingGC(T* t);
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(T* t);
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(const WeakHeapPtr<T*>& t);
+
+#endif // JSGC_HASH_TABLE_CHECKS
+
+} /* namespace gc */
+
+} /* namespace js */
+
+#endif /* gc_Marking_h */
diff --git a/js/src/gc/MaybeRooted.h b/js/src/gc/MaybeRooted.h
new file mode 100644
index 0000000000..fd615a1efb
--- /dev/null
+++ b/js/src/gc/MaybeRooted.h
@@ -0,0 +1,141 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Template types for use in generic code: to use Rooted/Handle/MutableHandle in
+ * cases where GC may occur, or to use mock versions of those types that perform
+ * no rooting or root list manipulation when GC cannot occur.
+ */
+
+#ifndef gc_MaybeRooted_h
+#define gc_MaybeRooted_h
+
+#include "mozilla/Attributes.h" // MOZ_IMPLICIT, MOZ_RAII
+
+#include <type_traits> // std::true_type
+
+#include "gc/Allocator.h" // js::AllowGC, js::CanGC, js::NoGC
+#include "js/ComparisonOperators.h" // JS::detail::DefineComparisonOps
+#include "js/RootingAPI.h" // js::{Rooted,MutableHandle}Base, JS::SafelyInitialized, DECLARE_POINTER_{CONSTREF,ASSIGN}_OPS, DECLARE_NONPOINTER_{,MUTABLE_}ACCESSOR_METHODS, JS::Rooted, JS::{,Mutable}Handle
+
+namespace js {
+
+/**
+ * Interface substitute for Rooted<T> which does not root the variable's
+ * memory.
+ */
+template <typename T>
+class MOZ_RAII FakeRooted : public RootedOperations<T, FakeRooted<T>> {
+ public:
+ using ElementType = T;
+
+ explicit FakeRooted(JSContext* cx)
+ : ptr(JS::SafelyInitialized<T>::create()) {}
+
+ FakeRooted(JSContext* cx, T initial) : ptr(initial) {}
+
+ DECLARE_POINTER_CONSTREF_OPS(T);
+ DECLARE_POINTER_ASSIGN_OPS(FakeRooted, T);
+ DECLARE_NONPOINTER_ACCESSOR_METHODS(ptr);
+ DECLARE_NONPOINTER_MUTABLE_ACCESSOR_METHODS(ptr);
+
+ private:
+ T ptr;
+
+ void set(const T& value) { ptr = value; }
+
+ FakeRooted(const FakeRooted&) = delete;
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::FakeRooted<T>> : std::true_type {
+ static const T& get(const js::FakeRooted<T>& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+/**
+ * Interface substitute for MutableHandle<T> which is not required to point to
+ * rooted memory.
+ */
+template <typename T>
+class FakeMutableHandle
+ : public js::MutableHandleOperations<T, FakeMutableHandle<T>> {
+ public:
+ using ElementType = T;
+
+ MOZ_IMPLICIT FakeMutableHandle(T* t) : ptr(t) {}
+
+ MOZ_IMPLICIT FakeMutableHandle(FakeRooted<T>* root) : ptr(root->address()) {}
+
+ void set(const T& v) { *ptr = v; }
+
+ DECLARE_POINTER_CONSTREF_OPS(T);
+ DECLARE_NONPOINTER_ACCESSOR_METHODS(*ptr);
+ DECLARE_NONPOINTER_MUTABLE_ACCESSOR_METHODS(*ptr);
+
+ private:
+ FakeMutableHandle() : ptr(nullptr) {}
+ DELETE_ASSIGNMENT_OPS(FakeMutableHandle, T);
+
+ T* ptr;
+};
+
+} // namespace js
+
+namespace JS {
+
+namespace detail {
+
+template <typename T>
+struct DefineComparisonOps<js::FakeMutableHandle<T>> : std::true_type {
+ static const T& get(const js::FakeMutableHandle<T>& v) { return v.get(); }
+};
+
+} // namespace detail
+
+} // namespace JS
+
+namespace js {
+
+/**
+ * Types for a variable that either should or shouldn't be rooted, depending on
+ * the template parameter allowGC. Used for implementing functions that can
+ * operate on either rooted or unrooted data.
+ */
+
+template <typename T, AllowGC allowGC>
+class MaybeRooted;
+
+template <typename T>
+class MaybeRooted<T, CanGC> {
+ public:
+ using HandleType = JS::Handle<T>;
+ using RootType = JS::Rooted<T>;
+ using MutableHandleType = JS::MutableHandle<T>;
+};
+
+template <typename T>
+class MaybeRooted<T, NoGC> {
+ public:
+ using HandleType = const T&;
+ using RootType = FakeRooted<T>;
+ using MutableHandleType = FakeMutableHandle<T>;
+};
+
+} // namespace js
+
+#endif // gc_MaybeRooted_h
diff --git a/js/src/gc/Memory.cpp b/js/src/gc/Memory.cpp
new file mode 100644
index 0000000000..c5bdaa14bf
--- /dev/null
+++ b/js/src/gc/Memory.cpp
@@ -0,0 +1,1050 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Memory.h"
+
+#include "mozilla/Atomics.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/RandomNum.h"
+#include "mozilla/TaggedAnonymousMemory.h"
+
+#include "jit/JitOptions.h"
+#include "js/HeapAPI.h"
+#include "js/Utility.h"
+#include "util/Memory.h"
+
+#ifdef XP_WIN
+
+# include "util/WindowsWrapper.h"
+# include <psapi.h>
+
+#else
+
+# include <algorithm>
+# include <errno.h>
+# include <unistd.h>
+
+# if !defined(__wasi__)
+# include <sys/mman.h>
+# include <sys/resource.h>
+# include <sys/stat.h>
+# include <sys/types.h>
+# endif // !defined(__wasi__)
+
+#endif // !XP_WIN
+
+namespace js {
+namespace gc {
+
+/*
+ * System allocation functions generally require the allocation size
+ * to be an integer multiple of the page size of the running process.
+ */
+static size_t pageSize = 0;
+
+/* The OS allocation granularity may not match the page size. */
+static size_t allocGranularity = 0;
+
+/* The number of bits used by addresses on this platform. */
+static size_t numAddressBits = 0;
+
+/* An estimate of the number of bytes available for virtual memory. */
+static size_t virtualMemoryLimit = size_t(-1);
+
+/*
+ * System allocation functions may hand out regions of memory in increasing or
+ * decreasing order. This ordering is used as a hint during chunk alignment to
+ * reduce the number of system calls. On systems with 48-bit addresses, our
+ * workarounds to obtain 47-bit pointers cause addresses to be handed out in
+ * increasing order.
+ *
+ * We do not use the growth direction on Windows, as constraints on VirtualAlloc
+ * would make its application failure prone and complex. Tests indicate that
+ * VirtualAlloc always hands out regions of memory in increasing order.
+ */
+#if defined(XP_DARWIN)
+static mozilla::Atomic<int, mozilla::Relaxed> growthDirection(1);
+#elif defined(XP_UNIX)
+static mozilla::Atomic<int, mozilla::Relaxed> growthDirection(0);
+#endif
+
+/*
+ * Data from OOM crashes shows there may be up to 24 chunk-sized but unusable
+ * chunks available in low memory situations. These chunks may all need to be
+ * used up before we gain access to remaining *alignable* chunk-sized regions,
+ * so we use a generous limit of 32 unusable chunks to ensure we reach them.
+ */
+static const int MaxLastDitchAttempts = 32;
+
+#ifdef JS_64BIT
+/*
+ * On some 64-bit platforms we can use a random, scattershot allocator that
+ * tries addresses from the available range at random. If the address range
+ * is large enough this will have a high chance of success and additionally
+ * makes the memory layout of our process less predictable.
+ *
+ * However, not all 64-bit platforms have a very large address range. For
+ * example, AArch64 on Linux defaults to using 39-bit addresses to limit the
+ * number of translation tables used. On such configurations the scattershot
+ * approach to allocation creates a conflict with our desire to reserve large
+ * regions of memory for applications like WebAssembly: Small allocations may
+ * inadvertently block off all available 4-6GiB regions, and conversely
+ * reserving such regions may lower the success rate for smaller allocations to
+ * unacceptable levels.
+ *
+ * So we make a compromise: Instead of using the scattershot on all 64-bit
+ * platforms, we only use it on platforms that meet a minimum requirement for
+ * the available address range. In addition we split the address range,
+ * reserving the upper half for huge allocations and the lower half for smaller
+ * allocations. We use a limit of 43 bits so that at least 42 bits are available
+ * for huge allocations - this matches the 8TiB per process address space limit
+ * that we're already subject to on Windows.
+ */
+static const size_t MinAddressBitsForRandomAlloc = 43;
+
+/* The lower limit for huge allocations. This is fairly arbitrary. */
+static const size_t HugeAllocationSize = 1024 * 1024 * 1024;
+
+/* The minimum and maximum valid addresses that can be allocated into. */
+static size_t minValidAddress = 0;
+static size_t maxValidAddress = 0;
+
+/* The upper limit for smaller allocations and the lower limit for huge ones. */
+static size_t hugeSplit = 0;
+#endif
+
+size_t SystemPageSize() { return pageSize; }
+
+size_t SystemAddressBits() { return numAddressBits; }
+
+size_t VirtualMemoryLimit() { return virtualMemoryLimit; }
+
+bool UsingScattershotAllocator() {
+#ifdef JS_64BIT
+ return numAddressBits >= MinAddressBitsForRandomAlloc;
+#else
+ return false;
+#endif
+}
+
+enum class Commit : bool {
+ No = false,
+ Yes = true,
+};
+
+#ifdef XP_WIN
+enum class PageAccess : DWORD {
+ None = PAGE_NOACCESS,
+ Read = PAGE_READONLY,
+ ReadWrite = PAGE_READWRITE,
+ Execute = PAGE_EXECUTE,
+ ReadExecute = PAGE_EXECUTE_READ,
+ ReadWriteExecute = PAGE_EXECUTE_READWRITE,
+};
+#elif defined(__wasi__)
+enum class PageAccess : int {
+ None = 0,
+ Read = 0,
+ ReadWrite = 0,
+ Execute = 0,
+ ReadExecute = 0,
+ ReadWriteExecute = 0,
+};
+#else
+enum class PageAccess : int {
+ None = PROT_NONE,
+ Read = PROT_READ,
+ ReadWrite = PROT_READ | PROT_WRITE,
+ Execute = PROT_EXEC,
+ ReadExecute = PROT_READ | PROT_EXEC,
+ ReadWriteExecute = PROT_READ | PROT_WRITE | PROT_EXEC,
+};
+#endif
+
+template <bool AlwaysGetNew = true>
+static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion,
+ size_t length, size_t alignment);
+
+#ifndef __wasi__
+static void* MapAlignedPagesSlow(size_t length, size_t alignment);
+#endif // wasi
+static void* MapAlignedPagesLastDitch(size_t length, size_t alignment);
+
+#ifdef JS_64BIT
+static void* MapAlignedPagesRandom(size_t length, size_t alignment);
+#endif
+
+void* TestMapAlignedPagesLastDitch(size_t length, size_t alignment) {
+ return MapAlignedPagesLastDitch(length, alignment);
+}
+
+bool DecommitEnabled() { return SystemPageSize() == PageSize; }
+
+/* Returns the offset from the nearest aligned address at or below |region|. */
+static inline size_t OffsetFromAligned(void* region, size_t alignment) {
+ return uintptr_t(region) % alignment;
+}
+
+template <Commit commit, PageAccess prot>
+static inline void* MapInternal(void* desired, size_t length) {
+ void* region = nullptr;
+#ifdef XP_WIN
+ DWORD flags =
+ (commit == Commit::Yes ? MEM_RESERVE | MEM_COMMIT : MEM_RESERVE);
+ region = VirtualAlloc(desired, length, flags, DWORD(prot));
+#elif defined(__wasi__)
+ if (int err = posix_memalign(&region, gc::SystemPageSize(), length)) {
+ MOZ_RELEASE_ASSERT(err == ENOMEM);
+ return nullptr;
+ }
+ if (region) {
+ memset(region, 0, length);
+ }
+#else
+ int flags = MAP_PRIVATE | MAP_ANON;
+ region = MozTaggedAnonymousMmap(desired, length, int(prot), flags, -1, 0,
+ "js-gc-heap");
+ if (region == MAP_FAILED) {
+ return nullptr;
+ }
+#endif
+ return region;
+}
+
+static inline void UnmapInternal(void* region, size_t length) {
+ MOZ_ASSERT(region && OffsetFromAligned(region, allocGranularity) == 0);
+ MOZ_ASSERT(length > 0 && length % pageSize == 0);
+
+#ifdef XP_WIN
+ MOZ_RELEASE_ASSERT(VirtualFree(region, 0, MEM_RELEASE) != 0);
+#elif defined(__wasi__)
+ free(region);
+#else
+ if (munmap(region, length)) {
+ MOZ_RELEASE_ASSERT(errno == ENOMEM);
+ }
+#endif
+}
+
+template <Commit commit = Commit::Yes, PageAccess prot = PageAccess::ReadWrite>
+static inline void* MapMemory(size_t length) {
+ MOZ_ASSERT(length > 0);
+
+ return MapInternal<commit, prot>(nullptr, length);
+}
+
+/*
+ * Attempts to map memory at the given address, but allows the system
+ * to return a different address that may still be suitable.
+ */
+template <Commit commit = Commit::Yes, PageAccess prot = PageAccess::ReadWrite>
+static inline void* MapMemoryAtFuzzy(void* desired, size_t length) {
+ MOZ_ASSERT(desired && OffsetFromAligned(desired, allocGranularity) == 0);
+ MOZ_ASSERT(length > 0);
+
+ // Note that some platforms treat the requested address as a hint, so the
+ // returned address might not match the requested address.
+ return MapInternal<commit, prot>(desired, length);
+}
+
+/*
+ * Attempts to map memory at the given address, returning nullptr if
+ * the system returns any address other than the requested one.
+ */
+template <Commit commit = Commit::Yes, PageAccess prot = PageAccess::ReadWrite>
+static inline void* MapMemoryAt(void* desired, size_t length) {
+ MOZ_ASSERT(desired && OffsetFromAligned(desired, allocGranularity) == 0);
+ MOZ_ASSERT(length > 0);
+
+ void* region = MapInternal<commit, prot>(desired, length);
+ if (!region) {
+ return nullptr;
+ }
+
+ // On some platforms mmap treats the desired address as a hint, so
+ // check that the address we got is the address we requested.
+ if (region != desired) {
+ UnmapInternal(region, length);
+ return nullptr;
+ }
+ return region;
+}
+
+#ifdef JS_64BIT
+
+/* Returns a random number in the given range. */
+static inline uint64_t GetNumberInRange(uint64_t minNum, uint64_t maxNum) {
+ const uint64_t MaxRand = UINT64_C(0xffffffffffffffff);
+ maxNum -= minNum;
+ uint64_t binSize = 1 + (MaxRand - maxNum) / (maxNum + 1);
+
+ uint64_t rndNum;
+ do {
+ mozilla::Maybe<uint64_t> result;
+ do {
+ result = mozilla::RandomUint64();
+ } while (!result);
+ rndNum = result.value() / binSize;
+ } while (rndNum > maxNum);
+
+ return minNum + rndNum;
+}
+
+# ifndef XP_WIN
+static inline uint64_t FindAddressLimitInner(size_t highBit, size_t tries);
+
+/*
+ * The address range available to applications depends on both hardware and
+ * kernel configuration. For example, AArch64 on Linux uses addresses with
+ * 39 significant bits by default, but can be configured to use addresses with
+ * 48 significant bits by enabling a 4th translation table. Unfortunately,
+ * there appears to be no standard way to query the limit at runtime
+ * (Windows exposes this via GetSystemInfo()).
+ *
+ * This function tries to find the address limit by performing a binary search
+ * on the index of the most significant set bit in the addresses it attempts to
+ * allocate. As the requested address is often treated as a hint by the
+ * operating system, we use the actual returned addresses to narrow the range.
+ * We return the number of bits of an address that may be set.
+ */
+static size_t FindAddressLimit() {
+ // Use 32 bits as a lower bound in case we keep getting nullptr.
+ uint64_t low = 31;
+ uint64_t highestSeen = (UINT64_C(1) << 32) - allocGranularity - 1;
+
+ // Exclude 48-bit and 47-bit addresses first.
+ uint64_t high = 47;
+ for (; high >= std::max(low, UINT64_C(46)); --high) {
+ highestSeen = std::max(FindAddressLimitInner(high, 4), highestSeen);
+ low = mozilla::FloorLog2(highestSeen);
+ }
+ // If those didn't work, perform a modified binary search.
+ while (high - 1 > low) {
+ uint64_t middle = low + (high - low) / 2;
+ highestSeen = std::max(FindAddressLimitInner(middle, 4), highestSeen);
+ low = mozilla::FloorLog2(highestSeen);
+ if (highestSeen < (UINT64_C(1) << middle)) {
+ high = middle;
+ }
+ }
+ // We can be sure of the lower bound, but check the upper bound again.
+ do {
+ high = low + 1;
+ highestSeen = std::max(FindAddressLimitInner(high, 8), highestSeen);
+ low = mozilla::FloorLog2(highestSeen);
+ } while (low >= high);
+
+ // `low` is the highest set bit, so `low + 1` is the number of bits.
+ return low + 1;
+}
+
+static inline uint64_t FindAddressLimitInner(size_t highBit, size_t tries) {
+ const size_t length = allocGranularity; // Used as both length and alignment.
+
+ uint64_t highestSeen = 0;
+ uint64_t startRaw = UINT64_C(1) << highBit;
+ uint64_t endRaw = 2 * startRaw - length - 1;
+ uint64_t start = (startRaw + length - 1) / length;
+ uint64_t end = (endRaw - (length - 1)) / length;
+ for (size_t i = 0; i < tries; ++i) {
+ uint64_t desired = length * GetNumberInRange(start, end);
+ void* address = MapMemoryAtFuzzy(reinterpret_cast<void*>(desired), length);
+ uint64_t actual = uint64_t(address);
+ if (address) {
+ UnmapInternal(address, length);
+ }
+ if (actual > highestSeen) {
+ highestSeen = actual;
+ if (actual >= startRaw) {
+ break;
+ }
+ }
+ }
+ return highestSeen;
+}
+# endif // !defined(XP_WIN)
+
+#endif // defined(JS_64BIT)
+
+void InitMemorySubsystem() {
+ if (pageSize == 0) {
+#ifdef XP_WIN
+ SYSTEM_INFO sysinfo;
+ GetSystemInfo(&sysinfo);
+ pageSize = sysinfo.dwPageSize;
+ allocGranularity = sysinfo.dwAllocationGranularity;
+#else
+ pageSize = size_t(sysconf(_SC_PAGESIZE));
+ allocGranularity = pageSize;
+#endif
+#ifdef JS_64BIT
+# ifdef XP_WIN
+ minValidAddress = size_t(sysinfo.lpMinimumApplicationAddress);
+ maxValidAddress = size_t(sysinfo.lpMaximumApplicationAddress);
+ numAddressBits = mozilla::FloorLog2(maxValidAddress) + 1;
+# else
+ // No standard way to determine these, so fall back to FindAddressLimit().
+ numAddressBits = FindAddressLimit();
+ minValidAddress = allocGranularity;
+ maxValidAddress = (UINT64_C(1) << numAddressBits) - 1 - allocGranularity;
+# endif
+ // Sanity check the address to ensure we don't use more than 47 bits.
+ uint64_t maxJSAddress = UINT64_C(0x00007fffffffffff) - allocGranularity;
+ if (maxValidAddress > maxJSAddress) {
+ maxValidAddress = maxJSAddress;
+ hugeSplit = UINT64_C(0x00003fffffffffff) - allocGranularity;
+ } else {
+ hugeSplit = (UINT64_C(1) << (numAddressBits - 1)) - 1 - allocGranularity;
+ }
+#else // !defined(JS_64BIT)
+ numAddressBits = 32;
+#endif
+#ifdef RLIMIT_AS
+ if (jit::HasJitBackend()) {
+ rlimit as_limit;
+ if (getrlimit(RLIMIT_AS, &as_limit) == 0 &&
+ as_limit.rlim_max != RLIM_INFINITY) {
+ virtualMemoryLimit = as_limit.rlim_max;
+ }
+ }
+#endif
+ }
+}
+
+#ifdef JS_64BIT
+/* The JS engine uses 47-bit pointers; all higher bits must be clear. */
+static inline bool IsInvalidRegion(void* region, size_t length) {
+ const uint64_t invalidPointerMask = UINT64_C(0xffff800000000000);
+ return (uintptr_t(region) + length - 1) & invalidPointerMask;
+}
+#endif
+
+void* MapAlignedPages(size_t length, size_t alignment) {
+ MOZ_RELEASE_ASSERT(length > 0 && alignment > 0);
+ MOZ_RELEASE_ASSERT(length % pageSize == 0);
+ MOZ_RELEASE_ASSERT(std::max(alignment, allocGranularity) %
+ std::min(alignment, allocGranularity) ==
+ 0);
+
+ // Smaller alignments aren't supported by the allocation functions.
+ if (alignment < allocGranularity) {
+ alignment = allocGranularity;
+ }
+
+#ifdef __wasi__
+ void* region = nullptr;
+ if (int err = posix_memalign(&region, alignment, length)) {
+ MOZ_ASSERT(err == ENOMEM);
+ return nullptr;
+ }
+ MOZ_ASSERT(region != nullptr);
+ memset(region, 0, length);
+ return region;
+#else
+
+# ifdef JS_64BIT
+ // Use the scattershot allocator if the address range is large enough.
+ if (UsingScattershotAllocator()) {
+ void* region = MapAlignedPagesRandom(length, alignment);
+
+ MOZ_RELEASE_ASSERT(!IsInvalidRegion(region, length));
+ MOZ_ASSERT(OffsetFromAligned(region, alignment) == 0);
+
+ return region;
+ }
+# endif
+
+ // Try to allocate the region. If the returned address is aligned,
+ // either we OOMed (region is nullptr) or we're done.
+ void* region = MapMemory(length);
+ if (OffsetFromAligned(region, alignment) == 0) {
+ return region;
+ }
+
+ // Try to align the region. On success, TryToAlignChunk() returns
+ // true and we can return the aligned region immediately.
+ void* retainedRegion;
+ if (TryToAlignChunk(&region, &retainedRegion, length, alignment)) {
+ MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0);
+ MOZ_ASSERT(!retainedRegion);
+ return region;
+ }
+
+ // On failure, the unaligned region is retained unless we OOMed. We don't
+ // use the retained region on this path (see the last ditch allocator).
+ if (retainedRegion) {
+ UnmapInternal(retainedRegion, length);
+ }
+
+ // If it fails to align the given region, TryToAlignChunk() returns the
+ // next valid region that we might be able to align (unless we OOMed).
+ if (region) {
+ MOZ_ASSERT(OffsetFromAligned(region, alignment) != 0);
+ UnmapInternal(region, length);
+ }
+
+ // Since we couldn't align the first region, fall back to allocating a
+ // region large enough that we can definitely align it.
+ region = MapAlignedPagesSlow(length, alignment);
+ if (!region) {
+ // If there wasn't enough contiguous address space left for that,
+ // try to find an alignable region using the last ditch allocator.
+ region = MapAlignedPagesLastDitch(length, alignment);
+ }
+
+ // At this point we should either have an aligned region or nullptr.
+ MOZ_ASSERT(OffsetFromAligned(region, alignment) == 0);
+ return region;
+#endif // !__wasi__
+}
+
+#ifdef JS_64BIT
+
+/*
+ * This allocator takes advantage of the large address range on some 64-bit
+ * platforms to allocate in a scattershot manner, choosing addresses at random
+ * from the range. By controlling the range we can avoid returning addresses
+ * that have more than 47 significant bits (as required by SpiderMonkey).
+ * This approach also has some other advantages over the methods employed by
+ * the other allocation functions in this file:
+ * 1) Allocations are extremely likely to succeed on the first try.
+ * 2) The randomness makes our memory layout becomes harder to predict.
+ * 3) The low probability of reusing regions guards against use-after-free.
+ *
+ * The main downside is that detecting physical OOM situations becomes more
+ * difficult; to guard against this, we occasionally try a regular allocation.
+ * In addition, sprinkling small allocations throughout the full address range
+ * might get in the way of large address space reservations such as those
+ * employed by WebAssembly. To avoid this (or the opposite problem of such
+ * reservations reducing the chance of success for smaller allocations) we
+ * split the address range in half, with one half reserved for huge allocations
+ * and the other for regular (usually chunk sized) allocations.
+ */
+static void* MapAlignedPagesRandom(size_t length, size_t alignment) {
+ uint64_t minNum, maxNum;
+ if (length < HugeAllocationSize) {
+ // Use the lower half of the range.
+ minNum = (minValidAddress + alignment - 1) / alignment;
+ maxNum = (hugeSplit - (length - 1)) / alignment;
+ } else {
+ // Use the upper half of the range.
+ minNum = (hugeSplit + 1 + alignment - 1) / alignment;
+ maxNum = (maxValidAddress - (length - 1)) / alignment;
+ }
+
+ // Try to allocate in random aligned locations.
+ void* region = nullptr;
+ for (size_t i = 1; i <= 1024; ++i) {
+ if (i & 0xf) {
+ uint64_t desired = alignment * GetNumberInRange(minNum, maxNum);
+ region = MapMemoryAtFuzzy(reinterpret_cast<void*>(desired), length);
+ if (!region) {
+ continue;
+ }
+ } else {
+ // Check for OOM.
+ region = MapMemory(length);
+ if (!region) {
+ return nullptr;
+ }
+ }
+ if (IsInvalidRegion(region, length)) {
+ UnmapInternal(region, length);
+ continue;
+ }
+ if (OffsetFromAligned(region, alignment) == 0) {
+ return region;
+ }
+ void* retainedRegion = nullptr;
+ if (TryToAlignChunk<false>(&region, &retainedRegion, length, alignment)) {
+ MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0);
+ MOZ_ASSERT(!retainedRegion);
+ return region;
+ }
+ MOZ_ASSERT(region && !retainedRegion);
+ UnmapInternal(region, length);
+ }
+
+ if (numAddressBits < 48) {
+ // Try the reliable fallback of overallocating.
+ // Note: This will not respect the address space split.
+ region = MapAlignedPagesSlow(length, alignment);
+ if (region) {
+ return region;
+ }
+ }
+ if (length < HugeAllocationSize) {
+ MOZ_CRASH("Couldn't allocate even after 1000 tries!");
+ }
+
+ return nullptr;
+}
+
+#endif // defined(JS_64BIT)
+
+#ifndef __wasi__
+static void* MapAlignedPagesSlow(size_t length, size_t alignment) {
+ void* alignedRegion = nullptr;
+ do {
+ size_t reserveLength = length + alignment - pageSize;
+# ifdef XP_WIN
+ // Don't commit the requested pages as we won't use the region directly.
+ void* region = MapMemory<Commit::No>(reserveLength);
+# else
+ void* region = MapMemory(reserveLength);
+# endif
+ if (!region) {
+ return nullptr;
+ }
+ alignedRegion =
+ reinterpret_cast<void*>(AlignBytes(uintptr_t(region), alignment));
+# ifdef XP_WIN
+ // Windows requires that map and unmap calls be matched, so deallocate
+ // and immediately reallocate at the desired (aligned) address.
+ UnmapInternal(region, reserveLength);
+ alignedRegion = MapMemoryAt(alignedRegion, length);
+# else
+ // munmap allows us to simply unmap the pages that don't interest us.
+ if (alignedRegion != region) {
+ UnmapInternal(region, uintptr_t(alignedRegion) - uintptr_t(region));
+ }
+ void* regionEnd =
+ reinterpret_cast<void*>(uintptr_t(region) + reserveLength);
+ void* alignedEnd =
+ reinterpret_cast<void*>(uintptr_t(alignedRegion) + length);
+ if (alignedEnd != regionEnd) {
+ UnmapInternal(alignedEnd, uintptr_t(regionEnd) - uintptr_t(alignedEnd));
+ }
+# endif
+ // On Windows we may have raced with another thread; if so, try again.
+ } while (!alignedRegion);
+
+ return alignedRegion;
+}
+#endif // wasi
+
+/*
+ * In a low memory or high fragmentation situation, alignable chunks of the
+ * desired length may still be available, even if there are no more contiguous
+ * free chunks that meet the |length + alignment - pageSize| requirement of
+ * MapAlignedPagesSlow. In this case, try harder to find an alignable chunk
+ * by temporarily holding onto the unaligned parts of each chunk until the
+ * allocator gives us a chunk that either is, or can be aligned.
+ */
+static void* MapAlignedPagesLastDitch(size_t length, size_t alignment) {
+ void* tempMaps[MaxLastDitchAttempts];
+ int attempt = 0;
+ void* region = MapMemory(length);
+ if (OffsetFromAligned(region, alignment) == 0) {
+ return region;
+ }
+ for (; attempt < MaxLastDitchAttempts; ++attempt) {
+ if (TryToAlignChunk(&region, tempMaps + attempt, length, alignment)) {
+ MOZ_ASSERT(region && OffsetFromAligned(region, alignment) == 0);
+ MOZ_ASSERT(!tempMaps[attempt]);
+ break; // Success!
+ }
+ if (!region || !tempMaps[attempt]) {
+ break; // We ran out of memory, so give up.
+ }
+ }
+ if (OffsetFromAligned(region, alignment)) {
+ UnmapInternal(region, length);
+ region = nullptr;
+ }
+ while (--attempt >= 0) {
+ UnmapInternal(tempMaps[attempt], length);
+ }
+ return region;
+}
+
+#ifdef XP_WIN
+
+/*
+ * On Windows, map and unmap calls must be matched, so we deallocate the
+ * unaligned chunk, then reallocate the unaligned part to block off the
+ * old address and force the allocator to give us a new one.
+ */
+template <bool>
+static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion,
+ size_t length, size_t alignment) {
+ void* region = *aRegion;
+ MOZ_ASSERT(region && OffsetFromAligned(region, alignment) != 0);
+
+ size_t retainedLength = 0;
+ void* retainedRegion = nullptr;
+ do {
+ size_t offset = OffsetFromAligned(region, alignment);
+ if (offset == 0) {
+ // If the address is aligned, either we hit OOM or we're done.
+ break;
+ }
+ UnmapInternal(region, length);
+ retainedLength = alignment - offset;
+ retainedRegion = MapMemoryAt<Commit::No>(region, retainedLength);
+ region = MapMemory(length);
+
+ // If retainedRegion is null here, we raced with another thread.
+ } while (!retainedRegion);
+
+ bool result = OffsetFromAligned(region, alignment) == 0;
+ if (result && retainedRegion) {
+ UnmapInternal(retainedRegion, retainedLength);
+ retainedRegion = nullptr;
+ }
+
+ *aRegion = region;
+ *aRetainedRegion = retainedRegion;
+ return region && result;
+}
+
+#else // !defined(XP_WIN)
+
+/*
+ * mmap calls don't have to be matched with calls to munmap, so we can unmap
+ * just the pages we don't need. However, as we don't know a priori if addresses
+ * are handed out in increasing or decreasing order, we have to try both
+ * directions (depending on the environment, one will always fail).
+ */
+template <bool AlwaysGetNew>
+static bool TryToAlignChunk(void** aRegion, void** aRetainedRegion,
+ size_t length, size_t alignment) {
+ void* regionStart = *aRegion;
+ MOZ_ASSERT(regionStart && OffsetFromAligned(regionStart, alignment) != 0);
+
+ bool addressesGrowUpward = growthDirection > 0;
+ bool directionUncertain = -8 < growthDirection && growthDirection <= 8;
+ size_t offsetLower = OffsetFromAligned(regionStart, alignment);
+ size_t offsetUpper = alignment - offsetLower;
+ for (size_t i = 0; i < 2; ++i) {
+ if (addressesGrowUpward) {
+ void* upperStart =
+ reinterpret_cast<void*>(uintptr_t(regionStart) + offsetUpper);
+ void* regionEnd =
+ reinterpret_cast<void*>(uintptr_t(regionStart) + length);
+ if (MapMemoryAt(regionEnd, offsetUpper)) {
+ UnmapInternal(regionStart, offsetUpper);
+ if (directionUncertain) {
+ ++growthDirection;
+ }
+ regionStart = upperStart;
+ break;
+ }
+ } else {
+ auto* lowerStart =
+ reinterpret_cast<void*>(uintptr_t(regionStart) - offsetLower);
+ auto* lowerEnd = reinterpret_cast<void*>(uintptr_t(lowerStart) + length);
+ if (MapMemoryAt(lowerStart, offsetLower)) {
+ UnmapInternal(lowerEnd, offsetLower);
+ if (directionUncertain) {
+ --growthDirection;
+ }
+ regionStart = lowerStart;
+ break;
+ }
+ }
+ // If we're confident in the growth direction, don't try the other.
+ if (!directionUncertain) {
+ break;
+ }
+ addressesGrowUpward = !addressesGrowUpward;
+ }
+
+ void* retainedRegion = nullptr;
+ bool result = OffsetFromAligned(regionStart, alignment) == 0;
+ if (AlwaysGetNew && !result) {
+ // If our current chunk cannot be aligned, just get a new one.
+ retainedRegion = regionStart;
+ regionStart = MapMemory(length);
+ // Our new region might happen to already be aligned.
+ result = OffsetFromAligned(regionStart, alignment) == 0;
+ if (result) {
+ UnmapInternal(retainedRegion, length);
+ retainedRegion = nullptr;
+ }
+ }
+
+ *aRegion = regionStart;
+ *aRetainedRegion = retainedRegion;
+ return regionStart && result;
+}
+
+#endif
+
+void UnmapPages(void* region, size_t length) {
+ MOZ_RELEASE_ASSERT(region &&
+ OffsetFromAligned(region, allocGranularity) == 0);
+ MOZ_RELEASE_ASSERT(length > 0 && length % pageSize == 0);
+
+ // ASan does not automatically unpoison memory, so we have to do this here.
+ MOZ_MAKE_MEM_UNDEFINED(region, length);
+
+ UnmapInternal(region, length);
+}
+
+static void CheckDecommit(void* region, size_t length) {
+ MOZ_RELEASE_ASSERT(region);
+ MOZ_RELEASE_ASSERT(length > 0);
+
+ // pageSize == ArenaSize doesn't necessarily hold, but this function is
+ // used by the GC to decommit unused Arenas, so we don't want to assert
+ // if pageSize > ArenaSize.
+ MOZ_ASSERT(OffsetFromAligned(region, ArenaSize) == 0);
+ MOZ_ASSERT(length % ArenaSize == 0);
+
+ MOZ_RELEASE_ASSERT(OffsetFromAligned(region, pageSize) == 0);
+ MOZ_RELEASE_ASSERT(length % pageSize == 0);
+}
+
+bool MarkPagesUnusedSoft(void* region, size_t length) {
+ MOZ_ASSERT(DecommitEnabled());
+ CheckDecommit(region, length);
+
+ MOZ_MAKE_MEM_NOACCESS(region, length);
+
+#if defined(XP_WIN)
+ return VirtualAlloc(region, length, MEM_RESET,
+ DWORD(PageAccess::ReadWrite)) == region;
+#elif defined(__wasi__)
+ return 0;
+#else
+ int status;
+ do {
+# if defined(XP_DARWIN)
+ status = madvise(region, length, MADV_FREE_REUSABLE);
+# elif defined(XP_SOLARIS)
+ status = posix_madvise(region, length, POSIX_MADV_DONTNEED);
+# else
+ status = madvise(region, length, MADV_DONTNEED);
+# endif
+ } while (status == -1 && errno == EAGAIN);
+ return status == 0;
+#endif
+}
+
+bool MarkPagesUnusedHard(void* region, size_t length) {
+ CheckDecommit(region, length);
+
+ MOZ_MAKE_MEM_NOACCESS(region, length);
+
+ if (!DecommitEnabled()) {
+ return true;
+ }
+
+#if defined(XP_WIN)
+ return VirtualFree(region, length, MEM_DECOMMIT);
+#else
+ return MarkPagesUnusedSoft(region, length);
+#endif
+}
+
+void MarkPagesInUseSoft(void* region, size_t length) {
+ MOZ_ASSERT(DecommitEnabled());
+ CheckDecommit(region, length);
+
+#if defined(XP_DARWIN)
+ while (madvise(region, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) {
+ }
+#endif
+
+ MOZ_MAKE_MEM_UNDEFINED(region, length);
+}
+
+bool MarkPagesInUseHard(void* region, size_t length) {
+ if (js::oom::ShouldFailWithOOM()) {
+ return false;
+ }
+
+ CheckDecommit(region, length);
+
+ MOZ_MAKE_MEM_UNDEFINED(region, length);
+
+ if (!DecommitEnabled()) {
+ return true;
+ }
+
+#if defined(XP_WIN)
+ return VirtualAlloc(region, length, MEM_COMMIT,
+ DWORD(PageAccess::ReadWrite)) == region;
+#else
+ return true;
+#endif
+}
+
+size_t GetPageFaultCount() {
+#ifdef XP_WIN
+ PROCESS_MEMORY_COUNTERS pmc;
+ if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) == 0) {
+ return 0;
+ }
+ return pmc.PageFaultCount;
+#elif defined(__wasi__)
+ return 0;
+#else
+ struct rusage usage;
+ int err = getrusage(RUSAGE_SELF, &usage);
+ if (err) {
+ return 0;
+ }
+ return usage.ru_majflt;
+#endif
+}
+
+void* AllocateMappedContent(int fd, size_t offset, size_t length,
+ size_t alignment) {
+#ifdef __wasi__
+ MOZ_CRASH("Not yet supported for WASI");
+#else
+ if (length == 0 || alignment == 0 || offset % alignment != 0 ||
+ std::max(alignment, allocGranularity) %
+ std::min(alignment, allocGranularity) !=
+ 0) {
+ return nullptr;
+ }
+
+ size_t alignedOffset = offset - (offset % allocGranularity);
+ size_t alignedLength = length + (offset % allocGranularity);
+
+ // We preallocate the mapping using MapAlignedPages, which expects
+ // the length parameter to be an integer multiple of the page size.
+ size_t mappedLength = alignedLength;
+ if (alignedLength % pageSize != 0) {
+ mappedLength += pageSize - alignedLength % pageSize;
+ }
+
+# ifdef XP_WIN
+ HANDLE hFile = reinterpret_cast<HANDLE>(intptr_t(fd));
+
+ // This call will fail if the file does not exist.
+ HANDLE hMap =
+ CreateFileMappingW(hFile, nullptr, PAGE_READONLY, 0, 0, nullptr);
+ if (!hMap) {
+ return nullptr;
+ }
+
+ DWORD offsetH = uint32_t(uint64_t(alignedOffset) >> 32);
+ DWORD offsetL = uint32_t(alignedOffset);
+
+ uint8_t* map = nullptr;
+ for (;;) {
+ // The value of a pointer is technically only defined while the region
+ // it points to is allocated, so explicitly treat this one as a number.
+ uintptr_t region = uintptr_t(MapAlignedPages(mappedLength, alignment));
+ if (region == 0) {
+ break;
+ }
+ UnmapInternal(reinterpret_cast<void*>(region), mappedLength);
+ // If the offset or length are out of bounds, this call will fail.
+ map = static_cast<uint8_t*>(
+ MapViewOfFileEx(hMap, FILE_MAP_COPY, offsetH, offsetL, alignedLength,
+ reinterpret_cast<void*>(region)));
+
+ // Retry if another thread mapped the address we were trying to use.
+ if (map || GetLastError() != ERROR_INVALID_ADDRESS) {
+ break;
+ }
+ }
+
+ // This just decreases the file mapping object's internal reference count;
+ // it won't actually be destroyed until we unmap the associated view.
+ CloseHandle(hMap);
+
+ if (!map) {
+ return nullptr;
+ }
+# else // !defined(XP_WIN)
+ // Sanity check the offset and length, as mmap does not do this for us.
+ struct stat st;
+ if (fstat(fd, &st) || offset >= uint64_t(st.st_size) ||
+ length > uint64_t(st.st_size) - offset) {
+ return nullptr;
+ }
+
+ void* region = MapAlignedPages(mappedLength, alignment);
+ if (!region) {
+ return nullptr;
+ }
+
+ // Calling mmap with MAP_FIXED will replace the previous mapping, allowing
+ // us to reuse the region we obtained without racing with other threads.
+ uint8_t* map =
+ static_cast<uint8_t*>(mmap(region, alignedLength, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_FIXED, fd, alignedOffset));
+ if (map == MAP_FAILED) {
+ UnmapInternal(region, mappedLength);
+ return nullptr;
+ }
+# endif
+
+# ifdef DEBUG
+ // Zero out data before and after the desired mapping to catch errors early.
+ if (offset != alignedOffset) {
+ memset(map, 0, offset - alignedOffset);
+ }
+ if (alignedLength % pageSize) {
+ memset(map + alignedLength, 0, pageSize - (alignedLength % pageSize));
+ }
+# endif
+
+ return map + (offset - alignedOffset);
+#endif // __wasi__
+}
+
+void DeallocateMappedContent(void* region, size_t length) {
+#ifdef __wasi__
+ MOZ_CRASH("Not yet supported for WASI");
+#else
+ if (!region) {
+ return;
+ }
+
+ // Due to bug 1502562, the following assertion does not currently hold.
+ // MOZ_RELEASE_ASSERT(length > 0);
+
+ // Calculate the address originally returned by the system call.
+ // This is needed because AllocateMappedContent returns a pointer
+ // that might be offset from the mapping, as the beginning of a
+ // mapping must be aligned with the allocation granularity.
+ uintptr_t map = uintptr_t(region) - (uintptr_t(region) % allocGranularity);
+# ifdef XP_WIN
+ MOZ_RELEASE_ASSERT(UnmapViewOfFile(reinterpret_cast<void*>(map)) != 0);
+# else
+ size_t alignedLength = length + (uintptr_t(region) % allocGranularity);
+ if (munmap(reinterpret_cast<void*>(map), alignedLength)) {
+ MOZ_RELEASE_ASSERT(errno == ENOMEM);
+ }
+# endif
+#endif // __wasi__
+}
+
+static inline void ProtectMemory(void* region, size_t length, PageAccess prot) {
+ MOZ_RELEASE_ASSERT(region && OffsetFromAligned(region, pageSize) == 0);
+ MOZ_RELEASE_ASSERT(length > 0 && length % pageSize == 0);
+#ifdef XP_WIN
+ DWORD oldProtect;
+ MOZ_RELEASE_ASSERT(VirtualProtect(region, length, DWORD(prot), &oldProtect) !=
+ 0);
+#elif defined(__wasi__)
+ /* nothing */
+#else
+ MOZ_RELEASE_ASSERT(mprotect(region, length, int(prot)) == 0);
+#endif
+}
+
+void ProtectPages(void* region, size_t length) {
+ ProtectMemory(region, length, PageAccess::None);
+}
+
+void MakePagesReadOnly(void* region, size_t length) {
+ ProtectMemory(region, length, PageAccess::Read);
+}
+
+void UnprotectPages(void* region, size_t length) {
+ ProtectMemory(region, length, PageAccess::ReadWrite);
+}
+
+} // namespace gc
+} // namespace js
diff --git a/js/src/gc/Memory.h b/js/src/gc/Memory.h
new file mode 100644
index 0000000000..e49a10f04b
--- /dev/null
+++ b/js/src/gc/Memory.h
@@ -0,0 +1,84 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Memory_h
+#define gc_Memory_h
+
+#include <stddef.h>
+
+namespace js {
+namespace gc {
+
+// Sanity check that our compiled configuration matches the currently
+// running instance and initialize any runtime data needed for allocation.
+void InitMemorySubsystem();
+
+// The page size as reported by the operating system.
+size_t SystemPageSize();
+
+// The number of bits that may be set in a valid address, as
+// reported by the operating system or measured at startup.
+size_t SystemAddressBits();
+
+// The number of bytes of virtual memory that may be allocated or mapped, as
+// reported by the operating system on certain platforms. If no limit was able
+// to be determined, then it will be size_t(-1).
+size_t VirtualMemoryLimit();
+
+// The scattershot allocator is used on platforms that have a large address
+// range. On these platforms we allocate at random addresses.
+bool UsingScattershotAllocator();
+
+// Allocate or deallocate pages from the system with the given alignment.
+// Pages will be read/write-able.
+void* MapAlignedPages(size_t length, size_t alignment);
+void UnmapPages(void* region, size_t length);
+
+// We can only decommit unused pages if the page size is less than or equal to
+// the hardcoded Arena size for the running process.
+bool DecommitEnabled();
+
+// Tell the OS that the given pages are not in use, so they should not be
+// written to a paging file. This may be a no-op on some platforms.
+bool MarkPagesUnusedSoft(void* region, size_t length);
+
+// Tell the OS that the given pages are not in use and it can decommit them
+// immediately. This may defer to MarkPagesUnusedSoft and must be paired with
+// MarkPagesInUse to use the pages again.
+bool MarkPagesUnusedHard(void* region, size_t length);
+
+// Undo |MarkPagesUnusedSoft|: tell the OS that the given pages are of interest
+// and should be paged in and out normally. This may be a no-op on some
+// platforms. May make pages read/write-able.
+void MarkPagesInUseSoft(void* region, size_t length);
+
+// Undo |MarkPagesUnusedHard|: tell the OS that the given pages are of interest
+// and should be paged in and out normally. This may be a no-op on some
+// platforms. Callers must check the result, false could mean that the pages
+// are not available. May make pages read/write.
+[[nodiscard]] bool MarkPagesInUseHard(void* region, size_t length);
+
+// Returns #(hard faults) + #(soft faults)
+size_t GetPageFaultCount();
+
+// Allocate memory mapped content.
+// The offset must be aligned according to alignment requirement.
+void* AllocateMappedContent(int fd, size_t offset, size_t length,
+ size_t alignment);
+
+// Deallocate memory mapped content.
+void DeallocateMappedContent(void* region, size_t length);
+
+void* TestMapAlignedPagesLastDitch(size_t size, size_t alignment);
+
+void ProtectPages(void* region, size_t length);
+void MakePagesReadOnly(void* region, size_t length);
+void UnprotectPages(void* region, size_t length);
+
+} // namespace gc
+} // namespace js
+
+#endif /* gc_Memory_h */
diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h
new file mode 100644
index 0000000000..48f25a536e
--- /dev/null
+++ b/js/src/gc/Nursery-inl.h
@@ -0,0 +1,172 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=4 sw=2 et tw=80 ft=cpp:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Nursery_inl_h
+#define gc_Nursery_inl_h
+
+#include "gc/Nursery.h"
+
+#include "gc/RelocationOverlay.h"
+#include "js/TracingAPI.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+
+#include "vm/JSContext-inl.h"
+
+inline JSRuntime* js::Nursery::runtime() const { return gc->rt; }
+
+template <typename T>
+bool js::Nursery::isInside(const SharedMem<T>& p) const {
+ return isInside(p.unwrap(/*safe - used for value in comparison above*/));
+}
+
+MOZ_ALWAYS_INLINE /* static */ bool js::Nursery::getForwardedPointer(
+ js::gc::Cell** ref) {
+ js::gc::Cell* cell = (*ref);
+ MOZ_ASSERT(IsInsideNursery(cell));
+ if (!cell->isForwarded()) {
+ return false;
+ }
+ const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(cell);
+ *ref = overlay->forwardingAddress();
+ return true;
+}
+
+inline void js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData,
+ void* newData, bool direct) {
+ if (trc->isTenuringTracer()) {
+ setForwardingPointerWhileTenuring(oldData, newData, direct);
+ }
+}
+
+inline void js::Nursery::setForwardingPointerWhileTenuring(void* oldData,
+ void* newData,
+ bool direct) {
+ if (isInside(oldData)) {
+ setForwardingPointer(oldData, newData, direct);
+ }
+}
+
+inline void js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots,
+ HeapSlot* newSlots,
+ uint32_t nslots) {
+ // Slot arrays always have enough space for a forwarding pointer, since the
+ // number of slots is never zero.
+ MOZ_ASSERT(nslots > 0);
+ setDirectForwardingPointer(oldSlots, newSlots);
+}
+
+inline void js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader,
+ ObjectElements* newHeader,
+ uint32_t capacity) {
+ // Only use a direct forwarding pointer if there is enough space for one.
+ setForwardingPointer(oldHeader->elements(), newHeader->elements(),
+ capacity > 0);
+}
+
+inline void js::Nursery::setForwardingPointer(void* oldData, void* newData,
+ bool direct) {
+ if (direct) {
+ setDirectForwardingPointer(oldData, newData);
+ return;
+ }
+
+ setIndirectForwardingPointer(oldData, newData);
+}
+
+inline void js::Nursery::setDirectForwardingPointer(void* oldData,
+ void* newData) {
+ MOZ_ASSERT(isInside(oldData));
+ MOZ_ASSERT(!isInside(newData));
+
+ new (oldData) BufferRelocationOverlay{newData};
+}
+
+namespace js {
+
+// The allocation methods below will not run the garbage collector. If the
+// nursery cannot accomodate the allocation, the malloc heap will be used
+// instead.
+
+template <typename T>
+static inline T* AllocateObjectBuffer(JSContext* cx, uint32_t count) {
+ size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value));
+ auto* buffer =
+ static_cast<T*>(cx->nursery().allocateBuffer(cx->zone(), nbytes));
+ if (!buffer) {
+ ReportOutOfMemory(cx);
+ }
+ return buffer;
+}
+
+template <typename T>
+static inline T* AllocateObjectBuffer(JSContext* cx, JSObject* obj,
+ uint32_t count) {
+ MOZ_ASSERT(cx->isMainThreadContext());
+
+ size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value));
+ auto* buffer =
+ static_cast<T*>(cx->nursery().allocateBuffer(cx->zone(), obj, nbytes));
+ if (!buffer) {
+ ReportOutOfMemory(cx);
+ }
+ return buffer;
+}
+
+// If this returns null then the old buffer will be left alone.
+template <typename T>
+static inline T* ReallocateObjectBuffer(JSContext* cx, JSObject* obj,
+ T* oldBuffer, uint32_t oldCount,
+ uint32_t newCount) {
+ MOZ_ASSERT(cx->isMainThreadContext());
+
+ T* buffer = static_cast<T*>(cx->nursery().reallocateBuffer(
+ obj->zone(), obj, oldBuffer, oldCount * sizeof(T), newCount * sizeof(T)));
+ if (!buffer) {
+ ReportOutOfMemory(cx);
+ }
+
+ return buffer;
+}
+
+static inline JS::BigInt::Digit* AllocateBigIntDigits(JSContext* cx,
+ JS::BigInt* bi,
+ uint32_t length) {
+ MOZ_ASSERT(cx->isMainThreadContext());
+
+ size_t nbytes = RoundUp(length * sizeof(JS::BigInt::Digit), sizeof(Value));
+ auto* digits =
+ static_cast<JS::BigInt::Digit*>(cx->nursery().allocateBuffer(bi, nbytes));
+ if (!digits) {
+ ReportOutOfMemory(cx);
+ }
+
+ return digits;
+}
+
+static inline JS::BigInt::Digit* ReallocateBigIntDigits(
+ JSContext* cx, JS::BigInt* bi, JS::BigInt::Digit* oldDigits,
+ uint32_t oldLength, uint32_t newLength) {
+ MOZ_ASSERT(cx->isMainThreadContext());
+
+ size_t oldBytes =
+ RoundUp(oldLength * sizeof(JS::BigInt::Digit), sizeof(Value));
+ size_t newBytes =
+ RoundUp(newLength * sizeof(JS::BigInt::Digit), sizeof(Value));
+
+ auto* digits = static_cast<JS::BigInt::Digit*>(cx->nursery().reallocateBuffer(
+ bi->zone(), bi, oldDigits, oldBytes, newBytes));
+ if (!digits) {
+ ReportOutOfMemory(cx);
+ }
+
+ return digits;
+}
+
+} // namespace js
+
+#endif /* gc_Nursery_inl_h */
diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp
new file mode 100644
index 0000000000..17831b7f97
--- /dev/null
+++ b/js/src/gc/Nursery.cpp
@@ -0,0 +1,2083 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sw=2 et tw=80:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Nursery-inl.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/IntegerPrintfMacros.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Sprintf.h"
+#include "mozilla/TimeStamp.h"
+
+#include <algorithm>
+#include <cmath>
+#include <utility>
+
+#include "builtin/MapObject.h"
+#include "debugger/DebugAPI.h"
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/GCParallelTask.h"
+#include "gc/GCProbes.h"
+#include "gc/Memory.h"
+#include "gc/PublicIterators.h"
+#include "gc/Tenuring.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRealm.h"
+#include "js/Printer.h"
+#include "util/DifferentialTesting.h"
+#include "util/GetPidProvider.h" // getpid()
+#include "util/Poison.h"
+#include "vm/JSONPrinter.h"
+#include "vm/Realm.h"
+#include "vm/Time.h"
+
+#include "gc/Heap-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/StableCellHasher-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::DebugOnly;
+using mozilla::PodCopy;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+namespace js {
+
+struct NurseryChunk : public ChunkBase {
+ char data[Nursery::NurseryChunkUsableSize];
+
+ static NurseryChunk* fromChunk(gc::TenuredChunk* chunk);
+
+ explicit NurseryChunk(JSRuntime* runtime)
+ : ChunkBase(runtime, &runtime->gc.storeBuffer()) {}
+
+ void poisonAndInit(JSRuntime* rt, size_t size = ChunkSize);
+ void poisonRange(size_t from, size_t size, uint8_t value,
+ MemCheckKind checkKind);
+ void poisonAfterEvict(size_t extent = ChunkSize);
+
+ // Mark pages from startOffset to the end of the chunk as unused. The start
+ // offset must be after the first page, which contains the chunk header and is
+ // not marked as unused.
+ void markPagesUnusedHard(size_t startOffset);
+
+ // Mark pages from the second page of the chunk to endOffset as in use,
+ // following a call to markPagesUnusedHard.
+ [[nodiscard]] bool markPagesInUseHard(size_t endOffset);
+
+ uintptr_t start() const { return uintptr_t(&data); }
+ uintptr_t end() const { return uintptr_t(this) + ChunkSize; }
+};
+static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize,
+ "Nursery chunk size must match gc::Chunk size.");
+
+class NurseryDecommitTask : public GCParallelTask {
+ public:
+ explicit NurseryDecommitTask(gc::GCRuntime* gc);
+ bool reserveSpaceForBytes(size_t nbytes);
+
+ bool isEmpty(const AutoLockHelperThreadState& lock) const;
+
+ void queueChunk(NurseryChunk* chunk, const AutoLockHelperThreadState& lock);
+ void queueRange(size_t newCapacity, NurseryChunk& chunk,
+ const AutoLockHelperThreadState& lock);
+
+ private:
+ using NurseryChunkVector = Vector<NurseryChunk*, 0, SystemAllocPolicy>;
+
+ void run(AutoLockHelperThreadState& lock) override;
+
+ NurseryChunkVector& chunksToDecommit() { return chunksToDecommit_.ref(); }
+ const NurseryChunkVector& chunksToDecommit() const {
+ return chunksToDecommit_.ref();
+ }
+
+ MainThreadOrGCTaskData<NurseryChunkVector> chunksToDecommit_;
+
+ MainThreadOrGCTaskData<NurseryChunk*> partialChunk;
+ MainThreadOrGCTaskData<size_t> partialCapacity;
+};
+
+} // namespace js
+
+inline void js::NurseryChunk::poisonAndInit(JSRuntime* rt, size_t size) {
+ MOZ_ASSERT(size >= sizeof(ChunkBase));
+ MOZ_ASSERT(size <= ChunkSize);
+ poisonRange(0, size, JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined);
+ new (this) NurseryChunk(rt);
+}
+
+inline void js::NurseryChunk::poisonRange(size_t from, size_t size,
+ uint8_t value,
+ MemCheckKind checkKind) {
+ MOZ_ASSERT(from + size <= ChunkSize);
+
+ auto* start = reinterpret_cast<uint8_t*>(this) + from;
+
+ // We can poison the same chunk more than once, so first make sure memory
+ // sanitizers will let us poison it.
+ MOZ_MAKE_MEM_UNDEFINED(start, size);
+ Poison(start, value, size, checkKind);
+}
+
+inline void js::NurseryChunk::poisonAfterEvict(size_t extent) {
+ MOZ_ASSERT(extent <= ChunkSize);
+ poisonRange(sizeof(ChunkBase), extent - sizeof(ChunkBase),
+ JS_SWEPT_NURSERY_PATTERN, MemCheckKind::MakeNoAccess);
+}
+
+inline void js::NurseryChunk::markPagesUnusedHard(size_t startOffset) {
+ MOZ_ASSERT(startOffset >= sizeof(ChunkBase)); // Don't touch the header.
+ MOZ_ASSERT(startOffset >= SystemPageSize());
+ MOZ_ASSERT(startOffset <= ChunkSize);
+ uintptr_t start = uintptr_t(this) + startOffset;
+ size_t length = ChunkSize - startOffset;
+ MarkPagesUnusedHard(reinterpret_cast<void*>(start), length);
+}
+
+inline bool js::NurseryChunk::markPagesInUseHard(size_t endOffset) {
+ MOZ_ASSERT(endOffset >= sizeof(ChunkBase));
+ MOZ_ASSERT(endOffset >= SystemPageSize());
+ MOZ_ASSERT(endOffset <= ChunkSize);
+ uintptr_t start = uintptr_t(this) + SystemPageSize();
+ size_t length = endOffset - SystemPageSize();
+ return MarkPagesInUseHard(reinterpret_cast<void*>(start), length);
+}
+
+// static
+inline js::NurseryChunk* js::NurseryChunk::fromChunk(TenuredChunk* chunk) {
+ return reinterpret_cast<NurseryChunk*>(chunk);
+}
+
+js::NurseryDecommitTask::NurseryDecommitTask(gc::GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::NONE) {
+ // This can occur outside GCs so doesn't have a stats phase.
+}
+
+bool js::NurseryDecommitTask::isEmpty(
+ const AutoLockHelperThreadState& lock) const {
+ return chunksToDecommit().empty() && !partialChunk;
+}
+
+bool js::NurseryDecommitTask::reserveSpaceForBytes(size_t nbytes) {
+ MOZ_ASSERT(isIdle());
+ size_t nchunks = HowMany(nbytes, ChunkSize);
+ return chunksToDecommit().reserve(nchunks);
+}
+
+void js::NurseryDecommitTask::queueChunk(
+ NurseryChunk* chunk, const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isIdle(lock));
+ MOZ_ALWAYS_TRUE(chunksToDecommit().append(chunk));
+}
+
+void js::NurseryDecommitTask::queueRange(
+ size_t newCapacity, NurseryChunk& newChunk,
+ const AutoLockHelperThreadState& lock) {
+ MOZ_ASSERT(isIdle(lock));
+ MOZ_ASSERT(!partialChunk);
+ MOZ_ASSERT(newCapacity < ChunkSize);
+ MOZ_ASSERT(newCapacity % SystemPageSize() == 0);
+
+ partialChunk = &newChunk;
+ partialCapacity = newCapacity;
+}
+
+void js::NurseryDecommitTask::run(AutoLockHelperThreadState& lock) {
+ while (!chunksToDecommit().empty()) {
+ NurseryChunk* nurseryChunk = chunksToDecommit().popCopy();
+ AutoUnlockHelperThreadState unlock(lock);
+ nurseryChunk->~NurseryChunk();
+ TenuredChunk* tenuredChunk = TenuredChunk::emplace(
+ nurseryChunk, gc, /* allMemoryCommitted = */ false);
+ AutoLockGC lock(gc);
+ gc->recycleChunk(tenuredChunk, lock);
+ }
+
+ if (partialChunk) {
+ {
+ AutoUnlockHelperThreadState unlock(lock);
+ partialChunk->markPagesUnusedHard(partialCapacity);
+ }
+ partialChunk = nullptr;
+ partialCapacity = 0;
+ }
+}
+
+js::Nursery::Nursery(GCRuntime* gc)
+ : position_(0),
+ currentEnd_(0),
+ gc(gc),
+ currentChunk_(0),
+ currentStartChunk_(0),
+ currentStartPosition_(0),
+ capacity_(0),
+ timeInChunkAlloc_(0),
+ enableProfiling_(false),
+ profileThreshold_(0),
+ canAllocateStrings_(true),
+ canAllocateBigInts_(true),
+ reportDeduplications_(false),
+ reportPretenuring_(false),
+ reportPretenuringThreshold_(0),
+ minorGCTriggerReason_(JS::GCReason::NO_REASON),
+ hasRecentGrowthData(false),
+ smoothedTargetSize(0.0) {
+ const char* env = getenv("MOZ_NURSERY_STRINGS");
+ if (env && *env) {
+ canAllocateStrings_ = (*env == '1');
+ }
+ env = getenv("MOZ_NURSERY_BIGINTS");
+ if (env && *env) {
+ canAllocateBigInts_ = (*env == '1');
+ }
+}
+
+static void PrintAndExit(const char* message) {
+ fprintf(stderr, "%s", message);
+ exit(0);
+}
+
+static const char* GetEnvVar(const char* name, const char* helpMessage) {
+ const char* value = getenv(name);
+ if (!value) {
+ return nullptr;
+ }
+
+ if (strcmp(value, "help") == 0) {
+ PrintAndExit(helpMessage);
+ }
+
+ return value;
+}
+
+static bool GetBoolEnvVar(const char* name, const char* helpMessage) {
+ const char* env = GetEnvVar(name, helpMessage);
+ return env && bool(atoi(env));
+}
+
+static void ReadReportPretenureEnv(const char* name, const char* helpMessage,
+ bool* enabled, size_t* threshold) {
+ *enabled = false;
+ *threshold = 0;
+
+ const char* env = GetEnvVar(name, helpMessage);
+ if (!env) {
+ return;
+ }
+
+ char* end;
+ *threshold = strtol(env, &end, 10);
+ if (end == env || *end) {
+ PrintAndExit(helpMessage);
+ }
+
+ *enabled = true;
+}
+
+bool js::Nursery::init(AutoLockGCBgAlloc& lock) {
+ ReadProfileEnv("JS_GC_PROFILE_NURSERY",
+ "Report minor GCs taking at least N microseconds.\n",
+ &enableProfiling_, &profileWorkers_, &profileThreshold_);
+
+ reportDeduplications_ = GetBoolEnvVar(
+ "JS_GC_REPORT_STATS",
+ "JS_GC_REPORT_STATS=1\n"
+ "\tAfter a minor GC, report how many strings were deduplicated.\n");
+
+ ReadReportPretenureEnv(
+ "JS_GC_REPORT_PRETENURE",
+ "JS_GC_REPORT_PRETENURE=N\n"
+ "\tAfter a minor GC, report information about pretenuring, including\n"
+ "\tallocation sites with at least N allocations.\n",
+ &reportPretenuring_, &reportPretenuringThreshold_);
+
+ decommitTask = MakeUnique<NurseryDecommitTask>(gc);
+ if (!decommitTask) {
+ return false;
+ }
+
+ if (!gc->storeBuffer().enable()) {
+ return false;
+ }
+
+ return initFirstChunk(lock);
+}
+
+js::Nursery::~Nursery() { disable(); }
+
+void js::Nursery::enable() {
+ MOZ_ASSERT(isEmpty());
+ MOZ_ASSERT(!gc->isVerifyPreBarriersEnabled());
+ if (isEnabled()) {
+ return;
+ }
+
+ {
+ AutoLockGCBgAlloc lock(gc);
+ if (!initFirstChunk(lock)) {
+ // If we fail to allocate memory, the nursery will not be enabled.
+ return;
+ }
+ }
+
+#ifdef JS_GC_ZEAL
+ if (gc->hasZealMode(ZealMode::GenerationalGC)) {
+ enterZealMode();
+ }
+#endif
+
+ updateAllZoneAllocFlags();
+
+ // This should always succeed after the first time it's called.
+ MOZ_ALWAYS_TRUE(gc->storeBuffer().enable());
+}
+
+bool js::Nursery::initFirstChunk(AutoLockGCBgAlloc& lock) {
+ MOZ_ASSERT(!isEnabled());
+
+ capacity_ = tunables().gcMinNurseryBytes();
+
+ if (!decommitTask->reserveSpaceForBytes(capacity_) ||
+ !allocateNextChunk(0, lock)) {
+ capacity_ = 0;
+ return false;
+ }
+
+ setCurrentChunk(0);
+ setStartPosition();
+ poisonAndInitCurrentChunk();
+
+ // Clear any information about previous collections.
+ clearRecentGrowthData();
+
+ return true;
+}
+
+void js::Nursery::disable() {
+ MOZ_ASSERT(isEmpty());
+ if (!isEnabled()) {
+ return;
+ }
+
+ // Free all chunks.
+ decommitTask->join();
+ freeChunksFrom(0);
+ decommitTask->runFromMainThread();
+
+ capacity_ = 0;
+
+ // We must reset currentEnd_ so that there is no space for anything in the
+ // nursery. JIT'd code uses this even if the nursery is disabled.
+ currentEnd_ = 0;
+ position_ = 0;
+ gc->storeBuffer().disable();
+
+ if (gc->wasInitialized()) {
+ // This assumes there is an atoms zone.
+ updateAllZoneAllocFlags();
+ }
+}
+
+void js::Nursery::enableStrings() {
+ MOZ_ASSERT(isEmpty());
+ canAllocateStrings_ = true;
+ updateAllZoneAllocFlags();
+}
+
+void js::Nursery::disableStrings() {
+ MOZ_ASSERT(isEmpty());
+ canAllocateStrings_ = false;
+ updateAllZoneAllocFlags();
+}
+
+void js::Nursery::enableBigInts() {
+ MOZ_ASSERT(isEmpty());
+ canAllocateBigInts_ = true;
+ updateAllZoneAllocFlags();
+}
+
+void js::Nursery::disableBigInts() {
+ MOZ_ASSERT(isEmpty());
+ canAllocateBigInts_ = false;
+ updateAllZoneAllocFlags();
+}
+
+void js::Nursery::updateAllZoneAllocFlags() {
+ // The alloc flags are not relevant for the atoms zone, and flushing
+ // jit-related information can be problematic for the atoms zone.
+ for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) {
+ updateAllocFlagsForZone(zone);
+ }
+}
+
+void js::Nursery::getAllocFlagsForZone(JS::Zone* zone, bool* allocObjectsOut,
+ bool* allocStringsOut,
+ bool* allocBigIntsOut) {
+ *allocObjectsOut = isEnabled();
+ *allocStringsOut =
+ isEnabled() && canAllocateStrings() && !zone->nurseryStringsDisabled;
+ *allocBigIntsOut =
+ isEnabled() && canAllocateBigInts() && !zone->nurseryBigIntsDisabled;
+}
+
+void js::Nursery::setAllocFlagsForZone(JS::Zone* zone) {
+ bool allocObjects;
+ bool allocStrings;
+ bool allocBigInts;
+
+ getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts);
+ zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts);
+}
+
+void js::Nursery::updateAllocFlagsForZone(JS::Zone* zone) {
+ bool allocObjects;
+ bool allocStrings;
+ bool allocBigInts;
+
+ getAllocFlagsForZone(zone, &allocObjects, &allocStrings, &allocBigInts);
+
+ if (allocObjects != zone->allocNurseryObjects() ||
+ allocStrings != zone->allocNurseryStrings() ||
+ allocBigInts != zone->allocNurseryBigInts()) {
+ CancelOffThreadIonCompile(zone);
+ zone->setNurseryAllocFlags(allocObjects, allocStrings, allocBigInts);
+ discardCodeAndSetJitFlagsForZone(zone);
+ }
+}
+
+void js::Nursery::discardCodeAndSetJitFlagsForZone(JS::Zone* zone) {
+ zone->forceDiscardJitCode(runtime()->gcContext());
+
+ for (RealmsInZoneIter r(zone); !r.done(); r.next()) {
+ if (jit::JitRealm* jitRealm = r->jitRealm()) {
+ jitRealm->discardStubs();
+ jitRealm->setStringsCanBeInNursery(zone->allocNurseryStrings());
+ }
+ }
+}
+
+bool js::Nursery::isEmpty() const {
+ if (!isEnabled()) {
+ return true;
+ }
+
+ if (!gc->hasZealMode(ZealMode::GenerationalGC)) {
+ MOZ_ASSERT(currentStartChunk_ == 0);
+ MOZ_ASSERT(currentStartPosition_ == chunk(0).start());
+ }
+ return position() == currentStartPosition_;
+}
+
+#ifdef JS_GC_ZEAL
+void js::Nursery::enterZealMode() {
+ if (!isEnabled()) {
+ return;
+ }
+
+ MOZ_ASSERT(isEmpty());
+
+ decommitTask->join();
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ if (isSubChunkMode()) {
+ {
+ if (!chunk(0).markPagesInUseHard(ChunkSize)) {
+ oomUnsafe.crash("Out of memory trying to extend chunk for zeal mode");
+ }
+ }
+
+ // It'd be simpler to poison the whole chunk, but we can't do that
+ // because the nursery might be partially used.
+ chunk(0).poisonRange(capacity_, ChunkSize - capacity_,
+ JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined);
+ }
+
+ capacity_ = RoundUp(tunables().gcMaxNurseryBytes(), ChunkSize);
+
+ if (!decommitTask->reserveSpaceForBytes(capacity_)) {
+ oomUnsafe.crash("Nursery::enterZealMode");
+ }
+
+ setCurrentEnd();
+}
+
+void js::Nursery::leaveZealMode() {
+ if (!isEnabled()) {
+ return;
+ }
+
+ MOZ_ASSERT(isEmpty());
+
+ setCurrentChunk(0);
+ setStartPosition();
+ poisonAndInitCurrentChunk();
+}
+#endif // JS_GC_ZEAL
+
+void* js::Nursery::allocateCell(gc::AllocSite* site, size_t size,
+ JS::TraceKind kind) {
+ // Ensure there's enough space to replace the contents with a
+ // RelocationOverlay.
+ MOZ_ASSERT(size >= sizeof(RelocationOverlay));
+ MOZ_ASSERT(size % CellAlignBytes == 0);
+ MOZ_ASSERT(size_t(kind) < NurseryTraceKinds);
+ MOZ_ASSERT_IF(kind == JS::TraceKind::String, canAllocateStrings());
+ MOZ_ASSERT_IF(kind == JS::TraceKind::BigInt, canAllocateBigInts());
+
+ void* ptr = allocate(sizeof(NurseryCellHeader) + size);
+ if (!ptr) {
+ return nullptr;
+ }
+
+ new (ptr) NurseryCellHeader(site, kind);
+
+ void* cell =
+ reinterpret_cast<void*>(uintptr_t(ptr) + sizeof(NurseryCellHeader));
+
+ // Update the allocation site. This code is also inlined in
+ // MacroAssembler::updateAllocSite.
+ uint32_t allocCount = site->incAllocCount();
+ if (allocCount == 1) {
+ pretenuringNursery.insertIntoAllocatedList(site);
+ } else {
+ MOZ_ASSERT_IF(site->isNormal(), site->isInAllocatedList());
+ }
+
+ gcprobes::NurseryAlloc(cell, kind);
+ return cell;
+}
+
+inline void* js::Nursery::allocate(size_t size) {
+ MOZ_ASSERT(isEnabled());
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+ MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_,
+ position() >= currentStartPosition_);
+ MOZ_ASSERT(position() % CellAlignBytes == 0);
+ MOZ_ASSERT(size % CellAlignBytes == 0);
+
+ if (MOZ_UNLIKELY(currentEnd() < position() + size)) {
+ return moveToNextChunkAndAllocate(size);
+ }
+
+ void* thing = (void*)position();
+ position_ = position() + size;
+
+ DebugOnlyPoison(thing, JS_ALLOCATED_NURSERY_PATTERN, size,
+ MemCheckKind::MakeUndefined);
+
+ return thing;
+}
+
+void* Nursery::moveToNextChunkAndAllocate(size_t size) {
+ MOZ_ASSERT(currentEnd() < position() + size);
+
+ unsigned chunkno = currentChunk_ + 1;
+ MOZ_ASSERT(chunkno <= maxChunkCount());
+ MOZ_ASSERT(chunkno <= allocatedChunkCount());
+ if (chunkno == maxChunkCount()) {
+ return nullptr;
+ }
+ if (chunkno == allocatedChunkCount()) {
+ TimeStamp start = TimeStamp::Now();
+ {
+ AutoLockGCBgAlloc lock(gc);
+ if (!allocateNextChunk(chunkno, lock)) {
+ return nullptr;
+ }
+ }
+ timeInChunkAlloc_ += TimeStamp::Now() - start;
+ MOZ_ASSERT(chunkno < allocatedChunkCount());
+ }
+ setCurrentChunk(chunkno);
+ poisonAndInitCurrentChunk();
+
+ // We know there's enough space to allocate now so we can call allocate()
+ // recursively.
+ MOZ_ASSERT(currentEnd() >= position() + size);
+ return allocate(size);
+}
+void* js::Nursery::allocateBuffer(Zone* zone, size_t nbytes) {
+ MOZ_ASSERT(nbytes > 0);
+
+ if (nbytes <= MaxNurseryBufferSize) {
+ void* buffer = allocate(nbytes);
+ if (buffer) {
+ return buffer;
+ }
+ }
+
+ void* buffer = zone->pod_malloc<uint8_t>(nbytes);
+ if (buffer && !registerMallocedBuffer(buffer, nbytes)) {
+ js_free(buffer);
+ return nullptr;
+ }
+ return buffer;
+}
+
+void* js::Nursery::allocateBuffer(Zone* zone, JSObject* obj, size_t nbytes) {
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(nbytes > 0);
+
+ if (!IsInsideNursery(obj)) {
+ return zone->pod_malloc<uint8_t>(nbytes);
+ }
+
+ return allocateBuffer(zone, nbytes);
+}
+
+void* js::Nursery::allocateBufferSameLocation(JSObject* obj, size_t nbytes) {
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(nbytes > 0);
+ MOZ_ASSERT(nbytes <= MaxNurseryBufferSize);
+
+ if (!IsInsideNursery(obj)) {
+ return obj->zone()->pod_malloc<uint8_t>(nbytes);
+ }
+
+ return allocate(nbytes);
+}
+
+void* js::Nursery::allocateZeroedBuffer(
+ Zone* zone, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) {
+ MOZ_ASSERT(nbytes > 0);
+
+ if (nbytes <= MaxNurseryBufferSize) {
+ void* buffer = allocate(nbytes);
+ if (buffer) {
+ memset(buffer, 0, nbytes);
+ return buffer;
+ }
+ }
+
+ void* buffer = zone->pod_arena_calloc<uint8_t>(arena, nbytes);
+ if (buffer && !registerMallocedBuffer(buffer, nbytes)) {
+ js_free(buffer);
+ return nullptr;
+ }
+ return buffer;
+}
+
+void* js::Nursery::allocateZeroedBuffer(
+ JSObject* obj, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) {
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(nbytes > 0);
+
+ if (!IsInsideNursery(obj)) {
+ return obj->zone()->pod_arena_calloc<uint8_t>(arena, nbytes);
+ }
+ return allocateZeroedBuffer(obj->zone(), nbytes, arena);
+}
+
+void* js::Nursery::reallocateBuffer(Zone* zone, Cell* cell, void* oldBuffer,
+ size_t oldBytes, size_t newBytes) {
+ if (!IsInsideNursery(cell)) {
+ MOZ_ASSERT(!isInside(oldBuffer));
+ return zone->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes);
+ }
+
+ if (!isInside(oldBuffer)) {
+ MOZ_ASSERT(mallocedBufferBytes >= oldBytes);
+ void* newBuffer =
+ zone->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes);
+ if (newBuffer) {
+ if (oldBuffer != newBuffer) {
+ MOZ_ALWAYS_TRUE(
+ mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer));
+ }
+ mallocedBufferBytes -= oldBytes;
+ mallocedBufferBytes += newBytes;
+ }
+ return newBuffer;
+ }
+
+ // The nursery cannot make use of the returned slots data.
+ if (newBytes < oldBytes) {
+ return oldBuffer;
+ }
+
+ void* newBuffer = allocateBuffer(zone, newBytes);
+ if (newBuffer) {
+ PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes);
+ }
+ return newBuffer;
+}
+
+void* js::Nursery::allocateBuffer(JS::BigInt* bi, size_t nbytes) {
+ MOZ_ASSERT(bi);
+ MOZ_ASSERT(nbytes > 0);
+
+ if (!IsInsideNursery(bi)) {
+ return bi->zone()->pod_malloc<uint8_t>(nbytes);
+ }
+ return allocateBuffer(bi->zone(), nbytes);
+}
+
+void js::Nursery::freeBuffer(void* buffer, size_t nbytes) {
+ if (!isInside(buffer)) {
+ removeMallocedBuffer(buffer, nbytes);
+ js_free(buffer);
+ }
+}
+
+#ifdef DEBUG
+/* static */
+inline bool Nursery::checkForwardingPointerLocation(void* ptr,
+ bool expectedInside) {
+ if (isInside(ptr) == expectedInside) {
+ return true;
+ }
+
+ // If a zero-capacity elements header lands right at the end of a chunk then
+ // elements data will appear to be in the next chunk. If we have a pointer to
+ // the very start of a chunk, check the previous chunk.
+ if ((uintptr_t(ptr) & ChunkMask) == 0 &&
+ isInside(reinterpret_cast<uint8_t*>(ptr) - 1) == expectedInside) {
+ return true;
+ }
+
+ return false;
+}
+#endif
+
+void Nursery::setIndirectForwardingPointer(void* oldData, void* newData) {
+ MOZ_ASSERT(checkForwardingPointerLocation(oldData, true));
+ MOZ_ASSERT(checkForwardingPointerLocation(newData, false));
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+#ifdef DEBUG
+ if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData)) {
+ MOZ_ASSERT(p->value() == newData);
+ }
+#endif
+ if (!forwardedBuffers.put(oldData, newData)) {
+ oomUnsafe.crash("Nursery::setForwardingPointer");
+ }
+}
+
+#ifdef DEBUG
+static bool IsWriteableAddress(void* ptr) {
+ auto* vPtr = reinterpret_cast<volatile uint64_t*>(ptr);
+ *vPtr = *vPtr;
+ return true;
+}
+#endif
+
+void js::Nursery::forwardBufferPointer(uintptr_t* pSlotsElems) {
+ // Read the current pointer value which may be one of:
+ // - Non-nursery pointer
+ // - Nursery-allocated buffer
+ // - A BufferRelocationOverlay inside the nursery
+ //
+ // Note: The buffer has already be relocated. We are just patching stale
+ // pointers now.
+ auto* buffer = reinterpret_cast<void*>(*pSlotsElems);
+
+ if (!isInside(buffer)) {
+ return;
+ }
+
+ // The new location for this buffer is either stored inline with it or in
+ // the forwardedBuffers table.
+ if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(buffer)) {
+ buffer = p->value();
+ // It's not valid to assert IsWriteableAddress for indirect forwarding
+ // pointers because the size of the allocation could be less than a word.
+ } else {
+ BufferRelocationOverlay* reloc =
+ static_cast<BufferRelocationOverlay*>(buffer);
+ buffer = *reloc;
+ MOZ_ASSERT(IsWriteableAddress(buffer));
+ }
+
+ MOZ_ASSERT(!isInside(buffer));
+ *pSlotsElems = reinterpret_cast<uintptr_t>(buffer);
+}
+
+inline double js::Nursery::calcPromotionRate(bool* validForTenuring) const {
+ MOZ_ASSERT(validForTenuring);
+
+ if (previousGC.nurseryUsedBytes == 0) {
+ *validForTenuring = false;
+ return 0.0;
+ }
+
+ double used = double(previousGC.nurseryUsedBytes);
+ double capacity = double(previousGC.nurseryCapacity);
+ double tenured = double(previousGC.tenuredBytes);
+
+ // We should only use the promotion rate to make tenuring decisions if it's
+ // likely to be valid. The criterion we use is that the nursery was at least
+ // 90% full.
+ *validForTenuring = used > capacity * 0.9;
+
+ return tenured / used;
+}
+
+void js::Nursery::renderProfileJSON(JSONPrinter& json) const {
+ if (!isEnabled()) {
+ json.beginObject();
+ json.property("status", "nursery disabled");
+ json.endObject();
+ return;
+ }
+
+ if (previousGC.reason == JS::GCReason::NO_REASON) {
+ // If the nursery was empty when the last minorGC was requested, then
+ // no nursery collection will have been performed but JSON may still be
+ // requested. (And as a public API, this function should not crash in
+ // such a case.)
+ json.beginObject();
+ json.property("status", "nursery empty");
+ json.endObject();
+ return;
+ }
+
+ json.beginObject();
+
+ json.property("status", "complete");
+
+ json.property("reason", JS::ExplainGCReason(previousGC.reason));
+ json.property("bytes_tenured", previousGC.tenuredBytes);
+ json.property("cells_tenured", previousGC.tenuredCells);
+ json.property("strings_tenured",
+ stats().getStat(gcstats::STAT_STRINGS_TENURED));
+ json.property("strings_deduplicated",
+ stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED));
+ json.property("bigints_tenured",
+ stats().getStat(gcstats::STAT_BIGINTS_TENURED));
+ json.property("bytes_used", previousGC.nurseryUsedBytes);
+ json.property("cur_capacity", previousGC.nurseryCapacity);
+ const size_t newCapacity = capacity();
+ if (newCapacity != previousGC.nurseryCapacity) {
+ json.property("new_capacity", newCapacity);
+ }
+ if (previousGC.nurseryCommitted != previousGC.nurseryCapacity) {
+ json.property("lazy_capacity", previousGC.nurseryCommitted);
+ }
+ if (!timeInChunkAlloc_.IsZero()) {
+ json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS);
+ }
+
+ // These counters only contain consistent data if the profiler is enabled,
+ // and then there's no guarentee.
+ if (runtime()->geckoProfiler().enabled()) {
+ json.property("cells_allocated_nursery",
+ pretenuringNursery.totalAllocCount());
+ json.property("cells_allocated_tenured",
+ stats().allocsSinceMinorGCTenured());
+ }
+
+ json.beginObjectProperty("phase_times");
+
+#define EXTRACT_NAME(name, text) #name,
+ static const char* const names[] = {
+ FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME)
+#undef EXTRACT_NAME
+ ""};
+
+ size_t i = 0;
+ for (auto time : profileDurations_) {
+ json.property(names[i++], time, json.MICROSECONDS);
+ }
+
+ json.endObject(); // timings value
+
+ json.endObject();
+}
+
+// The following macros define nursery GC profile metadata fields that are
+// printed before the timing information defined by
+// FOR_EACH_NURSERY_PROFILE_TIME.
+
+#define FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \
+ _("PID", 7, "%7zu", pid) \
+ _("Runtime", 14, "0x%12p", runtime)
+
+#define FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_) \
+ _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \
+ _("Reason", 20, "%-20.20s", reasonStr) \
+ _("PRate", 6, "%5.1f%%", promotionRatePercent) \
+ _("OldKB", 6, "%6zu", oldSizeKB) \
+ _("NewKB", 6, "%6zu", newSizeKB) \
+ _("Dedup", 6, "%6zu", dedupCount)
+
+#define FOR_EACH_NURSERY_PROFILE_METADATA(_) \
+ FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(_) \
+ FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(_)
+
+void js::Nursery::printCollectionProfile(JS::GCReason reason,
+ double promotionRate) {
+ stats().maybePrintProfileHeaders();
+
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) {
+ return;
+ }
+
+ size_t pid = getpid();
+ JSRuntime* runtime = gc->rt;
+ TimeDuration timestamp = collectionStartTime() - stats().creationTime();
+ const char* reasonStr = ExplainGCReason(reason);
+ double promotionRatePercent = promotionRate * 100;
+ size_t oldSizeKB = previousGC.nurseryCapacity / 1024;
+ size_t newSizeKB = capacity() / 1024;
+ size_t dedupCount = stats().getStat(gcstats::STAT_STRINGS_DEDUPLICATED);
+
+#define PRINT_FIELD_VALUE(_1, _2, format, value) \
+ if (!sprinter.jsprintf(" " format, value)) { \
+ return; \
+ }
+ FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_VALUE)
+#undef PRINT_FIELD_VALUE
+
+ printProfileDurations(profileDurations_, sprinter);
+
+ fputs(sprinter.string(), stats().profileFile());
+}
+
+void js::Nursery::printProfileHeader() {
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) {
+ return;
+ }
+
+#define PRINT_FIELD_NAME(name, width, _1, _2) \
+ if (!sprinter.jsprintf(" %-*s", width, name)) { \
+ return; \
+ }
+ FOR_EACH_NURSERY_PROFILE_METADATA(PRINT_FIELD_NAME)
+#undef PRINT_FIELD_NAME
+
+#define PRINT_PROFILE_NAME(_1, text) \
+ if (!sprinter.jsprintf(" %-6.6s", text)) { \
+ return; \
+ }
+ FOR_EACH_NURSERY_PROFILE_TIME(PRINT_PROFILE_NAME)
+#undef PRINT_PROFILE_NAME
+
+ if (!sprinter.put("\n")) {
+ return;
+ }
+
+ fputs(sprinter.string(), stats().profileFile());
+}
+
+// static
+bool js::Nursery::printProfileDurations(const ProfileDurations& times,
+ Sprinter& sprinter) {
+ for (auto time : times) {
+ int64_t micros = int64_t(time.ToMicroseconds());
+ if (!sprinter.jsprintf(" %6" PRIi64, micros)) {
+ return false;
+ }
+ }
+
+ return sprinter.put("\n");
+}
+
+static constexpr size_t NurserySliceMetadataFormatWidth() {
+ size_t fieldCount = 0;
+ size_t totalWidth = 0;
+
+#define UPDATE_COUNT_AND_WIDTH(_1, width, _2, _3) \
+ fieldCount++; \
+ totalWidth += width;
+ FOR_EACH_NURSERY_PROFILE_SLICE_METADATA(UPDATE_COUNT_AND_WIDTH)
+#undef UPDATE_COUNT_AND_WIDTH
+
+ // Add padding between fields.
+ totalWidth += fieldCount - 1;
+
+ return totalWidth;
+}
+
+void js::Nursery::printTotalProfileTimes() {
+ if (!enableProfiling_) {
+ return;
+ }
+
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(gcstats::MinorGCProfilePrefix)) {
+ return;
+ }
+
+ size_t pid = getpid();
+ JSRuntime* runtime = gc->rt;
+
+ char collections[32];
+ DebugOnly<int> r = SprintfLiteral(
+ collections, "TOTALS: %7" PRIu64 " collections:", gc->minorGCCount());
+ MOZ_ASSERT(r > 0 && r < int(sizeof(collections)));
+
+#define PRINT_FIELD_VALUE(_1, _2, format, value) \
+ if (!sprinter.jsprintf(" " format, value)) { \
+ return; \
+ }
+ FOR_EACH_NURSERY_PROFILE_COMMON_METADATA(PRINT_FIELD_VALUE)
+#undef PRINT_FIELD_VALUE
+
+ // Use whole width of per-slice metadata to print total slices so the profile
+ // totals that follow line up.
+ size_t width = NurserySliceMetadataFormatWidth();
+ if (!sprinter.jsprintf(" %-*s", int(width), collections)) {
+ return;
+ }
+
+ if (!printProfileDurations(totalDurations_, sprinter)) {
+ return;
+ }
+
+ fputs(sprinter.string(), stats().profileFile());
+}
+
+void js::Nursery::maybeClearProfileDurations() {
+ for (auto& duration : profileDurations_) {
+ duration = mozilla::TimeDuration();
+ }
+}
+
+inline void js::Nursery::startProfile(ProfileKey key) {
+ startTimes_[key] = TimeStamp::Now();
+}
+
+inline void js::Nursery::endProfile(ProfileKey key) {
+ profileDurations_[key] = TimeStamp::Now() - startTimes_[key];
+ totalDurations_[key] += profileDurations_[key];
+}
+
+inline TimeStamp js::Nursery::collectionStartTime() const {
+ return startTimes_[ProfileKey::Total];
+}
+
+inline TimeStamp js::Nursery::lastCollectionEndTime() const {
+ return previousGC.endTime;
+}
+
+bool js::Nursery::shouldCollect() const {
+ if (!isEnabled()) {
+ return false;
+ }
+
+ if (isEmpty() && capacity() == tunables().gcMinNurseryBytes()) {
+ return false;
+ }
+
+ if (minorGCRequested()) {
+ return true;
+ }
+
+ // Eagerly collect the nursery in idle time if it's nearly full.
+ if (isNearlyFull()) {
+ return true;
+ }
+
+ // If the nursery is not being collected often then it may be taking up more
+ // space than necessary.
+ return isUnderused();
+}
+
+inline bool js::Nursery::isNearlyFull() const {
+ bool belowBytesThreshold =
+ freeSpace() < tunables().nurseryFreeThresholdForIdleCollection();
+ bool belowFractionThreshold =
+ double(freeSpace()) / double(capacity()) <
+ tunables().nurseryFreeThresholdForIdleCollectionFraction();
+
+ // We want to use belowBytesThreshold when the nursery is sufficiently large,
+ // and belowFractionThreshold when it's small.
+ //
+ // When the nursery is small then belowBytesThreshold is a lower threshold
+ // (triggered earlier) than belowFractionThreshold. So if the fraction
+ // threshold is true, the bytes one will be true also. The opposite is true
+ // when the nursery is large.
+ //
+ // Therefore, by the time we cross the threshold we care about, we've already
+ // crossed the other one, and we can boolean AND to use either condition
+ // without encoding any "is the nursery big/small" test/threshold. The point
+ // at which they cross is when the nursery is: BytesThreshold /
+ // FractionThreshold large.
+ //
+ // With defaults that's:
+ //
+ // 1MB = 256KB / 0.25
+ //
+ return belowBytesThreshold && belowFractionThreshold;
+}
+
+inline bool js::Nursery::isUnderused() const {
+ if (js::SupportDifferentialTesting() || !previousGC.endTime) {
+ return false;
+ }
+
+ if (capacity() == tunables().gcMinNurseryBytes()) {
+ return false;
+ }
+
+ // If the nursery is above its minimum size, collect it every so often if we
+ // have idle time. This allows the nursery to shrink when it's not being
+ // used. There are other heuristics we could use for this, but this is the
+ // simplest.
+ TimeDuration timeSinceLastCollection = TimeStamp::Now() - previousGC.endTime;
+ return timeSinceLastCollection > tunables().nurseryTimeoutForIdleCollection();
+}
+
+// typeReason is the gcReason for specified type, for example,
+// FULL_CELL_PTR_OBJ_BUFFER is the gcReason for JSObject.
+static inline bool IsFullStoreBufferReason(JS::GCReason reason,
+ JS::GCReason typeReason) {
+ return reason == typeReason ||
+ reason == JS::GCReason::FULL_WHOLE_CELL_BUFFER ||
+ reason == JS::GCReason::FULL_GENERIC_BUFFER ||
+ reason == JS::GCReason::FULL_VALUE_BUFFER ||
+ reason == JS::GCReason::FULL_SLOT_BUFFER ||
+ reason == JS::GCReason::FULL_SHAPE_BUFFER;
+}
+
+void js::Nursery::collect(JS::GCOptions options, JS::GCReason reason) {
+ JSRuntime* rt = runtime();
+ MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
+
+ if (!isEnabled() || isEmpty()) {
+ // Our barriers are not always exact, and there may be entries in the
+ // storebuffer even when the nursery is disabled or empty. It's not safe
+ // to keep these entries as they may refer to tenured cells which may be
+ // freed after this point.
+ gc->storeBuffer().clear();
+
+ MOZ_ASSERT(!pretenuringNursery.hasAllocatedSites());
+ }
+
+ if (!isEnabled()) {
+ return;
+ }
+
+ AutoGCSession session(gc, JS::HeapState::MinorCollecting);
+
+ stats().beginNurseryCollection(reason);
+ gcprobes::MinorGCStart();
+
+ maybeClearProfileDurations();
+ startProfile(ProfileKey::Total);
+
+ previousGC.reason = JS::GCReason::NO_REASON;
+ previousGC.nurseryUsedBytes = usedSpace();
+ previousGC.nurseryCapacity = capacity();
+ previousGC.nurseryCommitted = committed();
+ previousGC.nurseryUsedChunkCount = currentChunk_ + 1;
+ previousGC.tenuredBytes = 0;
+ previousGC.tenuredCells = 0;
+
+ // If it isn't empty, it will call doCollection, and possibly after that
+ // isEmpty() will become true, so use another variable to keep track of the
+ // old empty state.
+ bool wasEmpty = isEmpty();
+ if (!wasEmpty) {
+ CollectionResult result = doCollection(session, options, reason);
+ // Don't include chunk headers when calculating nursery space, since this
+ // space does not represent data that can be tenured
+ MOZ_ASSERT(result.tenuredBytes <=
+ (previousGC.nurseryUsedBytes -
+ (sizeof(ChunkBase) * previousGC.nurseryUsedChunkCount)));
+
+ previousGC.reason = reason;
+ previousGC.tenuredBytes = result.tenuredBytes;
+ previousGC.tenuredCells = result.tenuredCells;
+ previousGC.nurseryUsedChunkCount = currentChunk_ + 1;
+ }
+
+ // Resize the nursery.
+ maybeResizeNursery(options, reason);
+
+ // Poison/initialise the first chunk.
+ if (previousGC.nurseryUsedBytes) {
+ // In most cases Nursery::clear() has not poisoned this chunk or marked it
+ // as NoAccess; so we only need to poison the region used during the last
+ // cycle. Also, if the heap was recently expanded we don't want to
+ // re-poison the new memory. In both cases we only need to poison until
+ // previousGC.nurseryUsedBytes.
+ //
+ // In cases where this is not true, like generational zeal mode or subchunk
+ // mode, poisonAndInitCurrentChunk() will ignore its parameter. It will
+ // also clamp the parameter.
+ poisonAndInitCurrentChunk(previousGC.nurseryUsedBytes);
+ }
+
+ bool validPromotionRate;
+ const double promotionRate = calcPromotionRate(&validPromotionRate);
+
+ startProfile(ProfileKey::Pretenure);
+ size_t sitesPretenured = 0;
+ if (!wasEmpty) {
+ sitesPretenured =
+ doPretenuring(rt, reason, validPromotionRate, promotionRate);
+ }
+ endProfile(ProfileKey::Pretenure);
+
+ // We ignore gcMaxBytes when allocating for minor collection. However, if we
+ // overflowed, we disable the nursery. The next time we allocate, we'll fail
+ // because bytes >= gcMaxBytes.
+ if (gc->heapSize.bytes() >= tunables().gcMaxBytes()) {
+ disable();
+ }
+
+ previousGC.endTime =
+ TimeStamp::Now(); // Must happen after maybeResizeNursery.
+ endProfile(ProfileKey::Total);
+ gc->incMinorGcNumber();
+
+ TimeDuration totalTime = profileDurations_[ProfileKey::Total];
+ sendTelemetry(reason, totalTime, wasEmpty, promotionRate, sitesPretenured);
+
+ stats().endNurseryCollection(reason); // Calls GCNurseryCollectionCallback.
+ gcprobes::MinorGCEnd();
+
+ timeInChunkAlloc_ = mozilla::TimeDuration();
+
+ js::StringStats prevStats = gc->stringStats;
+ js::StringStats& currStats = gc->stringStats;
+ currStats = js::StringStats();
+ for (ZonesIter zone(gc, WithAtoms); !zone.done(); zone.next()) {
+ currStats += zone->stringStats;
+ zone->previousGCStringStats = zone->stringStats;
+ }
+ stats().setStat(
+ gcstats::STAT_STRINGS_DEDUPLICATED,
+ currStats.deduplicatedStrings - prevStats.deduplicatedStrings);
+ if (ShouldPrintProfile(runtime(), enableProfiling_, profileWorkers_,
+ profileThreshold_, totalTime)) {
+ printCollectionProfile(reason, promotionRate);
+ }
+
+ if (reportDeduplications_) {
+ printDeduplicationData(prevStats, currStats);
+ }
+}
+
+void js::Nursery::sendTelemetry(JS::GCReason reason, TimeDuration totalTime,
+ bool wasEmpty, double promotionRate,
+ size_t sitesPretenured) {
+ JSRuntime* rt = runtime();
+ rt->metrics().GC_MINOR_REASON(uint32_t(reason));
+
+ // Long minor GCs are those that take more than 1ms.
+ bool wasLongMinorGC = totalTime.ToMilliseconds() > 1.0;
+ if (wasLongMinorGC) {
+ rt->metrics().GC_MINOR_REASON_LONG(uint32_t(reason));
+ }
+ rt->metrics().GC_MINOR_US(totalTime);
+ rt->metrics().GC_NURSERY_BYTES_2(committed());
+
+ if (!wasEmpty) {
+ rt->metrics().GC_PRETENURE_COUNT_2(sitesPretenured);
+ rt->metrics().GC_NURSERY_PROMOTION_RATE(promotionRate * 100);
+ }
+}
+
+void js::Nursery::printDeduplicationData(js::StringStats& prev,
+ js::StringStats& curr) {
+ if (curr.deduplicatedStrings > prev.deduplicatedStrings) {
+ fprintf(stderr,
+ "pid %zu: deduplicated %" PRIi64 " strings, %" PRIu64
+ " chars, %" PRIu64 " malloc bytes\n",
+ size_t(getpid()),
+ curr.deduplicatedStrings - prev.deduplicatedStrings,
+ curr.deduplicatedChars - prev.deduplicatedChars,
+ curr.deduplicatedBytes - prev.deduplicatedBytes);
+ }
+}
+
+void js::Nursery::freeTrailerBlocks(void) {
+ // This routine frees those blocks denoted by the set
+ //
+ // trailersAdded_ (all of it)
+ // - trailersRemoved_ (entries with index below trailersRemovedUsed_)
+ //
+ // For each block, places it back on the nursery's small-malloced-block pool
+ // by calling mallocedBlockCache_.free.
+
+ MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length());
+ MOZ_ASSERT(trailersRemovedUsed_ <= trailersRemoved_.length());
+
+ // Sort the removed entries.
+ std::sort(trailersRemoved_.begin(),
+ trailersRemoved_.begin() + trailersRemovedUsed_,
+ [](const void* block1, const void* block2) {
+ return uintptr_t(block1) < uintptr_t(block2);
+ });
+
+ // Use one of two schemes to enumerate the set subtraction.
+ if (trailersRemovedUsed_ < 1000) {
+ // If the number of removed items is relatively small, it isn't worth the
+ // cost of sorting `trailersAdded_`. Instead, walk through the vector in
+ // whatever order it is and use binary search to establish whether each
+ // item is present in trailersRemoved_[0 .. trailersRemovedUsed_ - 1].
+ const size_t nAdded = trailersAdded_.length();
+ for (size_t i = 0; i < nAdded; i++) {
+ const PointerAndUint7 block = trailersAdded_[i];
+ const void* blockPointer = block.pointer();
+ if (!std::binary_search(trailersRemoved_.begin(),
+ trailersRemoved_.begin() + trailersRemovedUsed_,
+ blockPointer)) {
+ mallocedBlockCache_.free(block);
+ }
+ }
+ } else {
+ // The general case, which is algorithmically safer for large inputs.
+ // Sort the added entries, and then walk through both them and the removed
+ // entries in lockstep.
+ std::sort(trailersAdded_.begin(), trailersAdded_.end(),
+ [](const PointerAndUint7& block1, const PointerAndUint7& block2) {
+ return uintptr_t(block1.pointer()) <
+ uintptr_t(block2.pointer());
+ });
+ // Enumerate the set subtraction. This is somewhat simplified by the fact
+ // that all elements of the removed set must also be present in the added
+ // set. (the "inclusion property").
+ const size_t nAdded = trailersAdded_.length();
+ const size_t nRemoved = trailersRemovedUsed_;
+ size_t iAdded;
+ size_t iRemoved = 0;
+ for (iAdded = 0; iAdded < nAdded; iAdded++) {
+ if (iRemoved == nRemoved) {
+ // We've run out of items to skip, so move on to the next loop.
+ break;
+ }
+ const PointerAndUint7 blockAdded = trailersAdded_[iAdded];
+ const void* blockRemoved = trailersRemoved_[iRemoved];
+ if (blockAdded.pointer() < blockRemoved) {
+ mallocedBlockCache_.free(blockAdded);
+ continue;
+ }
+ // If this doesn't hold
+ // (that is, if `blockAdded.pointer() > blockRemoved`),
+ // then the abovementioned inclusion property doesn't hold.
+ MOZ_RELEASE_ASSERT(blockAdded.pointer() == blockRemoved);
+ iRemoved++;
+ }
+ MOZ_ASSERT(iRemoved == nRemoved);
+ // We've used up the removed set, so now finish up the remainder of the
+ // added set.
+ for (/*keep going*/; iAdded < nAdded; iAdded++) {
+ const PointerAndUint7 block = trailersAdded_[iAdded];
+ mallocedBlockCache_.free(block);
+ }
+ }
+
+ // And empty out both sets, but preserve the underlying storage.
+ trailersAdded_.clear();
+ trailersRemoved_.clear();
+ trailersRemovedUsed_ = 0;
+ trailerBytes_ = 0;
+
+ // Discard blocks from the cache at 0.05% per megabyte of nursery capacity,
+ // that is, 0.8% of blocks for a 16-megabyte nursery. This allows the cache
+ // to gradually discard unneeded blocks in long running applications.
+ mallocedBlockCache_.preen(0.05 * float(capacity() / (1024 * 1024)));
+}
+
+size_t Nursery::sizeOfTrailerBlockSets(
+ mozilla::MallocSizeOf mallocSizeOf) const {
+ return trailersAdded_.sizeOfExcludingThis(mallocSizeOf) +
+ trailersRemoved_.sizeOfExcludingThis(mallocSizeOf);
+}
+
+js::Nursery::CollectionResult js::Nursery::doCollection(AutoGCSession& session,
+ JS::GCOptions options,
+ JS::GCReason reason) {
+ JSRuntime* rt = runtime();
+ AutoSetThreadIsPerformingGC performingGC(rt->gcContext());
+ AutoStopVerifyingBarriers av(rt, false);
+ AutoDisableProxyCheck disableStrictProxyChecking;
+ mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion;
+
+ // Move objects pointed to by roots from the nursery to the major heap.
+ TenuringTracer mover(rt, this);
+
+ // Trace everything considered as a root by a minor GC.
+ traceRoots(session, mover);
+
+ startProfile(ProfileKey::SweepCaches);
+ gc->purgeRuntimeForMinorGC();
+ endProfile(ProfileKey::SweepCaches);
+
+ // Most of the work is done here. This loop iterates over objects that have
+ // been moved to the major heap. If these objects have any outgoing pointers
+ // to the nursery, then those nursery objects get moved as well, until no
+ // objects are left to move. That is, we iterate to a fixed point.
+ startProfile(ProfileKey::CollectToObjFP);
+ mover.collectToObjectFixedPoint();
+ endProfile(ProfileKey::CollectToObjFP);
+
+ startProfile(ProfileKey::CollectToStrFP);
+ mover.collectToStringFixedPoint();
+ endProfile(ProfileKey::CollectToStrFP);
+
+ // Sweep to update any pointers to nursery objects that have now been
+ // tenured.
+ startProfile(ProfileKey::Sweep);
+ sweep();
+ endProfile(ProfileKey::Sweep);
+
+ // Update any slot or element pointers whose destination has been tenured.
+ startProfile(ProfileKey::UpdateJitActivations);
+ js::jit::UpdateJitActivationsForMinorGC(rt);
+ forwardedBuffers.clearAndCompact();
+ endProfile(ProfileKey::UpdateJitActivations);
+
+ startProfile(ProfileKey::ObjectsTenuredCallback);
+ gc->callObjectsTenuredCallback();
+ endProfile(ProfileKey::ObjectsTenuredCallback);
+
+ // Sweep.
+ startProfile(ProfileKey::FreeMallocedBuffers);
+ gc->queueBuffersForFreeAfterMinorGC(mallocedBuffers);
+ mallocedBufferBytes = 0;
+ endProfile(ProfileKey::FreeMallocedBuffers);
+
+ // Give trailer blocks associated with non-tenured Wasm{Struct,Array}Objects
+ // back to our `mallocedBlockCache_`.
+ startProfile(ProfileKey::FreeTrailerBlocks);
+ freeTrailerBlocks();
+ if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason)) {
+ mallocedBlockCache_.clear();
+ }
+ endProfile(ProfileKey::FreeTrailerBlocks);
+
+ startProfile(ProfileKey::ClearNursery);
+ clear();
+ endProfile(ProfileKey::ClearNursery);
+
+ startProfile(ProfileKey::ClearStoreBuffer);
+ gc->storeBuffer().clear();
+ endProfile(ProfileKey::ClearStoreBuffer);
+
+ // Purge the StringToAtomCache. This has to happen at the end because the
+ // cache is used when tenuring strings.
+ startProfile(ProfileKey::PurgeStringToAtomCache);
+ runtime()->caches().stringToAtomCache.purge();
+ endProfile(ProfileKey::PurgeStringToAtomCache);
+
+ // Make sure hashtables have been updated after the collection.
+ startProfile(ProfileKey::CheckHashTables);
+#ifdef JS_GC_ZEAL
+ if (gc->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) {
+ runtime()->caches().checkEvalCacheAfterMinorGC();
+ gc->checkHashTablesAfterMovingGC();
+ }
+#endif
+ endProfile(ProfileKey::CheckHashTables);
+
+ return {mover.getTenuredSize(), mover.getTenuredCells()};
+}
+
+void js::Nursery::traceRoots(AutoGCSession& session, TenuringTracer& mover) {
+ {
+ // Suppress the sampling profiler to prevent it observing moved functions.
+ AutoSuppressProfilerSampling suppressProfiler(
+ runtime()->mainContextFromOwnThread());
+
+ // Trace the store buffer. This must happen first.
+ StoreBuffer& sb = gc->storeBuffer();
+
+ // Strings in the whole cell buffer must be traced first, in order to mark
+ // tenured dependent strings' bases as non-deduplicatable. The rest of
+ // nursery collection (whole non-string cells, edges, etc.) can happen
+ // later.
+ startProfile(ProfileKey::TraceWholeCells);
+ sb.traceWholeCells(mover);
+ endProfile(ProfileKey::TraceWholeCells);
+
+ startProfile(ProfileKey::TraceValues);
+ sb.traceValues(mover);
+ endProfile(ProfileKey::TraceValues);
+
+ startProfile(ProfileKey::TraceCells);
+ sb.traceCells(mover);
+ endProfile(ProfileKey::TraceCells);
+
+ startProfile(ProfileKey::TraceSlots);
+ sb.traceSlots(mover);
+ endProfile(ProfileKey::TraceSlots);
+
+ startProfile(ProfileKey::TraceGenericEntries);
+ sb.traceGenericEntries(&mover);
+ endProfile(ProfileKey::TraceGenericEntries);
+
+ startProfile(ProfileKey::MarkRuntime);
+ gc->traceRuntimeForMinorGC(&mover, session);
+ endProfile(ProfileKey::MarkRuntime);
+ }
+
+ startProfile(ProfileKey::MarkDebugger);
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS);
+ DebugAPI::traceAllForMovingGC(&mover);
+ }
+ endProfile(ProfileKey::MarkDebugger);
+}
+
+size_t js::Nursery::doPretenuring(JSRuntime* rt, JS::GCReason reason,
+ bool validPromotionRate,
+ double promotionRate) {
+ size_t sitesPretenured = pretenuringNursery.doPretenuring(
+ gc, reason, validPromotionRate, promotionRate, reportPretenuring_,
+ reportPretenuringThreshold_);
+
+ bool highPromotionRate =
+ validPromotionRate && promotionRate > tunables().pretenureThreshold();
+
+ bool pretenureStr = false;
+ bool pretenureBigInt = false;
+ if (tunables().attemptPretenuring()) {
+ // Should we check for pretenuring regardless of GCReason?
+ // Use 3MB as the threshold so the pretenuring can be applied on Android.
+ bool pretenureAll =
+ highPromotionRate && previousGC.nurseryUsedBytes >= 3 * 1024 * 1024;
+
+ pretenureStr =
+ pretenureAll ||
+ IsFullStoreBufferReason(reason, JS::GCReason::FULL_CELL_PTR_STR_BUFFER);
+ pretenureBigInt =
+ pretenureAll || IsFullStoreBufferReason(
+ reason, JS::GCReason::FULL_CELL_PTR_BIGINT_BUFFER);
+ }
+
+ uint32_t numStringsTenured = 0;
+ uint32_t numBigIntsTenured = 0;
+ for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) {
+ // For some tests in JetStream2 and Kraken, the tenuredRate is high but the
+ // number of allocated strings is low. So we calculate the tenuredRate only
+ // if the number of string allocations is enough.
+ uint32_t zoneNurseryStrings =
+ zone->nurseryAllocCount(JS::TraceKind::String);
+ bool allocThreshold = zoneNurseryStrings > 30000;
+ uint64_t zoneTenuredStrings =
+ zone->stringStats.ref().liveNurseryStrings -
+ zone->previousGCStringStats.ref().liveNurseryStrings;
+ double tenuredRate =
+ allocThreshold ? double(zoneTenuredStrings) / double(zoneNurseryStrings)
+ : 0.0;
+ bool disableNurseryStrings =
+ pretenureStr && zone->allocNurseryStrings() &&
+ tenuredRate > tunables().pretenureStringThreshold();
+ bool disableNurseryBigInts = pretenureBigInt &&
+ zone->allocNurseryBigInts() &&
+ zone->tenuredBigInts >= 30 * 1000;
+ if (disableNurseryStrings || disableNurseryBigInts) {
+ if (disableNurseryStrings) {
+ zone->nurseryStringsDisabled = true;
+ }
+ if (disableNurseryBigInts) {
+ zone->nurseryBigIntsDisabled = true;
+ }
+ updateAllocFlagsForZone(zone);
+ }
+ numStringsTenured += zoneTenuredStrings;
+ numBigIntsTenured += zone->tenuredBigInts;
+ zone->tenuredBigInts = 0;
+ }
+ stats().setStat(gcstats::STAT_STRINGS_TENURED, numStringsTenured);
+ stats().setStat(gcstats::STAT_BIGINTS_TENURED, numBigIntsTenured);
+
+ return sitesPretenured;
+}
+
+bool js::Nursery::registerMallocedBuffer(void* buffer, size_t nbytes) {
+ MOZ_ASSERT(buffer);
+ MOZ_ASSERT(nbytes > 0);
+ if (!mallocedBuffers.putNew(buffer)) {
+ return false;
+ }
+
+ mallocedBufferBytes += nbytes;
+ if (MOZ_UNLIKELY(mallocedBufferBytes > capacity() * 8)) {
+ requestMinorGC(JS::GCReason::NURSERY_MALLOC_BUFFERS);
+ }
+
+ return true;
+}
+
+size_t Nursery::sizeOfMallocedBuffers(
+ mozilla::MallocSizeOf mallocSizeOf) const {
+ size_t total = 0;
+ for (BufferSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront()) {
+ total += mallocSizeOf(r.front());
+ }
+ total += mallocedBuffers.shallowSizeOfExcludingThis(mallocSizeOf);
+ return total;
+}
+
+void js::Nursery::sweep() {
+ // It's important that the context's GCUse is not Finalizing at this point,
+ // otherwise we will miscount memory attached to nursery objects with
+ // CellAllocPolicy.
+ AutoSetThreadIsSweeping setThreadSweeping(runtime()->gcContext());
+
+ MinorSweepingTracer trc(runtime());
+
+ // Sweep unique IDs first before we sweep any tables that may be keyed based
+ // on them.
+ for (Cell* cell : cellsWithUid_) {
+ auto* obj = static_cast<JSObject*>(cell);
+ if (!IsForwarded(obj)) {
+ gc::RemoveUniqueId(obj);
+ } else {
+ JSObject* dst = Forwarded(obj);
+ gc::TransferUniqueId(dst, obj);
+ }
+ }
+ cellsWithUid_.clear();
+
+ for (ZonesIter zone(runtime(), SkipAtoms); !zone.done(); zone.next()) {
+ zone->sweepAfterMinorGC(&trc);
+ }
+
+ sweepMapAndSetObjects();
+
+ runtime()->caches().sweepAfterMinorGC(&trc);
+}
+
+void js::Nursery::clear() {
+ // Poison the nursery contents so touching a freed object will crash.
+ unsigned firstClearChunk;
+ if (gc->hasZealMode(ZealMode::GenerationalGC)) {
+ // Poison all the chunks used in this cycle. The new start chunk is
+ // reposioned in Nursery::collect() but there's no point optimising that in
+ // this case.
+ firstClearChunk = currentStartChunk_;
+ } else {
+ // In normal mode we start at the second chunk, the first one will be used
+ // in the next cycle and poisoned in Nusery::collect();
+ MOZ_ASSERT(currentStartChunk_ == 0);
+ firstClearChunk = 1;
+ }
+ for (unsigned i = firstClearChunk; i < currentChunk_; ++i) {
+ chunk(i).poisonAfterEvict();
+ }
+ // Clear only the used part of the chunk because that's the part we touched,
+ // but only if it's not going to be re-used immediately (>= firstClearChunk).
+ if (currentChunk_ >= firstClearChunk) {
+ chunk(currentChunk_)
+ .poisonAfterEvict(position() - chunk(currentChunk_).start());
+ }
+
+ // Reset the start chunk & position if we're not in this zeal mode, or we're
+ // in it and close to the end of the nursery.
+ MOZ_ASSERT(maxChunkCount() > 0);
+ if (!gc->hasZealMode(ZealMode::GenerationalGC) ||
+ (gc->hasZealMode(ZealMode::GenerationalGC) &&
+ currentChunk_ + 1 == maxChunkCount())) {
+ setCurrentChunk(0);
+ }
+
+ // Set current start position for isEmpty checks.
+ setStartPosition();
+}
+
+size_t js::Nursery::spaceToEnd(unsigned chunkCount) const {
+ if (chunkCount == 0) {
+ return 0;
+ }
+
+ unsigned lastChunk = chunkCount - 1;
+
+ MOZ_ASSERT(lastChunk >= currentStartChunk_);
+ MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <=
+ NurseryChunkUsableSize);
+
+ size_t bytes;
+
+ if (chunkCount != 1) {
+ // In the general case we have to add:
+ // + the bytes used in the first
+ // chunk which may be less than the total size of a chunk since in some
+ // zeal modes we start the first chunk at some later position
+ // (currentStartPosition_).
+ // + the size of all the other chunks.
+ bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
+ ((lastChunk - currentStartChunk_) * ChunkSize);
+ } else {
+ // In sub-chunk mode, but it also works whenever chunkCount == 1, we need to
+ // use currentEnd_ since it may not refer to a full chunk.
+ bytes = currentEnd_ - currentStartPosition_;
+ }
+
+ MOZ_ASSERT(bytes <= maxChunkCount() * ChunkSize);
+
+ return bytes;
+}
+
+MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) {
+ MOZ_ASSERT(chunkno < allocatedChunkCount());
+
+ currentChunk_ = chunkno;
+ position_ = chunk(chunkno).start();
+ setCurrentEnd();
+}
+
+void js::Nursery::poisonAndInitCurrentChunk(size_t extent) {
+ if (gc->hasZealMode(ZealMode::GenerationalGC) || !isSubChunkMode()) {
+ chunk(currentChunk_).poisonAndInit(runtime());
+ } else {
+ extent = std::min(capacity_, extent);
+ chunk(currentChunk_).poisonAndInit(runtime(), extent);
+ }
+}
+
+MOZ_ALWAYS_INLINE void js::Nursery::setCurrentEnd() {
+ MOZ_ASSERT_IF(isSubChunkMode(),
+ currentChunk_ == 0 && currentEnd_ <= chunk(0).end());
+ currentEnd_ =
+ uintptr_t(&chunk(currentChunk_)) + std::min(capacity_, ChunkSize);
+}
+
+bool js::Nursery::allocateNextChunk(const unsigned chunkno,
+ AutoLockGCBgAlloc& lock) {
+ const unsigned priorCount = allocatedChunkCount();
+ const unsigned newCount = priorCount + 1;
+
+ MOZ_ASSERT((chunkno == currentChunk_ + 1) ||
+ (chunkno == 0 && allocatedChunkCount() == 0));
+ MOZ_ASSERT(chunkno == allocatedChunkCount());
+ MOZ_ASSERT(chunkno < HowMany(capacity(), ChunkSize));
+
+ if (!chunks_.resize(newCount)) {
+ return false;
+ }
+
+ TenuredChunk* newChunk;
+ newChunk = gc->getOrAllocChunk(lock);
+ if (!newChunk) {
+ chunks_.shrinkTo(priorCount);
+ return false;
+ }
+
+ chunks_[chunkno] = NurseryChunk::fromChunk(newChunk);
+ return true;
+}
+
+MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() {
+ currentStartChunk_ = currentChunk_;
+ currentStartPosition_ = position();
+}
+
+void js::Nursery::maybeResizeNursery(JS::GCOptions options,
+ JS::GCReason reason) {
+#ifdef JS_GC_ZEAL
+ // This zeal mode disabled nursery resizing.
+ if (gc->hasZealMode(ZealMode::GenerationalGC)) {
+ return;
+ }
+#endif
+
+ decommitTask->join();
+
+ size_t newCapacity = mozilla::Clamp(targetSize(options, reason),
+ tunables().gcMinNurseryBytes(),
+ tunables().gcMaxNurseryBytes());
+
+ MOZ_ASSERT(roundSize(newCapacity) == newCapacity);
+ MOZ_ASSERT(newCapacity >= SystemPageSize());
+
+ if (newCapacity > capacity()) {
+ growAllocableSpace(newCapacity);
+ } else if (newCapacity < capacity()) {
+ shrinkAllocableSpace(newCapacity);
+ }
+
+ AutoLockHelperThreadState lock;
+ if (!decommitTask->isEmpty(lock)) {
+ decommitTask->startOrRunIfIdle(lock);
+ }
+}
+
+static inline bool ClampDouble(double* value, double min, double max) {
+ MOZ_ASSERT(!std::isnan(*value) && !std::isnan(min) && !std::isnan(max));
+ MOZ_ASSERT(max >= min);
+
+ if (*value <= min) {
+ *value = min;
+ return true;
+ }
+
+ if (*value >= max) {
+ *value = max;
+ return true;
+ }
+
+ return false;
+}
+
+size_t js::Nursery::targetSize(JS::GCOptions options, JS::GCReason reason) {
+ // Shrink the nursery as much as possible if purging was requested or in low
+ // memory situations.
+ if (options == JS::GCOptions::Shrink || gc::IsOOMReason(reason) ||
+ gc->systemHasLowMemory()) {
+ clearRecentGrowthData();
+ return 0;
+ }
+
+ // Don't resize the nursery during shutdown.
+ if (options == JS::GCOptions::Shutdown) {
+ clearRecentGrowthData();
+ return capacity();
+ }
+
+ TimeStamp now = TimeStamp::Now();
+
+ // If the nursery is completely unused then minimise it.
+ if (hasRecentGrowthData && previousGC.nurseryUsedBytes == 0 &&
+ now - lastCollectionEndTime() >
+ tunables().nurseryTimeoutForIdleCollection() &&
+ !js::SupportDifferentialTesting()) {
+ clearRecentGrowthData();
+ return 0;
+ }
+
+ // Calculate the fraction of the nursery promoted out of its entire
+ // capacity. This gives better results than using the promotion rate (based on
+ // the amount of nursery used) in cases where we collect before the nursery is
+ // full.
+ double fractionPromoted =
+ double(previousGC.tenuredBytes) / double(previousGC.nurseryCapacity);
+
+ // Calculate the duty factor, the fraction of time spent collecting the
+ // nursery.
+ double dutyFactor = 0.0;
+ TimeDuration collectorTime = now - collectionStartTime();
+ if (hasRecentGrowthData && !js::SupportDifferentialTesting()) {
+ TimeDuration totalTime = now - lastCollectionEndTime();
+ dutyFactor = collectorTime.ToSeconds() / totalTime.ToSeconds();
+ }
+
+ // Calculate a growth factor to try to achieve target promotion rate and duty
+ // factor goals.
+ static const double PromotionGoal = 0.02;
+ static const double DutyFactorGoal = 0.01;
+ double promotionGrowth = fractionPromoted / PromotionGoal;
+ double dutyGrowth = dutyFactor / DutyFactorGoal;
+ double growthFactor = std::max(promotionGrowth, dutyGrowth);
+
+ // Decrease the growth factor to try to keep collections shorter than a target
+ // maximum time. Don't do this during page load.
+ static const double MaxTimeGoalMs = 4.0;
+ if (!gc->isInPageLoad() && !js::SupportDifferentialTesting()) {
+ double timeGrowth = MaxTimeGoalMs / collectorTime.ToMilliseconds();
+ growthFactor = std::min(growthFactor, timeGrowth);
+ }
+
+ // Limit the range of the growth factor to prevent transient high promotion
+ // rates from affecting the nursery size too far into the future.
+ static const double GrowthRange = 2.0;
+ bool wasClamped = ClampDouble(&growthFactor, 1.0 / GrowthRange, GrowthRange);
+
+ // Calculate the target size based on data from this collection.
+ double target = double(capacity()) * growthFactor;
+
+ // Use exponential smoothing on the target size to take into account data from
+ // recent previous collections.
+ if (hasRecentGrowthData &&
+ now - lastCollectionEndTime() < TimeDuration::FromMilliseconds(200) &&
+ !js::SupportDifferentialTesting()) {
+ // Pay more attention to large changes.
+ double fraction = wasClamped ? 0.5 : 0.25;
+ smoothedTargetSize =
+ (1 - fraction) * smoothedTargetSize + fraction * target;
+ } else {
+ smoothedTargetSize = target;
+ }
+ hasRecentGrowthData = true;
+
+ // Leave size untouched if we are close to the target.
+ static const double GoalWidth = 1.5;
+ growthFactor = smoothedTargetSize / double(capacity());
+ if (growthFactor > (1.0 / GoalWidth) && growthFactor < GoalWidth) {
+ return capacity();
+ }
+
+ return roundSize(size_t(smoothedTargetSize));
+}
+
+void js::Nursery::clearRecentGrowthData() {
+ if (js::SupportDifferentialTesting()) {
+ return;
+ }
+
+ hasRecentGrowthData = false;
+ smoothedTargetSize = 0.0;
+}
+
+/* static */
+size_t js::Nursery::roundSize(size_t size) {
+ size_t step = size >= ChunkSize ? ChunkSize : SystemPageSize();
+ return Round(size, step);
+}
+
+void js::Nursery::growAllocableSpace(size_t newCapacity) {
+ MOZ_ASSERT_IF(!isSubChunkMode(), newCapacity > currentChunk_ * ChunkSize);
+ MOZ_ASSERT(newCapacity <= tunables().gcMaxNurseryBytes());
+ MOZ_ASSERT(newCapacity > capacity());
+
+ if (!decommitTask->reserveSpaceForBytes(newCapacity)) {
+ return;
+ }
+
+ if (isSubChunkMode()) {
+ MOZ_ASSERT(currentChunk_ == 0);
+
+ // The remainder of the chunk may have been decommitted.
+ if (!chunk(0).markPagesInUseHard(std::min(newCapacity, ChunkSize))) {
+ // The OS won't give us the memory we need, we can't grow.
+ return;
+ }
+
+ // The capacity has changed and since we were in sub-chunk mode we need to
+ // update the poison values / asan information for the now-valid region of
+ // this chunk.
+ size_t size = std::min(newCapacity, ChunkSize) - capacity();
+ chunk(0).poisonRange(capacity(), size, JS_FRESH_NURSERY_PATTERN,
+ MemCheckKind::MakeUndefined);
+ }
+
+ capacity_ = newCapacity;
+
+ setCurrentEnd();
+}
+
+void js::Nursery::freeChunksFrom(const unsigned firstFreeChunk) {
+ MOZ_ASSERT(firstFreeChunk < chunks_.length());
+
+ // The loop below may need to skip the first chunk, so we may use this so we
+ // can modify it.
+ unsigned firstChunkToDecommit = firstFreeChunk;
+
+ if ((firstChunkToDecommit == 0) && isSubChunkMode()) {
+ // Part of the first chunk may be hard-decommitted, un-decommit it so that
+ // the GC's normal chunk-handling doesn't segfault.
+ MOZ_ASSERT(currentChunk_ == 0);
+ if (!chunk(0).markPagesInUseHard(ChunkSize)) {
+ // Free the chunk if we can't allocate its pages.
+ UnmapPages(static_cast<void*>(&chunk(0)), ChunkSize);
+ firstChunkToDecommit = 1;
+ }
+ }
+
+ {
+ AutoLockHelperThreadState lock;
+ for (size_t i = firstChunkToDecommit; i < chunks_.length(); i++) {
+ decommitTask->queueChunk(chunks_[i], lock);
+ }
+ }
+
+ chunks_.shrinkTo(firstFreeChunk);
+}
+
+void js::Nursery::shrinkAllocableSpace(size_t newCapacity) {
+#ifdef JS_GC_ZEAL
+ if (gc->hasZealMode(ZealMode::GenerationalGC)) {
+ return;
+ }
+#endif
+
+ // Don't shrink the nursery to zero (use Nursery::disable() instead)
+ // This can't happen due to the rounding-down performed above because of the
+ // clamping in maybeResizeNursery().
+ MOZ_ASSERT(newCapacity != 0);
+ // Don't attempt to shrink it to the same size.
+ if (newCapacity == capacity_) {
+ return;
+ }
+ MOZ_ASSERT(newCapacity < capacity_);
+
+ unsigned newCount = HowMany(newCapacity, ChunkSize);
+ if (newCount < allocatedChunkCount()) {
+ freeChunksFrom(newCount);
+ }
+
+ size_t oldCapacity = capacity_;
+ capacity_ = newCapacity;
+
+ setCurrentEnd();
+
+ if (isSubChunkMode()) {
+ MOZ_ASSERT(currentChunk_ == 0);
+ size_t size = std::min(oldCapacity, ChunkSize) - newCapacity;
+ chunk(0).poisonRange(newCapacity, size, JS_SWEPT_NURSERY_PATTERN,
+ MemCheckKind::MakeNoAccess);
+
+ AutoLockHelperThreadState lock;
+ decommitTask->queueRange(capacity_, chunk(0), lock);
+ }
+}
+
+uintptr_t js::Nursery::currentEnd() const {
+ // These are separate asserts because it can be useful to see which one
+ // failed.
+ MOZ_ASSERT_IF(isSubChunkMode(), currentChunk_ == 0);
+ MOZ_ASSERT_IF(isSubChunkMode(), currentEnd_ <= chunk(currentChunk_).end());
+ MOZ_ASSERT_IF(!isSubChunkMode(), currentEnd_ == chunk(currentChunk_).end());
+ MOZ_ASSERT(currentEnd_ != chunk(currentChunk_).start());
+ return currentEnd_;
+}
+
+gcstats::Statistics& js::Nursery::stats() const { return gc->stats(); }
+
+MOZ_ALWAYS_INLINE const js::gc::GCSchedulingTunables& js::Nursery::tunables()
+ const {
+ return gc->tunables;
+}
+
+bool js::Nursery::isSubChunkMode() const {
+ return capacity() <= NurseryChunkUsableSize;
+}
+
+void js::Nursery::sweepMapAndSetObjects() {
+ auto gcx = runtime()->gcContext();
+
+ for (auto mapobj : mapsWithNurseryMemory_) {
+ MapObject::sweepAfterMinorGC(gcx, mapobj);
+ }
+ mapsWithNurseryMemory_.clearAndFree();
+
+ for (auto setobj : setsWithNurseryMemory_) {
+ SetObject::sweepAfterMinorGC(gcx, setobj);
+ }
+ setsWithNurseryMemory_.clearAndFree();
+}
+
+void js::Nursery::joinDecommitTask() { decommitTask->join(); }
+
+JS_PUBLIC_API void JS::EnableNurseryStrings(JSContext* cx) {
+ AutoEmptyNursery empty(cx);
+ ReleaseAllJITCode(cx->gcContext());
+ cx->runtime()->gc.nursery().enableStrings();
+}
+
+JS_PUBLIC_API void JS::DisableNurseryStrings(JSContext* cx) {
+ AutoEmptyNursery empty(cx);
+ ReleaseAllJITCode(cx->gcContext());
+ cx->runtime()->gc.nursery().disableStrings();
+}
+
+JS_PUBLIC_API void JS::EnableNurseryBigInts(JSContext* cx) {
+ AutoEmptyNursery empty(cx);
+ ReleaseAllJITCode(cx->gcContext());
+ cx->runtime()->gc.nursery().enableBigInts();
+}
+
+JS_PUBLIC_API void JS::DisableNurseryBigInts(JSContext* cx) {
+ AutoEmptyNursery empty(cx);
+ ReleaseAllJITCode(cx->gcContext());
+ cx->runtime()->gc.nursery().disableBigInts();
+}
diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h
new file mode 100644
index 0000000000..3dde209d77
--- /dev/null
+++ b/js/src/gc/Nursery.h
@@ -0,0 +1,645 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sw=2 et tw=80:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Nursery_h
+#define gc_Nursery_h
+
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/Heap.h"
+#include "gc/MallocedBlockCache.h"
+#include "gc/Pretenuring.h"
+#include "js/AllocPolicy.h"
+#include "js/Class.h"
+#include "js/GCAPI.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Vector.h"
+
+#define FOR_EACH_NURSERY_PROFILE_TIME(_) \
+ /* Key Header text */ \
+ _(Total, "total") \
+ _(TraceValues, "mkVals") \
+ _(TraceCells, "mkClls") \
+ _(TraceSlots, "mkSlts") \
+ _(TraceWholeCells, "mcWCll") \
+ _(TraceGenericEntries, "mkGnrc") \
+ _(CheckHashTables, "ckTbls") \
+ _(MarkRuntime, "mkRntm") \
+ _(MarkDebugger, "mkDbgr") \
+ _(SweepCaches, "swpCch") \
+ _(CollectToObjFP, "colObj") \
+ _(CollectToStrFP, "colStr") \
+ _(ObjectsTenuredCallback, "tenCB") \
+ _(Sweep, "sweep") \
+ _(UpdateJitActivations, "updtIn") \
+ _(FreeMallocedBuffers, "frSlts") \
+ _(FreeTrailerBlocks, "frTrBs") \
+ _(ClearStoreBuffer, "clrSB") \
+ _(ClearNursery, "clear") \
+ _(PurgeStringToAtomCache, "pStoA") \
+ _(Pretenure, "pretnr")
+
+template <typename T>
+class SharedMem;
+
+namespace js {
+
+struct StringStats;
+class AutoLockGCBgAlloc;
+class ObjectElements;
+struct NurseryChunk;
+class HeapSlot;
+class JSONPrinter;
+class MapObject;
+class SetObject;
+class JS_PUBLIC_API Sprinter;
+
+namespace gc {
+class AutoGCSession;
+struct Cell;
+class GCSchedulingTunables;
+class TenuringTracer;
+} // namespace gc
+
+class Nursery {
+ public:
+ explicit Nursery(gc::GCRuntime* gc);
+ ~Nursery();
+
+ [[nodiscard]] bool init(AutoLockGCBgAlloc& lock);
+
+ // Number of allocated (ready to use) chunks.
+ unsigned allocatedChunkCount() const { return chunks_.length(); }
+
+ // Total number of chunks and the capacity of the nursery. Chunks will be
+ // lazilly allocated and added to the chunks array up to this limit, after
+ // that the nursery must be collected, this limit may be raised during
+ // collection.
+ unsigned maxChunkCount() const {
+ MOZ_ASSERT(capacity());
+ return HowMany(capacity(), gc::ChunkSize);
+ }
+
+ void enable();
+ void disable();
+ bool isEnabled() const { return capacity() != 0; }
+
+ void enableStrings();
+ void disableStrings();
+ bool canAllocateStrings() const { return canAllocateStrings_; }
+
+ void enableBigInts();
+ void disableBigInts();
+ bool canAllocateBigInts() const { return canAllocateBigInts_; }
+
+ // Return true if no allocations have been made since the last collection.
+ bool isEmpty() const;
+
+ // Check whether an arbitrary pointer is within the nursery. This is
+ // slower than IsInsideNursery(Cell*), but works on all types of pointers.
+ MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete;
+ MOZ_ALWAYS_INLINE bool isInside(const void* p) const {
+ for (auto chunk : chunks_) {
+ if (uintptr_t(p) - uintptr_t(chunk) < gc::ChunkSize) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <typename T>
+ inline bool isInside(const SharedMem<T>& p) const;
+
+ // Allocate and return a pointer to a new GC thing. Returns nullptr if the
+ // Nursery is full.
+ void* allocateCell(gc::AllocSite* site, size_t size, JS::TraceKind kind);
+
+ static size_t nurseryCellHeaderSize() {
+ return sizeof(gc::NurseryCellHeader);
+ }
+
+ // Allocate a buffer for a given zone, using the nursery if possible.
+ void* allocateBuffer(JS::Zone* zone, size_t nbytes);
+
+ // Allocate a buffer for a given object, using the nursery if possible and
+ // obj is in the nursery.
+ void* allocateBuffer(JS::Zone* zone, JSObject* obj, size_t nbytes);
+
+ // Allocate a buffer for a given object, always using the nursery if obj is
+ // in the nursery. The requested size must be less than or equal to
+ // MaxNurseryBufferSize.
+ void* allocateBufferSameLocation(JSObject* obj, size_t nbytes);
+
+ // Allocate a zero-initialized buffer for a given zone, using the nursery if
+ // possible. If the buffer isn't allocated in the nursery, the given arena is
+ // used.
+ void* allocateZeroedBuffer(JS::Zone* zone, size_t nbytes,
+ arena_id_t arena = js::MallocArena);
+
+ // Allocate a zero-initialized buffer for a given object, using the nursery if
+ // possible and obj is in the nursery. If the buffer isn't allocated in the
+ // nursery, the given arena is used.
+ void* allocateZeroedBuffer(JSObject* obj, size_t nbytes,
+ arena_id_t arena = js::MallocArena);
+
+ // Resize an existing buffer.
+ void* reallocateBuffer(JS::Zone* zone, gc::Cell* cell, void* oldBuffer,
+ size_t oldBytes, size_t newBytes);
+
+ // Allocate a digits buffer for a given BigInt, using the nursery if possible
+ // and |bi| is in the nursery.
+ void* allocateBuffer(JS::BigInt* bi, size_t nbytes);
+
+ // Free an object buffer.
+ void freeBuffer(void* buffer, size_t nbytes);
+
+ // The maximum number of bytes allowed to reside in nursery buffers.
+ static const size_t MaxNurseryBufferSize = 1024;
+
+ // Do a minor collection.
+ void collect(JS::GCOptions options, JS::GCReason reason);
+
+ // If the thing at |*ref| in the Nursery has been forwarded, set |*ref| to
+ // the new location and return true. Otherwise return false and leave
+ // |*ref| unset.
+ [[nodiscard]] MOZ_ALWAYS_INLINE static bool getForwardedPointer(
+ js::gc::Cell** ref);
+
+ // Forward a slots/elements pointer stored in an Ion frame.
+ void forwardBufferPointer(uintptr_t* pSlotsElems);
+
+ inline void maybeSetForwardingPointer(JSTracer* trc, void* oldData,
+ void* newData, bool direct);
+ inline void setForwardingPointerWhileTenuring(void* oldData, void* newData,
+ bool direct);
+
+ // Register a malloced buffer that is held by a nursery object, which
+ // should be freed at the end of a minor GC. Buffers are unregistered when
+ // their owning objects are tenured.
+ [[nodiscard]] bool registerMallocedBuffer(void* buffer, size_t nbytes);
+
+ // Mark a malloced buffer as no longer needing to be freed.
+ void removeMallocedBuffer(void* buffer, size_t nbytes) {
+ MOZ_ASSERT(mallocedBuffers.has(buffer));
+ MOZ_ASSERT(nbytes > 0);
+ MOZ_ASSERT(mallocedBufferBytes >= nbytes);
+ mallocedBuffers.remove(buffer);
+ mallocedBufferBytes -= nbytes;
+ }
+
+ // Mark a malloced buffer as no longer needing to be freed during minor
+ // GC. There's no need to account for the size here since all remaining
+ // buffers will soon be freed.
+ void removeMallocedBufferDuringMinorGC(void* buffer) {
+ MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+ MOZ_ASSERT(mallocedBuffers.has(buffer));
+ mallocedBuffers.remove(buffer);
+ }
+
+ [[nodiscard]] bool addedUniqueIdToCell(gc::Cell* cell) {
+ MOZ_ASSERT(IsInsideNursery(cell));
+ MOZ_ASSERT(isEnabled());
+ return cellsWithUid_.append(cell);
+ }
+
+ size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const;
+
+ // Wasm "trailer" (C++-heap-allocated) blocks.
+ //
+ // All involved blocks are allocated/deallocated via this nursery's
+ // `mallocedBlockCache_`. Hence we must store both the block address and
+ // its freelist ID, wrapped up in a PointerAndUint7.
+ //
+ // Trailer blocks registered here are added to `trailersAdded_`. Those that
+ // are later deregistered as a result of `obj_moved` calls that indicate
+ // tenuring, should be added to `trailersRemoved_`.
+ //
+ // Unfortunately ::unregisterTrailer cannot be allowed to OOM. To get
+ // around this we rely on the observation that all deregistered blocks
+ // should previously have been registered, so the deregistered set can never
+ // be larger than the registered set. Hence ::registerTrailer effectively
+ // preallocates space in `trailersRemoved_` so as to ensure that, in the
+ // worst case, all registered blocks can be handed to ::unregisterTrailer
+ // without needing to resize `trailersRemoved_` in ::unregisterTrailer.
+ //
+ // The downside is that most of the space in `trailersRemoved_` is wasted in
+ // the case where there are few blocks deregistered. This is unfortunate
+ // but it's hard to see how to avoid it.
+ //
+ // At the end of a minor collection, all blocks in the set `trailersAdded_ -
+ // trailersRemoved_[0 .. trailersRemovedUsed_ - 1]` are handed back to the
+ // `mallocedBlockCache_`.
+ [[nodiscard]] bool registerTrailer(PointerAndUint7 blockAndListID,
+ size_t nBytes) {
+ MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length());
+ MOZ_ASSERT(nBytes > 0);
+ if (MOZ_UNLIKELY(!trailersAdded_.append(blockAndListID))) {
+ return false;
+ }
+ if (MOZ_UNLIKELY(!trailersRemoved_.append(nullptr))) {
+ trailersAdded_.popBack();
+ return false;
+ }
+
+ // This is a clone of the logic in ::registerMallocedBuffer. It may be
+ // that some other heuristic is better, once we know more about the
+ // typical behaviour of wasm-GC applications.
+ trailerBytes_ += nBytes;
+ if (MOZ_UNLIKELY(trailerBytes_ > capacity() * 8)) {
+ requestMinorGC(JS::GCReason::NURSERY_TRAILERS);
+ }
+ return true;
+ }
+
+ void unregisterTrailer(void* block) {
+ MOZ_ASSERT(trailersRemovedUsed_ < trailersRemoved_.length());
+ trailersRemoved_[trailersRemovedUsed_] = block;
+ trailersRemovedUsed_++;
+ }
+
+ size_t sizeOfTrailerBlockSets(mozilla::MallocSizeOf mallocSizeOf) const;
+
+ // The number of bytes from the start position to the end of the nursery.
+ // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit()
+ // to calculate the nursery size, current lazy-allocated size or nursery
+ // limit respectively.
+ size_t spaceToEnd(unsigned chunkCount) const;
+
+ size_t capacity() const { return capacity_; }
+ size_t committed() const { return spaceToEnd(allocatedChunkCount()); }
+
+ // Used and free space both include chunk headers for that part of the
+ // nursery.
+ //
+ // usedSpace() + freeSpace() == capacity()
+ //
+ MOZ_ALWAYS_INLINE size_t usedSpace() const {
+ return capacity() - freeSpace();
+ }
+ MOZ_ALWAYS_INLINE size_t freeSpace() const {
+ MOZ_ASSERT(isEnabled());
+ MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
+ MOZ_ASSERT(currentChunk_ < maxChunkCount());
+ return (currentEnd_ - position_) +
+ (maxChunkCount() - currentChunk_ - 1) * gc::ChunkSize;
+ }
+
+#ifdef JS_GC_ZEAL
+ void enterZealMode();
+ void leaveZealMode();
+#endif
+
+ // Write profile time JSON on JSONPrinter.
+ void renderProfileJSON(JSONPrinter& json) const;
+
+ // Print header line for profile times.
+ void printProfileHeader();
+
+ // Print total profile times on shutdown.
+ void printTotalProfileTimes();
+
+ void* addressOfPosition() const { return (void**)&position_; }
+ static constexpr int32_t offsetOfCurrentEndFromPosition() {
+ return offsetof(Nursery, currentEnd_) - offsetof(Nursery, position_);
+ }
+
+ void* addressOfNurseryAllocatedSites() {
+ return pretenuringNursery.addressOfAllocatedSites();
+ }
+
+ void requestMinorGC(JS::GCReason reason) const;
+
+ bool minorGCRequested() const {
+ return minorGCTriggerReason_ != JS::GCReason::NO_REASON;
+ }
+ JS::GCReason minorGCTriggerReason() const { return minorGCTriggerReason_; }
+ void clearMinorGCRequest() {
+ minorGCTriggerReason_ = JS::GCReason::NO_REASON;
+ }
+
+ bool shouldCollect() const;
+ bool isNearlyFull() const;
+ bool isUnderused() const;
+
+ bool enableProfiling() const { return enableProfiling_; }
+
+ bool addMapWithNurseryMemory(MapObject* obj) {
+ MOZ_ASSERT_IF(!mapsWithNurseryMemory_.empty(),
+ mapsWithNurseryMemory_.back() != obj);
+ return mapsWithNurseryMemory_.append(obj);
+ }
+ bool addSetWithNurseryMemory(SetObject* obj) {
+ MOZ_ASSERT_IF(!setsWithNurseryMemory_.empty(),
+ setsWithNurseryMemory_.back() != obj);
+ return setsWithNurseryMemory_.append(obj);
+ }
+
+ // The amount of space in the mapped nursery available to allocations.
+ static const size_t NurseryChunkUsableSize =
+ gc::ChunkSize - sizeof(gc::ChunkBase);
+
+ void joinDecommitTask();
+
+ mozilla::TimeStamp collectionStartTime() {
+ return startTimes_[ProfileKey::Total];
+ }
+
+ bool canCreateAllocSite() { return pretenuringNursery.canCreateAllocSite(); }
+ void noteAllocSiteCreated() { pretenuringNursery.noteAllocSiteCreated(); }
+ bool reportPretenuring() const { return reportPretenuring_; }
+ void maybeStopPretenuring(gc::GCRuntime* gc) {
+ pretenuringNursery.maybeStopPretenuring(gc);
+ }
+
+ void setAllocFlagsForZone(JS::Zone* zone);
+
+ // Round a size in bytes to the nearest valid nursery size.
+ static size_t roundSize(size_t size);
+
+ // The malloc'd block cache.
+ gc::MallocedBlockCache& mallocedBlockCache() { return mallocedBlockCache_; }
+ size_t sizeOfMallocedBlockCache(mozilla::MallocSizeOf mallocSizeOf) const {
+ return mallocedBlockCache_.sizeOfExcludingThis(mallocSizeOf);
+ }
+
+ private:
+ // Fields used during allocation fast path are grouped first:
+
+ // Pointer to the first unallocated byte in the nursery.
+ uintptr_t position_;
+
+ // Pointer to the last byte of space in the current chunk.
+ uintptr_t currentEnd_;
+
+ // Other fields not necessarily used during allocation follow:
+
+ gc::GCRuntime* const gc;
+
+ // Vector of allocated chunks to allocate from.
+ Vector<NurseryChunk*, 0, SystemAllocPolicy> chunks_;
+
+ // The index of the chunk that is currently being allocated from.
+ uint32_t currentChunk_;
+
+ // These fields refer to the beginning of the nursery. They're normally 0
+ // and chunk(0).start() respectively. Except when a generational GC zeal
+ // mode is active, then they may be arbitrary (see Nursery::clear()).
+ uint32_t currentStartChunk_;
+ uintptr_t currentStartPosition_;
+
+ // The current nursery capacity measured in bytes. It may grow up to this
+ // value without a collection, allocating chunks on demand. This limit may be
+ // changed by maybeResizeNursery() each collection. It includes chunk headers.
+ size_t capacity_;
+
+ gc::PretenuringNursery pretenuringNursery;
+
+ mozilla::TimeDuration timeInChunkAlloc_;
+
+ // Report minor collections taking at least this long, if enabled.
+ bool enableProfiling_;
+ bool profileWorkers_;
+ mozilla::TimeDuration profileThreshold_;
+
+ // Whether we will nursery-allocate strings.
+ bool canAllocateStrings_;
+
+ // Whether we will nursery-allocate BigInts.
+ bool canAllocateBigInts_;
+
+ // Report how many strings were deduplicated.
+ bool reportDeduplications_;
+
+ // Whether to report information on pretenuring, and if so the allocation
+ // threshold at which to report details of each allocation site.
+ bool reportPretenuring_;
+ size_t reportPretenuringThreshold_;
+
+ // Whether and why a collection of this nursery has been requested. This is
+ // mutable as it is set by the store buffer, which otherwise cannot modify
+ // anything in the nursery.
+ mutable JS::GCReason minorGCTriggerReason_;
+
+ // Profiling data.
+
+ enum class ProfileKey {
+#define DEFINE_TIME_KEY(name, text) name,
+ FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY)
+#undef DEFINE_TIME_KEY
+ KeyCount
+ };
+
+ using ProfileTimes =
+ mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount,
+ mozilla::TimeStamp>;
+ using ProfileDurations =
+ mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount,
+ mozilla::TimeDuration>;
+
+ ProfileTimes startTimes_;
+ ProfileDurations profileDurations_;
+ ProfileDurations totalDurations_;
+
+ // Data about the previous collection.
+ struct PreviousGC {
+ JS::GCReason reason = JS::GCReason::NO_REASON;
+ size_t nurseryCapacity = 0;
+ size_t nurseryCommitted = 0;
+ size_t nurseryUsedBytes = 0;
+ size_t nurseryUsedChunkCount = 0;
+ size_t tenuredBytes = 0;
+ size_t tenuredCells = 0;
+ mozilla::TimeStamp endTime;
+ };
+ PreviousGC previousGC;
+
+ bool hasRecentGrowthData;
+ double smoothedTargetSize;
+
+ // Calculate the promotion rate of the most recent minor GC.
+ // The valid_for_tenuring parameter is used to return whether this
+ // promotion rate is accurate enough (the nursery was full enough) to be
+ // used for tenuring and other decisions.
+ //
+ // Must only be called if the previousGC data is initialised.
+ double calcPromotionRate(bool* validForTenuring) const;
+
+ // The set of externally malloced buffers potentially kept live by objects
+ // stored in the nursery. Any external buffers that do not belong to a
+ // tenured thing at the end of a minor GC must be freed.
+ using BufferRelocationOverlay = void*;
+ using BufferSet = HashSet<void*, PointerHasher<void*>, SystemAllocPolicy>;
+ BufferSet mallocedBuffers;
+ size_t mallocedBufferBytes = 0;
+
+ // Wasm "trailer" (C++-heap-allocated) blocks. See comments above on
+ // ::registerTrailer and ::unregisterTrailer.
+ Vector<PointerAndUint7, 0, SystemAllocPolicy> trailersAdded_;
+ Vector<void*, 0, SystemAllocPolicy> trailersRemoved_;
+ size_t trailersRemovedUsed_ = 0;
+ size_t trailerBytes_ = 0;
+
+ void freeTrailerBlocks();
+
+ // During a collection most hoisted slot and element buffers indicate their
+ // new location with a forwarding pointer at the base. This does not work
+ // for buffers whose length is less than pointer width, or when different
+ // buffers might overlap each other. For these, an entry in the following
+ // table is used.
+ typedef HashMap<void*, void*, PointerHasher<void*>, SystemAllocPolicy>
+ ForwardedBufferMap;
+ ForwardedBufferMap forwardedBuffers;
+
+ // When we assign a unique id to cell in the nursery, that almost always
+ // means that the cell will be in a hash table, and thus, held live,
+ // automatically moving the uid from the nursery to its new home in
+ // tenured. It is possible, if rare, for an object that acquired a uid to
+ // be dead before the next collection, in which case we need to know to
+ // remove it when we sweep.
+ //
+ // Note: we store the pointers as Cell* here, resulting in an ugly cast in
+ // sweep. This is because this structure is used to help implement
+ // stable object hashing and we have to break the cycle somehow.
+ using CellsWithUniqueIdVector = Vector<gc::Cell*, 8, SystemAllocPolicy>;
+ CellsWithUniqueIdVector cellsWithUid_;
+
+ // Lists of map and set objects allocated in the nursery or with iterators
+ // allocated there. Such objects need to be swept after minor GC.
+ Vector<MapObject*, 0, SystemAllocPolicy> mapsWithNurseryMemory_;
+ Vector<SetObject*, 0, SystemAllocPolicy> setsWithNurseryMemory_;
+
+ UniquePtr<NurseryDecommitTask> decommitTask;
+
+ // A cache of small C++-heap allocated blocks associated with this Nursery.
+ // This provided so as to provide cheap allocation/deallocation of
+ // out-of-line storage areas as used by WasmStructObject and
+ // WasmArrayObject, although the mechanism is general and not specific to
+ // these object types. Regarding lifetimes, because the cache holds only
+ // blocks that are not currently in use, it can be flushed at any point with
+ // no correctness impact, only a performance impact.
+ gc::MallocedBlockCache mallocedBlockCache_;
+
+ NurseryChunk& chunk(unsigned index) const { return *chunks_[index]; }
+
+ // Set the current chunk. This updates the currentChunk_, position_ and
+ // currentEnd_ values as appropriate. It'll also poison the chunk, either a
+ // portion of the chunk if it is already the current chunk, or the whole chunk
+ // if fullPoison is true or it is not the current chunk.
+ void setCurrentChunk(unsigned chunkno);
+
+ bool initFirstChunk(AutoLockGCBgAlloc& lock);
+
+ // extent is advisory, it will be ignored in sub-chunk and generational zeal
+ // modes. It will be clamped to Min(NurseryChunkUsableSize, capacity_).
+ void poisonAndInitCurrentChunk(size_t extent = gc::ChunkSize);
+
+ void setCurrentEnd();
+ void setStartPosition();
+
+ // Allocate the next chunk, or the first chunk for initialization.
+ // Callers will probably want to call setCurrentChunk(0) next.
+ [[nodiscard]] bool allocateNextChunk(unsigned chunkno,
+ AutoLockGCBgAlloc& lock);
+
+ MOZ_ALWAYS_INLINE uintptr_t currentEnd() const;
+
+ uintptr_t position() const { return position_; }
+
+ MOZ_ALWAYS_INLINE bool isSubChunkMode() const;
+
+ JSRuntime* runtime() const;
+ gcstats::Statistics& stats() const;
+
+ const js::gc::GCSchedulingTunables& tunables() const;
+
+ void getAllocFlagsForZone(JS::Zone* zone, bool* allocObjectsOut,
+ bool* allocStringsOut, bool* allocBigIntsOut);
+ void updateAllZoneAllocFlags();
+ void updateAllocFlagsForZone(JS::Zone* zone);
+ void discardCodeAndSetJitFlagsForZone(JS::Zone* zone);
+
+ // Common internal allocator function.
+ void* allocate(size_t size);
+
+ void* moveToNextChunkAndAllocate(size_t size);
+
+ struct CollectionResult {
+ size_t tenuredBytes;
+ size_t tenuredCells;
+ };
+ CollectionResult doCollection(gc::AutoGCSession& session,
+ JS::GCOptions options, JS::GCReason reason);
+ void traceRoots(gc::AutoGCSession& session, gc::TenuringTracer& mover);
+
+ size_t doPretenuring(JSRuntime* rt, JS::GCReason reason,
+ bool validPromotionRate, double promotionRate);
+
+ // Handle relocation of slots/elements pointers stored in Ion frames.
+ inline void setForwardingPointer(void* oldData, void* newData, bool direct);
+
+ inline void setDirectForwardingPointer(void* oldData, void* newData);
+ void setIndirectForwardingPointer(void* oldData, void* newData);
+
+ inline void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots,
+ uint32_t nslots);
+ inline void setElementsForwardingPointer(ObjectElements* oldHeader,
+ ObjectElements* newHeader,
+ uint32_t capacity);
+
+#ifdef DEBUG
+ bool checkForwardingPointerLocation(void* ptr, bool expectedInside);
+#endif
+
+ // Updates pointers to nursery objects that have been tenured and discards
+ // pointers to objects that have been freed.
+ void sweep();
+
+ // Reset the current chunk and position after a minor collection. Also poison
+ // the nursery on debug & nightly builds.
+ void clear();
+
+ void sweepMapAndSetObjects();
+
+ // Change the allocable space provided by the nursery.
+ void maybeResizeNursery(JS::GCOptions options, JS::GCReason reason);
+ size_t targetSize(JS::GCOptions options, JS::GCReason reason);
+ void clearRecentGrowthData();
+ void growAllocableSpace(size_t newCapacity);
+ void shrinkAllocableSpace(size_t newCapacity);
+ void minimizeAllocableSpace();
+
+ // Free the chunks starting at firstFreeChunk until the end of the chunks
+ // vector. Shrinks the vector but does not update maxChunkCount().
+ void freeChunksFrom(unsigned firstFreeChunk);
+
+ void sendTelemetry(JS::GCReason reason, mozilla::TimeDuration totalTime,
+ bool wasEmpty, double promotionRate,
+ size_t sitesPretenured);
+
+ void printCollectionProfile(JS::GCReason reason, double promotionRate);
+ void printDeduplicationData(js::StringStats& prev, js::StringStats& curr);
+
+ // Profile recording and printing.
+ void maybeClearProfileDurations();
+ void startProfile(ProfileKey key);
+ void endProfile(ProfileKey key);
+ static bool printProfileDurations(const ProfileDurations& times,
+ Sprinter& sprinter);
+
+ mozilla::TimeStamp collectionStartTime() const;
+ mozilla::TimeStamp lastCollectionEndTime() const;
+
+ friend class gc::GCRuntime;
+ friend class gc::TenuringTracer;
+ friend struct NurseryChunk;
+};
+
+} // namespace js
+
+#endif // gc_Nursery_h
diff --git a/js/src/gc/NurseryAwareHashMap.h b/js/src/gc/NurseryAwareHashMap.h
new file mode 100644
index 0000000000..35c2bebcea
--- /dev/null
+++ b/js/src/gc/NurseryAwareHashMap.h
@@ -0,0 +1,209 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_NurseryAwareHashMap_h
+#define gc_NurseryAwareHashMap_h
+
+#include "gc/Barrier.h"
+#include "gc/Tracer.h"
+#include "js/GCHashTable.h"
+#include "js/GCPolicyAPI.h"
+#include "js/HashTable.h"
+
+namespace js {
+
+namespace detail {
+
+// This class only handles the incremental case and does not deal with nursery
+// pointers. The only users should be for NurseryAwareHashMap; it is defined
+// externally because we need a GCPolicy for its use in the contained map.
+template <typename T>
+class UnsafeBareWeakHeapPtr : public ReadBarriered<T> {
+ public:
+ UnsafeBareWeakHeapPtr()
+ : ReadBarriered<T>(JS::SafelyInitialized<T>::create()) {}
+ MOZ_IMPLICIT UnsafeBareWeakHeapPtr(const T& v) : ReadBarriered<T>(v) {}
+ explicit UnsafeBareWeakHeapPtr(const UnsafeBareWeakHeapPtr& v)
+ : ReadBarriered<T>(v) {}
+ UnsafeBareWeakHeapPtr(UnsafeBareWeakHeapPtr&& v)
+ : ReadBarriered<T>(std::move(v)) {}
+
+ UnsafeBareWeakHeapPtr& operator=(const UnsafeBareWeakHeapPtr& v) {
+ this->value = v.value;
+ return *this;
+ }
+
+ UnsafeBareWeakHeapPtr& operator=(const T& v) {
+ this->value = v;
+ return *this;
+ }
+
+ const T get() const {
+ if (!InternalBarrierMethods<T>::isMarkable(this->value)) {
+ return JS::SafelyInitialized<T>::create();
+ }
+ this->read();
+ return this->value;
+ }
+
+ explicit operator bool() const { return bool(this->value); }
+
+ const T unbarrieredGet() const { return this->value; }
+ T* unsafeGet() { return &this->value; }
+ T const* unsafeGet() const { return &this->value; }
+};
+} // namespace detail
+
+enum : bool { DuplicatesNotPossible, DuplicatesPossible };
+
+// The "nursery aware" hash map is a special case of GCHashMap that is able to
+// treat nursery allocated members weakly during a minor GC: e.g. it allows for
+// nursery allocated objects to be collected during nursery GC where a normal
+// hash table treats such edges strongly.
+//
+// Doing this requires some strong constraints on what can be stored in this
+// table and how it can be accessed. At the moment, this table assumes that all
+// values contain a strong reference to the key. This limits its usefulness to
+// the CrossCompartmentMap at the moment, but might serve as a useful base for
+// other tables in future.
+template <typename Key, typename Value, typename AllocPolicy = TempAllocPolicy,
+ bool AllowDuplicates = DuplicatesNotPossible>
+class NurseryAwareHashMap {
+ using MapKey = UnsafeBarePtr<Key>;
+ using MapValue = detail::UnsafeBareWeakHeapPtr<Value>;
+ using HashPolicy = DefaultHasher<MapKey>;
+ using MapType = GCRekeyableHashMap<MapKey, MapValue, HashPolicy, AllocPolicy>;
+ MapType map;
+
+ // Keep a list of all keys for which key->isTenured() is false. This lets us
+ // avoid a full traversal of the map on each minor GC, keeping the minor GC
+ // times proportional to the nursery heap size.
+ Vector<Key, 0, AllocPolicy> nurseryEntries;
+
+ public:
+ using Lookup = typename MapType::Lookup;
+ using Ptr = typename MapType::Ptr;
+ using Range = typename MapType::Range;
+ using Entry = typename MapType::Entry;
+
+ explicit NurseryAwareHashMap(AllocPolicy a = AllocPolicy())
+ : map(a), nurseryEntries(std::move(a)) {}
+ explicit NurseryAwareHashMap(size_t length) : map(length) {}
+ NurseryAwareHashMap(AllocPolicy a, size_t length)
+ : map(a, length), nurseryEntries(std::move(a)) {}
+
+ bool empty() const { return map.empty(); }
+ Ptr lookup(const Lookup& l) const { return map.lookup(l); }
+ void remove(Ptr p) { map.remove(p); }
+ Range all() const { return map.all(); }
+ struct Enum : public MapType::Enum {
+ explicit Enum(NurseryAwareHashMap& namap) : MapType::Enum(namap.map) {}
+ };
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return map.shallowSizeOfExcludingThis(mallocSizeOf) +
+ nurseryEntries.sizeOfExcludingThis(mallocSizeOf);
+ }
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return map.shallowSizeOfIncludingThis(mallocSizeOf) +
+ nurseryEntries.sizeOfIncludingThis(mallocSizeOf);
+ }
+
+ [[nodiscard]] bool put(const Key& key, const Value& value) {
+ if ((!key->isTenured() || !value->isTenured()) &&
+ !nurseryEntries.append(key)) {
+ return false;
+ }
+
+ auto p = map.lookupForAdd(key);
+ if (p) {
+ p->value() = value;
+ return true;
+ }
+
+ return map.add(p, key, value);
+ }
+
+ void sweepAfterMinorGC(JSTracer* trc) {
+ for (auto& key : nurseryEntries) {
+ auto p = map.lookup(key);
+ if (!p) {
+ continue;
+ }
+
+ // Drop the entry if the value is not marked.
+ if (!JS::GCPolicy<MapValue>::traceWeak(trc, &p->value())) {
+ map.remove(p);
+ continue;
+ }
+
+ // Update and relocate the key, if the value is still needed.
+ //
+ // Non-string Values will contain a strong reference to Key, as per its
+ // use in the CrossCompartmentWrapperMap, so the key will never be dying
+ // here. Strings do *not* have any sort of pointer from wrapper to
+ // wrappee, as they are just copies. The wrapper map entry is merely used
+ // as a cache to avoid re-copying the string, and currently that entire
+ // cache is flushed on major GC.
+ MapKey copy(key);
+ if (!JS::GCPolicy<MapKey>::traceWeak(trc, &copy)) {
+ map.remove(p);
+ continue;
+ }
+
+ if (AllowDuplicates) {
+ // Drop duplicated keys.
+ //
+ // A key can be forwarded to another place. In this case, rekey the
+ // item. If two or more different keys are forwarded to the same new
+ // key, simply drop the later ones.
+ if (key == copy) {
+ // No rekey needed.
+ } else if (map.has(copy)) {
+ // Key was forwarded to the same place that another key was already
+ // forwarded to.
+ map.remove(p);
+ } else {
+ map.rekeyAs(key, copy, copy);
+ }
+ } else {
+ MOZ_ASSERT(key == copy || !map.has(copy));
+ map.rekeyIfMoved(key, copy);
+ }
+ }
+ nurseryEntries.clear();
+ }
+
+ void traceWeak(JSTracer* trc) { map.traceWeak(trc); }
+
+ void clear() {
+ map.clear();
+ nurseryEntries.clear();
+ }
+
+ bool hasNurseryEntries() const { return !nurseryEntries.empty(); }
+};
+
+} // namespace js
+
+namespace JS {
+
+template <typename T>
+struct GCPolicy<js::detail::UnsafeBareWeakHeapPtr<T>> {
+ static void trace(JSTracer* trc, js::detail::UnsafeBareWeakHeapPtr<T>* thingp,
+ const char* name) {
+ js::TraceEdge(trc, thingp, name);
+ }
+ static bool traceWeak(JSTracer* trc,
+ js::detail::UnsafeBareWeakHeapPtr<T>* thingp) {
+ return js::TraceWeakEdge(trc, thingp, "UnsafeBareWeakHeapPtr");
+ }
+};
+
+} // namespace JS
+
+namespace mozilla {} // namespace mozilla
+
+#endif // gc_NurseryAwareHashMap_h
diff --git a/js/src/gc/ObjectKind-inl.h b/js/src/gc/ObjectKind-inl.h
new file mode 100644
index 0000000000..a03b183447
--- /dev/null
+++ b/js/src/gc/ObjectKind-inl.h
@@ -0,0 +1,162 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal helper functions for getting the AllocKind used to allocate a
+ * JSObject and related information.
+ */
+
+#ifndef gc_ObjectKind_inl_h
+#define gc_ObjectKind_inl_h
+
+#include "util/Memory.h"
+#include "vm/NativeObject.h"
+
+namespace js {
+namespace gc {
+
+/* Capacity for slotsToThingKind */
+const size_t SLOTS_TO_THING_KIND_LIMIT = 17;
+
+extern const AllocKind slotsToThingKind[];
+
+/* Get the best kind to use when making an object with the given slot count. */
+static inline AllocKind GetGCObjectKind(size_t numSlots) {
+ if (numSlots >= SLOTS_TO_THING_KIND_LIMIT) {
+ return AllocKind::OBJECT16;
+ }
+ return slotsToThingKind[numSlots];
+}
+
+static inline AllocKind GetGCObjectKind(const JSClass* clasp) {
+ MOZ_ASSERT(!clasp->isProxyObject(),
+ "Proxies should use GetProxyGCObjectKind");
+ MOZ_ASSERT(!clasp->isJSFunction());
+
+ uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp);
+ return GetGCObjectKind(nslots);
+}
+
+static constexpr bool CanUseFixedElementsForArray(size_t numElements) {
+ if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT) {
+ return false;
+ }
+ size_t numSlots = numElements + ObjectElements::VALUES_PER_HEADER;
+ return numSlots < SLOTS_TO_THING_KIND_LIMIT;
+}
+
+/* As for GetGCObjectKind, but for dense array allocation. */
+static inline AllocKind GetGCArrayKind(size_t numElements) {
+ /*
+ * Dense arrays can use their fixed slots to hold their elements array
+ * (less two Values worth of ObjectElements header), but if more than the
+ * maximum number of fixed slots is needed then the fixed slots will be
+ * unused.
+ */
+ static_assert(ObjectElements::VALUES_PER_HEADER == 2);
+ if (!CanUseFixedElementsForArray(numElements)) {
+ return AllocKind::OBJECT2;
+ }
+ return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER];
+}
+
+static inline AllocKind GetGCObjectFixedSlotsKind(size_t numFixedSlots) {
+ MOZ_ASSERT(numFixedSlots < SLOTS_TO_THING_KIND_LIMIT);
+ return slotsToThingKind[numFixedSlots];
+}
+
+// Get the best kind to use when allocating an object that needs a specific
+// number of bytes.
+static inline AllocKind GetGCObjectKindForBytes(size_t nbytes) {
+ MOZ_ASSERT(nbytes <= JSObject::MAX_BYTE_SIZE);
+
+ if (nbytes <= sizeof(NativeObject)) {
+ return AllocKind::OBJECT0;
+ }
+ nbytes -= sizeof(NativeObject);
+
+ size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value);
+ MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value));
+ return GetGCObjectKind(dataSlots);
+}
+
+/* Get the number of fixed slots and initial capacity associated with a kind. */
+static constexpr inline size_t GetGCKindSlots(AllocKind thingKind) {
+ // Using a switch in hopes that thingKind will usually be a compile-time
+ // constant.
+ switch (thingKind) {
+ case AllocKind::OBJECT0:
+ case AllocKind::OBJECT0_BACKGROUND:
+ return 0;
+ case AllocKind::OBJECT2:
+ case AllocKind::OBJECT2_BACKGROUND:
+ return 2;
+ case AllocKind::FUNCTION:
+ case AllocKind::OBJECT4:
+ case AllocKind::OBJECT4_BACKGROUND:
+ return 4;
+ case AllocKind::FUNCTION_EXTENDED:
+ return 6;
+ case AllocKind::OBJECT8:
+ case AllocKind::OBJECT8_BACKGROUND:
+ return 8;
+ case AllocKind::OBJECT12:
+ case AllocKind::OBJECT12_BACKGROUND:
+ return 12;
+ case AllocKind::OBJECT16:
+ case AllocKind::OBJECT16_BACKGROUND:
+ return 16;
+ default:
+ MOZ_CRASH("Bad object alloc kind");
+ }
+}
+
+static inline size_t GetGCKindBytes(AllocKind thingKind) {
+ return sizeof(JSObject_Slots0) + GetGCKindSlots(thingKind) * sizeof(Value);
+}
+
+static inline bool CanUseBackgroundAllocKind(const JSClass* clasp) {
+ return !clasp->hasFinalize() || (clasp->flags & JSCLASS_BACKGROUND_FINALIZE);
+}
+
+static inline bool CanChangeToBackgroundAllocKind(AllocKind kind,
+ const JSClass* clasp) {
+ // If a foreground alloc kind is specified but the class has no finalizer or a
+ // finalizer that is safe to call on a different thread, we can change the
+ // alloc kind to one which is finalized on a background thread.
+ //
+ // For example, AllocKind::OBJECT0 calls the finalizer on the main thread, and
+ // AllocKind::OBJECT0_BACKGROUND calls the finalizer on the a helper thread.
+
+ MOZ_ASSERT(IsObjectAllocKind(kind));
+
+ if (IsBackgroundFinalized(kind)) {
+ return false; // This kind is already a background finalized kind.
+ }
+
+ return CanUseBackgroundAllocKind(clasp);
+}
+
+static inline AllocKind ForegroundToBackgroundAllocKind(AllocKind fgKind) {
+ MOZ_ASSERT(IsObjectAllocKind(fgKind));
+ MOZ_ASSERT(IsForegroundFinalized(fgKind));
+
+ // For objects, each background alloc kind is defined just after the
+ // corresponding foreground alloc kind so we can convert between them by
+ // incrementing or decrementing as appropriate.
+ AllocKind bgKind = AllocKind(size_t(fgKind) + 1);
+
+ MOZ_ASSERT(IsObjectAllocKind(bgKind));
+ MOZ_ASSERT(IsBackgroundFinalized(bgKind));
+ MOZ_ASSERT(GetGCKindSlots(bgKind) == GetGCKindSlots(fgKind));
+
+ return bgKind;
+}
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_ObjectKind_inl_h
diff --git a/js/src/gc/ParallelMarking.cpp b/js/src/gc/ParallelMarking.cpp
new file mode 100644
index 0000000000..67c29c02d7
--- /dev/null
+++ b/js/src/gc/ParallelMarking.cpp
@@ -0,0 +1,359 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/ParallelMarking.h"
+
+#include "gc/GCLock.h"
+#include "gc/ParallelWork.h"
+#include "vm/GeckoProfiler.h"
+#include "vm/HelperThreadState.h"
+#include "vm/Runtime.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::Maybe;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+class AutoAddTimeDuration {
+ TimeStamp start;
+ TimeDuration& result;
+
+ public:
+ explicit AutoAddTimeDuration(TimeDuration& result)
+ : start(TimeStamp::Now()), result(result) {}
+ ~AutoAddTimeDuration() { result += TimeSince(start); }
+};
+
+ParallelMarker::ParallelMarker(GCRuntime* gc) : gc(gc) {}
+
+size_t ParallelMarker::workerCount() const { return gc->markers.length(); }
+
+bool ParallelMarker::mark(SliceBudget& sliceBudget) {
+#ifdef DEBUG
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(workerCount() <= HelperThreadState().maxGCParallelThreads(lock));
+
+ // TODO: Even if the thread limits checked above are correct, there may not
+ // be enough threads available to start our mark tasks immediately due to
+ // other runtimes in the same process running GC.
+ }
+#endif
+
+ if (markOneColor(MarkColor::Black, sliceBudget) == NotFinished) {
+ return false;
+ }
+ MOZ_ASSERT(!hasWork(MarkColor::Black));
+
+ if (markOneColor(MarkColor::Gray, sliceBudget) == NotFinished) {
+ return false;
+ }
+ MOZ_ASSERT(!hasWork(MarkColor::Gray));
+
+ // Handle any delayed marking, which is not performed in parallel.
+ if (gc->hasDelayedMarking()) {
+ gc->markAllDelayedChildren(ReportMarkTime);
+ }
+
+ return true;
+}
+
+bool ParallelMarker::markOneColor(MarkColor color, SliceBudget& sliceBudget) {
+ // Run a marking slice and return whether the stack is now empty.
+
+ if (!hasWork(color)) {
+ return true;
+ }
+
+ gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::PARALLEL_MARK);
+
+ MOZ_ASSERT(workerCount() <= MaxParallelWorkers);
+ mozilla::Maybe<ParallelMarkTask> tasks[MaxParallelWorkers];
+
+ for (size_t i = 0; i < workerCount(); i++) {
+ GCMarker* marker = gc->markers[i].get();
+ tasks[i].emplace(this, marker, color, sliceBudget);
+
+ // Attempt to populate empty mark stacks.
+ //
+ // TODO: When tuning for more than two markers we may need to adopt a more
+ // sophisticated approach.
+ if (!marker->hasEntriesForCurrentColor() && gc->marker().canDonateWork()) {
+ GCMarker::moveWork(marker, &gc->marker());
+ }
+ }
+
+ {
+ AutoLockGC lock(gc);
+
+ activeTasks = 0;
+ for (size_t i = 0; i < workerCount(); i++) {
+ ParallelMarkTask& task = *tasks[i];
+ if (task.hasWork()) {
+ incActiveTasks(&task, lock);
+ }
+ }
+ }
+
+ {
+ AutoLockHelperThreadState lock;
+
+ // There should always be enough parallel tasks to run our marking work.
+ MOZ_RELEASE_ASSERT(HelperThreadState().getGCParallelThreadCount(lock) >=
+ workerCount());
+
+ for (size_t i = 0; i < workerCount(); i++) {
+ gc->startTask(*tasks[i], lock);
+ }
+
+ for (size_t i = 0; i < workerCount(); i++) {
+ gc->joinTask(*tasks[i], lock);
+ }
+ }
+
+#ifdef DEBUG
+ {
+ AutoLockGC lock(gc);
+ MOZ_ASSERT(waitingTasks.ref().isEmpty());
+ MOZ_ASSERT(waitingTaskCount == 0);
+ MOZ_ASSERT(activeTasks == 0);
+ }
+#endif
+
+ return !hasWork(color);
+}
+
+bool ParallelMarker::hasWork(MarkColor color) const {
+ for (const auto& marker : gc->markers) {
+ if (marker->hasEntries(color)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+ParallelMarkTask::ParallelMarkTask(ParallelMarker* pm, GCMarker* marker,
+ MarkColor color, const SliceBudget& budget)
+ : GCParallelTask(pm->gc, gcstats::PhaseKind::PARALLEL_MARK, GCUse::Marking),
+ pm(pm),
+ marker(marker),
+ color(*marker, color),
+ budget(budget) {
+ marker->enterParallelMarkingMode(pm);
+}
+
+ParallelMarkTask::~ParallelMarkTask() {
+ MOZ_ASSERT(!isWaiting.refNoCheck());
+ marker->leaveParallelMarkingMode();
+}
+
+bool ParallelMarkTask::hasWork() const {
+ return marker->hasEntriesForCurrentColor();
+}
+
+void ParallelMarkTask::recordDuration() {
+ gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK,
+ duration());
+ gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK_MARK,
+ markTime.ref());
+ gc->stats().recordParallelPhase(gcstats::PhaseKind::PARALLEL_MARK_WAIT,
+ waitTime.ref());
+}
+
+void ParallelMarkTask::run(AutoLockHelperThreadState& lock) {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ AutoLockGC gcLock(pm->gc);
+
+ markOrRequestWork(gcLock);
+
+ MOZ_ASSERT(!isWaiting);
+}
+
+void ParallelMarkTask::markOrRequestWork(AutoLockGC& lock) {
+ for (;;) {
+ if (hasWork()) {
+ if (!tryMarking(lock)) {
+ return;
+ }
+ } else {
+ if (!requestWork(lock)) {
+ return;
+ }
+ }
+ }
+}
+
+bool ParallelMarkTask::tryMarking(AutoLockGC& lock) {
+ MOZ_ASSERT(hasWork());
+ MOZ_ASSERT(marker->isParallelMarking());
+
+ // Mark until budget exceeded or we run out of work.
+ bool finished;
+ {
+ AutoUnlockGC unlock(lock);
+
+ AutoAddTimeDuration time(markTime.ref());
+ finished = marker->markCurrentColorInParallel(budget);
+ }
+
+ MOZ_ASSERT_IF(finished, !hasWork());
+ pm->decActiveTasks(this, lock);
+
+ return finished;
+}
+
+bool ParallelMarkTask::requestWork(AutoLockGC& lock) {
+ MOZ_ASSERT(!hasWork());
+
+ if (!pm->hasActiveTasks(lock)) {
+ return false; // All other tasks are empty. We're finished.
+ }
+
+ budget.stepAndForceCheck();
+ if (budget.isOverBudget()) {
+ return false; // Over budget or interrupted.
+ }
+
+ // Add ourselves to the waiting list and wait for another task to give us
+ // work. The task with work calls ParallelMarker::donateWorkFrom.
+ waitUntilResumed(lock);
+
+ return true;
+}
+
+void ParallelMarkTask::waitUntilResumed(AutoLockGC& lock) {
+ GeckoProfilerRuntime& profiler = gc->rt->geckoProfiler();
+ if (profiler.enabled()) {
+ profiler.markEvent("Parallel marking wait start", "");
+ }
+
+ pm->addTaskToWaitingList(this, lock);
+
+ // Set isWaiting flag and wait for another thread to clear it and resume us.
+ MOZ_ASSERT(!isWaiting);
+ isWaiting = true;
+
+ AutoAddTimeDuration time(waitTime.ref());
+
+ do {
+ MOZ_ASSERT(pm->hasActiveTasks(lock));
+ resumed.wait(lock.guard());
+ } while (isWaiting);
+
+ MOZ_ASSERT(!pm->isTaskInWaitingList(this, lock));
+
+ if (profiler.enabled()) {
+ profiler.markEvent("Parallel marking wait end", "");
+ }
+}
+
+void ParallelMarkTask::resume() {
+ {
+ AutoLockGC lock(gc);
+ MOZ_ASSERT(isWaiting);
+
+ isWaiting = false;
+
+ // Increment the active task count before donateWorkFrom() returns so this
+ // can't reach zero before the waiting task runs again.
+ if (hasWork()) {
+ pm->incActiveTasks(this, lock);
+ }
+ }
+
+ resumed.notify_all();
+}
+
+void ParallelMarkTask::resumeOnFinish(const AutoLockGC& lock) {
+ MOZ_ASSERT(isWaiting);
+ MOZ_ASSERT(!hasWork());
+
+ isWaiting = false;
+ resumed.notify_all();
+}
+
+void ParallelMarker::addTaskToWaitingList(ParallelMarkTask* task,
+ const AutoLockGC& lock) {
+ MOZ_ASSERT(!task->hasWork());
+ MOZ_ASSERT(hasActiveTasks(lock));
+ MOZ_ASSERT(!isTaskInWaitingList(task, lock));
+ MOZ_ASSERT(waitingTaskCount < workerCount() - 1);
+
+ waitingTasks.ref().pushBack(task);
+ waitingTaskCount++;
+}
+
+#ifdef DEBUG
+bool ParallelMarker::isTaskInWaitingList(const ParallelMarkTask* task,
+ const AutoLockGC& lock) const {
+ // The const cast is because ElementProbablyInList is not const.
+ return const_cast<ParallelMarkTaskList&>(waitingTasks.ref())
+ .ElementProbablyInList(const_cast<ParallelMarkTask*>(task));
+}
+#endif
+
+void ParallelMarker::incActiveTasks(ParallelMarkTask* task,
+ const AutoLockGC& lock) {
+ MOZ_ASSERT(task->hasWork());
+ MOZ_ASSERT(activeTasks < workerCount());
+
+ activeTasks++;
+}
+
+void ParallelMarker::decActiveTasks(ParallelMarkTask* task,
+ const AutoLockGC& lock) {
+ MOZ_ASSERT(activeTasks != 0);
+
+ activeTasks--;
+
+ if (activeTasks == 0) {
+ while (!waitingTasks.ref().isEmpty()) {
+ ParallelMarkTask* task = waitingTasks.ref().popFront();
+ MOZ_ASSERT(waitingTaskCount != 0);
+ waitingTaskCount--;
+ task->resumeOnFinish(lock);
+ }
+ }
+}
+
+void ParallelMarker::donateWorkFrom(GCMarker* src) {
+ if (!gc->tryLockGC()) {
+ return;
+ }
+
+ // Check there are tasks waiting for work while holding the lock.
+ if (waitingTaskCount == 0) {
+ gc->unlockGC();
+ return;
+ }
+
+ // Take the first waiting task off the list.
+ ParallelMarkTask* waitingTask = waitingTasks.ref().popFront();
+ waitingTaskCount--;
+
+ // |task| is not running so it's safe to move work to it.
+ MOZ_ASSERT(waitingTask->isWaiting);
+
+ gc->unlockGC();
+
+ // Move some work from this thread's mark stack to the waiting task.
+ MOZ_ASSERT(!waitingTask->hasWork());
+ GCMarker::moveWork(waitingTask->marker, src);
+
+ gc->stats().count(gcstats::COUNT_PARALLEL_MARK_INTERRUPTIONS);
+
+ GeckoProfilerRuntime& profiler = gc->rt->geckoProfiler();
+ if (profiler.enabled()) {
+ profiler.markEvent("Parallel marking donated work", "");
+ }
+
+ // Resume waiting task.
+ waitingTask->resume();
+}
diff --git a/js/src/gc/ParallelMarking.h b/js/src/gc/ParallelMarking.h
new file mode 100644
index 0000000000..46d8f381c6
--- /dev/null
+++ b/js/src/gc/ParallelMarking.h
@@ -0,0 +1,123 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_ParallelMarking_h
+#define gc_ParallelMarking_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/DoublyLinkedList.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/GCMarker.h"
+#include "gc/GCParallelTask.h"
+#include "js/HeapAPI.h"
+#include "js/SliceBudget.h"
+#include "threading/ConditionVariable.h"
+#include "threading/ProtectedData.h"
+
+namespace js {
+
+class AutoLockGC;
+class AutoLockHelperThreadState;
+class AutoUnlockGC;
+
+namespace gc {
+
+class ParallelMarkTask;
+
+// Per-runtime parallel marking state.
+//
+// This class is used on the main thread and coordinates parallel marking using
+// several helper threads running ParallelMarkTasks.
+//
+// This uses a work-requesting approach. Threads mark until they run out of
+// work and then add themselves to a list of waiting tasks and block. Running
+// tasks with enough work may donate work to a waiting task and resume it.
+class MOZ_STACK_CLASS ParallelMarker {
+ public:
+ explicit ParallelMarker(GCRuntime* gc);
+
+ bool mark(SliceBudget& sliceBudget);
+
+ using AtomicCount = mozilla::Atomic<uint32_t, mozilla::Relaxed>;
+ AtomicCount& waitingTaskCountRef() { return waitingTaskCount; }
+ bool hasWaitingTasks() { return waitingTaskCount != 0; }
+ void donateWorkFrom(GCMarker* src);
+
+ private:
+ bool markOneColor(MarkColor color, SliceBudget& sliceBudget);
+
+ bool hasWork(MarkColor color) const;
+
+ void addTask(ParallelMarkTask* task, const AutoLockGC& lock);
+
+ void addTaskToWaitingList(ParallelMarkTask* task, const AutoLockGC& lock);
+#ifdef DEBUG
+ bool isTaskInWaitingList(const ParallelMarkTask* task,
+ const AutoLockGC& lock) const;
+#endif
+
+ bool hasActiveTasks(const AutoLockGC& lock) const { return activeTasks; }
+ void incActiveTasks(ParallelMarkTask* task, const AutoLockGC& lock);
+ void decActiveTasks(ParallelMarkTask* task, const AutoLockGC& lock);
+
+ size_t workerCount() const;
+
+ friend class ParallelMarkTask;
+
+ GCRuntime* const gc;
+
+ using ParallelMarkTaskList = mozilla::DoublyLinkedList<ParallelMarkTask>;
+ GCLockData<ParallelMarkTaskList> waitingTasks;
+ AtomicCount waitingTaskCount;
+
+ GCLockData<size_t> activeTasks;
+};
+
+// A helper thread task that performs parallel marking.
+class alignas(TypicalCacheLineSize) ParallelMarkTask
+ : public GCParallelTask,
+ public mozilla::DoublyLinkedListElement<ParallelMarkTask> {
+ public:
+ friend class ParallelMarker;
+
+ ParallelMarkTask(ParallelMarker* pm, GCMarker* marker, MarkColor color,
+ const SliceBudget& budget);
+ ~ParallelMarkTask();
+
+ void run(AutoLockHelperThreadState& lock) override;
+
+ void recordDuration() override;
+
+ private:
+ void markOrRequestWork(AutoLockGC& lock);
+ bool tryMarking(AutoLockGC& lock);
+ bool requestWork(AutoLockGC& lock);
+
+ void waitUntilResumed(AutoLockGC& lock);
+ void resume();
+ void resumeOnFinish(const AutoLockGC& lock);
+
+ bool hasWork() const;
+
+ // The following fields are only accessed by the marker thread:
+ ParallelMarker* const pm;
+ GCMarker* const marker;
+ AutoSetMarkColor color;
+ SliceBudget budget;
+ ConditionVariable resumed;
+
+ GCLockData<bool> isWaiting;
+
+ // Length of time this task spent blocked waiting for work.
+ MainThreadOrGCTaskData<mozilla::TimeDuration> markTime;
+ MainThreadOrGCTaskData<mozilla::TimeDuration> waitTime;
+};
+
+} // namespace gc
+} // namespace js
+
+#endif /* gc_ParallelMarking_h */
diff --git a/js/src/gc/ParallelWork.h b/js/src/gc/ParallelWork.h
new file mode 100644
index 0000000000..db97c75009
--- /dev/null
+++ b/js/src/gc/ParallelWork.h
@@ -0,0 +1,136 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_ParallelWork_h
+#define gc_ParallelWork_h
+
+#include "mozilla/Maybe.h"
+
+#include <algorithm>
+
+#include "gc/GCParallelTask.h"
+#include "gc/GCRuntime.h"
+#include "js/SliceBudget.h"
+#include "vm/HelperThreads.h"
+
+namespace js {
+
+namespace gcstats {
+enum class PhaseKind : uint8_t;
+}
+
+namespace gc {
+
+template <typename WorkItem>
+using ParallelWorkFunc = size_t (*)(GCRuntime*, const WorkItem&);
+
+// A GCParallelTask task that executes WorkItems from a WorkItemIterator.
+//
+// The WorkItemIterator class must supply done(), next() and get() methods. The
+// get() method must return WorkItems objects.
+template <typename WorkItem, typename WorkItemIterator>
+class ParallelWorker : public GCParallelTask {
+ public:
+ using WorkFunc = ParallelWorkFunc<WorkItem>;
+
+ ParallelWorker(GCRuntime* gc, gcstats::PhaseKind phaseKind, GCUse use,
+ WorkFunc func, WorkItemIterator& work,
+ const SliceBudget& budget, AutoLockHelperThreadState& lock)
+ : GCParallelTask(gc, phaseKind, use),
+ func_(func),
+ work_(work),
+ budget_(budget),
+ item_(work.get()) {
+ // Consume a work item on creation so that we can stop creating workers if
+ // the number of workers exceeds the number of work items.
+ work.next();
+ }
+
+ void run(AutoLockHelperThreadState& lock) {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ for (;;) {
+ size_t steps = func_(gc, item_);
+ budget_.step(std::max(steps, size_t(1)));
+ if (budget_.isOverBudget()) {
+ break;
+ }
+
+ AutoLockHelperThreadState lock;
+ if (work().done()) {
+ break;
+ }
+
+ item_ = work().get();
+ work().next();
+ }
+ }
+
+ private:
+ WorkItemIterator& work() { return work_.ref(); }
+
+ // A function to execute work items on the helper thread.
+ WorkFunc func_;
+
+ // An iterator which produces work items to execute.
+ HelperThreadLockData<WorkItemIterator&> work_;
+
+ // The budget that determines how long to run for.
+ SliceBudget budget_;
+
+ // The next work item to process.
+ WorkItem item_;
+};
+
+static constexpr size_t MaxParallelWorkers = 8;
+
+// An RAII class that starts a number of ParallelWorkers and waits for them to
+// finish.
+template <typename WorkItem, typename WorkItemIterator>
+class MOZ_RAII AutoRunParallelWork {
+ public:
+ using Worker = ParallelWorker<WorkItem, WorkItemIterator>;
+ using WorkFunc = ParallelWorkFunc<WorkItem>;
+
+ AutoRunParallelWork(GCRuntime* gc, WorkFunc func,
+ gcstats::PhaseKind phaseKind, GCUse use,
+ WorkItemIterator& work, const SliceBudget& budget,
+ AutoLockHelperThreadState& lock)
+ : gc(gc), phaseKind(phaseKind), lock(lock), tasksStarted(0) {
+ size_t workerCount = gc->parallelWorkerCount();
+ MOZ_ASSERT(workerCount <= MaxParallelWorkers);
+ MOZ_ASSERT_IF(workerCount == 0, work.done());
+
+ for (size_t i = 0; i < workerCount && !work.done(); i++) {
+ tasks[i].emplace(gc, phaseKind, use, func, work, budget, lock);
+ gc->startTask(*tasks[i], lock);
+ tasksStarted++;
+ }
+ }
+
+ ~AutoRunParallelWork() {
+ gHelperThreadLock.assertOwnedByCurrentThread();
+
+ for (size_t i = 0; i < tasksStarted; i++) {
+ gc->joinTask(*tasks[i], lock);
+ }
+ for (size_t i = tasksStarted; i < MaxParallelWorkers; i++) {
+ MOZ_ASSERT(tasks[i].isNothing());
+ }
+ }
+
+ private:
+ GCRuntime* gc;
+ gcstats::PhaseKind phaseKind;
+ AutoLockHelperThreadState& lock;
+ size_t tasksStarted;
+ mozilla::Maybe<Worker> tasks[MaxParallelWorkers];
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_ParallelWork_h */
diff --git a/js/src/gc/Policy.h b/js/src/gc/Policy.h
new file mode 100644
index 0000000000..290e4ed177
--- /dev/null
+++ b/js/src/gc/Policy.h
@@ -0,0 +1,101 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* JS Garbage Collector. */
+
+#ifndef gc_Policy_h
+#define gc_Policy_h
+
+#include <type_traits>
+
+#include "gc/Barrier.h"
+#include "gc/Tracer.h"
+#include "js/GCPolicyAPI.h"
+
+namespace js {
+
+// Define the GCPolicy for all internal pointers.
+template <typename T>
+struct InternalGCPointerPolicy : public JS::GCPointerPolicy<T> {
+ using Type = std::remove_pointer_t<T>;
+
+#define IS_BASE_OF_OR(_1, BaseType, _2, _3) std::is_base_of_v<BaseType, Type> ||
+ static_assert(
+ JS_FOR_EACH_TRACEKIND(IS_BASE_OF_OR) false,
+ "InternalGCPointerPolicy must only be used for GC thing pointers");
+#undef IS_BASE_OF_OR
+
+ static void trace(JSTracer* trc, T* vp, const char* name) {
+ // It's not safe to trace unbarriered pointers except as part of root
+ // marking. If you get an assertion here you probably need to add a barrier,
+ // e.g. HeapPtr<T>.
+ TraceNullableRoot(trc, vp, name);
+ }
+};
+
+} // namespace js
+
+namespace JS {
+
+// Internally, all pointer types are treated as pointers to GC things by
+// default.
+template <typename T>
+struct GCPolicy<T*> : public js::InternalGCPointerPolicy<T*> {};
+template <typename T>
+struct GCPolicy<T* const> : public js::InternalGCPointerPolicy<T* const> {};
+
+template <typename T>
+struct GCPolicy<js::HeapPtr<T>> {
+ static void trace(JSTracer* trc, js::HeapPtr<T>* thingp, const char* name) {
+ js::TraceNullableEdge(trc, thingp, name);
+ }
+ static bool traceWeak(JSTracer* trc, js::HeapPtr<T>* thingp) {
+ return js::TraceWeakEdge(trc, thingp, "HeapPtr");
+ }
+};
+
+template <typename T>
+struct GCPolicy<js::PreBarriered<T>> {
+ static void trace(JSTracer* trc, js::PreBarriered<T>* thingp,
+ const char* name) {
+ js::TraceNullableEdge(trc, thingp, name);
+ }
+};
+
+template <typename T>
+struct GCPolicy<js::WeakHeapPtr<T>> {
+ static void trace(JSTracer* trc, js::WeakHeapPtr<T>* thingp,
+ const char* name) {
+ js::TraceEdge(trc, thingp, name);
+ }
+ static bool traceWeak(JSTracer* trc, js::WeakHeapPtr<T>* thingp) {
+ return js::TraceWeakEdge(trc, thingp, "traceWeak");
+ }
+};
+
+template <typename T>
+struct GCPolicy<js::UnsafeBarePtr<T>> {
+ static bool traceWeak(JSTracer* trc, js::UnsafeBarePtr<T>* vp) {
+ if (*vp) {
+ return js::TraceManuallyBarrieredWeakEdge(trc, vp->unbarrieredAddress(),
+ "UnsafeBarePtr");
+ }
+ return true;
+ }
+};
+
+template <>
+struct GCPolicy<JS::GCCellPtr> {
+ static void trace(JSTracer* trc, JS::GCCellPtr* thingp, const char* name) {
+ // It's not safe to trace unbarriered pointers except as part of root
+ // marking.
+ js::TraceGCCellPtrRoot(trc, thingp, name);
+ }
+};
+
+} // namespace JS
+
+#endif // gc_Policy_h
diff --git a/js/src/gc/Pretenuring.cpp b/js/src/gc/Pretenuring.cpp
new file mode 100644
index 0000000000..5703663e47
--- /dev/null
+++ b/js/src/gc/Pretenuring.cpp
@@ -0,0 +1,459 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sw=2 et tw=80:
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Pretenuring.h"
+
+#include "mozilla/Sprintf.h"
+
+#include "gc/GCInternals.h"
+#include "gc/PublicIterators.h"
+#include "jit/Invalidation.h"
+
+#include "gc/PrivateIterators-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+// The number of nursery allocations at which to pay attention to an allocation
+// site. This must be large enough to ensure we have enough information to infer
+// the lifetime and also large enough to avoid pretenuring low volume allocation
+// sites.
+static constexpr size_t AllocSiteAttentionThreshold = 500;
+
+// The maximum number of alloc sites to create between each minor
+// collection. Stop tracking allocation after this limit is reached. This
+// prevents unbounded time traversing the list during minor GC.
+static constexpr size_t MaxAllocSitesPerMinorGC = 500;
+
+// The maximum number of times to invalidate JIT code for a site. After this we
+// leave the site's state as Unknown and don't pretenure allocations.
+// Note we use 4 bits to store the invalidation count.
+static constexpr size_t MaxInvalidationCount = 5;
+
+// The minimum number of allocated cells needed to determine the survival rate
+// of cells in newly created arenas.
+static constexpr size_t MinCellsRequiredForSurvivalRate = 100;
+
+// The young survival rate below which a major collection is determined to have
+// a low young survival rate.
+static constexpr double LowYoungSurvivalThreshold = 0.05;
+
+// The number of consecutive major collections with a low young survival rate
+// that must occur before recovery is attempted.
+static constexpr size_t LowYoungSurvivalCountBeforeRecovery = 2;
+
+// The proportion of the nursery that must be tenured above which a minor
+// collection may be determined to have a high nursery survival rate.
+static constexpr double HighNurserySurvivalPromotionThreshold = 0.6;
+
+// The number of nursery allocations made by optimized JIT code that must be
+// tenured above which a minor collection may be determined to have a high
+// nursery survival rate.
+static constexpr size_t HighNurserySurvivalOptimizedAllocThreshold = 10000;
+
+// The number of consecutive minor collections with a high nursery survival rate
+// that must occur before recovery is attempted.
+static constexpr size_t HighNurserySurvivalCountBeforeRecovery = 2;
+
+AllocSite* const AllocSite::EndSentinel = reinterpret_cast<AllocSite*>(1);
+JSScript* const AllocSite::WasmScript =
+ reinterpret_cast<JSScript*>(AllocSite::STATE_MASK + 1);
+
+static bool SiteBasedPretenuringEnabled = true;
+
+JS_PUBLIC_API void JS::SetSiteBasedPretenuringEnabled(bool enable) {
+ SiteBasedPretenuringEnabled = enable;
+}
+
+bool PretenuringNursery::canCreateAllocSite() {
+ MOZ_ASSERT(allocSitesCreated <= MaxAllocSitesPerMinorGC);
+ return SiteBasedPretenuringEnabled &&
+ allocSitesCreated < MaxAllocSitesPerMinorGC;
+}
+
+size_t PretenuringNursery::doPretenuring(GCRuntime* gc, JS::GCReason reason,
+ bool validPromotionRate,
+ double promotionRate, bool reportInfo,
+ size_t reportThreshold) {
+ size_t sitesActive = 0;
+ size_t sitesPretenured = 0;
+ size_t sitesInvalidated = 0;
+ size_t zonesWithHighNurserySurvival = 0;
+
+ // Zero allocation counts.
+ totalAllocCount_ = 0;
+ for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) {
+ for (auto& count : zone->pretenuring.nurseryAllocCounts) {
+ count = 0;
+ }
+ }
+
+ // Check whether previously optimized code has changed its behaviour and
+ // needs to be recompiled so that it can pretenure its allocations.
+ if (validPromotionRate) {
+ for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) {
+ bool highNurserySurvivalRate =
+ promotionRate > HighNurserySurvivalPromotionThreshold &&
+ zone->optimizedAllocSite()->nurseryTenuredCount >=
+ HighNurserySurvivalOptimizedAllocThreshold;
+ zone->pretenuring.noteHighNurserySurvivalRate(highNurserySurvivalRate);
+ if (highNurserySurvivalRate) {
+ zonesWithHighNurserySurvival++;
+ }
+ }
+ }
+
+ if (reportInfo) {
+ AllocSite::printInfoHeader(reason, promotionRate);
+ }
+
+ AllocSite* site = allocatedSites;
+ allocatedSites = AllocSite::EndSentinel;
+ while (site != AllocSite::EndSentinel) {
+ AllocSite* next = site->nextNurseryAllocated;
+ site->nextNurseryAllocated = nullptr;
+
+ MOZ_ASSERT_IF(site->isNormal(),
+ site->nurseryAllocCount >= site->nurseryTenuredCount);
+
+ if (site->isNormal()) {
+ processSite(gc, site, sitesActive, sitesPretenured, sitesInvalidated,
+ reportInfo, reportThreshold);
+ }
+
+ site = next;
+ }
+
+ // Catch-all sites don't end up on the list if they are only used from
+ // optimized JIT code, so process them here.
+ for (ZonesIter zone(gc, SkipAtoms); !zone.done(); zone.next()) {
+ for (auto& site : zone->pretenuring.unknownAllocSites) {
+ processCatchAllSite(&site, reportInfo, reportThreshold);
+ }
+ processCatchAllSite(zone->optimizedAllocSite(), reportInfo,
+ reportThreshold);
+ }
+
+ if (reportInfo) {
+ AllocSite::printInfoFooter(allocSitesCreated, sitesActive, sitesPretenured,
+ sitesInvalidated);
+ if (zonesWithHighNurserySurvival) {
+ fprintf(stderr, " %zu zones with high nursery survival rate\n",
+ zonesWithHighNurserySurvival);
+ }
+ }
+
+ allocSitesCreated = 0;
+
+ return sitesPretenured;
+}
+
+void PretenuringNursery::processSite(GCRuntime* gc, AllocSite* site,
+ size_t& sitesActive,
+ size_t& sitesPretenured,
+ size_t& sitesInvalidated, bool reportInfo,
+ size_t reportThreshold) {
+ sitesActive++;
+
+ updateAllocCounts(site);
+
+ bool hasPromotionRate = false;
+ double promotionRate = 0.0;
+ bool wasInvalidated = false;
+ if (site->nurseryAllocCount > AllocSiteAttentionThreshold) {
+ promotionRate =
+ double(site->nurseryTenuredCount) / double(site->nurseryAllocCount);
+ hasPromotionRate = true;
+
+ AllocSite::State prevState = site->state();
+ site->updateStateOnMinorGC(promotionRate);
+ AllocSite::State newState = site->state();
+
+ if (prevState == AllocSite::State::Unknown &&
+ newState == AllocSite::State::LongLived) {
+ sitesPretenured++;
+
+ // We can optimize JIT code before we realise that a site should be
+ // pretenured. Make sure we invalidate any existing optimized code.
+ if (site->hasScript()) {
+ wasInvalidated = site->invalidateScript(gc);
+ if (wasInvalidated) {
+ sitesInvalidated++;
+ }
+ }
+ }
+ }
+
+ if (reportInfo && site->allocCount() >= reportThreshold) {
+ site->printInfo(hasPromotionRate, promotionRate, wasInvalidated);
+ }
+
+ site->resetNurseryAllocations();
+}
+
+void PretenuringNursery::processCatchAllSite(AllocSite* site, bool reportInfo,
+ size_t reportThreshold) {
+ if (!site->hasNurseryAllocations()) {
+ return;
+ }
+
+ updateAllocCounts(site);
+
+ if (reportInfo && site->allocCount() >= reportThreshold) {
+ site->printInfo(false, 0.0, false);
+ }
+
+ site->resetNurseryAllocations();
+}
+
+void PretenuringNursery::updateAllocCounts(AllocSite* site) {
+ JS::TraceKind kind = site->traceKind();
+ totalAllocCount_ += site->nurseryAllocCount;
+ PretenuringZone& zone = site->zone()->pretenuring;
+ zone.nurseryAllocCount(kind) += site->nurseryAllocCount;
+}
+
+bool AllocSite::invalidateScript(GCRuntime* gc) {
+ CancelOffThreadIonCompile(script());
+
+ if (!script()->hasIonScript()) {
+ return false;
+ }
+
+ if (invalidationLimitReached()) {
+ MOZ_ASSERT(state() == State::Unknown);
+ return false;
+ }
+
+ invalidationCount++;
+ if (invalidationLimitReached()) {
+ setState(State::Unknown);
+ }
+
+ JSContext* cx = gc->rt->mainContextFromOwnThread();
+ jit::Invalidate(cx, script(),
+ /* resetUses = */ false,
+ /* cancelOffThread = */ true);
+ return true;
+}
+
+bool AllocSite::invalidationLimitReached() const {
+ MOZ_ASSERT(invalidationCount <= MaxInvalidationCount);
+ return invalidationCount == MaxInvalidationCount;
+}
+
+void PretenuringNursery::maybeStopPretenuring(GCRuntime* gc) {
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ double rate;
+ if (zone->pretenuring.calculateYoungTenuredSurvivalRate(&rate)) {
+ bool lowYoungSurvivalRate = rate < LowYoungSurvivalThreshold;
+ zone->pretenuring.noteLowYoungTenuredSurvivalRate(lowYoungSurvivalRate);
+ }
+ }
+}
+
+AllocSite::Kind AllocSite::kind() const {
+ if (isNormal()) {
+ return Kind::Normal;
+ }
+
+ if (this == zone()->optimizedAllocSite()) {
+ return Kind::Optimized;
+ }
+
+ MOZ_ASSERT(this == zone()->unknownAllocSite(traceKind()));
+ return Kind::Unknown;
+}
+
+void AllocSite::updateStateOnMinorGC(double promotionRate) {
+ // The state changes based on whether the promotion rate is deemed high
+ // (greater that 90%):
+ //
+ // high high
+ // ------------------> ------------------>
+ // ShortLived Unknown LongLived
+ // <------------------ <------------------
+ // !high !high
+ //
+ // The nursery is used to allocate if the site's state is Unknown or
+ // ShortLived. There are no direct transition between ShortLived and LongLived
+ // to avoid pretenuring sites that we've recently observed being short-lived.
+
+ if (invalidationLimitReached()) {
+ MOZ_ASSERT(state() == State::Unknown);
+ return;
+ }
+
+ bool highPromotionRate = promotionRate >= 0.9;
+
+ switch (state()) {
+ case State::Unknown:
+ if (highPromotionRate) {
+ setState(State::LongLived);
+ } else {
+ setState(State::ShortLived);
+ }
+ break;
+
+ case State::ShortLived: {
+ if (highPromotionRate) {
+ setState(State::Unknown);
+ }
+ break;
+ }
+
+ case State::LongLived: {
+ if (!highPromotionRate) {
+ setState(State::Unknown);
+ }
+ break;
+ }
+ }
+}
+
+bool AllocSite::maybeResetState() {
+ if (invalidationLimitReached()) {
+ MOZ_ASSERT(state() == State::Unknown);
+ return false;
+ }
+
+ invalidationCount++;
+ setState(State::Unknown);
+ return true;
+}
+
+void AllocSite::trace(JSTracer* trc) {
+ if (JSScript* s = script()) {
+ TraceManuallyBarrieredEdge(trc, &s, "AllocSite script");
+ if (s != script()) {
+ setScript(s);
+ }
+ }
+}
+
+bool PretenuringZone::calculateYoungTenuredSurvivalRate(double* rateOut) {
+ MOZ_ASSERT(allocCountInNewlyCreatedArenas >=
+ survivorCountInNewlyCreatedArenas);
+ if (allocCountInNewlyCreatedArenas < MinCellsRequiredForSurvivalRate) {
+ return false;
+ }
+
+ *rateOut = double(survivorCountInNewlyCreatedArenas) /
+ double(allocCountInNewlyCreatedArenas);
+ return true;
+}
+
+void PretenuringZone::noteLowYoungTenuredSurvivalRate(
+ bool lowYoungSurvivalRate) {
+ if (lowYoungSurvivalRate) {
+ lowYoungTenuredSurvivalCount++;
+ } else {
+ lowYoungTenuredSurvivalCount = 0;
+ }
+}
+
+void PretenuringZone::noteHighNurserySurvivalRate(
+ bool highNurserySurvivalRate) {
+ if (highNurserySurvivalRate) {
+ highNurserySurvivalCount++;
+ } else {
+ highNurserySurvivalCount = 0;
+ }
+}
+
+bool PretenuringZone::shouldResetNurseryAllocSites() {
+ bool shouldReset =
+ highNurserySurvivalCount >= HighNurserySurvivalCountBeforeRecovery;
+ if (shouldReset) {
+ highNurserySurvivalCount = 0;
+ }
+ return shouldReset;
+}
+
+bool PretenuringZone::shouldResetPretenuredAllocSites() {
+ bool shouldReset =
+ lowYoungTenuredSurvivalCount >= LowYoungSurvivalCountBeforeRecovery;
+ if (shouldReset) {
+ lowYoungTenuredSurvivalCount = 0;
+ }
+ return shouldReset;
+}
+
+/* static */
+void AllocSite::printInfoHeader(JS::GCReason reason, double promotionRate) {
+ fprintf(stderr,
+ "Pretenuring info after %s minor GC with %4.1f%% promotion rate:\n",
+ ExplainGCReason(reason), promotionRate * 100.0);
+}
+
+/* static */
+void AllocSite::printInfoFooter(size_t sitesCreated, size_t sitesActive,
+ size_t sitesPretenured,
+ size_t sitesInvalidated) {
+ fprintf(stderr,
+ " %zu alloc sites created, %zu active, %zu pretenured, %zu "
+ "invalidated\n",
+ sitesCreated, sitesActive, sitesPretenured, sitesInvalidated);
+}
+
+void AllocSite::printInfo(bool hasPromotionRate, double promotionRate,
+ bool wasInvalidated) const {
+ // Zone.
+ fprintf(stderr, " %p %p", this, zone());
+
+ // Script, or which kind of catch-all site this is.
+ if (!hasScript()) {
+ fprintf(stderr, " %16s",
+ kind() == Kind::Optimized
+ ? "optimized"
+ : (kind() == Kind::Normal ? "normal" : "unknown"));
+ } else {
+ fprintf(stderr, " %16p", script());
+ }
+
+ // Nursery allocation count, missing for optimized sites.
+ char buffer[16] = {'\0'};
+ if (kind() != Kind::Optimized) {
+ SprintfLiteral(buffer, "%8" PRIu32, nurseryAllocCount);
+ }
+ fprintf(stderr, " %8s", buffer);
+
+ // Nursery tenure count.
+ fprintf(stderr, " %8" PRIu32, nurseryTenuredCount);
+
+ // Promotion rate, if there were enough allocations.
+ buffer[0] = '\0';
+ if (hasPromotionRate) {
+ SprintfLiteral(buffer, "%5.1f%%", std::min(1.0, promotionRate) * 100);
+ }
+ fprintf(stderr, " %6s", buffer);
+
+ // Current state for sites associated with a script.
+ const char* state = isNormal() ? stateName() : "";
+ fprintf(stderr, " %10s", state);
+
+ // Whether the associated script was invalidated.
+ if (wasInvalidated) {
+ fprintf(stderr, " invalidated");
+ }
+
+ fprintf(stderr, "\n");
+}
+
+const char* AllocSite::stateName() const {
+ switch (state()) {
+ case State::ShortLived:
+ return "ShortLived";
+ case State::Unknown:
+ return "Unknown";
+ case State::LongLived:
+ return "LongLived";
+ }
+
+ MOZ_CRASH("Unknown state");
+}
diff --git a/js/src/gc/Pretenuring.h b/js/src/gc/Pretenuring.h
new file mode 100644
index 0000000000..ed156cbbd0
--- /dev/null
+++ b/js/src/gc/Pretenuring.h
@@ -0,0 +1,348 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Pretenuring.
+ *
+ * Some kinds of GC cells can be allocated in either the nursery or the tenured
+ * heap. The pretenuring system decides where to allocate such cells based on
+ * their expected lifetime with the aim of minimising total collection time.
+ *
+ * Lifetime is predicted based on data gathered about the cells' allocation
+ * site. This data is gathered in the middle JIT tiers, after code has stopped
+ * executing in the interpreter and before we generate fully optimized code.
+ */
+
+#ifndef gc_Pretenuring_h
+#define gc_Pretenuring_h
+
+#include <algorithm>
+
+#include "gc/AllocKind.h"
+#include "js/TypeDecls.h"
+
+class JS_PUBLIC_API JSTracer;
+
+namespace JS {
+enum class GCReason;
+} // namespace JS
+
+namespace js {
+namespace gc {
+
+class GCRuntime;
+class PretenuringNursery;
+
+// Number of trace kinds supportd by the nursery. These are arranged at the
+// start of JS::TraceKind.
+static constexpr size_t NurseryTraceKinds = 3;
+
+enum class CatchAllAllocSite { Unknown, Optimized };
+
+// Information about an allocation site.
+//
+// Nursery cells contain a pointer to one of these in their cell header (stored
+// before the cell). The site can relate to either a specific JS bytecode
+// instruction, a specific WebAssembly type, or can be a catch-all instance for
+// unknown sites or JS JIT optimized code.
+class AllocSite {
+ public:
+ enum class State : uint32_t { ShortLived = 0, Unknown = 1, LongLived = 2 };
+
+ // The JIT depends on being able to tell the states apart by checking a single
+ // bit.
+ static constexpr int32_t LONG_LIVED_BIT = int32_t(State::LongLived);
+ static_assert((LONG_LIVED_BIT & int32_t(State::Unknown)) == 0);
+ static_assert((AllocSite::LONG_LIVED_BIT & int32_t(State::ShortLived)) == 0);
+
+ private:
+ JS::Zone* zone_ = nullptr;
+
+ // Word storing JSScript pointer and site state.
+ //
+ // The script pointer is the script that owns this allocation site, a special
+ // sentinel script for wasm sites, or null for unknown sites. This is used
+ // when we need to invalidate the script.
+ uintptr_t scriptAndState = uintptr_t(State::Unknown);
+ static constexpr uintptr_t STATE_MASK = BitMask(2);
+
+ // Next pointer forming a linked list of sites at which nursery allocation
+ // happened since the last nursery collection.
+ AllocSite* nextNurseryAllocated = nullptr;
+
+ // Number of nursery allocations at this site since last nursery collection.
+ uint32_t nurseryAllocCount = 0;
+
+ // Number of nursery allocations that survived. Used during collection.
+ uint32_t nurseryTenuredCount : 24;
+
+ // Number of times the script has been invalidated.
+ uint32_t invalidationCount : 4;
+
+ // The trace kind of the allocation. Only kinds up to NurseryTraceKinds are
+ // allowed.
+ uint32_t traceKind_ : 4;
+
+ static AllocSite* const EndSentinel;
+
+ // Sentinel script for wasm sites.
+ static JSScript* const WasmScript;
+
+ friend class PretenuringZone;
+ friend class PretenuringNursery;
+
+ uintptr_t rawScript() const { return scriptAndState & ~STATE_MASK; }
+
+ public:
+ AllocSite() : nurseryTenuredCount(0), invalidationCount(0), traceKind_(0) {}
+
+ // Create a dummy site to use for unknown allocations.
+ explicit AllocSite(JS::Zone* zone, JS::TraceKind kind)
+ : zone_(zone),
+ nurseryTenuredCount(0),
+ invalidationCount(0),
+ traceKind_(uint32_t(kind)) {
+ MOZ_ASSERT(traceKind_ < NurseryTraceKinds);
+ }
+
+ // Create a site for an opcode in the given script.
+ AllocSite(JS::Zone* zone, JSScript* script, JS::TraceKind kind)
+ : AllocSite(zone, kind) {
+ MOZ_ASSERT(script != WasmScript);
+ setScript(script);
+ }
+
+ void initUnknownSite(JS::Zone* zone, JS::TraceKind kind) {
+ MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown));
+ zone_ = zone;
+ nurseryTenuredCount = 0;
+ invalidationCount = 0;
+ traceKind_ = uint32_t(kind);
+ MOZ_ASSERT(traceKind_ < NurseryTraceKinds);
+ }
+
+ // Initialize a site to be a wasm site.
+ void initWasm(JS::Zone* zone) {
+ MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown));
+ zone_ = zone;
+ setScript(WasmScript);
+ nurseryTenuredCount = 0;
+ invalidationCount = 0;
+ traceKind_ = uint32_t(JS::TraceKind::Object);
+ }
+
+ JS::Zone* zone() const { return zone_; }
+
+ JS::TraceKind traceKind() const { return JS::TraceKind(traceKind_); }
+
+ State state() const { return State(scriptAndState & STATE_MASK); }
+
+ // Whether this site has a script associated with it. This is not true if
+ // this site is for a wasm site.
+ bool hasScript() const { return rawScript() != uintptr_t(WasmScript); }
+ JSScript* script() const {
+ MOZ_ASSERT(hasScript());
+ return reinterpret_cast<JSScript*>(rawScript());
+ }
+
+ // Whether this site is not an unknown or optimized site.
+ bool isNormal() const { return rawScript() != 0; }
+
+ enum class Kind : uint32_t { Normal, Unknown, Optimized };
+ Kind kind() const;
+
+ bool isInAllocatedList() const { return nextNurseryAllocated; }
+
+ // Whether allocations at this site should be allocated in the nursery or the
+ // tenured heap.
+ Heap initialHeap() const {
+ return state() == State::LongLived ? Heap::Tenured : Heap::Default;
+ }
+
+ bool hasNurseryAllocations() const {
+ return nurseryAllocCount != 0 || nurseryTenuredCount != 0;
+ }
+ void resetNurseryAllocations() {
+ nurseryAllocCount = 0;
+ nurseryTenuredCount = 0;
+ }
+
+ uint32_t incAllocCount() { return ++nurseryAllocCount; }
+ uint32_t* nurseryAllocCountAddress() { return &nurseryAllocCount; }
+
+ void incTenuredCount() {
+ // The nursery is not large enough for this to overflow.
+ nurseryTenuredCount++;
+ MOZ_ASSERT(nurseryTenuredCount != 0);
+ }
+
+ size_t allocCount() const {
+ return std::max(nurseryAllocCount, nurseryTenuredCount);
+ }
+
+ void updateStateOnMinorGC(double promotionRate);
+
+ // Reset the state to 'Unknown' unless we have reached the invalidation limit
+ // for this site. Return whether the state was reset.
+ bool maybeResetState();
+
+ bool invalidationLimitReached() const;
+ bool invalidateScript(GCRuntime* gc);
+
+ void trace(JSTracer* trc);
+
+ static void printInfoHeader(JS::GCReason reason, double promotionRate);
+ static void printInfoFooter(size_t sitesCreated, size_t sitesActive,
+ size_t sitesPretenured, size_t sitesInvalidated);
+ void printInfo(bool hasPromotionRate, double promotionRate,
+ bool wasInvalidated) const;
+
+ static constexpr size_t offsetOfScriptAndState() {
+ return offsetof(AllocSite, scriptAndState);
+ }
+ static constexpr size_t offsetOfNurseryAllocCount() {
+ return offsetof(AllocSite, nurseryAllocCount);
+ }
+ static constexpr size_t offsetOfNextNurseryAllocated() {
+ return offsetof(AllocSite, nextNurseryAllocated);
+ }
+
+ private:
+ void setScript(JSScript* newScript) {
+ MOZ_ASSERT((uintptr_t(newScript) & STATE_MASK) == 0);
+ scriptAndState = uintptr_t(newScript) | uintptr_t(state());
+ }
+
+ void setState(State newState) {
+ MOZ_ASSERT((uintptr_t(newState) & ~STATE_MASK) == 0);
+ scriptAndState = rawScript() | uintptr_t(newState);
+ }
+
+ const char* stateName() const;
+};
+
+// Pretenuring information stored per zone.
+class PretenuringZone {
+ public:
+ // Catch-all allocation site instance used when the actual site is unknown, or
+ // when optimized JIT code allocates a GC thing that's not handled by the
+ // pretenuring system.
+ AllocSite unknownAllocSites[NurseryTraceKinds];
+
+ // Catch-all allocation instance used by optimized JIT code when allocating GC
+ // things that are handled by the pretenuring system. Allocation counts are
+ // not recorded by optimized JIT code.
+ AllocSite optimizedAllocSite;
+
+ // Count of tenured cell allocations made between each major collection and
+ // how many survived.
+ uint32_t allocCountInNewlyCreatedArenas = 0;
+ uint32_t survivorCountInNewlyCreatedArenas = 0;
+
+ // Count of successive collections that had a low young tenured survival
+ // rate. Used to discard optimized code if we get the pretenuring decision
+ // wrong.
+ uint32_t lowYoungTenuredSurvivalCount = 0;
+
+ // Count of successive nursery collections that had a high survival rate for
+ // objects allocated by optimized code. Used to discard optimized code if we
+ // get the pretenuring decision wrong.
+ uint32_t highNurserySurvivalCount = 0;
+
+ // Total allocation count by trace kind (ignoring optimized
+ // allocations). Calculated during nursery collection.
+ uint32_t nurseryAllocCounts[NurseryTraceKinds] = {0};
+
+ explicit PretenuringZone(JS::Zone* zone)
+ : optimizedAllocSite(zone, JS::TraceKind::Object) {
+ for (uint32_t i = 0; i < NurseryTraceKinds; i++) {
+ unknownAllocSites[i].initUnknownSite(zone, JS::TraceKind(i));
+ }
+ }
+
+ AllocSite& unknownAllocSite(JS::TraceKind kind) {
+ size_t i = size_t(kind);
+ MOZ_ASSERT(i < NurseryTraceKinds);
+ return unknownAllocSites[i];
+ }
+
+ void clearCellCountsInNewlyCreatedArenas() {
+ allocCountInNewlyCreatedArenas = 0;
+ survivorCountInNewlyCreatedArenas = 0;
+ }
+ void updateCellCountsInNewlyCreatedArenas(uint32_t allocCount,
+ uint32_t survivorCount) {
+ allocCountInNewlyCreatedArenas += allocCount;
+ survivorCountInNewlyCreatedArenas += survivorCount;
+ }
+
+ bool calculateYoungTenuredSurvivalRate(double* rateOut);
+
+ void noteLowYoungTenuredSurvivalRate(bool lowYoungSurvivalRate);
+ void noteHighNurserySurvivalRate(bool highNurserySurvivalRate);
+
+ // Recovery: if code behaviour change we may need to reset allocation site
+ // state and invalidate JIT code.
+ bool shouldResetNurseryAllocSites();
+ bool shouldResetPretenuredAllocSites();
+
+ uint32_t& nurseryAllocCount(JS::TraceKind kind) {
+ size_t i = size_t(kind);
+ MOZ_ASSERT(i < NurseryTraceKinds);
+ return nurseryAllocCounts[i];
+ }
+ uint32_t nurseryAllocCount(JS::TraceKind kind) const {
+ return const_cast<PretenuringZone*>(this)->nurseryAllocCount(kind);
+ }
+};
+
+// Pretenuring information stored as part of the the GC nursery.
+class PretenuringNursery {
+ gc::AllocSite* allocatedSites;
+
+ size_t allocSitesCreated = 0;
+
+ uint32_t totalAllocCount_ = 0;
+
+ public:
+ PretenuringNursery() : allocatedSites(AllocSite::EndSentinel) {}
+
+ bool hasAllocatedSites() const {
+ return allocatedSites != AllocSite::EndSentinel;
+ }
+
+ bool canCreateAllocSite();
+ void noteAllocSiteCreated() { allocSitesCreated++; }
+
+ void insertIntoAllocatedList(AllocSite* site) {
+ MOZ_ASSERT(!site->isInAllocatedList());
+ site->nextNurseryAllocated = allocatedSites;
+ allocatedSites = site;
+ }
+
+ size_t doPretenuring(GCRuntime* gc, JS::GCReason reason,
+ bool validPromotionRate, double promotionRate,
+ bool reportInfo, size_t reportThreshold);
+
+ void maybeStopPretenuring(GCRuntime* gc);
+
+ uint32_t totalAllocCount() const { return totalAllocCount_; }
+
+ void* addressOfAllocatedSites() { return &allocatedSites; }
+
+ private:
+ void processSite(GCRuntime* gc, AllocSite* site, size_t& sitesActive,
+ size_t& sitesPretenured, size_t& sitesInvalidated,
+ bool reportInfo, size_t reportThreshold);
+ void processCatchAllSite(AllocSite* site, bool reportInfo,
+ size_t reportThreshold);
+ void updateAllocCounts(AllocSite* site);
+};
+
+} // namespace gc
+} // namespace js
+
+#endif /* gc_Pretenuring_h */
diff --git a/js/src/gc/PrivateIterators-inl.h b/js/src/gc/PrivateIterators-inl.h
new file mode 100644
index 0000000000..359e6b22b7
--- /dev/null
+++ b/js/src/gc/PrivateIterators-inl.h
@@ -0,0 +1,164 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal iterators for various data structures.
+ */
+
+#ifndef gc_PrivateIterators_inl_h
+#define gc_PrivateIterators_inl_h
+
+#include "gc/PublicIterators.h"
+
+#include "gc/GC-inl.h"
+
+namespace js {
+namespace gc {
+
+class ArenaCellIterUnderGC : public ArenaCellIter {
+ public:
+ explicit ArenaCellIterUnderGC(Arena* arena) : ArenaCellIter(arena) {
+ MOZ_ASSERT(CurrentThreadIsPerformingGC());
+ }
+};
+
+class ArenaCellIterUnderFinalize : public ArenaCellIter {
+ public:
+ explicit ArenaCellIterUnderFinalize(Arena* arena) : ArenaCellIter(arena) {
+ MOZ_ASSERT(CurrentThreadIsGCFinalizing());
+ }
+};
+
+class GrayObjectIter : public ZoneAllCellIter<js::gc::TenuredCell> {
+ public:
+ explicit GrayObjectIter(JS::Zone* zone, AllocKind kind)
+ : ZoneAllCellIter<js::gc::TenuredCell>() {
+ initForTenuredIteration(zone, kind);
+ }
+
+ JSObject* get() const {
+ return ZoneAllCellIter<js::gc::TenuredCell>::get<JSObject>();
+ }
+ operator JSObject*() const { return get(); }
+ JSObject* operator->() const { return get(); }
+};
+
+class GCZonesIter {
+ AllZonesIter zone;
+
+ public:
+ explicit GCZonesIter(GCRuntime* gc) : zone(gc) {
+ MOZ_ASSERT(gc->heapState() != JS::HeapState::Idle);
+ if (!done() && !zone->wasGCStarted()) {
+ next();
+ }
+ }
+ explicit GCZonesIter(JSRuntime* rt) : GCZonesIter(&rt->gc) {}
+
+ bool done() const { return zone.done(); }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ do {
+ zone.next();
+ } while (!zone.done() && !zone->wasGCStarted());
+ }
+
+ JS::Zone* get() const {
+ MOZ_ASSERT(!done());
+ return zone;
+ }
+
+ operator JS::Zone*() const { return get(); }
+ JS::Zone* operator->() const { return get(); }
+};
+
+using GCCompartmentsIter =
+ CompartmentsOrRealmsIterT<GCZonesIter, CompartmentsInZoneIter>;
+using GCRealmsIter = CompartmentsOrRealmsIterT<GCZonesIter, RealmsInZoneIter>;
+
+/* Iterates over all zones in the current sweep group. */
+class SweepGroupZonesIter {
+ JS::Zone* current;
+
+ public:
+ explicit SweepGroupZonesIter(GCRuntime* gc)
+ : current(gc->getCurrentSweepGroup()) {
+ MOZ_ASSERT(CurrentThreadIsPerformingGC());
+ }
+ explicit SweepGroupZonesIter(JSRuntime* rt) : SweepGroupZonesIter(&rt->gc) {}
+
+ bool done() const { return !current; }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ current = current->nextNodeInGroup();
+ }
+
+ JS::Zone* get() const {
+ MOZ_ASSERT(!done());
+ return current;
+ }
+
+ operator JS::Zone*() const { return get(); }
+ JS::Zone* operator->() const { return get(); }
+};
+
+using SweepGroupCompartmentsIter =
+ CompartmentsOrRealmsIterT<SweepGroupZonesIter, CompartmentsInZoneIter>;
+using SweepGroupRealmsIter =
+ CompartmentsOrRealmsIterT<SweepGroupZonesIter, RealmsInZoneIter>;
+
+// Iterate the free cells in an arena. See also ArenaCellIter which iterates
+// the allocated cells.
+class ArenaFreeCellIter {
+ Arena* arena;
+ size_t thingSize;
+ FreeSpan span;
+ uint_fast16_t thing;
+
+ public:
+ explicit ArenaFreeCellIter(Arena* arena)
+ : arena(arena),
+ thingSize(arena->getThingSize()),
+ span(*arena->getFirstFreeSpan()),
+ thing(span.first) {
+ MOZ_ASSERT(arena);
+ MOZ_ASSERT(thing < ArenaSize);
+ }
+
+ bool done() const {
+ MOZ_ASSERT(thing < ArenaSize);
+ return !thing;
+ }
+
+ TenuredCell* get() const {
+ MOZ_ASSERT(!done());
+ return reinterpret_cast<TenuredCell*>(uintptr_t(arena) + thing);
+ }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ MOZ_ASSERT(thing >= span.first && thing <= span.last);
+
+ if (thing == span.last) {
+ span = *span.nextSpan(arena);
+ thing = span.first;
+ } else {
+ thing += thingSize;
+ }
+
+ MOZ_ASSERT(thing < ArenaSize);
+ }
+
+ operator TenuredCell*() const { return get(); }
+ TenuredCell* operator->() const { return get(); }
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_PrivateIterators_inl_h
diff --git a/js/src/gc/PublicIterators.cpp b/js/src/gc/PublicIterators.cpp
new file mode 100644
index 0000000000..582a21ddf3
--- /dev/null
+++ b/js/src/gc/PublicIterators.cpp
@@ -0,0 +1,272 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "vm/Realm.h"
+#include "vm/Runtime.h"
+
+#include "gc/PrivateIterators-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+static void IterateRealmsArenasCellsUnbarriered(
+ JSContext* cx, Zone* zone, void* data,
+ JS::IterateRealmCallback realmCallback, IterateArenaCallback arenaCallback,
+ IterateCellCallback cellCallback, const JS::AutoRequireNoGC& nogc) {
+ {
+ Rooted<Realm*> realm(cx);
+ for (RealmsInZoneIter r(zone); !r.done(); r.next()) {
+ realm = r;
+ (*realmCallback)(cx, data, realm, nogc);
+ }
+ }
+
+ for (auto thingKind : AllAllocKinds()) {
+ JS::TraceKind traceKind = MapAllocToTraceKind(thingKind);
+ size_t thingSize = Arena::thingSize(thingKind);
+
+ for (ArenaIter aiter(zone, thingKind); !aiter.done(); aiter.next()) {
+ Arena* arena = aiter.get();
+ (*arenaCallback)(cx->runtime(), data, arena, traceKind, thingSize, nogc);
+ for (ArenaCellIter cell(arena); !cell.done(); cell.next()) {
+ (*cellCallback)(cx->runtime(), data, JS::GCCellPtr(cell, traceKind),
+ thingSize, nogc);
+ }
+ }
+ }
+}
+
+void js::IterateHeapUnbarriered(JSContext* cx, void* data,
+ IterateZoneCallback zoneCallback,
+ JS::IterateRealmCallback realmCallback,
+ IterateArenaCallback arenaCallback,
+ IterateCellCallback cellCallback) {
+ AutoPrepareForTracing prep(cx);
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ auto iterateZone = [&](Zone* zone) -> void {
+ (*zoneCallback)(cx->runtime(), data, zone, nogc);
+ IterateRealmsArenasCellsUnbarriered(cx, zone, data, realmCallback,
+ arenaCallback, cellCallback, nogc);
+ };
+
+ // Include the shared atoms zone if present.
+ if (Zone* zone = cx->runtime()->gc.maybeSharedAtomsZone()) {
+ iterateZone(zone);
+ }
+
+ for (ZonesIter zone(cx->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ iterateZone(zone);
+ }
+}
+
+void js::IterateHeapUnbarrieredForZone(JSContext* cx, Zone* zone, void* data,
+ IterateZoneCallback zoneCallback,
+ JS::IterateRealmCallback realmCallback,
+ IterateArenaCallback arenaCallback,
+ IterateCellCallback cellCallback) {
+ AutoPrepareForTracing prep(cx);
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ (*zoneCallback)(cx->runtime(), data, zone, nogc);
+ IterateRealmsArenasCellsUnbarriered(cx, zone, data, realmCallback,
+ arenaCallback, cellCallback, nogc);
+}
+
+void js::IterateChunks(JSContext* cx, void* data,
+ IterateChunkCallback chunkCallback) {
+ AutoPrepareForTracing prep(cx);
+ AutoLockGC lock(cx->runtime());
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ for (auto chunk = cx->runtime()->gc.allNonEmptyChunks(lock); !chunk.done();
+ chunk.next()) {
+ chunkCallback(cx->runtime(), data, chunk, nogc);
+ }
+}
+
+static void TraverseInnerLazyScriptsForLazyScript(
+ JSContext* cx, void* data, BaseScript* enclosingScript,
+ IterateScriptCallback lazyScriptCallback, const JS::AutoRequireNoGC& nogc) {
+ for (JS::GCCellPtr gcThing : enclosingScript->gcthings()) {
+ if (!gcThing.is<JSObject>()) {
+ continue;
+ }
+ JSObject* obj = &gcThing.as<JSObject>();
+
+ MOZ_ASSERT(obj->is<JSFunction>(),
+ "All objects in lazy scripts should be functions");
+ JSFunction* fun = &obj->as<JSFunction>();
+
+ if (!fun->hasBaseScript()) {
+ // Ignore asm.js.
+ continue;
+ }
+ MOZ_ASSERT(fun->baseScript());
+ if (!fun->baseScript()) {
+ // If the function doesn't have script, ignore it.
+ continue;
+ }
+
+ if (fun->hasBytecode()) {
+ // Ignore non lazy function.
+ continue;
+ }
+
+ // If the function is "ghost", we shouldn't expose it to the debugger.
+ //
+ // See GHOST_FUNCTION in FunctionFlags.h for more details.
+ if (fun->isGhost()) {
+ continue;
+ }
+
+ BaseScript* script = fun->baseScript();
+ MOZ_ASSERT_IF(script->hasEnclosingScript(),
+ script->enclosingScript() == enclosingScript);
+
+ lazyScriptCallback(cx->runtime(), data, script, nogc);
+
+ TraverseInnerLazyScriptsForLazyScript(cx, data, script, lazyScriptCallback,
+ nogc);
+ }
+}
+
+static inline void DoScriptCallback(JSContext* cx, void* data,
+ BaseScript* script,
+ IterateScriptCallback callback,
+ const JS::AutoRequireNoGC& nogc) {
+ // Exclude any scripts that may be the result of a failed compile. Check that
+ // script either has bytecode or is ready to delazify.
+ //
+ // This excludes lazy scripts that do not have an enclosing scope because we
+ // cannot distinguish a failed compile fragment from a lazy script with a lazy
+ // parent.
+ if (!script->hasBytecode() && !script->isReadyForDelazification()) {
+ return;
+ }
+
+ // Invoke callback.
+ callback(cx->runtime(), data, script, nogc);
+
+ // The check above excluded lazy scripts with lazy parents, so explicitly
+ // visit inner scripts now if we are lazy with a successfully compiled parent.
+ if (!script->hasBytecode()) {
+ TraverseInnerLazyScriptsForLazyScript(cx, data, script, callback, nogc);
+ }
+}
+
+void js::IterateScripts(JSContext* cx, Realm* realm, void* data,
+ IterateScriptCallback scriptCallback) {
+ MOZ_ASSERT(!cx->suppressGC);
+ AutoEmptyNurseryAndPrepareForTracing prep(cx);
+ JS::AutoSuppressGCAnalysis nogc;
+
+ if (realm) {
+ Zone* zone = realm->zone();
+ for (auto iter = zone->cellIter<BaseScript>(prep); !iter.done();
+ iter.next()) {
+ if (iter->realm() != realm) {
+ continue;
+ }
+ DoScriptCallback(cx, data, iter.get(), scriptCallback, nogc);
+ }
+ } else {
+ for (ZonesIter zone(cx->runtime(), SkipAtoms); !zone.done(); zone.next()) {
+ for (auto iter = zone->cellIter<BaseScript>(prep); !iter.done();
+ iter.next()) {
+ DoScriptCallback(cx, data, iter.get(), scriptCallback, nogc);
+ }
+ }
+ }
+}
+
+void js::IterateGrayObjects(Zone* zone, IterateGCThingCallback cellCallback,
+ void* data) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+
+ JSContext* cx = TlsContext.get();
+ AutoPrepareForTracing prep(cx);
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ for (auto kind : ObjectAllocKinds()) {
+ for (GrayObjectIter obj(zone, kind); !obj.done(); obj.next()) {
+ if (obj->asTenured().isMarkedGray()) {
+ cellCallback(data, JS::GCCellPtr(obj.get()), nogc);
+ }
+ }
+ }
+}
+
+JS_PUBLIC_API void JS_IterateCompartments(
+ JSContext* cx, void* data,
+ JSIterateCompartmentCallback compartmentCallback) {
+ AutoTraceSession session(cx->runtime());
+
+ for (CompartmentsIter c(cx->runtime()); !c.done(); c.next()) {
+ if ((*compartmentCallback)(cx, data, c) ==
+ JS::CompartmentIterResult::Stop) {
+ break;
+ }
+ }
+}
+
+JS_PUBLIC_API void JS_IterateCompartmentsInZone(
+ JSContext* cx, JS::Zone* zone, void* data,
+ JSIterateCompartmentCallback compartmentCallback) {
+ AutoTraceSession session(cx->runtime());
+
+ for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
+ if ((*compartmentCallback)(cx, data, c) ==
+ JS::CompartmentIterResult::Stop) {
+ break;
+ }
+ }
+}
+
+JS_PUBLIC_API void JS::IterateRealms(JSContext* cx, void* data,
+ JS::IterateRealmCallback realmCallback) {
+ AutoTraceSession session(cx->runtime());
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ Rooted<Realm*> realm(cx);
+ for (RealmsIter r(cx->runtime()); !r.done(); r.next()) {
+ realm = r;
+ (*realmCallback)(cx, data, realm, nogc);
+ }
+}
+
+JS_PUBLIC_API void JS::IterateRealmsWithPrincipals(
+ JSContext* cx, JSPrincipals* principals, void* data,
+ JS::IterateRealmCallback realmCallback) {
+ MOZ_ASSERT(principals);
+
+ AutoTraceSession session(cx->runtime());
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ Rooted<Realm*> realm(cx);
+ for (RealmsIter r(cx->runtime()); !r.done(); r.next()) {
+ if (r->principals() != principals) {
+ continue;
+ }
+ realm = r;
+ (*realmCallback)(cx, data, realm, nogc);
+ }
+}
+
+JS_PUBLIC_API void JS::IterateRealmsInCompartment(
+ JSContext* cx, JS::Compartment* compartment, void* data,
+ JS::IterateRealmCallback realmCallback) {
+ AutoTraceSession session(cx->runtime());
+ JS::AutoSuppressGCAnalysis nogc(cx);
+
+ Rooted<Realm*> realm(cx);
+ for (RealmsInCompartmentIter r(compartment); !r.done(); r.next()) {
+ realm = r;
+ (*realmCallback)(cx, data, realm, nogc);
+ }
+}
diff --git a/js/src/gc/PublicIterators.h b/js/src/gc/PublicIterators.h
new file mode 100644
index 0000000000..d1072cfe98
--- /dev/null
+++ b/js/src/gc/PublicIterators.h
@@ -0,0 +1,158 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Iterators for various data structures.
+ */
+
+#ifndef gc_PublicIterators_h
+#define gc_PublicIterators_h
+
+#include "jstypes.h"
+#include "gc/GCRuntime.h"
+#include "gc/IteratorUtils.h"
+#include "gc/Zone.h"
+#include "vm/Compartment.h"
+#include "vm/Runtime.h"
+
+namespace JS {
+class JS_PUBLIC_API Realm;
+}
+
+namespace js {
+
+enum ZoneSelector { WithAtoms, SkipAtoms };
+
+// Iterate over all zones in the runtime. May or may not include the atoms zone.
+class ZonesIter {
+ gc::AutoEnterIteration iterMarker;
+ JS::Zone** it;
+ JS::Zone** const end;
+
+ public:
+ ZonesIter(gc::GCRuntime* gc, ZoneSelector selector)
+ : iterMarker(gc), it(gc->zones().begin()), end(gc->zones().end()) {
+ if (selector == SkipAtoms) {
+ MOZ_ASSERT(get()->isAtomsZone());
+ next();
+ }
+ }
+ ZonesIter(JSRuntime* rt, ZoneSelector selector)
+ : ZonesIter(&rt->gc, selector) {}
+
+ bool done() const { return it == end; }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ it++;
+ }
+
+ JS::Zone* get() const {
+ MOZ_ASSERT(!done());
+ return *it;
+ }
+
+ operator JS::Zone*() const { return get(); }
+ JS::Zone* operator->() const { return get(); }
+};
+
+// Iterate over all zones in the runtime apart from the atoms zone.
+class NonAtomZonesIter : public ZonesIter {
+ public:
+ explicit NonAtomZonesIter(gc::GCRuntime* gc) : ZonesIter(gc, SkipAtoms) {}
+ explicit NonAtomZonesIter(JSRuntime* rt) : NonAtomZonesIter(&rt->gc) {}
+};
+
+// Iterate over all zones in the runtime, except those which may be in use by
+// parse threads.
+class AllZonesIter : public ZonesIter {
+ public:
+ explicit AllZonesIter(gc::GCRuntime* gc) : ZonesIter(gc, WithAtoms) {}
+ explicit AllZonesIter(JSRuntime* rt) : AllZonesIter(&rt->gc) {}
+};
+
+struct CompartmentsInZoneIter {
+ explicit CompartmentsInZoneIter(JS::Zone* zone) : zone(zone) {
+ it = zone->compartments().begin();
+ }
+
+ bool done() const {
+ MOZ_ASSERT(it);
+ return it < zone->compartments().begin() ||
+ it >= zone->compartments().end();
+ }
+ void next() {
+ MOZ_ASSERT(!done());
+ it++;
+ }
+
+ JS::Compartment* get() const {
+ MOZ_ASSERT(it);
+ return *it;
+ }
+
+ operator JS::Compartment*() const { return get(); }
+ JS::Compartment* operator->() const { return get(); }
+
+ private:
+ JS::Zone* zone;
+ JS::Compartment** it;
+};
+
+class RealmsInCompartmentIter {
+ JS::Compartment* comp;
+ JS::Realm** it;
+
+ public:
+ explicit RealmsInCompartmentIter(JS::Compartment* comp) : comp(comp) {
+ it = comp->realms().begin();
+ MOZ_ASSERT(!done(), "Compartments must have at least one realm");
+ }
+
+ bool done() const {
+ MOZ_ASSERT(it);
+ return it < comp->realms().begin() || it >= comp->realms().end();
+ }
+ void next() {
+ MOZ_ASSERT(!done());
+ it++;
+ }
+
+ JS::Realm* get() const {
+ MOZ_ASSERT(!done());
+ return *it;
+ }
+
+ operator JS::Realm*() const { return get(); }
+ JS::Realm* operator->() const { return get(); }
+};
+
+using RealmsInZoneIter =
+ NestedIterator<CompartmentsInZoneIter, RealmsInCompartmentIter>;
+
+// This iterator iterates over all the compartments or realms in a given set of
+// zones. The set of zones is determined by iterating ZoneIterT. The set of
+// compartments or realms is determined by InnerIterT.
+template <class ZonesIterT, class InnerIterT>
+class CompartmentsOrRealmsIterT
+ : public NestedIterator<ZonesIterT, InnerIterT> {
+ gc::AutoEnterIteration iterMarker;
+
+ public:
+ explicit CompartmentsOrRealmsIterT(gc::GCRuntime* gc)
+ : NestedIterator<ZonesIterT, InnerIterT>(gc), iterMarker(gc) {}
+ explicit CompartmentsOrRealmsIterT(JSRuntime* rt)
+ : CompartmentsOrRealmsIterT(&rt->gc) {}
+};
+
+using CompartmentsIter =
+ CompartmentsOrRealmsIterT<NonAtomZonesIter, CompartmentsInZoneIter>;
+using RealmsIter =
+ CompartmentsOrRealmsIterT<NonAtomZonesIter, RealmsInZoneIter>;
+
+} // namespace js
+
+#endif // gc_PublicIterators_h
diff --git a/js/src/gc/RelocationOverlay.h b/js/src/gc/RelocationOverlay.h
new file mode 100644
index 0000000000..047e763d44
--- /dev/null
+++ b/js/src/gc/RelocationOverlay.h
@@ -0,0 +1,66 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definition of relocation overlay used while moving cells.
+ */
+
+#ifndef gc_RelocationOverlay_h
+#define gc_RelocationOverlay_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "gc/Cell.h"
+
+namespace js {
+namespace gc {
+
+/*
+ * This structure overlays a Cell that has been moved and provides a way to find
+ * its new location. It's used during generational and compacting GC.
+ */
+class RelocationOverlay : public Cell {
+ public:
+ /* The location the cell has been moved to, stored in the cell header. */
+ Cell* forwardingAddress() const {
+ MOZ_ASSERT(isForwarded());
+ return reinterpret_cast<Cell*>(header_.getForwardingAddress());
+ }
+
+ protected:
+ /* A list entry to track all relocated things. */
+ RelocationOverlay* next_;
+
+ explicit RelocationOverlay(Cell* dst);
+
+ public:
+ static const RelocationOverlay* fromCell(const Cell* cell) {
+ return static_cast<const RelocationOverlay*>(cell);
+ }
+
+ static RelocationOverlay* fromCell(Cell* cell) {
+ return static_cast<RelocationOverlay*>(cell);
+ }
+
+ static RelocationOverlay* forwardCell(Cell* src, Cell* dst);
+
+ void setNext(RelocationOverlay* next) {
+ MOZ_ASSERT(isForwarded());
+ next_ = next;
+ }
+
+ RelocationOverlay* next() const {
+ MOZ_ASSERT(isForwarded());
+ return next_;
+ }
+};
+
+} // namespace gc
+} // namespace js
+
+#endif /* gc_RelocationOverlay_h */
diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp
new file mode 100644
index 0000000000..c397711933
--- /dev/null
+++ b/js/src/gc/RootMarking.cpp
@@ -0,0 +1,467 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef MOZ_VALGRIND
+# include <valgrind/memcheck.h>
+#endif
+
+#include "jstypes.h"
+
+#include "debugger/DebugAPI.h"
+#include "gc/ClearEdgesTracer.h"
+#include "gc/GCInternals.h"
+#include "gc/PublicIterators.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "js/ValueArray.h"
+#include "vm/BigIntType.h"
+#include "vm/Compartment.h"
+#include "vm/HelperThreadState.h"
+#include "vm/JSContext.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::LinkedList;
+
+using JS::AutoGCRooter;
+
+using RootRange = RootedValueMap::Range;
+using RootEntry = RootedValueMap::Entry;
+using RootEnum = RootedValueMap::Enum;
+
+template <typename Base, typename T>
+inline void TypedRootedGCThingBase<Base, T>::trace(JSTracer* trc,
+ const char* name) {
+ auto* self = this->template derived<T>();
+ TraceNullableRoot(trc, self->address(), name);
+}
+
+template <typename T>
+static inline void TraceExactStackRootList(JSTracer* trc,
+ StackRootedBase* listHead,
+ const char* name) {
+ // Check size of Rooted<T> does not increase.
+ static_assert(sizeof(Rooted<T>) == sizeof(T) + 2 * sizeof(uintptr_t));
+
+ for (StackRootedBase* root = listHead; root; root = root->previous()) {
+ static_cast<Rooted<T>*>(root)->trace(trc, name);
+ }
+}
+
+static inline void TraceExactStackRootTraceableList(JSTracer* trc,
+ StackRootedBase* listHead,
+ const char* name) {
+ for (StackRootedBase* root = listHead; root; root = root->previous()) {
+ static_cast<StackRootedTraceableBase*>(root)->trace(trc, name);
+ }
+}
+
+static inline void TraceStackRoots(JSTracer* trc,
+ JS::RootedListHeads& stackRoots) {
+#define TRACE_ROOTS(name, type, _, _1) \
+ TraceExactStackRootList<type*>(trc, stackRoots[JS::RootKind::name], \
+ "exact-" #name);
+ JS_FOR_EACH_TRACEKIND(TRACE_ROOTS)
+#undef TRACE_ROOTS
+ TraceExactStackRootList<jsid>(trc, stackRoots[JS::RootKind::Id], "exact-id");
+ TraceExactStackRootList<Value>(trc, stackRoots[JS::RootKind::Value],
+ "exact-value");
+
+ // RootedTraceable uses virtual dispatch.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ TraceExactStackRootTraceableList(trc, stackRoots[JS::RootKind::Traceable],
+ "Traceable");
+}
+
+void JS::RootingContext::traceStackRoots(JSTracer* trc) {
+ TraceStackRoots(trc, stackRoots_);
+}
+
+static void TraceExactStackRoots(JSContext* cx, JSTracer* trc) {
+ cx->traceStackRoots(trc);
+}
+
+template <typename T>
+static inline void TracePersistentRootedList(
+ JSTracer* trc, LinkedList<PersistentRootedBase>& list, const char* name) {
+ for (PersistentRootedBase* root : list) {
+ static_cast<PersistentRooted<T>*>(root)->trace(trc, name);
+ }
+}
+
+static inline void TracePersistentRootedTraceableList(
+ JSTracer* trc, LinkedList<PersistentRootedBase>& list, const char* name) {
+ for (PersistentRootedBase* root : list) {
+ static_cast<PersistentRootedTraceableBase*>(root)->trace(trc, name);
+ }
+}
+
+void JSRuntime::tracePersistentRoots(JSTracer* trc) {
+#define TRACE_ROOTS(name, type, _, _1) \
+ TracePersistentRootedList<type*>(trc, heapRoots.ref()[JS::RootKind::name], \
+ "persistent-" #name);
+ JS_FOR_EACH_TRACEKIND(TRACE_ROOTS)
+#undef TRACE_ROOTS
+ TracePersistentRootedList<jsid>(trc, heapRoots.ref()[JS::RootKind::Id],
+ "persistent-id");
+ TracePersistentRootedList<Value>(trc, heapRoots.ref()[JS::RootKind::Value],
+ "persistent-value");
+
+ // RootedTraceable uses virtual dispatch.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ TracePersistentRootedTraceableList(
+ trc, heapRoots.ref()[JS::RootKind::Traceable], "persistent-traceable");
+}
+
+static void TracePersistentRooted(JSRuntime* rt, JSTracer* trc) {
+ rt->tracePersistentRoots(trc);
+}
+
+template <typename T>
+static void FinishPersistentRootedChain(
+ LinkedList<PersistentRootedBase>& list) {
+ while (!list.isEmpty()) {
+ static_cast<PersistentRooted<T>*>(list.getFirst())->reset();
+ }
+}
+
+void JSRuntime::finishPersistentRoots() {
+#define FINISH_ROOT_LIST(name, type, _, _1) \
+ FinishPersistentRootedChain<type*>(heapRoots.ref()[JS::RootKind::name]);
+ JS_FOR_EACH_TRACEKIND(FINISH_ROOT_LIST)
+#undef FINISH_ROOT_LIST
+ FinishPersistentRootedChain<jsid>(heapRoots.ref()[JS::RootKind::Id]);
+ FinishPersistentRootedChain<Value>(heapRoots.ref()[JS::RootKind::Value]);
+
+ // Note that we do not finalize the Traceable list as we do not know how to
+ // safely clear members. We instead assert that none escape the RootLists.
+ // See the comment on RootLists::~RootLists for details.
+}
+
+JS_PUBLIC_API void js::TraceValueArray(JSTracer* trc, size_t length,
+ Value* elements) {
+ TraceRootRange(trc, length, elements, "JS::RootedValueArray");
+}
+
+void AutoGCRooter::trace(JSTracer* trc) {
+ switch (kind_) {
+ case Kind::Wrapper:
+ static_cast<AutoWrapperRooter*>(this)->trace(trc);
+ break;
+
+ case Kind::WrapperVector:
+ static_cast<AutoWrapperVector*>(this)->trace(trc);
+ break;
+
+ case Kind::Custom:
+ static_cast<JS::CustomAutoRooter*>(this)->trace(trc);
+ break;
+
+ default:
+ MOZ_CRASH("Bad AutoGCRooter::Kind");
+ break;
+ }
+}
+
+void AutoWrapperRooter::trace(JSTracer* trc) {
+ /*
+ * We need to use TraceManuallyBarrieredEdge here because we trace wrapper
+ * roots in every slice. This is because of some rule-breaking in
+ * RemapAllWrappersForObject; see comment there.
+ */
+ TraceManuallyBarrieredEdge(trc, &value.get(), "js::AutoWrapperRooter.value");
+}
+
+void AutoWrapperVector::trace(JSTracer* trc) {
+ /*
+ * We need to use TraceManuallyBarrieredEdge here because we trace wrapper
+ * roots in every slice. This is because of some rule-breaking in
+ * RemapAllWrappersForObject; see comment there.
+ */
+ for (WrapperValue& value : *this) {
+ TraceManuallyBarrieredEdge(trc, &value.get(),
+ "js::AutoWrapperVector.vector");
+ }
+}
+
+void JS::RootingContext::traceAllGCRooters(JSTracer* trc) {
+ for (AutoGCRooter* list : autoGCRooters_) {
+ traceGCRooterList(trc, list);
+ }
+}
+
+void JS::RootingContext::traceWrapperGCRooters(JSTracer* trc) {
+ traceGCRooterList(trc, autoGCRooters_[AutoGCRooter::Kind::Wrapper]);
+ traceGCRooterList(trc, autoGCRooters_[AutoGCRooter::Kind::WrapperVector]);
+}
+
+/* static */
+inline void JS::RootingContext::traceGCRooterList(JSTracer* trc,
+ AutoGCRooter* head) {
+ for (AutoGCRooter* rooter = head; rooter; rooter = rooter->down) {
+ rooter->trace(trc);
+ }
+}
+
+void PropertyDescriptor::trace(JSTracer* trc) {
+ TraceRoot(trc, &value_, "Descriptor::value");
+ if (getter_) {
+ TraceRoot(trc, &getter_, "Descriptor::getter");
+ }
+ if (setter_) {
+ TraceRoot(trc, &setter_, "Descriptor::setter");
+ }
+}
+
+void js::gc::GCRuntime::traceRuntimeForMajorGC(JSTracer* trc,
+ AutoGCSession& session) {
+ MOZ_ASSERT(!TlsContext.get()->suppressGC);
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS);
+
+ // We only need to trace atoms when we're marking; atoms are never moved by
+ // compacting GC.
+ if (atomsZone()->isGCMarking()) {
+ traceRuntimeAtoms(trc);
+ }
+
+ {
+ // Trace incoming cross compartment edges from uncollected compartments,
+ // skipping gray edges which are traced later.
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_CCWS);
+ Compartment::traceIncomingCrossCompartmentEdgesForZoneGC(
+ trc, Compartment::NonGrayEdges);
+ }
+
+ traceRuntimeCommon(trc, MarkRuntime);
+}
+
+void js::gc::GCRuntime::traceRuntimeForMinorGC(JSTracer* trc,
+ AutoGCSession& session) {
+ MOZ_ASSERT(!TlsContext.get()->suppressGC);
+
+ // Note that we *must* trace the runtime during the SHUTDOWN_GC's minor GC
+ // despite having called FinishRoots already. This is because FinishRoots
+ // does not clear the crossCompartmentWrapper map. It cannot do this
+ // because Proxy's trace for CrossCompartmentWrappers asserts presence in
+ // the map. And we can reach its trace function despite having finished the
+ // roots via the edges stored by the pre-barrier verifier when we finish
+ // the verifier for the last time.
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS);
+
+ traceRuntimeCommon(trc, TraceRuntime);
+}
+
+void js::TraceRuntime(JSTracer* trc) {
+ MOZ_ASSERT(!trc->isMarkingTracer());
+
+ JSRuntime* rt = trc->runtime();
+ AutoEmptyNurseryAndPrepareForTracing prep(rt->mainContextFromOwnThread());
+ gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP);
+ rt->gc.traceRuntime(trc, prep);
+}
+
+void js::TraceRuntimeWithoutEviction(JSTracer* trc) {
+ MOZ_ASSERT(!trc->isMarkingTracer());
+
+ JSRuntime* rt = trc->runtime();
+ AutoTraceSession session(rt);
+ gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP);
+ rt->gc.traceRuntime(trc, session);
+}
+
+void js::gc::GCRuntime::traceRuntime(JSTracer* trc, AutoTraceSession& session) {
+ MOZ_ASSERT(!rt->isBeingDestroyed());
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS);
+
+ traceRuntimeAtoms(trc);
+ traceRuntimeCommon(trc, TraceRuntime);
+}
+
+void js::gc::GCRuntime::traceRuntimeAtoms(JSTracer* trc) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_RUNTIME_DATA);
+ TraceAtoms(trc);
+ jit::JitRuntime::TraceAtomZoneRoots(trc);
+}
+
+void js::gc::GCRuntime::traceRuntimeCommon(JSTracer* trc,
+ TraceOrMarkRuntime traceOrMark) {
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_STACK);
+
+ JSContext* cx = rt->mainContextFromOwnThread();
+
+ // Trace active interpreter and JIT stack roots.
+ TraceInterpreterActivations(cx, trc);
+ jit::TraceJitActivations(cx, trc);
+
+ // Trace legacy C stack roots.
+ cx->traceAllGCRooters(trc);
+
+ // Trace C stack roots.
+ TraceExactStackRoots(cx, trc);
+
+ for (RootRange r = rootsHash.ref().all(); !r.empty(); r.popFront()) {
+ const RootEntry& entry = r.front();
+ TraceRoot(trc, entry.key(), entry.value());
+ }
+ }
+
+ // Trace runtime global roots.
+ TracePersistentRooted(rt, trc);
+
+#ifdef JS_HAS_INTL_API
+ // Trace the shared Intl data.
+ rt->traceSharedIntlData(trc);
+#endif
+
+ // Trace the JSContext.
+ rt->mainContextFromOwnThread()->trace(trc);
+
+ // Trace all realm roots, but not the realm itself; it is traced via the
+ // parent pointer if traceRoots actually traces anything.
+ for (RealmsIter r(rt); !r.done(); r.next()) {
+ r->traceRoots(trc, traceOrMark);
+ }
+
+ if (!JS::RuntimeHeapIsMinorCollecting()) {
+ // Trace the self-hosting stencil. The contents of this are always tenured.
+ rt->traceSelfHostingStencil(trc);
+
+ for (ZonesIter zone(this, ZoneSelector::SkipAtoms); !zone.done();
+ zone.next()) {
+ zone->traceRootsInMajorGC(trc);
+ }
+
+ // Trace interpreter entry code generated with --emit-interpreter-entry
+ if (rt->hasJitRuntime() && rt->jitRuntime()->hasInterpreterEntryMap()) {
+ rt->jitRuntime()->getInterpreterEntryMap()->traceTrampolineCode(trc);
+ }
+ }
+
+ // Trace helper thread roots.
+ HelperThreadState().trace(trc);
+
+ // Trace Debugger.Frames that have live hooks, since dropping them would be
+ // observable. In effect, they are rooted by the stack frames.
+ DebugAPI::traceFramesWithLiveHooks(trc);
+
+ // Trace the embedding's black and gray roots.
+ if (!JS::RuntimeHeapIsMinorCollecting()) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_EMBEDDING);
+
+ /*
+ * The embedding can register additional roots here.
+ *
+ * We don't need to trace these in a minor GC because all pointers into
+ * the nursery should be in the store buffer, and we want to avoid the
+ * time taken to trace all these roots.
+ */
+ traceEmbeddingBlackRoots(trc);
+
+ /* During GC, we don't trace gray roots at this stage. */
+ if (traceOrMark == TraceRuntime) {
+ traceEmbeddingGrayRoots(trc);
+ }
+ }
+
+ traceKeptObjects(trc);
+}
+
+void GCRuntime::traceEmbeddingBlackRoots(JSTracer* trc) {
+ // The analysis doesn't like the function pointer below.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ for (size_t i = 0; i < blackRootTracers.ref().length(); i++) {
+ const Callback<JSTraceDataOp>& e = blackRootTracers.ref()[i];
+ (*e.op)(trc, e.data);
+ }
+}
+
+void GCRuntime::traceEmbeddingGrayRoots(JSTracer* trc) {
+ SliceBudget budget = SliceBudget::unlimited();
+ MOZ_ALWAYS_TRUE(traceEmbeddingGrayRoots(trc, budget) == Finished);
+}
+
+IncrementalProgress GCRuntime::traceEmbeddingGrayRoots(JSTracer* trc,
+ SliceBudget& budget) {
+ // The analysis doesn't like the function pointer below.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ const auto& callback = grayRootTracer.ref();
+ if (!callback.op) {
+ return Finished;
+ }
+
+ return callback.op(trc, budget, callback.data) ? Finished : NotFinished;
+}
+
+#ifdef DEBUG
+class AssertNoRootsTracer final : public JS::CallbackTracer {
+ void onChild(JS::GCCellPtr thing, const char* name) override {
+ MOZ_CRASH("There should not be any roots during runtime shutdown");
+ }
+
+ public:
+ // This skips tracking WeakMap entries because they are not roots.
+ explicit AssertNoRootsTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt, JS::TracerKind::Callback,
+ JS::WeakMapTraceAction::Skip) {}
+};
+#endif // DEBUG
+
+void js::gc::GCRuntime::finishRoots() {
+ AutoNoteSingleThreadedRegion anstr;
+
+ rt->finishAtoms();
+ restoreSharedAtomsZone();
+
+ rootsHash.ref().clear();
+
+ rt->finishPersistentRoots();
+
+ rt->finishSelfHosting();
+
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ zone->finishRoots();
+ }
+
+#ifdef JS_GC_ZEAL
+ clearSelectedForMarking();
+#endif
+
+ // Clear out the interpreter entry map before the final gc.
+ ClearInterpreterEntryMap(rt);
+
+ // Clear any remaining roots from the embedding (as otherwise they will be
+ // left dangling after we shut down) and remove the callbacks.
+ ClearEdgesTracer trc(rt);
+ traceEmbeddingBlackRoots(&trc);
+ traceEmbeddingGrayRoots(&trc);
+ clearBlackAndGrayRootTracers();
+}
+
+void js::gc::GCRuntime::checkNoRuntimeRoots(AutoGCSession& session) {
+#ifdef DEBUG
+ AssertNoRootsTracer trc(rt);
+ traceRuntimeForMajorGC(&trc, session);
+#endif // DEBUG
+}
+
+JS_PUBLIC_API void JS::AddPersistentRoot(JS::RootingContext* cx, RootKind kind,
+ PersistentRootedBase* root) {
+ JSRuntime* rt = static_cast<JSContext*>(cx)->runtime();
+ rt->heapRoots.ref()[kind].insertBack(root);
+}
+
+JS_PUBLIC_API void JS::AddPersistentRoot(JSRuntime* rt, RootKind kind,
+ PersistentRootedBase* root) {
+ rt->heapRoots.ref()[kind].insertBack(root);
+}
diff --git a/js/src/gc/Scheduling.cpp b/js/src/gc/Scheduling.cpp
new file mode 100644
index 0000000000..ec03c85f8d
--- /dev/null
+++ b/js/src/gc/Scheduling.cpp
@@ -0,0 +1,873 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Scheduling.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/TimeStamp.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "gc/Memory.h"
+#include "gc/Nursery.h"
+#include "gc/RelocationOverlay.h"
+#include "gc/ZoneAllocator.h"
+#include "util/DifferentialTesting.h"
+#include "vm/MutexIDs.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::CheckedInt;
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+/*
+ * We may start to collect a zone before its trigger threshold is reached if
+ * GCRuntime::maybeGC() is called for that zone or we start collecting other
+ * zones. These eager threshold factors are not configurable.
+ */
+static constexpr double HighFrequencyEagerAllocTriggerFactor = 0.85;
+static constexpr double LowFrequencyEagerAllocTriggerFactor = 0.9;
+
+/*
+ * Don't allow heap growth factors to be set so low that eager collections could
+ * reduce the trigger threshold.
+ */
+static constexpr double MinHeapGrowthFactor =
+ 1.0f / std::min(HighFrequencyEagerAllocTriggerFactor,
+ LowFrequencyEagerAllocTriggerFactor);
+
+// Limit various parameters to reasonable levels to catch errors.
+static constexpr double MaxHeapGrowthFactor = 100;
+static constexpr size_t MaxNurseryBytesParam = 128 * 1024 * 1024;
+
+namespace {
+
+// Helper classes to marshal GC parameter values to/from uint32_t.
+
+template <typename T>
+struct ConvertGeneric {
+ static uint32_t toUint32(T value) {
+ static_assert(std::is_arithmetic_v<T>);
+ if constexpr (std::is_signed_v<T>) {
+ MOZ_ASSERT(value >= 0);
+ }
+ if constexpr (!std::is_same_v<T, bool> &&
+ std::numeric_limits<T>::max() >
+ std::numeric_limits<uint32_t>::max()) {
+ MOZ_ASSERT(value <= UINT32_MAX);
+ }
+ return uint32_t(value);
+ }
+ static Maybe<T> fromUint32(uint32_t param) {
+ // Currently we use explicit conversion and don't range check.
+ return Some(T(param));
+ }
+};
+
+using ConvertBool = ConvertGeneric<bool>;
+using ConvertSize = ConvertGeneric<size_t>;
+using ConvertDouble = ConvertGeneric<double>;
+
+struct ConvertTimes100 {
+ static uint32_t toUint32(double value) { return uint32_t(value * 100.0); }
+ static Maybe<double> fromUint32(uint32_t param) {
+ return Some(double(param) / 100.0);
+ }
+};
+
+struct ConvertNurseryBytes : ConvertSize {
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ return Some(Nursery::roundSize(param));
+ }
+};
+
+struct ConvertKB {
+ static uint32_t toUint32(size_t value) { return value / 1024; }
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ // Parameters which represent heap sizes in bytes are restricted to values
+ // which can be represented on 32 bit platforms.
+ CheckedInt<uint32_t> size = CheckedInt<uint32_t>(param) * 1024;
+ return size.isValid() ? Some(size_t(size.value())) : Nothing();
+ }
+};
+
+struct ConvertMB {
+ static uint32_t toUint32(size_t value) { return value / (1024 * 1024); }
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ // Parameters which represent heap sizes in bytes are restricted to values
+ // which can be represented on 32 bit platforms.
+ CheckedInt<uint32_t> size = CheckedInt<uint32_t>(param) * 1024 * 1024;
+ return size.isValid() ? Some(size_t(size.value())) : Nothing();
+ }
+};
+
+struct ConvertMillis {
+ static uint32_t toUint32(TimeDuration value) {
+ return uint32_t(value.ToMilliseconds());
+ }
+ static Maybe<TimeDuration> fromUint32(uint32_t param) {
+ return Some(TimeDuration::FromMilliseconds(param));
+ }
+};
+
+struct ConvertSeconds {
+ static uint32_t toUint32(TimeDuration value) {
+ return uint32_t(value.ToSeconds());
+ }
+ static Maybe<TimeDuration> fromUint32(uint32_t param) {
+ return Some(TimeDuration::FromSeconds(param));
+ }
+};
+
+} // anonymous namespace
+
+// Helper functions to check GC parameter values
+
+template <typename T>
+static bool NoCheck(T value) {
+ return true;
+}
+
+template <typename T>
+static bool CheckNonZero(T value) {
+ return value != 0;
+}
+
+static bool CheckNurserySize(size_t bytes) {
+ return bytes >= SystemPageSize() && bytes <= MaxNurseryBytesParam;
+}
+
+static bool CheckHeapGrowth(double growth) {
+ return growth >= MinHeapGrowthFactor && growth <= MaxHeapGrowthFactor;
+}
+
+static bool CheckIncrementalLimit(double factor) {
+ return factor >= 1.0 && factor <= MaxHeapGrowthFactor;
+}
+
+static bool CheckNonZeroUnitRange(double value) {
+ return value > 0.0 && value <= 100.0;
+}
+
+GCSchedulingTunables::GCSchedulingTunables() {
+#define INIT_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ name##_ = default; \
+ MOZ_ASSERT(check(name##_));
+ FOR_EACH_GC_TUNABLE(INIT_TUNABLE_FIELD)
+#undef INIT_TUNABLE_FIELD
+
+ checkInvariants();
+}
+
+uint32_t GCSchedulingTunables::getParameter(JSGCParamKey key) {
+ switch (key) {
+#define GET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: \
+ return convert::toUint32(name##_);
+ FOR_EACH_GC_TUNABLE(GET_TUNABLE_FIELD)
+#undef GET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown parameter key");
+ }
+}
+
+bool GCSchedulingTunables::setParameter(JSGCParamKey key, uint32_t value) {
+ auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); });
+
+ switch (key) {
+#define SET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: { \
+ Maybe<type> converted = convert::fromUint32(value); \
+ if (!converted || !check(converted.value())) { \
+ return false; \
+ } \
+ name##_ = converted.value(); \
+ break; \
+ }
+ FOR_EACH_GC_TUNABLE(SET_TUNABLE_FIELD)
+#undef SET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown GC parameter.");
+ }
+
+ maintainInvariantsAfterUpdate(key);
+ return true;
+}
+
+void GCSchedulingTunables::resetParameter(JSGCParamKey key) {
+ auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); });
+
+ switch (key) {
+#define RESET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: \
+ name##_ = default; \
+ MOZ_ASSERT(check(name##_)); \
+ break;
+ FOR_EACH_GC_TUNABLE(RESET_TUNABLE_FIELD)
+#undef RESET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown GC parameter.");
+ }
+
+ maintainInvariantsAfterUpdate(key);
+}
+
+void GCSchedulingTunables::maintainInvariantsAfterUpdate(JSGCParamKey updated) {
+ switch (updated) {
+ case JSGC_MIN_NURSERY_BYTES:
+ if (gcMaxNurseryBytes_ < gcMinNurseryBytes_) {
+ gcMaxNurseryBytes_ = gcMinNurseryBytes_;
+ }
+ break;
+ case JSGC_MAX_NURSERY_BYTES:
+ if (gcMinNurseryBytes_ > gcMaxNurseryBytes_) {
+ gcMinNurseryBytes_ = gcMaxNurseryBytes_;
+ }
+ break;
+ case JSGC_SMALL_HEAP_SIZE_MAX:
+ if (smallHeapSizeMaxBytes_ >= largeHeapSizeMinBytes_) {
+ largeHeapSizeMinBytes_ = smallHeapSizeMaxBytes_ + 1;
+ }
+ break;
+ case JSGC_LARGE_HEAP_SIZE_MIN:
+ if (largeHeapSizeMinBytes_ <= smallHeapSizeMaxBytes_) {
+ smallHeapSizeMaxBytes_ = largeHeapSizeMinBytes_ - 1;
+ }
+ break;
+ case JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH:
+ if (highFrequencySmallHeapGrowth_ < highFrequencyLargeHeapGrowth_) {
+ highFrequencyLargeHeapGrowth_ = highFrequencySmallHeapGrowth_;
+ }
+ break;
+ case JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH:
+ if (highFrequencyLargeHeapGrowth_ > highFrequencySmallHeapGrowth_) {
+ highFrequencySmallHeapGrowth_ = highFrequencyLargeHeapGrowth_;
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+void GCSchedulingTunables::checkInvariants() {
+ MOZ_ASSERT(gcMinNurseryBytes_ == Nursery::roundSize(gcMinNurseryBytes_));
+ MOZ_ASSERT(gcMaxNurseryBytes_ == Nursery::roundSize(gcMaxNurseryBytes_));
+ MOZ_ASSERT(gcMinNurseryBytes_ <= gcMaxNurseryBytes_);
+ MOZ_ASSERT(gcMinNurseryBytes_ >= SystemPageSize());
+ MOZ_ASSERT(gcMaxNurseryBytes_ <= MaxNurseryBytesParam);
+
+ MOZ_ASSERT(largeHeapSizeMinBytes_ > smallHeapSizeMaxBytes_);
+
+ MOZ_ASSERT(lowFrequencyHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(lowFrequencyHeapGrowth_ <= MaxHeapGrowthFactor);
+
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencyLargeHeapGrowth_ <= highFrequencySmallHeapGrowth_);
+ MOZ_ASSERT(highFrequencyLargeHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor);
+}
+
+void GCSchedulingState::updateHighFrequencyMode(
+ const mozilla::TimeStamp& lastGCTime, const mozilla::TimeStamp& currentTime,
+ const GCSchedulingTunables& tunables) {
+ if (js::SupportDifferentialTesting()) {
+ return;
+ }
+
+ inHighFrequencyGCMode_ =
+ !lastGCTime.IsNull() &&
+ lastGCTime + tunables.highFrequencyThreshold() > currentTime;
+}
+
+void GCSchedulingState::updateHighFrequencyModeForReason(JS::GCReason reason) {
+ // These reason indicate that the embedding isn't triggering GC slices often
+ // enough and allocation rate is high.
+ if (reason == JS::GCReason::ALLOC_TRIGGER ||
+ reason == JS::GCReason::TOO_MUCH_MALLOC) {
+ inHighFrequencyGCMode_ = true;
+ }
+}
+
+static constexpr size_t BytesPerMB = 1024 * 1024;
+static constexpr double CollectionRateSmoothingFactor = 0.5;
+static constexpr double AllocationRateSmoothingFactor = 0.5;
+
+static double ExponentialMovingAverage(double prevAverage, double newData,
+ double smoothingFactor) {
+ MOZ_ASSERT(smoothingFactor > 0.0 && smoothingFactor <= 1.0);
+ return smoothingFactor * newData + (1.0 - smoothingFactor) * prevAverage;
+}
+
+void js::ZoneAllocator::updateCollectionRate(
+ mozilla::TimeDuration mainThreadGCTime, size_t initialBytesForAllZones) {
+ MOZ_ASSERT(initialBytesForAllZones != 0);
+ MOZ_ASSERT(gcHeapSize.initialBytes() <= initialBytesForAllZones);
+
+ double zoneFraction =
+ double(gcHeapSize.initialBytes()) / double(initialBytesForAllZones);
+ double zoneDuration = mainThreadGCTime.ToSeconds() * zoneFraction +
+ perZoneGCTime.ref().ToSeconds();
+ double collectionRate =
+ double(gcHeapSize.initialBytes()) / (zoneDuration * BytesPerMB);
+
+ if (!smoothedCollectionRate.ref()) {
+ smoothedCollectionRate = Some(collectionRate);
+ } else {
+ double prevRate = smoothedCollectionRate.ref().value();
+ smoothedCollectionRate = Some(ExponentialMovingAverage(
+ prevRate, collectionRate, CollectionRateSmoothingFactor));
+ }
+}
+
+void js::ZoneAllocator::updateAllocationRate(TimeDuration mutatorTime) {
+ // To get the total size allocated since the last collection we have to
+ // take account of how much memory got freed in the meantime.
+ size_t freedBytes = gcHeapSize.freedBytes();
+
+ size_t sizeIncludingFreedBytes = gcHeapSize.bytes() + freedBytes;
+
+ MOZ_ASSERT(prevGCHeapSize <= sizeIncludingFreedBytes);
+ size_t allocatedBytes = sizeIncludingFreedBytes - prevGCHeapSize;
+
+ double allocationRate =
+ double(allocatedBytes) / (mutatorTime.ToSeconds() * BytesPerMB);
+
+ if (!smoothedAllocationRate.ref()) {
+ smoothedAllocationRate = Some(allocationRate);
+ } else {
+ double prevRate = smoothedAllocationRate.ref().value();
+ smoothedAllocationRate = Some(ExponentialMovingAverage(
+ prevRate, allocationRate, AllocationRateSmoothingFactor));
+ }
+
+ gcHeapSize.clearFreedBytes();
+ prevGCHeapSize = gcHeapSize.bytes();
+}
+
+// GC thresholds may exceed the range of size_t on 32-bit platforms, so these
+// are calculated using 64-bit integers and clamped.
+static inline size_t ToClampedSize(uint64_t bytes) {
+ return std::min(bytes, uint64_t(SIZE_MAX));
+}
+
+void HeapThreshold::setIncrementalLimitFromStartBytes(
+ size_t retainedBytes, const GCSchedulingTunables& tunables) {
+ // Calculate the incremental limit for a heap based on its size and start
+ // threshold.
+ //
+ // This effectively classifies the heap size into small, medium or large, and
+ // uses the small heap incremental limit paramer, the large heap incremental
+ // limit parameter or an interpolation between them.
+ //
+ // The incremental limit is always set greater than the start threshold by at
+ // least the maximum nursery size to reduce the chance that tenuring a full
+ // nursery will send us straight into non-incremental collection.
+
+ MOZ_ASSERT(tunables.smallHeapIncrementalLimit() >=
+ tunables.largeHeapIncrementalLimit());
+
+ double factor = LinearInterpolate(
+ retainedBytes, tunables.smallHeapSizeMaxBytes(),
+ tunables.smallHeapIncrementalLimit(), tunables.largeHeapSizeMinBytes(),
+ tunables.largeHeapIncrementalLimit());
+
+ uint64_t bytes =
+ std::max(uint64_t(double(startBytes_) * factor),
+ uint64_t(startBytes_) + tunables.gcMaxNurseryBytes());
+ incrementalLimitBytes_ = ToClampedSize(bytes);
+ MOZ_ASSERT(incrementalLimitBytes_ >= startBytes_);
+
+ // Maintain the invariant that the slice threshold is always less than the
+ // incremental limit when adjusting GC parameters.
+ if (hasSliceThreshold() && sliceBytes() > incrementalLimitBytes()) {
+ sliceBytes_ = incrementalLimitBytes();
+ }
+}
+
+double HeapThreshold::eagerAllocTrigger(bool highFrequencyGC) const {
+ double eagerTriggerFactor = highFrequencyGC
+ ? HighFrequencyEagerAllocTriggerFactor
+ : LowFrequencyEagerAllocTriggerFactor;
+ return eagerTriggerFactor * startBytes();
+}
+
+void HeapThreshold::setSliceThreshold(ZoneAllocator* zone,
+ const HeapSize& heapSize,
+ const GCSchedulingTunables& tunables,
+ bool waitingOnBGTask) {
+ // Set the allocation threshold at which to trigger the a GC slice in an
+ // ongoing incremental collection. This is used to ensure progress in
+ // allocation heavy code that may not return to the main event loop.
+ //
+ // The threshold is based on the JSGC_ZONE_ALLOC_DELAY_KB parameter, but this
+ // is reduced to increase the slice frequency as we approach the incremental
+ // limit, in the hope that we never reach it. If collector is waiting for a
+ // background task to complete, don't trigger any slices until we reach the
+ // urgent threshold.
+
+ size_t bytesRemaining = incrementalBytesRemaining(heapSize);
+ bool isUrgent = bytesRemaining < tunables.urgentThresholdBytes();
+
+ size_t delayBeforeNextSlice = tunables.zoneAllocDelayBytes();
+ if (isUrgent) {
+ double fractionRemaining =
+ double(bytesRemaining) / double(tunables.urgentThresholdBytes());
+ delayBeforeNextSlice =
+ size_t(double(delayBeforeNextSlice) * fractionRemaining);
+ MOZ_ASSERT(delayBeforeNextSlice <= tunables.zoneAllocDelayBytes());
+ } else if (waitingOnBGTask) {
+ delayBeforeNextSlice = bytesRemaining - tunables.urgentThresholdBytes();
+ }
+
+ sliceBytes_ = ToClampedSize(
+ std::min(uint64_t(heapSize.bytes()) + uint64_t(delayBeforeNextSlice),
+ uint64_t(incrementalLimitBytes_)));
+}
+
+size_t HeapThreshold::incrementalBytesRemaining(
+ const HeapSize& heapSize) const {
+ if (heapSize.bytes() >= incrementalLimitBytes_) {
+ return 0;
+ }
+
+ return incrementalLimitBytes_ - heapSize.bytes();
+}
+
+/* static */
+double HeapThreshold::computeZoneHeapGrowthFactorForHeapSize(
+ size_t lastBytes, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state) {
+ // For small zones, our collection heuristics do not matter much: favor
+ // something simple in this case.
+ if (lastBytes < 1 * 1024 * 1024) {
+ return tunables.lowFrequencyHeapGrowth();
+ }
+
+ // The heap growth factor depends on the heap size after a GC and the GC
+ // frequency. If GC's are not triggering in rapid succession, use a lower
+ // threshold so that we will collect garbage sooner.
+ if (!state.inHighFrequencyGCMode()) {
+ return tunables.lowFrequencyHeapGrowth();
+ }
+
+ // For high frequency GCs we let the heap grow depending on whether we
+ // classify the heap as small, medium or large. There are parameters for small
+ // and large heap sizes and linear interpolation is used between them for
+ // medium sized heaps.
+
+ MOZ_ASSERT(tunables.smallHeapSizeMaxBytes() <=
+ tunables.largeHeapSizeMinBytes());
+ MOZ_ASSERT(tunables.highFrequencyLargeHeapGrowth() <=
+ tunables.highFrequencySmallHeapGrowth());
+
+ return LinearInterpolate(lastBytes, tunables.smallHeapSizeMaxBytes(),
+ tunables.highFrequencySmallHeapGrowth(),
+ tunables.largeHeapSizeMinBytes(),
+ tunables.highFrequencyLargeHeapGrowth());
+}
+
+/* static */
+size_t GCHeapThreshold::computeZoneTriggerBytes(
+ double growthFactor, size_t lastBytes,
+ const GCSchedulingTunables& tunables) {
+ size_t base = std::max(lastBytes, tunables.gcZoneAllocThresholdBase());
+ double trigger = double(base) * growthFactor;
+ double triggerMax =
+ double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit();
+ return ToClampedSize(std::min(triggerMax, trigger));
+}
+
+// Parameters for balanced heap limits computation.
+
+// The W0 parameter. How much memory can be traversed in the minimum collection
+// time.
+static constexpr double BalancedHeapBaseMB = 5.0;
+
+// The minimum heap limit. Do not constrain the heap to any less than this size.
+static constexpr double MinBalancedHeapLimitMB = 10.0;
+
+// The minimum amount of additional space to allow beyond the retained size.
+static constexpr double MinBalancedHeadroomMB = 3.0;
+
+// The maximum factor by which to expand the heap beyond the retained size.
+static constexpr double MaxHeapGrowth = 3.0;
+
+// The default allocation rate in MB/s allocated by the mutator to use before we
+// have an estimate. Used to set the heap limit for zones that have not yet been
+// collected.
+static constexpr double DefaultAllocationRate = 0.0;
+
+// The s0 parameter. The default collection rate in MB/s to use before we have
+// an estimate. Used to set the heap limit for zones that have not yet been
+// collected.
+static constexpr double DefaultCollectionRate = 200.0;
+
+double GCHeapThreshold::computeBalancedHeapLimit(
+ size_t lastBytes, double allocationRate, double collectionRate,
+ const GCSchedulingTunables& tunables) {
+ MOZ_ASSERT(tunables.balancedHeapLimitsEnabled());
+
+ // Optimal heap limits as described in https://arxiv.org/abs/2204.10455
+
+ double W = double(lastBytes) / BytesPerMB; // Retained size / MB.
+ double W0 = BalancedHeapBaseMB;
+ double d = tunables.heapGrowthFactor(); // Rearranged constant 'c'.
+ double g = allocationRate;
+ double s = collectionRate;
+ double f = d * sqrt((W + W0) * (g / s));
+ double M = W + std::min(f, MaxHeapGrowth * W);
+ M = std::max({MinBalancedHeapLimitMB, W + MinBalancedHeadroomMB, M});
+
+ return M * double(BytesPerMB);
+}
+
+void GCHeapThreshold::updateStartThreshold(
+ size_t lastBytes, mozilla::Maybe<double> allocationRate,
+ mozilla::Maybe<double> collectionRate, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state, bool isAtomsZone) {
+ if (!tunables.balancedHeapLimitsEnabled()) {
+ double growthFactor =
+ computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state);
+
+ startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes, tunables);
+ } else {
+ double threshold = computeBalancedHeapLimit(
+ lastBytes, allocationRate.valueOr(DefaultAllocationRate),
+ collectionRate.valueOr(DefaultCollectionRate), tunables);
+
+ double triggerMax =
+ double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit();
+
+ startBytes_ = ToClampedSize(uint64_t(std::min(triggerMax, threshold)));
+ }
+
+ setIncrementalLimitFromStartBytes(lastBytes, tunables);
+}
+
+/* static */
+size_t MallocHeapThreshold::computeZoneTriggerBytes(double growthFactor,
+ size_t lastBytes,
+ size_t baseBytes) {
+ return ToClampedSize(double(std::max(lastBytes, baseBytes)) * growthFactor);
+}
+
+void MallocHeapThreshold::updateStartThreshold(
+ size_t lastBytes, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state) {
+ double growthFactor =
+ computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state);
+
+ startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes,
+ tunables.mallocThresholdBase());
+
+ setIncrementalLimitFromStartBytes(lastBytes, tunables);
+}
+
+#ifdef DEBUG
+
+static const char* MemoryUseName(MemoryUse use) {
+ switch (use) {
+# define DEFINE_CASE(Name) \
+ case MemoryUse::Name: \
+ return #Name;
+ JS_FOR_EACH_MEMORY_USE(DEFINE_CASE)
+# undef DEFINE_CASE
+ }
+
+ MOZ_CRASH("Unknown memory use");
+}
+
+MemoryTracker::MemoryTracker() : mutex(mutexid::MemoryTracker) {}
+
+void MemoryTracker::checkEmptyOnDestroy() {
+ bool ok = true;
+
+ if (!gcMap.empty()) {
+ ok = false;
+ fprintf(stderr, "Missing calls to JS::RemoveAssociatedMemory:\n");
+ for (auto r = gcMap.all(); !r.empty(); r.popFront()) {
+ fprintf(stderr, " %p 0x%zx %s\n", r.front().key().ptr(),
+ r.front().value(), MemoryUseName(r.front().key().use()));
+ }
+ }
+
+ if (!nonGCMap.empty()) {
+ ok = false;
+ fprintf(stderr, "Missing calls to Zone::decNonGCMemory:\n");
+ for (auto r = nonGCMap.all(); !r.empty(); r.popFront()) {
+ fprintf(stderr, " %p 0x%zx\n", r.front().key().ptr(), r.front().value());
+ }
+ }
+
+ MOZ_ASSERT(ok);
+}
+
+/* static */
+inline bool MemoryTracker::isGCMemoryUse(MemoryUse use) {
+ // Most memory uses are for memory associated with GC things but some are for
+ // memory associated with non-GC thing pointers.
+ return !isNonGCMemoryUse(use);
+}
+
+/* static */
+inline bool MemoryTracker::isNonGCMemoryUse(MemoryUse use) {
+ return use == MemoryUse::TrackedAllocPolicy;
+}
+
+/* static */
+inline bool MemoryTracker::allowMultipleAssociations(MemoryUse use) {
+ // For most uses only one association is possible for each GC thing. Allow a
+ // one-to-many relationship only where necessary.
+ return isNonGCMemoryUse(use) || use == MemoryUse::RegExpSharedBytecode ||
+ use == MemoryUse::BreakpointSite || use == MemoryUse::Breakpoint ||
+ use == MemoryUse::ForOfPICStub || use == MemoryUse::ICUObject;
+}
+
+void MemoryTracker::trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(cell->isTenured());
+ MOZ_ASSERT(isGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<Cell> key{cell, use};
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ auto ptr = gcMap.lookupForAdd(key);
+ if (ptr) {
+ if (!allowMultipleAssociations(use)) {
+ MOZ_CRASH_UNSAFE_PRINTF("Association already present: %p 0x%zx %s", cell,
+ nbytes, MemoryUseName(use));
+ }
+ ptr->value() += nbytes;
+ return;
+ }
+
+ if (!gcMap.add(ptr, key, nbytes)) {
+ oomUnsafe.crash("MemoryTracker::trackGCMemory");
+ }
+}
+
+void MemoryTracker::untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(cell->isTenured());
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<Cell> key{cell, use};
+ auto ptr = gcMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("Association not found: %p 0x%zx %s", cell, nbytes,
+ MemoryUseName(use));
+ }
+
+ if (!allowMultipleAssociations(use) && ptr->value() != nbytes) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "Association for %p %s has different size: "
+ "expected 0x%zx but got 0x%zx",
+ cell, MemoryUseName(use), ptr->value(), nbytes);
+ }
+
+ if (nbytes > ptr->value()) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "Association for %p %s size is too large: "
+ "expected at most 0x%zx but got 0x%zx",
+ cell, MemoryUseName(use), ptr->value(), nbytes);
+ }
+
+ ptr->value() -= nbytes;
+
+ if (ptr->value() == 0) {
+ gcMap.remove(ptr);
+ }
+}
+
+void MemoryTracker::swapGCMemory(Cell* a, Cell* b, MemoryUse use) {
+ Key<Cell> ka{a, use};
+ Key<Cell> kb{b, use};
+
+ LockGuard<Mutex> lock(mutex);
+
+ size_t sa = getAndRemoveEntry(ka, lock);
+ size_t sb = getAndRemoveEntry(kb, lock);
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ if ((sa && b->isTenured() && !gcMap.put(kb, sa)) ||
+ (sb && a->isTenured() && !gcMap.put(ka, sb))) {
+ oomUnsafe.crash("MemoryTracker::swapGCMemory");
+ }
+}
+
+size_t MemoryTracker::getAndRemoveEntry(const Key<Cell>& key,
+ LockGuard<Mutex>& lock) {
+ auto ptr = gcMap.lookup(key);
+ if (!ptr) {
+ return 0;
+ }
+
+ size_t size = ptr->value();
+ gcMap.remove(ptr);
+ return size;
+}
+
+void MemoryTracker::registerNonGCMemory(void* mem, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookupForAdd(key);
+ if (ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s assocaition %p already registered",
+ MemoryUseName(use), mem);
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!nonGCMap.add(ptr, key, 0)) {
+ oomUnsafe.crash("MemoryTracker::registerNonGCMemory");
+ }
+}
+
+void MemoryTracker::unregisterNonGCMemory(void* mem, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ if (ptr->value() != 0) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "%s association %p still has 0x%zx bytes associated",
+ MemoryUseName(use), mem, ptr->value());
+ }
+
+ nonGCMap.remove(ptr);
+}
+
+void MemoryTracker::moveNonGCMemory(void* dst, void* src, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> srcKey{src, use};
+ auto srcPtr = nonGCMap.lookup(srcKey);
+ if (!srcPtr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use),
+ src);
+ }
+
+ size_t nbytes = srcPtr->value();
+ nonGCMap.remove(srcPtr);
+
+ Key<void> dstKey{dst, use};
+ auto dstPtr = nonGCMap.lookupForAdd(dstKey);
+ if (dstPtr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s %p already registered", MemoryUseName(use),
+ dst);
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!nonGCMap.add(dstPtr, dstKey, nbytes)) {
+ oomUnsafe.crash("MemoryTracker::moveNonGCMemory");
+ }
+}
+
+void MemoryTracker::incNonGCMemory(void* mem, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(isNonGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ ptr->value() += nbytes;
+}
+
+void MemoryTracker::decNonGCMemory(void* mem, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(isNonGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ size_t& value = ptr->value();
+ if (nbytes > value) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "%s allocation %p is too large: "
+ "expected at most 0x%zx but got 0x%zx bytes",
+ MemoryUseName(use), mem, value, nbytes);
+ }
+
+ value -= nbytes;
+}
+
+void MemoryTracker::fixupAfterMovingGC() {
+ // Update the table after we move GC things. We don't use StableCellHasher
+ // because that would create a difference between debug and release builds.
+ for (GCMap::Enum e(gcMap); !e.empty(); e.popFront()) {
+ const auto& key = e.front().key();
+ Cell* cell = key.ptr();
+ if (cell->isForwarded()) {
+ cell = gc::RelocationOverlay::fromCell(cell)->forwardingAddress();
+ e.rekeyFront(Key<Cell>{cell, key.use()});
+ }
+ }
+}
+
+template <typename Ptr>
+inline MemoryTracker::Key<Ptr>::Key(Ptr* ptr, MemoryUse use)
+ : ptr_(uint64_t(ptr)), use_(uint64_t(use)) {
+# ifdef JS_64BIT
+ static_assert(sizeof(Key) == 8,
+ "MemoryTracker::Key should be packed into 8 bytes");
+# endif
+ MOZ_ASSERT(this->ptr() == ptr);
+ MOZ_ASSERT(this->use() == use);
+}
+
+template <typename Ptr>
+inline Ptr* MemoryTracker::Key<Ptr>::ptr() const {
+ return reinterpret_cast<Ptr*>(ptr_);
+}
+template <typename Ptr>
+inline MemoryUse MemoryTracker::Key<Ptr>::use() const {
+ return static_cast<MemoryUse>(use_);
+}
+
+template <typename Ptr>
+inline HashNumber MemoryTracker::Hasher<Ptr>::hash(const Lookup& l) {
+ return mozilla::HashGeneric(DefaultHasher<Ptr*>::hash(l.ptr()),
+ DefaultHasher<unsigned>::hash(unsigned(l.use())));
+}
+
+template <typename Ptr>
+inline bool MemoryTracker::Hasher<Ptr>::match(const KeyT& k, const Lookup& l) {
+ return k.ptr() == l.ptr() && k.use() == l.use();
+}
+
+template <typename Ptr>
+inline void MemoryTracker::Hasher<Ptr>::rekey(KeyT& k, const KeyT& newKey) {
+ k = newKey;
+}
+
+#endif // DEBUG
diff --git a/js/src/gc/Scheduling.h b/js/src/gc/Scheduling.h
new file mode 100644
index 0000000000..94b06bc786
--- /dev/null
+++ b/js/src/gc/Scheduling.h
@@ -0,0 +1,917 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * [SMDOC] GC Scheduling
+ *
+ * GC Scheduling Overview
+ * ======================
+ *
+ * See also GC scheduling from Firefox's perspective here:
+ * https://searchfox.org/mozilla-central/source/dom/base/CCGCScheduler.cpp
+ *
+ * Scheduling GC's in SpiderMonkey/Firefox is tremendously complicated because
+ * of the large number of subtle, cross-cutting, and widely dispersed factors
+ * that must be taken into account. A summary of some of the more important
+ * factors follows.
+ *
+ * Cost factors:
+ *
+ * * GC too soon and we'll revisit an object graph almost identical to the
+ * one we just visited; since we are unlikely to find new garbage, the
+ * traversal will be largely overhead. We rely heavily on external factors
+ * to signal us that we are likely to find lots of garbage: e.g. "a tab
+ * just got closed".
+ *
+ * * GC too late and we'll run out of memory to allocate (e.g. Out-Of-Memory,
+ * hereafter simply abbreviated to OOM). If this happens inside
+ * SpiderMonkey we may be able to recover, but most embedder allocations
+ * will simply crash on OOM, even if the GC has plenty of free memory it
+ * could surrender.
+ *
+ * * Memory fragmentation: if we fill the process with GC allocations, a
+ * request for a large block of contiguous memory may fail because no
+ * contiguous block is free, despite having enough memory available to
+ * service the request.
+ *
+ * * Management overhead: if our GC heap becomes large, we create extra
+ * overhead when managing the GC's structures, even if the allocations are
+ * mostly unused.
+ *
+ * Heap Management Factors:
+ *
+ * * GC memory: The GC has its own allocator that it uses to make fixed size
+ * allocations for GC managed things. In cases where the GC thing requires
+ * larger or variable sized memory to implement itself, it is responsible
+ * for using the system heap.
+ *
+ * * C Heap Memory: Rather than allowing for large or variable allocations,
+ * the SpiderMonkey GC allows GC things to hold pointers to C heap memory.
+ * It is the responsibility of the thing to free this memory with a custom
+ * finalizer (with the sole exception of NativeObject, which knows about
+ * slots and elements for performance reasons). C heap memory has different
+ * performance and overhead tradeoffs than GC internal memory, which need
+ * to be considered with scheduling a GC.
+ *
+ * Application Factors:
+ *
+ * * Most applications allocate heavily at startup, then enter a processing
+ * stage where memory utilization remains roughly fixed with a slower
+ * allocation rate. This is not always the case, however, so while we may
+ * optimize for this pattern, we must be able to handle arbitrary
+ * allocation patterns.
+ *
+ * Other factors:
+ *
+ * * Other memory: This is memory allocated outside the purview of the GC.
+ * Data mapped by the system for code libraries, data allocated by those
+ * libraries, data in the JSRuntime that is used to manage the engine,
+ * memory used by the embedding that is not attached to a GC thing, memory
+ * used by unrelated processes running on the hardware that use space we
+ * could otherwise use for allocation, etc. While we don't have to manage
+ * it, we do have to take it into account when scheduling since it affects
+ * when we will OOM.
+ *
+ * * Physical Reality: All real machines have limits on the number of bits
+ * that they are physically able to store. While modern operating systems
+ * can generally make additional space available with swapping, at some
+ * point there are simply no more bits to allocate. There is also the
+ * factor of address space limitations, particularly on 32bit machines.
+ *
+ * * Platform Factors: Each OS makes use of wildly different memory
+ * management techniques. These differences result in different performance
+ * tradeoffs, different fragmentation patterns, and different hard limits
+ * on the amount of physical and/or virtual memory that we can use before
+ * OOMing.
+ *
+ *
+ * Reasons for scheduling GC
+ * -------------------------
+ *
+ * While code generally takes the above factors into account in only an ad-hoc
+ * fashion, the API forces the user to pick a "reason" for the GC. We have a
+ * bunch of JS::GCReason reasons in GCAPI.h. These fall into a few categories
+ * that generally coincide with one or more of the above factors.
+ *
+ * Embedding reasons:
+ *
+ * 1) Do a GC now because the embedding knows something useful about the
+ * zone's memory retention state. These are GCReasons like LOAD_END,
+ * PAGE_HIDE, SET_NEW_DOCUMENT, DOM_UTILS. Mostly, Gecko uses these to
+ * indicate that a significant fraction of the scheduled zone's memory is
+ * probably reclaimable.
+ *
+ * 2) Do some known amount of GC work now because the embedding knows now is
+ * a good time to do a long, unblockable operation of a known duration.
+ * These are INTER_SLICE_GC and REFRESH_FRAME.
+ *
+ * Correctness reasons:
+ *
+ * 3) Do a GC now because correctness depends on some GC property. For
+ * example, CC_FORCED is where the embedding requires the mark bits to be
+ * set correctly. Also, EVICT_NURSERY where we need to work on the tenured
+ * heap.
+ *
+ * 4) Do a GC because we are shutting down: e.g. SHUTDOWN_CC or DESTROY_*.
+ *
+ * 5) Do a GC because a compartment was accessed between GC slices when we
+ * would have otherwise discarded it. We have to do a second GC to clean
+ * it up: e.g. COMPARTMENT_REVIVED.
+ *
+ * Emergency Reasons:
+ *
+ * 6) Do an all-zones, non-incremental GC now because the embedding knows it
+ * cannot wait: e.g. MEM_PRESSURE.
+ *
+ * 7) OOM when fetching a new Chunk results in a LAST_DITCH GC.
+ *
+ * Heap Size Limitation Reasons:
+ *
+ * 8) Do an incremental, zonal GC with reason MAYBEGC when we discover that
+ * the gc's allocated size is approaching the current trigger. This is
+ * called MAYBEGC because we make this check in the MaybeGC function.
+ * MaybeGC gets called at the top of the main event loop. Normally, it is
+ * expected that this callback will keep the heap size limited. It is
+ * relatively inexpensive, because it is invoked with no JS running and
+ * thus few stack roots to scan. For this reason, the GC's "trigger" bytes
+ * is less than the GC's "max" bytes as used by the trigger below.
+ *
+ * 9) Do an incremental, zonal GC with reason MAYBEGC when we go to allocate
+ * a new GC thing and find that the GC heap size has grown beyond the
+ * configured maximum (JSGC_MAX_BYTES). We trigger this GC by returning
+ * nullptr and then calling maybeGC at the top level of the allocator.
+ * This is then guaranteed to fail the "size greater than trigger" check
+ * above, since trigger is always less than max. After performing the GC,
+ * the allocator unconditionally returns nullptr to force an OOM exception
+ * is raised by the script.
+ *
+ * Note that this differs from a LAST_DITCH GC where we actually run out
+ * of memory (i.e., a call to a system allocator fails) when trying to
+ * allocate. Unlike above, LAST_DITCH GC only happens when we are really
+ * out of memory, not just when we cross an arbitrary trigger; despite
+ * this, it may still return an allocation at the end and allow the script
+ * to continue, if the LAST_DITCH GC was able to free up enough memory.
+ *
+ * 10) Do a GC under reason ALLOC_TRIGGER when we are over the GC heap trigger
+ * limit, but in the allocator rather than in a random call to maybeGC.
+ * This occurs if we allocate too much before returning to the event loop
+ * and calling maybeGC; this is extremely common in benchmarks and
+ * long-running Worker computations. Note that this uses a wildly
+ * different mechanism from the above in that it sets the interrupt flag
+ * and does the GC at the next loop head, before the next alloc, or
+ * maybeGC. The reason for this is that this check is made after the
+ * allocation and we cannot GC with an uninitialized thing in the heap.
+ *
+ * 11) Do an incremental, zonal GC with reason TOO_MUCH_MALLOC when the total
+ * amount of malloced memory is greater than the malloc trigger limit for the
+ * zone.
+ *
+ *
+ * Size Limitation Triggers Explanation
+ * ------------------------------------
+ *
+ * The GC internally is entirely unaware of the context of the execution of
+ * the mutator. It sees only:
+ *
+ * A) Allocated size: this is the amount of memory currently requested by the
+ * mutator. This quantity is monotonically increasing: i.e. the allocation
+ * rate is always >= 0. It is also easy for the system to track.
+ *
+ * B) Retained size: this is the amount of memory that the mutator can
+ * currently reach. Said another way, it is the size of the heap
+ * immediately after a GC (modulo background sweeping). This size is very
+ * costly to know exactly and also extremely hard to estimate with any
+ * fidelity.
+ *
+ * For reference, a common allocated vs. retained graph might look like:
+ *
+ * | ** **
+ * | ** ** * **
+ * | ** * ** * **
+ * | * ** * ** * **
+ * | ** ** * ** * **
+ * s| * * ** ** + + **
+ * i| * * * + + + + +
+ * z| * * * + + + + +
+ * e| * **+
+ * | * +
+ * | * +
+ * | * +
+ * | * +
+ * | * +
+ * |*+
+ * +--------------------------------------------------
+ * time
+ * *** = allocated
+ * +++ = retained
+ *
+ * Note that this is a bit of a simplification
+ * because in reality we track malloc and GC heap
+ * sizes separately and have a different level of
+ * granularity and accuracy on each heap.
+ *
+ * This presents some obvious implications for Mark-and-Sweep collectors.
+ * Namely:
+ * -> t[marking] ~= size[retained]
+ * -> t[sweeping] ~= size[allocated] - size[retained]
+ *
+ * In a non-incremental collector, maintaining low latency and high
+ * responsiveness requires that total GC times be as low as possible. Thus,
+ * in order to stay responsive when we did not have a fully incremental
+ * collector, our GC triggers were focused on minimizing collection time.
+ * Furthermore, since size[retained] is not under control of the GC, all the
+ * GC could do to control collection times was reduce sweep times by
+ * minimizing size[allocated], per the equation above.
+ *
+ * The result of the above is GC triggers that focus on size[allocated] to
+ * the exclusion of other important factors and default heuristics that are
+ * not optimal for a fully incremental collector. On the other hand, this is
+ * not all bad: minimizing size[allocated] also minimizes the chance of OOM
+ * and sweeping remains one of the hardest areas to further incrementalize.
+ *
+ * EAGER_ALLOC_TRIGGER
+ * -------------------
+ * Occurs when we return to the event loop and find our heap is getting
+ * largish, but before t[marking] OR t[sweeping] is too large for a
+ * responsive non-incremental GC. This is intended to be the common case
+ * in normal web applications: e.g. we just finished an event handler and
+ * the few objects we allocated when computing the new whatzitz have
+ * pushed us slightly over the limit. After this GC we rescale the new
+ * EAGER_ALLOC_TRIGGER trigger to 150% of size[retained] so that our
+ * non-incremental GC times will always be proportional to this size
+ * rather than being dominated by sweeping.
+ *
+ * As a concession to mutators that allocate heavily during their startup
+ * phase, we have a highFrequencyGCMode that ups the growth rate to 300%
+ * of the current size[retained] so that we'll do fewer longer GCs at the
+ * end of the mutator startup rather than more, smaller GCs.
+ *
+ * Assumptions:
+ * -> Responsiveness is proportional to t[marking] + t[sweeping].
+ * -> size[retained] is proportional only to GC allocations.
+ *
+ * ALLOC_TRIGGER (non-incremental)
+ * -------------------------------
+ * If we do not return to the event loop before getting all the way to our
+ * gc trigger bytes then MAYBEGC will never fire. To avoid OOMing, we
+ * succeed the current allocation and set the script interrupt so that we
+ * will (hopefully) do a GC before we overflow our max and have to raise
+ * an OOM exception for the script.
+ *
+ * Assumptions:
+ * -> Common web scripts will return to the event loop before using
+ * 10% of the current triggerBytes worth of GC memory.
+ *
+ * ALLOC_TRIGGER (incremental)
+ * ---------------------------
+ * In practice the above trigger is rough: if a website is just on the
+ * cusp, sometimes it will trigger a non-incremental GC moments before
+ * returning to the event loop, where it could have done an incremental
+ * GC. Thus, we recently added an incremental version of the above with a
+ * substantially lower threshold, so that we have a soft limit here. If
+ * IGC can collect faster than the allocator generates garbage, even if
+ * the allocator does not return to the event loop frequently, we should
+ * not have to fall back to a non-incremental GC.
+ *
+ * INCREMENTAL_TOO_SLOW
+ * --------------------
+ * Do a full, non-incremental GC if we overflow ALLOC_TRIGGER during an
+ * incremental GC. When in the middle of an incremental GC, we suppress
+ * our other triggers, so we need a way to backstop the IGC if the
+ * mutator allocates faster than the IGC can clean things up.
+ *
+ * TOO_MUCH_MALLOC
+ * ---------------
+ * Performs a GC before size[allocated] - size[retained] gets too large
+ * for non-incremental sweeping to be fast in the case that we have
+ * significantly more malloc allocation than GC allocation. This is meant
+ * to complement MAYBEGC triggers. We track this by counting malloced
+ * bytes; the counter gets reset at every GC since we do not always have a
+ * size at the time we call free. Because of this, the malloc heuristic
+ * is, unfortunately, not usefully able to augment our other GC heap
+ * triggers and is limited to this singular heuristic.
+ *
+ * Assumptions:
+ * -> EITHER size[allocated_by_malloc] ~= size[allocated_by_GC]
+ * OR time[sweeping] ~= size[allocated_by_malloc]
+ * -> size[retained] @ t0 ~= size[retained] @ t1
+ * i.e. That the mutator is in steady-state operation.
+ *
+ * LAST_DITCH_GC
+ * -------------
+ * Does a GC because we are out of memory.
+ *
+ * Assumptions:
+ * -> size[retained] < size[available_memory]
+ */
+
+#ifndef gc_Scheduling_h
+#define gc_Scheduling_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include "gc/GCEnum.h"
+#include "js/AllocPolicy.h"
+#include "js/GCAPI.h"
+#include "js/HashTable.h"
+#include "js/HeapAPI.h"
+#include "threading/ProtectedData.h"
+
+// Macro to define scheduling tunables for GC parameters. Expands its argument
+// repeatedly with the following arguments:
+// - key: the JSGCParamKey value for this parameter
+// - type: the storage type
+// - name: the name of GCSchedulingTunables getter method
+// - convert: a helper class defined in Scheduling.cpp that provides
+// conversion methods
+// - check: a helper function defined in Scheduling.cppto check the value is
+// valid
+// - default: the initial value and that assigned by resetParameter
+#define FOR_EACH_GC_TUNABLE(_) \
+ /* \
+ * JSGC_MAX_BYTES \
+ * \
+ * Maximum nominal heap before last ditch GC. \
+ */ \
+ _(JSGC_MAX_BYTES, size_t, gcMaxBytes, ConvertSize, NoCheck, 0xffffffff) \
+ \
+ /* \
+ * JSGC_MIN_NURSERY_BYTES \
+ * JSGC_MAX_NURSERY_BYTES \
+ * \
+ * Minimum and maximum nursery size for each runtime. \
+ */ \
+ _(JSGC_MIN_NURSERY_BYTES, size_t, gcMinNurseryBytes, ConvertNurseryBytes, \
+ CheckNurserySize, 256 * 1024) \
+ _(JSGC_MAX_NURSERY_BYTES, size_t, gcMaxNurseryBytes, ConvertNurseryBytes, \
+ CheckNurserySize, JS::DefaultNurseryMaxBytes) \
+ \
+ /* \
+ * JSGC_ALLOCATION_THRESHOLD \
+ * \
+ * \
+ * The base value used to compute zone->threshold.bytes(). When \
+ * gcHeapSize.bytes() exceeds threshold.bytes() for a zone, the zone may be \
+ * scheduled for a GC, depending on the exact circumstances. \
+ */ \
+ _(JSGC_ALLOCATION_THRESHOLD, size_t, gcZoneAllocThresholdBase, ConvertMB, \
+ NoCheck, 27 * 1024 * 1024) \
+ \
+ /* \
+ * JSGC_SMALL_HEAP_SIZE_MAX \
+ * JSGC_LARGE_HEAP_SIZE_MIN \
+ * \
+ * Used to classify heap sizes into one of small, medium or large. This \
+ * affects the calcuation of the incremental GC trigger and the heap growth \
+ * factor in high frequency GC mode. \
+ */ \
+ _(JSGC_SMALL_HEAP_SIZE_MAX, size_t, smallHeapSizeMaxBytes, ConvertMB, \
+ NoCheck, 100 * 1024 * 1024) \
+ _(JSGC_LARGE_HEAP_SIZE_MIN, size_t, largeHeapSizeMinBytes, ConvertMB, \
+ CheckNonZero, 500 * 1024 * 1024) \
+ \
+ /* \
+ * JSGC_SMALL_HEAP_INCREMENTAL_LIMIT \
+ * JSGC_LARGE_HEAP_INCREMENTAL_LIMIT \
+ * \
+ * Multiple of threshold.bytes() which triggers a non-incremental GC. \
+ * \
+ * The small heap limit must be greater than 1.3 to maintain performance on \
+ * splay-latency. \
+ */ \
+ _(JSGC_SMALL_HEAP_INCREMENTAL_LIMIT, double, smallHeapIncrementalLimit, \
+ ConvertTimes100, CheckIncrementalLimit, 1.50) \
+ _(JSGC_LARGE_HEAP_INCREMENTAL_LIMIT, double, largeHeapIncrementalLimit, \
+ ConvertTimes100, CheckIncrementalLimit, 1.10) \
+ \
+ /* \
+ * JSGC_HIGH_FREQUENCY_TIME_LIMIT \
+ * \
+ * We enter high-frequency mode if we GC a twice within this many \
+ * millisconds. \
+ */ \
+ _(JSGC_HIGH_FREQUENCY_TIME_LIMIT, mozilla::TimeDuration, \
+ highFrequencyThreshold, ConvertMillis, NoCheck, \
+ mozilla::TimeDuration::FromSeconds(1)) \
+ \
+ /* \
+ * JSGC_LOW_FREQUENCY_HEAP_GROWTH \
+ * \
+ * When not in |highFrequencyGC| mode, this is the global (stored per-zone) \
+ * "HeapGrowthFactor". \
+ */ \
+ _(JSGC_LOW_FREQUENCY_HEAP_GROWTH, double, lowFrequencyHeapGrowth, \
+ ConvertTimes100, CheckHeapGrowth, 1.5) \
+ \
+ /* \
+ * JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH \
+ * JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH \
+ * \
+ * When in the |highFrequencyGC| mode, these parameterize the per-zone \
+ * "HeapGrowthFactor" computation. \
+ */ \
+ _(JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH, double, \
+ highFrequencySmallHeapGrowth, ConvertTimes100, CheckHeapGrowth, 3.0) \
+ _(JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH, double, \
+ highFrequencyLargeHeapGrowth, ConvertTimes100, CheckHeapGrowth, 1.5) \
+ \
+ /* \
+ * JSGC_MALLOC_THRESHOLD_BASE \
+ * \
+ * The base value used to compute the GC trigger for malloc allocated \
+ * memory. \
+ */ \
+ _(JSGC_MALLOC_THRESHOLD_BASE, size_t, mallocThresholdBase, ConvertMB, \
+ NoCheck, 38 * 1024 * 1024) \
+ \
+ /* \
+ * Number of bytes to allocate between incremental slices in GCs triggered \
+ * by the zone allocation threshold. \
+ */ \
+ _(JSGC_ZONE_ALLOC_DELAY_KB, size_t, zoneAllocDelayBytes, ConvertKB, \
+ CheckNonZero, 1024 * 1024) \
+ \
+ /* \
+ * JSGC_URGENT_THRESHOLD_MB \
+ * \
+ * The point before reaching the non-incremental limit at which to start \
+ * increasing the slice budget and frequency of allocation triggered slices. \
+ */ \
+ _(JSGC_URGENT_THRESHOLD_MB, size_t, urgentThresholdBytes, ConvertMB, \
+ NoCheck, 16 * 1024 * 1024) \
+ \
+ /* \
+ * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION \
+ * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_FRACTION \
+ * JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS \
+ * \
+ * Attempt to run a minor GC in the idle time if the free space falls below \
+ * this threshold or if it hasn't been collected for too long. The absolute \
+ * threshold is used when the nursery is large and the percentage when it is \
+ * small. See Nursery::shouldCollect(). \
+ */ \
+ _(JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION, size_t, \
+ nurseryFreeThresholdForIdleCollection, ConvertSize, NoCheck, \
+ ChunkSize / 4) \
+ _(JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_PERCENT, double, \
+ nurseryFreeThresholdForIdleCollectionFraction, ConvertTimes100, \
+ CheckNonZeroUnitRange, 0.25) \
+ _(JSGC_NURSERY_TIMEOUT_FOR_IDLE_COLLECTION_MS, mozilla::TimeDuration, \
+ nurseryTimeoutForIdleCollection, ConvertMillis, NoCheck, \
+ mozilla::TimeDuration::FromSeconds(5)) \
+ \
+ /* \
+ * JSGC_BALANCED_HEAP_LIMITS_ENABLED \
+ * JSGC_HEAP_GROWTH_FACTOR \
+ */ \
+ _(JSGC_BALANCED_HEAP_LIMITS_ENABLED, bool, balancedHeapLimitsEnabled, \
+ ConvertBool, NoCheck, false) \
+ _(JSGC_HEAP_GROWTH_FACTOR, double, heapGrowthFactor, ConvertDouble, NoCheck, \
+ 50.0) \
+ \
+ /* \
+ * JSGC_PRETENURE_THRESHOLD \
+ * \
+ * Fraction of objects tenured to trigger pretenuring (between 0 and 1). If \
+ * this fraction is met, the GC proceeds to calculate which objects will be \
+ * tenured. If this is 1.0f (actually if it is not < 1.0f) then pretenuring \
+ * is disabled. \
+ */ \
+ _(JSGC_PRETENURE_THRESHOLD, double, pretenureThreshold, ConvertTimes100, \
+ CheckNonZeroUnitRange, 0.6) \
+ \
+ /* \
+ * JSGC_PRETENURE_STRING_THRESHOLD \
+ * \
+ * If the percentage of the tenured strings exceeds this threshold, string \
+ * will be allocated in tenured heap instead. (Default is allocated in \
+ * nursery.) \
+ */ \
+ _(JSGC_PRETENURE_STRING_THRESHOLD, double, pretenureStringThreshold, \
+ ConvertTimes100, CheckNonZeroUnitRange, 0.55) \
+ \
+ /* \
+ * JSGC_STOP_PRETENURE_STRING_THRESHOLD \
+ * \
+ * If the finalization rate of the tenured strings exceeds this threshold, \
+ * string will be allocated in nursery. \
+ */ \
+ _(JSGC_STOP_PRETENURE_STRING_THRESHOLD, double, \
+ stopPretenureStringThreshold, ConvertTimes100, CheckNonZeroUnitRange, 0.9) \
+ \
+ /* \
+ * JSGC_MIN_LAST_DITCH_GC_PERIOD \
+ * \
+ * Last ditch GC is skipped if allocation failure occurs less than this many \
+ * seconds from the previous one. \
+ */ \
+ _(JSGC_MIN_LAST_DITCH_GC_PERIOD, mozilla::TimeDuration, \
+ minLastDitchGCPeriod, ConvertSeconds, NoCheck, \
+ TimeDuration::FromSeconds(60)) \
+ \
+ /* \
+ * JSGC_PARALLEL_MARKING_THRESHOLD_KB \
+ */ \
+ _(JSGC_PARALLEL_MARKING_THRESHOLD_KB, size_t, parallelMarkingThresholdBytes, \
+ ConvertKB, NoCheck, 10 * 1024 * 1024)
+
+namespace js {
+
+class ZoneAllocator;
+
+namespace gc {
+
+struct Cell;
+
+/*
+ * Default settings for tuning the GC. Some of these can be set at runtime,
+ * This list is not complete, some tuning parameters are not listed here.
+ *
+ * If you change the values here, please also consider changing them in
+ * modules/libpref/init/all.js where they are duplicated for the Firefox
+ * preferences.
+ */
+namespace TuningDefaults {
+
+/* JSGC_MIN_EMPTY_CHUNK_COUNT */
+static const uint32_t MinEmptyChunkCount = 1;
+
+/* JSGC_MAX_EMPTY_CHUNK_COUNT */
+static const uint32_t MaxEmptyChunkCount = 30;
+
+/* JSGC_SLICE_TIME_BUDGET_MS */
+static const int64_t DefaultTimeBudgetMS = 0; // Unlimited by default.
+
+/* JSGC_INCREMENTAL_GC_ENABLED */
+static const bool IncrementalGCEnabled = false;
+
+/* JSGC_PER_ZONE_GC_ENABLED */
+static const bool PerZoneGCEnabled = false;
+
+/* JSGC_COMPACTING_ENABLED */
+static const bool CompactingEnabled = true;
+
+/* JSGC_PARALLEL_MARKING_ENABLED */
+static const bool ParallelMarkingEnabled = false;
+
+/* JSGC_INCREMENTAL_WEAKMAP_ENABLED */
+static const bool IncrementalWeakMapMarkingEnabled = true;
+
+/* JSGC_HELPER_THREAD_RATIO */
+static const double HelperThreadRatio = 0.5;
+
+/* JSGC_MAX_HELPER_THREADS */
+static const size_t MaxHelperThreads = 8;
+
+} // namespace TuningDefaults
+
+/*
+ * Encapsulates all of the GC tunables. These are effectively constant and
+ * should only be modified by setParameter.
+ */
+class GCSchedulingTunables {
+#define DEFINE_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ MainThreadOrGCTaskData<type> name##_;
+ FOR_EACH_GC_TUNABLE(DEFINE_TUNABLE_FIELD)
+#undef DEFINE_TUNABLE_FIELD
+
+ public:
+ GCSchedulingTunables();
+
+#define DEFINE_TUNABLE_ACCESSOR(key, type, name, convert, check, default) \
+ type name() const { return name##_; }
+ FOR_EACH_GC_TUNABLE(DEFINE_TUNABLE_ACCESSOR)
+#undef DEFINE_TUNABLE_ACCESSOR
+
+ bool attemptPretenuring() const { return pretenureThreshold_ < 1.0; }
+
+ uint32_t getParameter(JSGCParamKey key);
+ [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value);
+ void resetParameter(JSGCParamKey key);
+
+ private:
+ void maintainInvariantsAfterUpdate(JSGCParamKey updated);
+ void checkInvariants();
+};
+
+class GCSchedulingState {
+ /*
+ * Influences how we schedule and run GC's in several subtle ways. The most
+ * important factor is in how it controls the "HeapGrowthFactor". The
+ * growth factor is a measure of how large (as a percentage of the last GC)
+ * the heap is allowed to grow before we try to schedule another GC.
+ */
+ mozilla::Atomic<bool, mozilla::ReleaseAcquire> inHighFrequencyGCMode_;
+
+ public:
+ GCSchedulingState() : inHighFrequencyGCMode_(false) {}
+
+ bool inHighFrequencyGCMode() const { return inHighFrequencyGCMode_; }
+
+ void updateHighFrequencyMode(const mozilla::TimeStamp& lastGCTime,
+ const mozilla::TimeStamp& currentTime,
+ const GCSchedulingTunables& tunables);
+ void updateHighFrequencyModeForReason(JS::GCReason reason);
+};
+
+struct TriggerResult {
+ bool shouldTrigger;
+ size_t usedBytes;
+ size_t thresholdBytes;
+};
+
+using AtomicByteCount = mozilla::Atomic<size_t, mozilla::ReleaseAcquire>;
+
+/*
+ * Tracks the size of allocated data. This is used for both GC and malloc data.
+ * It automatically maintains the memory usage relationship between parent and
+ * child instances, i.e. between those in a GCRuntime and its Zones.
+ */
+class HeapSize {
+ /*
+ * The number of bytes in use. For GC heaps this is approximate to the nearest
+ * ArenaSize. It is atomic because it is updated by both the active and GC
+ * helper threads.
+ */
+ AtomicByteCount bytes_;
+
+ /*
+ * The number of bytes in use at the start of the last collection.
+ */
+ MainThreadData<size_t> initialBytes_;
+
+ /*
+ * The number of bytes retained after the last collection. This is updated
+ * dynamically during incremental GC. It does not include allocations that
+ * happen during a GC.
+ */
+ AtomicByteCount retainedBytes_;
+
+ public:
+ explicit HeapSize() {
+ MOZ_ASSERT(bytes_ == 0);
+ MOZ_ASSERT(retainedBytes_ == 0);
+ }
+
+ size_t bytes() const { return bytes_; }
+ size_t initialBytes() const { return initialBytes_; }
+ size_t retainedBytes() const { return retainedBytes_; }
+
+ void updateOnGCStart() { retainedBytes_ = initialBytes_ = bytes(); }
+
+ void addGCArena() { addBytes(ArenaSize); }
+ void removeGCArena() {
+ MOZ_ASSERT(retainedBytes_ >= ArenaSize);
+ removeBytes(ArenaSize, true /* only sweeping removes arenas */);
+ MOZ_ASSERT(retainedBytes_ <= bytes_);
+ }
+
+ void addBytes(size_t nbytes) {
+ mozilla::DebugOnly<size_t> initialBytes(bytes_);
+ MOZ_ASSERT(initialBytes + nbytes > initialBytes);
+ bytes_ += nbytes;
+ }
+ void removeBytes(size_t nbytes, bool updateRetainedSize) {
+ if (updateRetainedSize) {
+ MOZ_ASSERT(retainedBytes_ >= nbytes);
+ retainedBytes_ -= nbytes;
+ }
+ MOZ_ASSERT(bytes_ >= nbytes);
+ bytes_ -= nbytes;
+ }
+};
+
+/*
+ * Like HeapSize, but also updates a 'parent' HeapSize. Used for per-zone heap
+ * size data that also updates a runtime-wide parent.
+ */
+class HeapSizeChild : public HeapSize {
+ public:
+ void addGCArena(HeapSize& parent) {
+ HeapSize::addGCArena();
+ parent.addGCArena();
+ }
+
+ void removeGCArena(HeapSize& parent) {
+ HeapSize::removeGCArena();
+ parent.removeGCArena();
+ }
+
+ void addBytes(size_t nbytes, HeapSize& parent) {
+ HeapSize::addBytes(nbytes);
+ parent.addBytes(nbytes);
+ }
+
+ void removeBytes(size_t nbytes, bool updateRetainedSize, HeapSize& parent) {
+ HeapSize::removeBytes(nbytes, updateRetainedSize);
+ parent.removeBytes(nbytes, updateRetainedSize);
+ }
+};
+
+class PerZoneGCHeapSize : public HeapSizeChild {
+ public:
+ size_t freedBytes() const { return freedBytes_; }
+ void clearFreedBytes() { freedBytes_ = 0; }
+
+ void removeGCArena(HeapSize& parent) {
+ HeapSizeChild::removeGCArena(parent);
+ freedBytes_ += ArenaSize;
+ }
+
+ void removeBytes(size_t nbytes, bool updateRetainedSize, HeapSize& parent) {
+ HeapSizeChild::removeBytes(nbytes, updateRetainedSize, parent);
+ freedBytes_ += nbytes;
+ }
+
+ private:
+ AtomicByteCount freedBytes_;
+};
+
+// Heap size thresholds used to trigger GC. This is an abstract base class for
+// GC heap and malloc thresholds defined below.
+class HeapThreshold {
+ protected:
+ HeapThreshold()
+ : startBytes_(SIZE_MAX),
+ incrementalLimitBytes_(SIZE_MAX),
+ sliceBytes_(SIZE_MAX) {}
+
+ // The threshold at which to start a new incremental collection.
+ //
+ // This can be read off main thread during collection, for example by sweep
+ // tasks that resize tables.
+ MainThreadOrGCTaskData<size_t> startBytes_;
+
+ // The threshold at which start a new non-incremental collection or finish an
+ // ongoing collection non-incrementally.
+ MainThreadData<size_t> incrementalLimitBytes_;
+
+ // The threshold at which to trigger a slice during an ongoing incremental
+ // collection.
+ MainThreadData<size_t> sliceBytes_;
+
+ public:
+ size_t startBytes() const { return startBytes_; }
+ size_t sliceBytes() const { return sliceBytes_; }
+ size_t incrementalLimitBytes() const { return incrementalLimitBytes_; }
+ double eagerAllocTrigger(bool highFrequencyGC) const;
+ size_t incrementalBytesRemaining(const HeapSize& heapSize) const;
+
+ void setSliceThreshold(ZoneAllocator* zone, const HeapSize& heapSize,
+ const GCSchedulingTunables& tunables,
+ bool waitingOnBGTask);
+ void clearSliceThreshold() { sliceBytes_ = SIZE_MAX; }
+ bool hasSliceThreshold() const { return sliceBytes_ != SIZE_MAX; }
+
+ protected:
+ static double computeZoneHeapGrowthFactorForHeapSize(
+ size_t lastBytes, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state);
+
+ void setIncrementalLimitFromStartBytes(size_t retainedBytes,
+ const GCSchedulingTunables& tunables);
+};
+
+// A heap threshold that is based on a multiple of the retained size after the
+// last collection adjusted based on collection frequency and retained
+// size. This is used to determine when to do a zone GC based on GC heap size.
+class GCHeapThreshold : public HeapThreshold {
+ public:
+ void updateStartThreshold(size_t lastBytes,
+ mozilla::Maybe<double> allocationRate,
+ mozilla::Maybe<double> collectionRate,
+ const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state, bool isAtomsZone);
+
+ private:
+ // This is our original algorithm for calculating heap limits.
+ static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes,
+ const GCSchedulingTunables& tunables);
+
+ // This is the algorithm described in the optimal heap limits paper.
+ static double computeBalancedHeapLimit(size_t lastBytes,
+ double allocationRate,
+ double collectionRate,
+ const GCSchedulingTunables& tunables);
+};
+
+// A heap threshold that is calculated as a constant multiple of the retained
+// size after the last collection. This is used to determines when to do a zone
+// GC based on malloc data.
+class MallocHeapThreshold : public HeapThreshold {
+ public:
+ void updateStartThreshold(size_t lastBytes,
+ const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state);
+
+ private:
+ static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes,
+ size_t baseBytes);
+};
+
+// A fixed threshold that's used to determine when we need to do a zone GC based
+// on allocated JIT code.
+class JitHeapThreshold : public HeapThreshold {
+ public:
+ explicit JitHeapThreshold(size_t bytes) { startBytes_ = bytes; }
+};
+
+#ifdef DEBUG
+
+// Counts memory associated with GC things in a zone.
+//
+// This records details of the cell (or non-cell pointer) the memory allocation
+// is associated with to check the correctness of the information provided. This
+// is not present in opt builds.
+class MemoryTracker {
+ public:
+ MemoryTracker();
+ void fixupAfterMovingGC();
+ void checkEmptyOnDestroy();
+
+ // Track memory by associated GC thing pointer.
+ void trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use);
+ void untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use);
+ void swapGCMemory(Cell* a, Cell* b, MemoryUse use);
+
+ // Track memory by associated non-GC thing pointer.
+ void registerNonGCMemory(void* ptr, MemoryUse use);
+ void unregisterNonGCMemory(void* ptr, MemoryUse use);
+ void moveNonGCMemory(void* dst, void* src, MemoryUse use);
+ void incNonGCMemory(void* ptr, size_t nbytes, MemoryUse use);
+ void decNonGCMemory(void* ptr, size_t nbytes, MemoryUse use);
+
+ private:
+ template <typename Ptr>
+ struct Key {
+ Key(Ptr* ptr, MemoryUse use);
+ Ptr* ptr() const;
+ MemoryUse use() const;
+
+ private:
+# ifdef JS_64BIT
+ // Pack this into a single word on 64 bit platforms.
+ uintptr_t ptr_ : 56;
+ uintptr_t use_ : 8;
+# else
+ uintptr_t ptr_ : 32;
+ uintptr_t use_ : 8;
+# endif
+ };
+
+ template <typename Ptr>
+ struct Hasher {
+ using KeyT = Key<Ptr>;
+ using Lookup = KeyT;
+ static HashNumber hash(const Lookup& l);
+ static bool match(const KeyT& key, const Lookup& l);
+ static void rekey(KeyT& k, const KeyT& newKey);
+ };
+
+ template <typename Ptr>
+ using Map = HashMap<Key<Ptr>, size_t, Hasher<Ptr>, SystemAllocPolicy>;
+ using GCMap = Map<Cell>;
+ using NonGCMap = Map<void>;
+
+ static bool isGCMemoryUse(MemoryUse use);
+ static bool isNonGCMemoryUse(MemoryUse use);
+ static bool allowMultipleAssociations(MemoryUse use);
+
+ size_t getAndRemoveEntry(const Key<Cell>& key, LockGuard<Mutex>& lock);
+
+ Mutex mutex MOZ_UNANNOTATED;
+
+ // Map containing the allocated size associated with (cell, use) pairs.
+ GCMap gcMap;
+
+ // Map containing the allocated size associated (non-cell pointer, use) pairs.
+ NonGCMap nonGCMap;
+};
+
+#endif // DEBUG
+
+static inline double LinearInterpolate(double x, double x0, double y0,
+ double x1, double y1) {
+ MOZ_ASSERT(x0 < x1);
+
+ if (x < x0) {
+ return y0;
+ }
+
+ if (x < x1) {
+ return y0 + (y1 - y0) * ((x - x0) / (x1 - x0));
+ }
+
+ return y1;
+}
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_Scheduling_h
diff --git a/js/src/gc/StableCellHasher-inl.h b/js/src/gc/StableCellHasher-inl.h
new file mode 100644
index 0000000000..f76748afef
--- /dev/null
+++ b/js/src/gc/StableCellHasher-inl.h
@@ -0,0 +1,245 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_StableCellHasher_inl_h
+#define gc_StableCellHasher_inl_h
+
+#include "gc/StableCellHasher.h"
+
+#include "mozilla/HashFunctions.h"
+
+#include "gc/Cell.h"
+#include "gc/Marking.h"
+#include "gc/Zone.h"
+#include "vm/JSObject.h"
+#include "vm/NativeObject.h"
+#include "vm/Runtime.h"
+
+namespace js {
+namespace gc {
+
+extern uint64_t NextCellUniqueId(JSRuntime* rt);
+
+inline bool MaybeGetUniqueId(Cell* cell, uint64_t* uidp) {
+ MOZ_ASSERT(uidp);
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) ||
+ CurrentThreadIsPerformingGC());
+
+ if (cell->is<JSObject>()) {
+ JSObject* obj = cell->as<JSObject>();
+ if (obj->is<NativeObject>()) {
+ auto* nobj = &obj->as<NativeObject>();
+ if (!nobj->hasUniqueId()) {
+ return false;
+ }
+
+ *uidp = nobj->uniqueId();
+ return true;
+ }
+ }
+
+ // Get an existing uid, if one has been set.
+ auto p = cell->zone()->uniqueIds().readonlyThreadsafeLookup(cell);
+ if (!p) {
+ return false;
+ }
+
+ *uidp = p->value();
+
+ return true;
+}
+
+extern bool CreateUniqueIdForNativeObject(NativeObject* obj, uint64_t* uidp);
+extern bool CreateUniqueIdForNonNativeObject(Cell* cell, UniqueIdMap::AddPtr,
+ uint64_t* uidp);
+
+inline bool GetOrCreateUniqueId(Cell* cell, uint64_t* uidp) {
+ MOZ_ASSERT(uidp);
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) ||
+ CurrentThreadIsPerformingGC());
+
+ if (cell->is<JSObject>()) {
+ JSObject* obj = cell->as<JSObject>();
+ if (obj->is<NativeObject>()) {
+ auto* nobj = &obj->as<NativeObject>();
+ if (nobj->hasUniqueId()) {
+ *uidp = nobj->uniqueId();
+ return true;
+ }
+
+ return CreateUniqueIdForNativeObject(nobj, uidp);
+ }
+ }
+
+ // Get an existing uid, if one has been set.
+ auto p = cell->zone()->uniqueIds().lookupForAdd(cell);
+ if (p) {
+ *uidp = p->value();
+ return true;
+ }
+
+ return CreateUniqueIdForNonNativeObject(cell, p, uidp);
+}
+
+inline bool SetOrUpdateUniqueId(JSContext* cx, Cell* cell, uint64_t uid) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()));
+
+ if (cell->is<JSObject>()) {
+ JSObject* obj = cell->as<JSObject>();
+ if (obj->is<NativeObject>()) {
+ auto* nobj = &obj->as<NativeObject>();
+ return nobj->setOrUpdateUniqueId(cx, uid);
+ }
+ }
+
+ // If the cell was in the nursery, hopefully unlikely, then we need to
+ // tell the nursery about it so that it can sweep the uid if the thing
+ // does not get tenured.
+ JSRuntime* runtime = cell->runtimeFromMainThread();
+ if (IsInsideNursery(cell) &&
+ !runtime->gc.nursery().addedUniqueIdToCell(cell)) {
+ return false;
+ }
+
+ return cell->zone()->uniqueIds().put(cell, uid);
+}
+
+inline uint64_t GetUniqueIdInfallible(Cell* cell) {
+ uint64_t uid;
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!GetOrCreateUniqueId(cell, &uid)) {
+ oomUnsafe.crash("failed to allocate uid");
+ }
+ return uid;
+}
+
+inline bool HasUniqueId(Cell* cell) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()) ||
+ CurrentThreadIsPerformingGC());
+
+ if (cell->is<JSObject>()) {
+ JSObject* obj = cell->as<JSObject>();
+ if (obj->is<NativeObject>()) {
+ return obj->as<NativeObject>().hasUniqueId();
+ }
+ }
+
+ return cell->zone()->uniqueIds().has(cell);
+}
+
+inline void TransferUniqueId(Cell* tgt, Cell* src) {
+ MOZ_ASSERT(src != tgt);
+ MOZ_ASSERT(!IsInsideNursery(tgt));
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(tgt->runtimeFromAnyThread()));
+ MOZ_ASSERT(src->zone() == tgt->zone());
+
+ Zone* zone = tgt->zone();
+ MOZ_ASSERT(!zone->uniqueIds().has(tgt));
+ zone->uniqueIds().rekeyIfMoved(src, tgt);
+}
+
+inline void RemoveUniqueId(Cell* cell) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cell->runtimeFromAnyThread()));
+ // The cell may no longer be in the hash table if it was swapped with a
+ // NativeObject.
+ cell->zone()->uniqueIds().remove(cell);
+}
+
+} // namespace gc
+
+static inline js::HashNumber UniqueIdToHash(uint64_t uid) {
+ // This uses the default hasher which returns the lower 32 bits of 64 bit
+ // integers as the hash code. This is OK because he result will be scrambled
+ // later by ScrambleHashCode().
+ return DefaultHasher<uint64_t>::hash(uid);
+}
+
+template <typename T>
+/* static */ bool StableCellHasher<T>::maybeGetHash(const Lookup& l,
+ HashNumber* hashOut) {
+ if (!l) {
+ *hashOut = 0;
+ return true;
+ }
+
+ uint64_t uid;
+ if (!gc::MaybeGetUniqueId(l, &uid)) {
+ return false;
+ }
+
+ *hashOut = UniqueIdToHash(uid);
+ return true;
+}
+
+template <typename T>
+/* static */ bool StableCellHasher<T>::ensureHash(const Lookup& l,
+ HashNumber* hashOut) {
+ if (!l) {
+ *hashOut = 0;
+ return true;
+ }
+
+ uint64_t uid;
+ if (!gc::GetOrCreateUniqueId(l, &uid)) {
+ return false;
+ }
+
+ *hashOut = UniqueIdToHash(uid);
+ return true;
+}
+
+template <typename T>
+/* static */ HashNumber StableCellHasher<T>::hash(const Lookup& l) {
+ if (!l) {
+ return 0;
+ }
+
+ // We have to access the zone from-any-thread here: a worker thread may be
+ // cloning a self-hosted object from the main runtime's self- hosting zone
+ // into another runtime. The zone's uid lock will protect against multiple
+ // workers doing this simultaneously.
+ MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) ||
+ CurrentThreadIsPerformingGC());
+
+ return UniqueIdToHash(gc::GetUniqueIdInfallible(l));
+}
+
+template <typename T>
+/* static */ bool StableCellHasher<T>::match(const Key& k, const Lookup& l) {
+ if (k == l) {
+ return true;
+ }
+
+ if (!k || !l) {
+ return false;
+ }
+
+ MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) ||
+ CurrentThreadIsPerformingGC());
+
+#ifdef DEBUG
+ // Incremental table sweeping means that existing table entries may no
+ // longer have unique IDs. We fail the match in that case and the entry is
+ // removed from the table later on.
+ if (!gc::HasUniqueId(k)) {
+ Key key = k;
+ MOZ_ASSERT(IsAboutToBeFinalizedUnbarriered(key));
+ }
+ MOZ_ASSERT(gc::HasUniqueId(l));
+#endif
+
+ uint64_t keyId;
+ if (!gc::MaybeGetUniqueId(k, &keyId)) {
+ // Key is dead and cannot match lookup which must be live.
+ return false;
+ }
+
+ return keyId == gc::GetUniqueIdInfallible(l);
+}
+
+} // namespace js
+
+#endif // gc_StableCellHasher_inl_h
diff --git a/js/src/gc/StableCellHasher.h b/js/src/gc/StableCellHasher.h
new file mode 100644
index 0000000000..80b1856976
--- /dev/null
+++ b/js/src/gc/StableCellHasher.h
@@ -0,0 +1,46 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_StableCellHasher_h
+#define gc_StableCellHasher_h
+
+#include "js/RootingAPI.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+
+// StableCellHasher itself is defined in js/public/RootingAPI.h.
+
+namespace gc {
+
+struct Cell;
+
+// Gets an existing UID in |uidp| if one exists.
+[[nodiscard]] bool MaybeGetUniqueId(Cell* cell, uint64_t* uidp);
+
+// Puts an existing UID in |uidp|, or creates a new UID for this Cell and
+// puts that into |uidp|. Returns false on OOM.
+[[nodiscard]] bool GetOrCreateUniqueId(Cell* cell, uint64_t* uidp);
+
+uint64_t GetUniqueIdInfallible(Cell* cell);
+
+// Return true if this cell has a UID associated with it.
+[[nodiscard]] bool HasUniqueId(Cell* cell);
+
+// Transfer an id from another cell. This must only be called on behalf of a
+// moving GC. This method is infallible.
+void TransferUniqueId(Cell* tgt, Cell* src);
+
+// Remove any unique id associated with this Cell.
+void RemoveUniqueId(Cell* cell);
+
+// Used to restore unique ID after JSObject::swap.
+bool SetOrUpdateUniqueId(JSContext* cx, Cell* cell, uint64_t uid);
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_StableCellHasher_h
diff --git a/js/src/gc/Statistics.cpp b/js/src/gc/Statistics.cpp
new file mode 100644
index 0000000000..66b1c40673
--- /dev/null
+++ b/js/src/gc/Statistics.cpp
@@ -0,0 +1,1811 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Statistics.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Sprintf.h"
+#include "mozilla/TimeStamp.h"
+
+#include <algorithm>
+#include <stdarg.h>
+#include <stdio.h>
+#include <type_traits>
+
+#include "gc/GC.h"
+#include "gc/GCInternals.h"
+#include "gc/Memory.h"
+#include "js/Printer.h"
+#include "util/GetPidProvider.h"
+#include "util/Text.h"
+#include "vm/JSONPrinter.h"
+#include "vm/Runtime.h"
+#include "vm/Time.h"
+
+#include "gc/PrivateIterators-inl.h"
+
+using namespace js;
+using namespace js::gc;
+using namespace js::gcstats;
+
+using mozilla::DebugOnly;
+using mozilla::EnumeratedArray;
+using mozilla::Maybe;
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+
+static const size_t BYTES_PER_MB = 1024 * 1024;
+
+/*
+ * If this fails, then you can either delete this assertion and allow all
+ * larger-numbered reasons to pile up in the last telemetry bucket, or switch
+ * to GC_REASON_3 and bump the max value.
+ */
+static_assert(JS::GCReason::NUM_TELEMETRY_REASONS >= JS::GCReason::NUM_REASONS);
+
+static inline auto AllPhaseKinds() {
+ return mozilla::MakeEnumeratedRange(PhaseKind::FIRST, PhaseKind::LIMIT);
+}
+
+static inline auto MajorGCPhaseKinds() {
+ return mozilla::MakeEnumeratedRange(PhaseKind::GC_BEGIN,
+ PhaseKind(size_t(PhaseKind::GC_END) + 1));
+}
+
+static const char* ExplainGCOptions(JS::GCOptions options) {
+ switch (options) {
+ case JS::GCOptions::Normal:
+ return "Normal";
+ case JS::GCOptions::Shrink:
+ return "Shrink";
+ case JS::GCOptions::Shutdown:
+ return "Shutdown";
+ }
+
+ MOZ_CRASH("Unexpected GCOptions value");
+}
+
+JS_PUBLIC_API const char* JS::ExplainGCReason(JS::GCReason reason) {
+ switch (reason) {
+#define SWITCH_REASON(name, _) \
+ case JS::GCReason::name: \
+ return #name;
+ GCREASONS(SWITCH_REASON)
+#undef SWITCH_REASON
+
+ case JS::GCReason::NO_REASON:
+ return "NO_REASON";
+
+ default:
+ MOZ_CRASH("bad GC reason");
+ }
+}
+
+JS_PUBLIC_API bool JS::InternalGCReason(JS::GCReason reason) {
+ return reason < JS::GCReason::FIRST_FIREFOX_REASON;
+}
+
+const char* js::gcstats::ExplainAbortReason(GCAbortReason reason) {
+ switch (reason) {
+#define SWITCH_REASON(name, _) \
+ case GCAbortReason::name: \
+ return #name;
+ GC_ABORT_REASONS(SWITCH_REASON)
+
+ default:
+ MOZ_CRASH("bad GC abort reason");
+#undef SWITCH_REASON
+ }
+}
+
+static FILE* MaybeOpenFileFromEnv(const char* env,
+ FILE* defaultFile = nullptr) {
+ const char* value = getenv(env);
+ if (!value) {
+ return defaultFile;
+ }
+
+ FILE* file;
+ if (strcmp(value, "none") == 0) {
+ file = nullptr;
+ } else if (strcmp(value, "stdout") == 0) {
+ file = stdout;
+ } else if (strcmp(value, "stderr") == 0) {
+ file = stderr;
+ } else {
+ char path[300];
+ if (value[0] != '/') {
+ const char* dir = getenv("MOZ_UPLOAD_DIR");
+ if (dir) {
+ SprintfLiteral(path, "%s/%s", dir, value);
+ value = path;
+ }
+ }
+
+ file = fopen(value, "a");
+ if (!file || setvbuf(file, nullptr, _IOLBF, 256) != 0) {
+ perror("Error opening log file");
+ MOZ_CRASH("Failed to open log file.");
+ }
+ }
+
+ return file;
+}
+
+struct PhaseKindInfo {
+ Phase firstPhase;
+ uint8_t telemetryBucket;
+ const char* name;
+};
+
+// PhaseInfo objects form a tree.
+struct PhaseInfo {
+ Phase parent;
+ Phase firstChild;
+ Phase nextSibling;
+ Phase nextWithPhaseKind;
+ PhaseKind phaseKind;
+ uint8_t depth;
+ const char* name;
+ const char* path;
+};
+
+// A table of PhaseInfo indexed by Phase.
+using PhaseTable = EnumeratedArray<Phase, Phase::LIMIT, PhaseInfo>;
+
+// A table of PhaseKindInfo indexed by PhaseKind.
+using PhaseKindTable =
+ EnumeratedArray<PhaseKind, PhaseKind::LIMIT, PhaseKindInfo>;
+
+#include "gc/StatsPhasesGenerated.inc"
+
+// Iterate the phases in a phase kind.
+class PhaseIter {
+ Phase phase;
+
+ public:
+ explicit PhaseIter(PhaseKind kind) : phase(phaseKinds[kind].firstPhase) {}
+ bool done() const { return phase == Phase::NONE; }
+ void next() { phase = phases[phase].nextWithPhaseKind; }
+ Phase get() const { return phase; }
+ operator Phase() const { return phase; }
+};
+
+static double t(TimeDuration duration) { return duration.ToMilliseconds(); }
+
+inline JSContext* Statistics::context() {
+ return gc->rt->mainContextFromOwnThread();
+}
+
+inline Phase Statistics::currentPhase() const {
+ return phaseStack.empty() ? Phase::NONE : phaseStack.back();
+}
+
+PhaseKind Statistics::currentPhaseKind() const {
+ // Public API to get the current phase kind, suppressing the synthetic
+ // PhaseKind::MUTATOR phase.
+
+ Phase phase = currentPhase();
+ MOZ_ASSERT_IF(phase == Phase::MUTATOR, phaseStack.length() == 1);
+ if (phase == Phase::NONE || phase == Phase::MUTATOR) {
+ return PhaseKind::NONE;
+ }
+
+ return phases[phase].phaseKind;
+}
+
+static Phase LookupPhaseWithParent(PhaseKind phaseKind, Phase parentPhase) {
+ for (PhaseIter phase(phaseKind); !phase.done(); phase.next()) {
+ if (phases[phase].parent == parentPhase) {
+ return phase;
+ }
+ }
+
+ return Phase::NONE;
+}
+
+static const char* PhaseKindName(PhaseKind kind) {
+ if (kind == PhaseKind::NONE) {
+ return "NONE";
+ }
+
+ return phaseKinds[kind].name;
+}
+
+Phase Statistics::lookupChildPhase(PhaseKind phaseKind) const {
+ if (phaseKind == PhaseKind::IMPLICIT_SUSPENSION) {
+ return Phase::IMPLICIT_SUSPENSION;
+ }
+ if (phaseKind == PhaseKind::EXPLICIT_SUSPENSION) {
+ return Phase::EXPLICIT_SUSPENSION;
+ }
+
+ MOZ_ASSERT(phaseKind < PhaseKind::LIMIT);
+
+ // Search all expanded phases that correspond to the required
+ // phase to find the one whose parent is the current expanded phase.
+ Phase phase = LookupPhaseWithParent(phaseKind, currentPhase());
+
+ if (phase == Phase::NONE) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "Child phase kind %s not found under current phase kind %s",
+ PhaseKindName(phaseKind), PhaseKindName(currentPhaseKind()));
+ }
+
+ return phase;
+}
+
+inline auto AllPhases() {
+ return mozilla::MakeEnumeratedRange(Phase::FIRST, Phase::LIMIT);
+}
+
+void Statistics::gcDuration(TimeDuration* total, TimeDuration* maxPause) const {
+ *total = *maxPause = 0;
+ for (auto& slice : slices_) {
+ *total += slice.duration();
+ if (slice.duration() > *maxPause) {
+ *maxPause = slice.duration();
+ }
+ }
+ if (*maxPause > maxPauseInInterval) {
+ maxPauseInInterval = *maxPause;
+ }
+}
+
+void Statistics::sccDurations(TimeDuration* total,
+ TimeDuration* maxPause) const {
+ *total = *maxPause = 0;
+ for (size_t i = 0; i < sccTimes.length(); i++) {
+ *total += sccTimes[i];
+ *maxPause = std::max(*maxPause, sccTimes[i]);
+ }
+}
+
+typedef Vector<UniqueChars, 8, SystemAllocPolicy> FragmentVector;
+
+static UniqueChars Join(const FragmentVector& fragments,
+ const char* separator = "") {
+ const size_t separatorLength = strlen(separator);
+ size_t length = 0;
+ for (size_t i = 0; i < fragments.length(); ++i) {
+ length += fragments[i] ? strlen(fragments[i].get()) : 0;
+ if (i < (fragments.length() - 1)) {
+ length += separatorLength;
+ }
+ }
+
+ char* joined = js_pod_malloc<char>(length + 1);
+ if (!joined) {
+ return UniqueChars();
+ }
+
+ joined[length] = '\0';
+ char* cursor = joined;
+ for (size_t i = 0; i < fragments.length(); ++i) {
+ if (fragments[i]) {
+ strcpy(cursor, fragments[i].get());
+ }
+ cursor += fragments[i] ? strlen(fragments[i].get()) : 0;
+ if (i < (fragments.length() - 1)) {
+ if (separatorLength) {
+ strcpy(cursor, separator);
+ }
+ cursor += separatorLength;
+ }
+ }
+
+ return UniqueChars(joined);
+}
+
+static TimeDuration SumChildTimes(Phase phase,
+ const Statistics::PhaseTimes& phaseTimes) {
+ TimeDuration total = 0;
+ for (phase = phases[phase].firstChild; phase != Phase::NONE;
+ phase = phases[phase].nextSibling) {
+ total += phaseTimes[phase];
+ }
+ return total;
+}
+
+UniqueChars Statistics::formatCompactSliceMessage() const {
+ // Skip if we OOM'ed.
+ if (slices_.length() == 0) {
+ return UniqueChars(nullptr);
+ }
+
+ const size_t index = slices_.length() - 1;
+ const SliceData& slice = slices_.back();
+
+ char budgetDescription[200];
+ slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1);
+
+ const char* format =
+ "GC Slice %u - Pause: %.3fms of %s budget (@ %.3fms); Reason: %s; Reset: "
+ "%s%s; Times: ";
+ char buffer[1024];
+ SprintfLiteral(buffer, format, index, t(slice.duration()), budgetDescription,
+ t(slice.start - slices_[0].start),
+ ExplainGCReason(slice.reason),
+ slice.wasReset() ? "yes - " : "no",
+ slice.wasReset() ? ExplainAbortReason(slice.resetReason) : "");
+
+ FragmentVector fragments;
+ if (!fragments.append(DuplicateString(buffer)) ||
+ !fragments.append(
+ formatCompactSlicePhaseTimes(slices_[index].phaseTimes))) {
+ return UniqueChars(nullptr);
+ }
+ return Join(fragments);
+}
+
+UniqueChars Statistics::formatCompactSummaryMessage() const {
+ FragmentVector fragments;
+ if (!fragments.append(DuplicateString("Summary - "))) {
+ return UniqueChars(nullptr);
+ }
+
+ TimeDuration total, longest;
+ gcDuration(&total, &longest);
+
+ const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20));
+ const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50));
+
+ char buffer[1024];
+ if (!nonincremental()) {
+ SprintfLiteral(buffer,
+ "Max Pause: %.3fms; MMU 20ms: %.1f%%; MMU 50ms: %.1f%%; "
+ "Total: %.3fms; ",
+ t(longest), mmu20 * 100., mmu50 * 100., t(total));
+ } else {
+ SprintfLiteral(buffer, "Non-Incremental: %.3fms (%s); ", t(total),
+ ExplainAbortReason(nonincrementalReason_));
+ }
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+
+ SprintfLiteral(buffer,
+ "Zones: %d of %d (-%d); Compartments: %d of %d (-%d); "
+ "HeapSize: %.3f MiB; "
+ "HeapChange (abs): %+d (%u); ",
+ zoneStats.collectedZoneCount, zoneStats.zoneCount,
+ zoneStats.sweptZoneCount, zoneStats.collectedCompartmentCount,
+ zoneStats.compartmentCount, zoneStats.sweptCompartmentCount,
+ double(preTotalHeapBytes) / BYTES_PER_MB,
+ int32_t(counts[COUNT_NEW_CHUNK] - counts[COUNT_DESTROY_CHUNK]),
+ counts[COUNT_NEW_CHUNK] + counts[COUNT_DESTROY_CHUNK]);
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+
+ MOZ_ASSERT_IF(counts[COUNT_ARENA_RELOCATED],
+ gcOptions == JS::GCOptions::Shrink);
+ if (gcOptions == JS::GCOptions::Shrink) {
+ SprintfLiteral(
+ buffer, "Kind: %s; Relocated: %.3f MiB; ", ExplainGCOptions(gcOptions),
+ double(ArenaSize * counts[COUNT_ARENA_RELOCATED]) / BYTES_PER_MB);
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+ }
+
+ return Join(fragments);
+}
+
+UniqueChars Statistics::formatCompactSlicePhaseTimes(
+ const PhaseTimes& phaseTimes) const {
+ static const TimeDuration MaxUnaccountedTime =
+ TimeDuration::FromMicroseconds(100);
+
+ FragmentVector fragments;
+ char buffer[128];
+ for (auto phase : AllPhases()) {
+ DebugOnly<uint8_t> level = phases[phase].depth;
+ MOZ_ASSERT(level < 4);
+
+ TimeDuration ownTime = phaseTimes[phase];
+ TimeDuration childTime = SumChildTimes(phase, phaseTimes);
+ if (ownTime > MaxUnaccountedTime) {
+ SprintfLiteral(buffer, "%s: %.3fms", phases[phase].name, t(ownTime));
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+
+ if (childTime && (ownTime - childTime) > MaxUnaccountedTime) {
+ MOZ_ASSERT(level < 3);
+ SprintfLiteral(buffer, "%s: %.3fms", "Other", t(ownTime - childTime));
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+ }
+ }
+ }
+ return Join(fragments, ", ");
+}
+
+UniqueChars Statistics::formatDetailedMessage() const {
+ FragmentVector fragments;
+
+ if (!fragments.append(formatDetailedDescription())) {
+ return UniqueChars(nullptr);
+ }
+
+ if (!slices_.empty()) {
+ for (unsigned i = 0; i < slices_.length(); i++) {
+ if (!fragments.append(formatDetailedSliceDescription(i, slices_[i]))) {
+ return UniqueChars(nullptr);
+ }
+ if (!fragments.append(formatDetailedPhaseTimes(slices_[i].phaseTimes))) {
+ return UniqueChars(nullptr);
+ }
+ }
+ }
+ if (!fragments.append(formatDetailedTotals())) {
+ return UniqueChars(nullptr);
+ }
+ if (!fragments.append(formatDetailedPhaseTimes(phaseTimes))) {
+ return UniqueChars(nullptr);
+ }
+
+ return Join(fragments);
+}
+
+UniqueChars Statistics::formatDetailedDescription() const {
+ TimeDuration sccTotal, sccLongest;
+ sccDurations(&sccTotal, &sccLongest);
+
+ const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20));
+ const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50));
+
+ const char* format =
+ "=================================================================\n\
+ Invocation Kind: %s\n\
+ Reason: %s\n\
+ Incremental: %s%s\n\
+ Zones Collected: %d of %d (-%d)\n\
+ Compartments Collected: %d of %d (-%d)\n\
+ MinorGCs since last GC: %d\n\
+ Store Buffer Overflows: %d\n\
+ MMU 20ms:%.1f%%; 50ms:%.1f%%\n\
+ SCC Sweep Total (MaxPause): %.3fms (%.3fms)\n\
+ HeapSize: %.3f MiB\n\
+ Chunk Delta (magnitude): %+d (%d)\n\
+ Arenas Relocated: %.3f MiB\n\
+";
+
+ char buffer[1024];
+ SprintfLiteral(
+ buffer, format, ExplainGCOptions(gcOptions),
+ ExplainGCReason(slices_[0].reason), nonincremental() ? "no - " : "yes",
+ nonincremental() ? ExplainAbortReason(nonincrementalReason_) : "",
+ zoneStats.collectedZoneCount, zoneStats.zoneCount,
+ zoneStats.sweptZoneCount, zoneStats.collectedCompartmentCount,
+ zoneStats.compartmentCount, zoneStats.sweptCompartmentCount,
+ getCount(COUNT_MINOR_GC), getCount(COUNT_STOREBUFFER_OVERFLOW),
+ mmu20 * 100., mmu50 * 100., t(sccTotal), t(sccLongest),
+ double(preTotalHeapBytes) / BYTES_PER_MB,
+ getCount(COUNT_NEW_CHUNK) - getCount(COUNT_DESTROY_CHUNK),
+ getCount(COUNT_NEW_CHUNK) + getCount(COUNT_DESTROY_CHUNK),
+ double(ArenaSize * getCount(COUNT_ARENA_RELOCATED)) / BYTES_PER_MB);
+
+ return DuplicateString(buffer);
+}
+
+UniqueChars Statistics::formatDetailedSliceDescription(
+ unsigned i, const SliceData& slice) const {
+ char budgetDescription[200];
+ slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1);
+
+ const char* format =
+ "\
+ ---- Slice %u ----\n\
+ Reason: %s\n\
+ Trigger: %s\n\
+ Reset: %s%s\n\
+ State: %s -> %s\n\
+ Page Faults: %" PRIu64
+ "\n\
+ Pause: %.3fms of %s budget (@ %.3fms)\n\
+";
+
+ char triggerBuffer[100] = "n/a";
+ if (slice.trigger) {
+ Trigger trigger = slice.trigger.value();
+ SprintfLiteral(triggerBuffer, "%.3f MiB of %.3f MiB threshold\n",
+ double(trigger.amount) / BYTES_PER_MB,
+ double(trigger.threshold) / BYTES_PER_MB);
+ }
+
+ char buffer[1024];
+ SprintfLiteral(
+ buffer, format, i, ExplainGCReason(slice.reason), triggerBuffer,
+ slice.wasReset() ? "yes - " : "no",
+ slice.wasReset() ? ExplainAbortReason(slice.resetReason) : "",
+ gc::StateName(slice.initialState), gc::StateName(slice.finalState),
+ uint64_t(slice.endFaults - slice.startFaults), t(slice.duration()),
+ budgetDescription, t(slice.start - slices_[0].start));
+ return DuplicateString(buffer);
+}
+
+static bool IncludePhase(TimeDuration duration) {
+ // Don't include durations that will print as "0.000ms".
+ return duration.ToMilliseconds() >= 0.001;
+}
+
+UniqueChars Statistics::formatDetailedPhaseTimes(
+ const PhaseTimes& phaseTimes) const {
+ static const TimeDuration MaxUnaccountedChildTime =
+ TimeDuration::FromMicroseconds(50);
+
+ FragmentVector fragments;
+ char buffer[128];
+ for (auto phase : AllPhases()) {
+ uint8_t level = phases[phase].depth;
+ TimeDuration ownTime = phaseTimes[phase];
+ TimeDuration childTime = SumChildTimes(phase, phaseTimes);
+ if (IncludePhase(ownTime)) {
+ SprintfLiteral(buffer, " %*s%s: %.3fms\n", level * 2, "",
+ phases[phase].name, t(ownTime));
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+
+ if (childTime && (ownTime - childTime) > MaxUnaccountedChildTime) {
+ SprintfLiteral(buffer, " %*s%s: %.3fms\n", (level + 1) * 2, "",
+ "Other", t(ownTime - childTime));
+ if (!fragments.append(DuplicateString(buffer))) {
+ return UniqueChars(nullptr);
+ }
+ }
+ }
+ }
+ return Join(fragments);
+}
+
+UniqueChars Statistics::formatDetailedTotals() const {
+ TimeDuration total, longest;
+ gcDuration(&total, &longest);
+
+ const char* format =
+ "\
+ ---- Totals ----\n\
+ Total Time: %.3fms\n\
+ Max Pause: %.3fms\n\
+";
+ char buffer[1024];
+ SprintfLiteral(buffer, format, t(total), t(longest));
+ return DuplicateString(buffer);
+}
+
+void Statistics::formatJsonSlice(size_t sliceNum, JSONPrinter& json) const {
+ /*
+ * We number each of the slice properties to keep the code in
+ * GCTelemetry.jsm in sync. See MAX_SLICE_KEYS.
+ */
+ json.beginObject();
+ formatJsonSliceDescription(sliceNum, slices_[sliceNum], json); // # 1-11
+
+ json.beginObjectProperty("times"); // # 12
+ formatJsonPhaseTimes(slices_[sliceNum].phaseTimes, json);
+ json.endObject();
+
+ json.endObject();
+}
+
+UniqueChars Statistics::renderJsonSlice(size_t sliceNum) const {
+ Sprinter printer(nullptr, false);
+ if (!printer.init()) {
+ return UniqueChars(nullptr);
+ }
+ JSONPrinter json(printer, false);
+
+ formatJsonSlice(sliceNum, json);
+ return printer.release();
+}
+
+UniqueChars Statistics::renderNurseryJson() const {
+ Sprinter printer(nullptr, false);
+ if (!printer.init()) {
+ return UniqueChars(nullptr);
+ }
+ JSONPrinter json(printer, false);
+ gc->nursery().renderProfileJSON(json);
+ return printer.release();
+}
+
+#ifdef DEBUG
+void Statistics::log(const char* fmt, ...) {
+ va_list args;
+ va_start(args, fmt);
+ if (gcDebugFile) {
+ TimeDuration sinceStart = TimeStamp::Now() - TimeStamp::FirstTimeStamp();
+ fprintf(gcDebugFile, "%12.3f: ", sinceStart.ToMicroseconds());
+ vfprintf(gcDebugFile, fmt, args);
+ fprintf(gcDebugFile, "\n");
+ fflush(gcDebugFile);
+ }
+ va_end(args);
+}
+#endif
+
+UniqueChars Statistics::renderJsonMessage() const {
+ /*
+ * The format of the JSON message is specified by the GCMajorMarkerPayload
+ * type in profiler.firefox.com
+ * https://github.com/firefox-devtools/profiler/blob/master/src/types/markers.js#L62
+ *
+ * All the properties listed here are created within the timings property
+ * of the GCMajor marker.
+ */
+ if (aborted) {
+ return DuplicateString("{status:\"aborted\"}"); // May return nullptr
+ }
+
+ Sprinter printer(nullptr, false);
+ if (!printer.init()) {
+ return UniqueChars(nullptr);
+ }
+ JSONPrinter json(printer, false);
+
+ json.beginObject();
+ json.property("status", "completed");
+ formatJsonDescription(json);
+
+ json.beginObjectProperty("totals");
+ formatJsonPhaseTimes(phaseTimes, json);
+ json.endObject();
+
+ json.endObject();
+
+ return printer.release();
+}
+
+void Statistics::formatJsonDescription(JSONPrinter& json) const {
+ // If you change JSON properties here, please update:
+ // Firefox Profiler:
+ // https://github.com/firefox-devtools/profiler
+
+ TimeDuration total, longest;
+ gcDuration(&total, &longest);
+ json.property("max_pause", longest, JSONPrinter::MILLISECONDS);
+ json.property("total_time", total, JSONPrinter::MILLISECONDS);
+ // We might be able to omit reason if profiler.firefox.com was able to retrive
+ // it from the first slice. But it doesn't do this yet.
+ json.property("reason", ExplainGCReason(slices_[0].reason));
+ json.property("zones_collected", zoneStats.collectedZoneCount);
+ json.property("total_zones", zoneStats.zoneCount);
+ json.property("total_compartments", zoneStats.compartmentCount);
+ json.property("minor_gcs", getCount(COUNT_MINOR_GC));
+ json.property("minor_gc_number", gc->minorGCCount());
+ json.property("major_gc_number", gc->majorGCCount());
+ uint32_t storebufferOverflows = getCount(COUNT_STOREBUFFER_OVERFLOW);
+ if (storebufferOverflows) {
+ json.property("store_buffer_overflows", storebufferOverflows);
+ }
+ json.property("slices", slices_.length());
+
+ const double mmu20 = computeMMU(TimeDuration::FromMilliseconds(20));
+ const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50));
+ json.property("mmu_20ms", int(mmu20 * 100));
+ json.property("mmu_50ms", int(mmu50 * 100));
+
+ TimeDuration sccTotal, sccLongest;
+ sccDurations(&sccTotal, &sccLongest);
+ json.property("scc_sweep_total", sccTotal, JSONPrinter::MILLISECONDS);
+ json.property("scc_sweep_max_pause", sccLongest, JSONPrinter::MILLISECONDS);
+
+ if (nonincrementalReason_ != GCAbortReason::None) {
+ json.property("nonincremental_reason",
+ ExplainAbortReason(nonincrementalReason_));
+ }
+ json.property("allocated_bytes", preTotalHeapBytes);
+ json.property("post_heap_size", postTotalHeapBytes);
+
+ uint32_t addedChunks = getCount(COUNT_NEW_CHUNK);
+ if (addedChunks) {
+ json.property("added_chunks", addedChunks);
+ }
+ uint32_t removedChunks = getCount(COUNT_DESTROY_CHUNK);
+ if (removedChunks) {
+ json.property("removed_chunks", removedChunks);
+ }
+ json.property("major_gc_number", startingMajorGCNumber);
+ json.property("minor_gc_number", startingMinorGCNumber);
+ json.property("slice_number", startingSliceNumber);
+}
+
+void Statistics::formatJsonSliceDescription(unsigned i, const SliceData& slice,
+ JSONPrinter& json) const {
+ // If you change JSON properties here, please update:
+ // Firefox Profiler:
+ // https://github.com/firefox-devtools/profiler
+ //
+ char budgetDescription[200];
+ slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1);
+ TimeStamp originTime = TimeStamp::ProcessCreation();
+
+ json.property("slice", i);
+ json.property("pause", slice.duration(), JSONPrinter::MILLISECONDS);
+ json.property("reason", ExplainGCReason(slice.reason));
+ json.property("initial_state", gc::StateName(slice.initialState));
+ json.property("final_state", gc::StateName(slice.finalState));
+ json.property("budget", budgetDescription);
+ json.property("major_gc_number", startingMajorGCNumber);
+ if (slice.trigger) {
+ Trigger trigger = slice.trigger.value();
+ json.property("trigger_amount", trigger.amount);
+ json.property("trigger_threshold", trigger.threshold);
+ }
+ int64_t numFaults = slice.endFaults - slice.startFaults;
+ if (numFaults != 0) {
+ json.property("page_faults", numFaults);
+ }
+ json.property("start_timestamp", slice.start - originTime,
+ JSONPrinter::SECONDS);
+}
+
+void Statistics::formatJsonPhaseTimes(const PhaseTimes& phaseTimes,
+ JSONPrinter& json) const {
+ for (auto phase : AllPhases()) {
+ TimeDuration ownTime = phaseTimes[phase];
+ if (!ownTime.IsZero()) {
+ json.property(phases[phase].path, ownTime, JSONPrinter::MILLISECONDS);
+ }
+ }
+}
+
+Statistics::Statistics(GCRuntime* gc)
+ : gc(gc),
+ gcTimerFile(nullptr),
+ gcDebugFile(nullptr),
+ nonincrementalReason_(GCAbortReason::None),
+ creationTime_(TimeStamp::Now()),
+ tenuredAllocsSinceMinorGC(0),
+ preTotalHeapBytes(0),
+ postTotalHeapBytes(0),
+ preCollectedHeapBytes(0),
+ startingMinorGCNumber(0),
+ startingMajorGCNumber(0),
+ startingSliceNumber(0),
+ maxPauseInInterval(0),
+ sliceCallback(nullptr),
+ nurseryCollectionCallback(nullptr),
+ aborted(false),
+ enableProfiling_(false),
+ sliceCount_(0) {
+ for (auto& count : counts) {
+ count = 0;
+ }
+
+ for (auto& stat : stats) {
+ stat = 0;
+ }
+
+#ifdef DEBUG
+ for (const auto& duration : totalTimes_) {
+ using ElementType = std::remove_reference_t<decltype(duration)>;
+ static_assert(!std::is_trivially_constructible_v<ElementType>,
+ "Statistics::Statistics will only initialize "
+ "totalTimes_'s elements if their default constructor is "
+ "non-trivial");
+ MOZ_ASSERT(duration.IsZero(),
+ "totalTimes_ default-initialization should have "
+ "default-initialized every element of totalTimes_ to zero");
+ }
+#endif
+
+ MOZ_ALWAYS_TRUE(phaseStack.reserve(MAX_PHASE_NESTING));
+ MOZ_ALWAYS_TRUE(suspendedPhases.reserve(MAX_SUSPENDED_PHASES));
+
+ gcTimerFile = MaybeOpenFileFromEnv("MOZ_GCTIMER");
+ gcDebugFile = MaybeOpenFileFromEnv("JS_GC_DEBUG");
+ gcProfileFile = MaybeOpenFileFromEnv("JS_GC_PROFILE_FILE", stderr);
+
+ gc::ReadProfileEnv("JS_GC_PROFILE",
+ "Report major GCs taking more than N milliseconds for "
+ "all or just the main runtime\n",
+ &enableProfiling_, &profileWorkers_, &profileThreshold_);
+}
+
+Statistics::~Statistics() {
+ if (gcTimerFile && gcTimerFile != stdout && gcTimerFile != stderr) {
+ fclose(gcTimerFile);
+ }
+ if (gcDebugFile && gcDebugFile != stdout && gcDebugFile != stderr) {
+ fclose(gcDebugFile);
+ }
+}
+
+/* static */
+bool Statistics::initialize() {
+#ifdef DEBUG
+ // Sanity check generated tables.
+ for (auto i : AllPhases()) {
+ auto parent = phases[i].parent;
+ if (parent != Phase::NONE) {
+ MOZ_ASSERT(phases[i].depth == phases[parent].depth + 1);
+ }
+ auto firstChild = phases[i].firstChild;
+ if (firstChild != Phase::NONE) {
+ MOZ_ASSERT(i == phases[firstChild].parent);
+ MOZ_ASSERT(phases[i].depth == phases[firstChild].depth - 1);
+ }
+ auto nextSibling = phases[i].nextSibling;
+ if (nextSibling != Phase::NONE) {
+ MOZ_ASSERT(parent == phases[nextSibling].parent);
+ MOZ_ASSERT(phases[i].depth == phases[nextSibling].depth);
+ }
+ auto nextWithPhaseKind = phases[i].nextWithPhaseKind;
+ if (nextWithPhaseKind != Phase::NONE) {
+ MOZ_ASSERT(phases[i].phaseKind == phases[nextWithPhaseKind].phaseKind);
+ MOZ_ASSERT(parent != phases[nextWithPhaseKind].parent);
+ }
+ }
+ for (auto i : AllPhaseKinds()) {
+ MOZ_ASSERT(phases[phaseKinds[i].firstPhase].phaseKind == i);
+ for (auto j : AllPhaseKinds()) {
+ MOZ_ASSERT_IF(i != j, phaseKinds[i].telemetryBucket !=
+ phaseKinds[j].telemetryBucket);
+ }
+ }
+#endif
+
+ return true;
+}
+
+JS::GCSliceCallback Statistics::setSliceCallback(
+ JS::GCSliceCallback newCallback) {
+ JS::GCSliceCallback oldCallback = sliceCallback;
+ sliceCallback = newCallback;
+ return oldCallback;
+}
+
+JS::GCNurseryCollectionCallback Statistics::setNurseryCollectionCallback(
+ JS::GCNurseryCollectionCallback newCallback) {
+ auto oldCallback = nurseryCollectionCallback;
+ nurseryCollectionCallback = newCallback;
+ return oldCallback;
+}
+
+TimeDuration Statistics::clearMaxGCPauseAccumulator() {
+ TimeDuration prior = maxPauseInInterval;
+ maxPauseInInterval = 0;
+ return prior;
+}
+
+TimeDuration Statistics::getMaxGCPauseSinceClear() {
+ return maxPauseInInterval;
+}
+
+// Sum up the time for a phase, including instances of the phase with different
+// parents.
+static TimeDuration SumPhase(PhaseKind phaseKind,
+ const Statistics::PhaseTimes& times) {
+ TimeDuration sum;
+ for (PhaseIter phase(phaseKind); !phase.done(); phase.next()) {
+ sum += times[phase];
+ }
+ return sum;
+}
+
+static bool CheckSelfTime(Phase parent, Phase child,
+ const Statistics::PhaseTimes& times,
+ const Statistics::PhaseTimes& selfTimes,
+ TimeDuration childTime) {
+ if (selfTimes[parent] < childTime) {
+ fprintf(
+ stderr,
+ "Parent %s time = %.3fms with %.3fms remaining, child %s time %.3fms\n",
+ phases[parent].name, times[parent].ToMilliseconds(),
+ selfTimes[parent].ToMilliseconds(), phases[child].name,
+ childTime.ToMilliseconds());
+ fflush(stderr);
+ return false;
+ }
+
+ return true;
+}
+
+static PhaseKind FindLongestPhaseKind(const Statistics::PhaseKindTimes& times) {
+ TimeDuration longestTime;
+ PhaseKind phaseKind = PhaseKind::NONE;
+ for (auto i : MajorGCPhaseKinds()) {
+ if (times[i] > longestTime) {
+ longestTime = times[i];
+ phaseKind = i;
+ }
+ }
+
+ return phaseKind;
+}
+
+static PhaseKind LongestPhaseSelfTimeInMajorGC(
+ const Statistics::PhaseTimes& times) {
+ // Start with total times per expanded phase, including children's times.
+ Statistics::PhaseTimes selfTimes(times);
+
+ // We have the total time spent in each phase, including descendant times.
+ // Loop over the children and subtract their times from their parent's self
+ // time.
+ for (auto i : AllPhases()) {
+ Phase parent = phases[i].parent;
+ if (parent != Phase::NONE) {
+ bool ok = CheckSelfTime(parent, i, times, selfTimes, times[i]);
+
+ // This happens very occasionally in release builds and frequently
+ // in Windows debug builds. Skip collecting longest phase telemetry
+ // if it does.
+#ifndef XP_WIN
+ MOZ_ASSERT(ok, "Inconsistent time data; see bug 1400153");
+#endif
+ if (!ok) {
+ return PhaseKind::NONE;
+ }
+
+ selfTimes[parent] -= times[i];
+ }
+ }
+
+ // Sum expanded phases corresponding to the same phase.
+ Statistics::PhaseKindTimes phaseKindTimes;
+ for (auto i : AllPhaseKinds()) {
+ phaseKindTimes[i] = SumPhase(i, selfTimes);
+ }
+
+ return FindLongestPhaseKind(phaseKindTimes);
+}
+
+void Statistics::printStats() {
+ if (aborted) {
+ fprintf(gcTimerFile,
+ "OOM during GC statistics collection. The report is unavailable "
+ "for this GC.\n");
+ } else {
+ UniqueChars msg = formatDetailedMessage();
+ if (msg) {
+ double secSinceStart =
+ (slices_[0].start - TimeStamp::ProcessCreation()).ToSeconds();
+ fprintf(gcTimerFile, "GC(T+%.3fs) %s\n", secSinceStart, msg.get());
+ }
+ }
+ fflush(gcTimerFile);
+}
+
+void Statistics::beginGC(JS::GCOptions options, const TimeStamp& currentTime) {
+ slices_.clearAndFree();
+ sccTimes.clearAndFree();
+ gcOptions = options;
+ nonincrementalReason_ = GCAbortReason::None;
+
+ preTotalHeapBytes = gc->heapSize.bytes();
+
+ preCollectedHeapBytes = 0;
+
+ startingMajorGCNumber = gc->majorGCCount();
+ startingSliceNumber = gc->gcNumber();
+
+ if (gc->lastGCEndTime()) {
+ timeSinceLastGC = currentTime - gc->lastGCEndTime();
+ }
+
+ totalGCTime_ = TimeDuration();
+}
+
+void Statistics::measureInitialHeapSize() {
+ MOZ_ASSERT(preCollectedHeapBytes == 0);
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ preCollectedHeapBytes += zone->gcHeapSize.bytes();
+ }
+}
+
+void Statistics::endGC() {
+ postTotalHeapBytes = gc->heapSize.bytes();
+
+ sendGCTelemetry();
+}
+
+TimeDuration Statistics::sumTotalParallelTime(PhaseKind phaseKind) const {
+ TimeDuration total;
+ for (const SliceData& slice : slices_) {
+ total += slice.totalParallelTimes[phaseKind];
+ }
+ return total;
+}
+
+void Statistics::sendGCTelemetry() {
+ JSRuntime* runtime = gc->rt;
+ // NOTE: "Compartmental" is term that was deprecated with the
+ // introduction of zone-based GC, but the old telemetry probe
+ // continues to be used.
+ runtime->metrics().GC_IS_COMPARTMENTAL(!gc->fullGCRequested);
+ runtime->metrics().GC_ZONE_COUNT(zoneStats.zoneCount);
+ runtime->metrics().GC_ZONES_COLLECTED(zoneStats.collectedZoneCount);
+
+ TimeDuration prepareTotal = phaseTimes[Phase::PREPARE];
+ TimeDuration markTotal = SumPhase(PhaseKind::MARK, phaseTimes);
+ TimeDuration markRootsTotal = SumPhase(PhaseKind::MARK_ROOTS, phaseTimes);
+
+ // Gray and weak marking time is counted under MARK_WEAK and not MARK_GRAY.
+ TimeDuration markWeakTotal = SumPhase(PhaseKind::MARK_WEAK, phaseTimes);
+ TimeDuration markGrayNotWeak =
+ SumPhase(PhaseKind::MARK_GRAY, phaseTimes) +
+ SumPhase(PhaseKind::MARK_INCOMING_GRAY, phaseTimes);
+ TimeDuration markGrayWeak = SumPhase(PhaseKind::MARK_GRAY_WEAK, phaseTimes);
+ TimeDuration markGrayTotal = markGrayNotWeak + markGrayWeak;
+ TimeDuration markNotGrayOrWeak = markTotal - markGrayNotWeak - markWeakTotal;
+ if (markNotGrayOrWeak < TimeDuration::FromMilliseconds(0)) {
+ markNotGrayOrWeak = TimeDuration();
+ }
+
+ size_t markCount = getCount(COUNT_CELLS_MARKED);
+
+ runtime->metrics().GC_PREPARE_MS(prepareTotal);
+ runtime->metrics().GC_MARK_MS(markNotGrayOrWeak);
+ if (markTotal >= TimeDuration::FromMicroseconds(1)) {
+ double markRate = double(markCount) / t(markTotal);
+ runtime->metrics().GC_MARK_RATE_2(uint32_t(markRate));
+ }
+ runtime->metrics().GC_SWEEP_MS(phaseTimes[Phase::SWEEP]);
+ if (gc->didCompactZones()) {
+ runtime->metrics().GC_COMPACT_MS(phaseTimes[Phase::COMPACT]);
+ }
+ runtime->metrics().GC_MARK_ROOTS_US(markRootsTotal);
+ runtime->metrics().GC_MARK_GRAY_MS_2(markGrayTotal);
+ runtime->metrics().GC_MARK_WEAK_MS(markWeakTotal);
+ runtime->metrics().GC_NON_INCREMENTAL(nonincremental());
+ if (nonincremental()) {
+ runtime->metrics().GC_NON_INCREMENTAL_REASON(
+ uint32_t(nonincrementalReason_));
+ }
+
+#ifdef DEBUG
+ // Reset happens non-incrementally, so only the last slice can be reset.
+ for (size_t i = 0; i < slices_.length() - 1; i++) {
+ MOZ_ASSERT(!slices_[i].wasReset());
+ }
+#endif
+ const auto& lastSlice = slices_.back();
+ runtime->metrics().GC_RESET(lastSlice.wasReset());
+ if (lastSlice.wasReset()) {
+ runtime->metrics().GC_RESET_REASON(uint32_t(lastSlice.resetReason));
+ }
+
+ TimeDuration total, longest;
+ gcDuration(&total, &longest);
+
+ runtime->metrics().GC_MS(total);
+ runtime->metrics().GC_MAX_PAUSE_MS_2(longest);
+
+ const double mmu50 = computeMMU(TimeDuration::FromMilliseconds(50));
+ runtime->metrics().GC_MMU_50(mmu50 * 100);
+
+ // Record scheduling telemetry for the main runtime but not for workers, which
+ // are scheduled differently.
+ if (!runtime->parentRuntime && timeSinceLastGC) {
+ runtime->metrics().GC_TIME_BETWEEN_S(timeSinceLastGC);
+ if (!nonincremental()) {
+ runtime->metrics().GC_SLICE_COUNT(slices_.length());
+ }
+ }
+
+ if (!lastSlice.wasReset() && preCollectedHeapBytes != 0) {
+ size_t bytesSurvived = 0;
+ for (ZonesIter zone(runtime, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->wasCollected()) {
+ bytesSurvived += zone->gcHeapSize.retainedBytes();
+ }
+ }
+
+ MOZ_ASSERT(preCollectedHeapBytes >= bytesSurvived);
+ double survivalRate =
+ 100.0 * double(bytesSurvived) / double(preCollectedHeapBytes);
+ runtime->metrics().GC_TENURED_SURVIVAL_RATE(uint32_t(survivalRate));
+
+ // Calculate 'effectiveness' in MB / second, on main thread only for now.
+ if (!runtime->parentRuntime) {
+ size_t bytesFreed = preCollectedHeapBytes - bytesSurvived;
+ TimeDuration clampedTotal =
+ TimeDuration::Max(total, TimeDuration::FromMilliseconds(1));
+ double effectiveness =
+ (double(bytesFreed) / BYTES_PER_MB) / clampedTotal.ToSeconds();
+ runtime->metrics().GC_EFFECTIVENESS(uint32_t(effectiveness));
+ }
+ }
+
+ // Parallel marking stats.
+ if (gc->isParallelMarkingEnabled()) {
+ TimeDuration wallTime = SumPhase(PhaseKind::PARALLEL_MARK, phaseTimes);
+ TimeDuration parallelRunTime =
+ sumTotalParallelTime(PhaseKind::PARALLEL_MARK) -
+ sumTotalParallelTime(PhaseKind::PARALLEL_MARK_WAIT);
+ TimeDuration parallelMarkTime =
+ sumTotalParallelTime(PhaseKind::PARALLEL_MARK_MARK);
+ if (wallTime && parallelMarkTime) {
+ uint32_t threadCount = gc->markers.length();
+ double speedup = parallelMarkTime / wallTime;
+ double utilization = parallelRunTime / (wallTime * threadCount);
+ runtime->metrics().GC_PARALLEL_MARK_SPEEDUP(uint32_t(speedup * 100.0));
+ runtime->metrics().GC_PARALLEL_MARK_UTILIZATION(
+ std::clamp<uint32_t>(utilization * 100.0, 0, 100));
+ runtime->metrics().GC_PARALLEL_MARK_INTERRUPTIONS(
+ getCount(COUNT_PARALLEL_MARK_INTERRUPTIONS));
+ }
+ }
+}
+
+void Statistics::beginNurseryCollection(JS::GCReason reason) {
+ count(COUNT_MINOR_GC);
+ startingMinorGCNumber = gc->minorGCCount();
+ if (nurseryCollectionCallback) {
+ (*nurseryCollectionCallback)(
+ context(), JS::GCNurseryProgress::GC_NURSERY_COLLECTION_START, reason);
+ }
+}
+
+void Statistics::endNurseryCollection(JS::GCReason reason) {
+ if (nurseryCollectionCallback) {
+ (*nurseryCollectionCallback)(
+ context(), JS::GCNurseryProgress::GC_NURSERY_COLLECTION_END, reason);
+ }
+
+ tenuredAllocsSinceMinorGC = 0;
+}
+
+Statistics::SliceData::SliceData(const SliceBudget& budget,
+ Maybe<Trigger> trigger, JS::GCReason reason,
+ TimeStamp start, size_t startFaults,
+ gc::State initialState)
+ : budget(budget),
+ reason(reason),
+ trigger(trigger),
+ initialState(initialState),
+ start(start),
+ startFaults(startFaults) {}
+
+void Statistics::beginSlice(const ZoneGCStats& zoneStats, JS::GCOptions options,
+ const SliceBudget& budget, JS::GCReason reason,
+ bool budgetWasIncreased) {
+ MOZ_ASSERT(phaseStack.empty() ||
+ (phaseStack.length() == 1 && phaseStack[0] == Phase::MUTATOR));
+
+ this->zoneStats = zoneStats;
+
+ TimeStamp currentTime = TimeStamp::Now();
+
+ bool first = !gc->isIncrementalGCInProgress();
+ if (first) {
+ beginGC(options, currentTime);
+ }
+
+ JSRuntime* runtime = gc->rt;
+ if (!runtime->parentRuntime && !slices_.empty()) {
+ TimeDuration timeSinceLastSlice = currentTime - slices_.back().end;
+ runtime->metrics().GC_TIME_BETWEEN_SLICES_MS(timeSinceLastSlice);
+ }
+
+ Maybe<Trigger> trigger = recordedTrigger;
+ recordedTrigger.reset();
+
+ if (!slices_.emplaceBack(budget, trigger, reason, currentTime,
+ GetPageFaultCount(), gc->state())) {
+ // If we are OOM, set a flag to indicate we have missing slice data.
+ aborted = true;
+ return;
+ }
+
+ runtime->metrics().GC_REASON_2(uint32_t(reason));
+ runtime->metrics().GC_BUDGET_WAS_INCREASED(budgetWasIncreased);
+
+ // Slice callbacks should only fire for the outermost level.
+ if (sliceCallback) {
+ JSContext* cx = context();
+ JS::GCDescription desc(!gc->fullGCRequested, false, options, reason);
+ if (first) {
+ (*sliceCallback)(cx, JS::GC_CYCLE_BEGIN, desc);
+ }
+ (*sliceCallback)(cx, JS::GC_SLICE_BEGIN, desc);
+ }
+
+ log("begin slice");
+}
+
+void Statistics::endSlice() {
+ MOZ_ASSERT(phaseStack.empty() ||
+ (phaseStack.length() == 1 && phaseStack[0] == Phase::MUTATOR));
+
+ if (!aborted) {
+ auto& slice = slices_.back();
+ slice.end = TimeStamp::Now();
+ slice.endFaults = GetPageFaultCount();
+ slice.finalState = gc->state();
+
+ log("end slice");
+
+ sendSliceTelemetry(slice);
+
+ sliceCount_++;
+
+ totalGCTime_ += slice.end - slice.start;
+ }
+
+ bool last = !gc->isIncrementalGCInProgress();
+ if (last) {
+ if (gcTimerFile) {
+ printStats();
+ }
+
+ if (!aborted) {
+ endGC();
+ }
+ }
+
+ if (!aborted &&
+ ShouldPrintProfile(gc->rt, enableProfiling_, profileWorkers_,
+ profileThreshold_, slices_.back().duration())) {
+ printSliceProfile();
+ }
+
+ // Slice callbacks should only fire for the outermost level.
+ if (!aborted) {
+ if (sliceCallback) {
+ JSContext* cx = context();
+ JS::GCDescription desc(!gc->fullGCRequested, last, gcOptions,
+ slices_.back().reason);
+ (*sliceCallback)(cx, JS::GC_SLICE_END, desc);
+ if (last) {
+ (*sliceCallback)(cx, JS::GC_CYCLE_END, desc);
+ }
+ }
+ }
+
+ // Do this after the slice callback since it uses these values.
+ if (last) {
+ for (auto& count : counts) {
+ count = 0;
+ }
+
+ // Clear the timers at the end of a GC, preserving the data for
+ // PhaseKind::MUTATOR.
+ auto mutatorStartTime = phaseStartTimes[Phase::MUTATOR];
+ auto mutatorTime = phaseTimes[Phase::MUTATOR];
+
+ phaseStartTimes = PhaseTimeStamps();
+#ifdef DEBUG
+ phaseEndTimes = PhaseTimeStamps();
+#endif
+ phaseTimes = PhaseTimes();
+
+ phaseStartTimes[Phase::MUTATOR] = mutatorStartTime;
+ phaseTimes[Phase::MUTATOR] = mutatorTime;
+ }
+
+ aborted = false;
+}
+
+void Statistics::sendSliceTelemetry(const SliceData& slice) {
+ JSRuntime* runtime = gc->rt;
+ TimeDuration sliceTime = slice.end - slice.start;
+ runtime->metrics().GC_SLICE_MS(sliceTime);
+
+ if (slice.budget.isTimeBudget()) {
+ TimeDuration budgetDuration = slice.budget.timeBudgetDuration();
+ runtime->metrics().GC_BUDGET_MS_2(budgetDuration);
+
+ if (IsCurrentlyAnimating(runtime->lastAnimationTime, slice.end)) {
+ runtime->metrics().GC_ANIMATION_MS(sliceTime);
+ }
+
+ bool wasLongSlice = false;
+ if (sliceTime > budgetDuration) {
+ // Record how long we went over budget.
+ TimeDuration overrun = sliceTime - budgetDuration;
+ runtime->metrics().GC_BUDGET_OVERRUN(overrun);
+
+ // Long GC slices are those that go 50% or 5ms over their budget.
+ wasLongSlice = (overrun > TimeDuration::FromMilliseconds(5)) ||
+ (overrun > (budgetDuration / int64_t(2)));
+
+ // Record the longest phase in any long slice.
+ if (wasLongSlice) {
+ PhaseKind longest = LongestPhaseSelfTimeInMajorGC(slice.phaseTimes);
+ reportLongestPhaseInMajorGC(longest, [runtime](auto sample) {
+ runtime->metrics().GC_SLOW_PHASE(sample);
+ });
+
+ // If the longest phase was waiting for parallel tasks then record the
+ // longest task.
+ if (longest == PhaseKind::JOIN_PARALLEL_TASKS) {
+ PhaseKind longestParallel =
+ FindLongestPhaseKind(slice.maxParallelTimes);
+ reportLongestPhaseInMajorGC(longestParallel, [runtime](auto sample) {
+ runtime->metrics().GC_SLOW_TASK(sample);
+ });
+ }
+ }
+ }
+
+ // Record `wasLongSlice` for all TimeBudget slices.
+ runtime->metrics().GC_SLICE_WAS_LONG(wasLongSlice);
+ }
+}
+
+template <typename Fn>
+void Statistics::reportLongestPhaseInMajorGC(PhaseKind longest, Fn reportFn) {
+ if (longest != PhaseKind::NONE) {
+ uint8_t bucket = phaseKinds[longest].telemetryBucket;
+ reportFn(bucket);
+ }
+}
+
+bool Statistics::startTimingMutator() {
+ if (phaseStack.length() != 0) {
+ // Should only be called from outside of GC.
+ MOZ_ASSERT(phaseStack.length() == 1);
+ MOZ_ASSERT(phaseStack[0] == Phase::MUTATOR);
+ return false;
+ }
+
+ MOZ_ASSERT(suspendedPhases.empty());
+
+ timedGCTime = 0;
+ phaseStartTimes[Phase::MUTATOR] = TimeStamp();
+ phaseTimes[Phase::MUTATOR] = 0;
+ timedGCStart = TimeStamp();
+
+ beginPhase(PhaseKind::MUTATOR);
+ return true;
+}
+
+bool Statistics::stopTimingMutator(double& mutator_ms, double& gc_ms) {
+ // This should only be called from outside of GC, while timing the mutator.
+ if (phaseStack.length() != 1 || phaseStack[0] != Phase::MUTATOR) {
+ return false;
+ }
+
+ endPhase(PhaseKind::MUTATOR);
+ mutator_ms = t(phaseTimes[Phase::MUTATOR]);
+ gc_ms = t(timedGCTime);
+
+ return true;
+}
+
+void Statistics::suspendPhases(PhaseKind suspension) {
+ MOZ_ASSERT(suspension == PhaseKind::EXPLICIT_SUSPENSION ||
+ suspension == PhaseKind::IMPLICIT_SUSPENSION);
+ while (!phaseStack.empty()) {
+ MOZ_ASSERT(suspendedPhases.length() < MAX_SUSPENDED_PHASES);
+ Phase parent = phaseStack.back();
+ suspendedPhases.infallibleAppend(parent);
+ recordPhaseEnd(parent);
+ }
+ suspendedPhases.infallibleAppend(lookupChildPhase(suspension));
+}
+
+void Statistics::resumePhases() {
+ MOZ_ASSERT(suspendedPhases.back() == Phase::EXPLICIT_SUSPENSION ||
+ suspendedPhases.back() == Phase::IMPLICIT_SUSPENSION);
+ suspendedPhases.popBack();
+
+ while (!suspendedPhases.empty() &&
+ suspendedPhases.back() != Phase::EXPLICIT_SUSPENSION &&
+ suspendedPhases.back() != Phase::IMPLICIT_SUSPENSION) {
+ Phase resumePhase = suspendedPhases.popCopy();
+ if (resumePhase == Phase::MUTATOR) {
+ timedGCTime += TimeStamp::Now() - timedGCStart;
+ }
+ recordPhaseBegin(resumePhase);
+ }
+}
+
+void Statistics::beginPhase(PhaseKind phaseKind) {
+ // No longer timing these phases. We should never see these.
+ MOZ_ASSERT(phaseKind != PhaseKind::GC_BEGIN &&
+ phaseKind != PhaseKind::GC_END);
+
+ // PhaseKind::MUTATOR is suspended while performing GC.
+ if (currentPhase() == Phase::MUTATOR) {
+ suspendPhases(PhaseKind::IMPLICIT_SUSPENSION);
+ }
+
+ recordPhaseBegin(lookupChildPhase(phaseKind));
+}
+
+void Statistics::recordPhaseBegin(Phase phase) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt));
+
+ // Guard against any other re-entry.
+ MOZ_ASSERT(!phaseStartTimes[phase]);
+
+ MOZ_ASSERT(phaseStack.length() < MAX_PHASE_NESTING);
+
+ Phase current = currentPhase();
+ MOZ_ASSERT(phases[phase].parent == current);
+
+ TimeStamp now = TimeStamp::Now();
+
+ if (current != Phase::NONE) {
+ MOZ_ASSERT(now >= phaseStartTimes[currentPhase()],
+ "Inconsistent time data; see bug 1400153");
+ if (now < phaseStartTimes[currentPhase()]) {
+ now = phaseStartTimes[currentPhase()];
+ aborted = true;
+ }
+ }
+
+ phaseStack.infallibleAppend(phase);
+ phaseStartTimes[phase] = now;
+ log("begin: %s", phases[phase].path);
+}
+
+void Statistics::recordPhaseEnd(Phase phase) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt));
+
+ MOZ_ASSERT(phaseStartTimes[phase]);
+
+ TimeStamp now = TimeStamp::Now();
+
+ // Make sure this phase ends after it starts.
+ MOZ_ASSERT(now >= phaseStartTimes[phase],
+ "Inconsistent time data; see bug 1400153");
+
+#ifdef DEBUG
+ // Make sure this phase ends after all of its children. Note that some
+ // children might not have run in this instance, in which case they will
+ // have run in a previous instance of this parent or not at all.
+ for (Phase kid = phases[phase].firstChild; kid != Phase::NONE;
+ kid = phases[kid].nextSibling) {
+ if (phaseEndTimes[kid].IsNull()) {
+ continue;
+ }
+ if (phaseEndTimes[kid] > now) {
+ fprintf(stderr,
+ "Parent %s ended at %.3fms, before child %s ended at %.3fms?\n",
+ phases[phase].name, t(now - TimeStamp::FirstTimeStamp()),
+ phases[kid].name,
+ t(phaseEndTimes[kid] - TimeStamp::FirstTimeStamp()));
+ }
+ MOZ_ASSERT(phaseEndTimes[kid] <= now,
+ "Inconsistent time data; see bug 1400153");
+ }
+#endif
+
+ if (now < phaseStartTimes[phase]) {
+ now = phaseStartTimes[phase];
+ aborted = true;
+ }
+
+ if (phase == Phase::MUTATOR) {
+ timedGCStart = now;
+ }
+
+ phaseStack.popBack();
+
+ TimeDuration t = now - phaseStartTimes[phase];
+ if (!slices_.empty()) {
+ slices_.back().phaseTimes[phase] += t;
+ }
+ phaseTimes[phase] += t;
+ phaseStartTimes[phase] = TimeStamp();
+
+#ifdef DEBUG
+ phaseEndTimes[phase] = now;
+ log("end: %s", phases[phase].path);
+#endif
+}
+
+void Statistics::endPhase(PhaseKind phaseKind) {
+ Phase phase = currentPhase();
+ MOZ_ASSERT(phase != Phase::NONE);
+ MOZ_ASSERT(phases[phase].phaseKind == phaseKind);
+
+ recordPhaseEnd(phase);
+
+ // When emptying the stack, we may need to return to timing the mutator
+ // (PhaseKind::MUTATOR).
+ if (phaseStack.empty() && !suspendedPhases.empty() &&
+ suspendedPhases.back() == Phase::IMPLICIT_SUSPENSION) {
+ resumePhases();
+ }
+}
+
+void Statistics::recordParallelPhase(PhaseKind phaseKind,
+ TimeDuration duration) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(gc->rt));
+
+ if (aborted) {
+ return;
+ }
+
+ slices_.back().totalParallelTimes[phaseKind] += duration;
+
+ // Also record the maximum task time for each phase. Don't record times for
+ // parent phases.
+ TimeDuration& maxTime = slices_.back().maxParallelTimes[phaseKind];
+ maxTime = std::max(maxTime, duration);
+}
+
+TimeStamp Statistics::beginSCC() { return TimeStamp::Now(); }
+
+void Statistics::endSCC(unsigned scc, TimeStamp start) {
+ if (scc >= sccTimes.length() && !sccTimes.resize(scc + 1)) {
+ return;
+ }
+
+ sccTimes[scc] += TimeStamp::Now() - start;
+}
+
+/*
+ * MMU (minimum mutator utilization) is a measure of how much garbage collection
+ * is affecting the responsiveness of the system. MMU measurements are given
+ * with respect to a certain window size. If we report MMU(50ms) = 80%, then
+ * that means that, for any 50ms window of time, at least 80% of the window is
+ * devoted to the mutator. In other words, the GC is running for at most 20% of
+ * the window, or 10ms. The GC can run multiple slices during the 50ms window
+ * as long as the total time it spends is at most 10ms.
+ */
+double Statistics::computeMMU(TimeDuration window) const {
+ MOZ_ASSERT(!slices_.empty());
+
+ TimeDuration gc = slices_[0].end - slices_[0].start;
+ TimeDuration gcMax = gc;
+
+ if (gc >= window) {
+ return 0.0;
+ }
+
+ int startIndex = 0;
+ for (size_t endIndex = 1; endIndex < slices_.length(); endIndex++) {
+ auto* startSlice = &slices_[startIndex];
+ auto& endSlice = slices_[endIndex];
+ gc += endSlice.end - endSlice.start;
+
+ while (endSlice.end - startSlice->end >= window) {
+ gc -= startSlice->end - startSlice->start;
+ startSlice = &slices_[++startIndex];
+ }
+
+ TimeDuration cur = gc;
+ if (endSlice.end - startSlice->start > window) {
+ cur -= (endSlice.end - startSlice->start - window);
+ }
+ if (cur > gcMax) {
+ gcMax = cur;
+ }
+ }
+
+ return double((window - gcMax) / window);
+}
+
+void Statistics::maybePrintProfileHeaders() {
+ static int printedHeader = 0;
+ if ((printedHeader++ % 200) == 0) {
+ printProfileHeader();
+ if (gc->nursery().enableProfiling()) {
+ gc->nursery().printProfileHeader();
+ }
+ }
+}
+
+// The following macros define GC profile metadata fields that are printed
+// before the timing information defined by FOR_EACH_GC_PROFILE_TIME.
+
+#define FOR_EACH_GC_PROFILE_COMMON_METADATA(_) \
+ _("PID", 7, "%7zu", pid) \
+ _("Runtime", 14, "0x%12p", runtime)
+
+#define FOR_EACH_GC_PROFILE_SLICE_METADATA(_) \
+ _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \
+ _("Reason", 20, "%-20.20s", reason) \
+ _("States", 6, "%6s", formatGCStates(slice)) \
+ _("FSNR", 4, "%4s", formatGCFlags(slice)) \
+ _("SizeKB", 8, "%8zu", sizeKB) \
+ _("Budget", 6, "%6s", formatBudget(slice))
+
+#define FOR_EACH_GC_PROFILE_METADATA(_) \
+ FOR_EACH_GC_PROFILE_COMMON_METADATA(_) \
+ FOR_EACH_GC_PROFILE_SLICE_METADATA(_)
+
+void Statistics::printProfileHeader() {
+ if (!enableProfiling_) {
+ return;
+ }
+
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) {
+ return;
+ }
+
+#define PRINT_METADATA_NAME(name, width, _1, _2) \
+ if (!sprinter.jsprintf(" %-*s", width, name)) { \
+ return; \
+ }
+ FOR_EACH_GC_PROFILE_METADATA(PRINT_METADATA_NAME)
+#undef PRINT_METADATA_NAME
+
+#define PRINT_PROFILE_NAME(_1, text, _2) \
+ if (!sprinter.jsprintf(" %-6.6s", text)) { \
+ return; \
+ }
+ FOR_EACH_GC_PROFILE_TIME(PRINT_PROFILE_NAME)
+#undef PRINT_PROFILE_NAME
+
+ if (!sprinter.put("\n")) {
+ return;
+ }
+
+ fputs(sprinter.string(), profileFile());
+}
+
+static TimeDuration SumAllPhaseKinds(const Statistics::PhaseKindTimes& times) {
+ TimeDuration sum;
+ for (PhaseKind kind : AllPhaseKinds()) {
+ sum += times[kind];
+ }
+ return sum;
+}
+
+void Statistics::printSliceProfile() {
+ maybePrintProfileHeaders();
+
+ const SliceData& slice = slices_.back();
+ ProfileDurations times = getProfileTimes(slice);
+ updateTotalProfileTimes(times);
+
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) {
+ return;
+ }
+
+ size_t pid = getpid();
+ JSRuntime* runtime = gc->rt;
+ TimeDuration timestamp = slice.end - creationTime();
+ const char* reason = ExplainGCReason(slice.reason);
+ size_t sizeKB = gc->heapSize.bytes() / 1024;
+
+#define PRINT_FIELD_VALUE(_1, _2, format, value) \
+ if (!sprinter.jsprintf(" " format, value)) { \
+ return; \
+ }
+ FOR_EACH_GC_PROFILE_METADATA(PRINT_FIELD_VALUE)
+#undef PRINT_FIELD_VALUE
+
+ if (!printProfileTimes(times, sprinter)) {
+ return;
+ }
+
+ fputs(sprinter.string(), profileFile());
+}
+
+Statistics::ProfileDurations Statistics::getProfileTimes(
+ const SliceData& slice) const {
+ ProfileDurations times;
+
+ times[ProfileKey::Total] = slice.duration();
+ times[ProfileKey::Background] = SumAllPhaseKinds(slice.totalParallelTimes);
+
+#define GET_PROFILE_TIME(name, text, phase) \
+ if (phase != PhaseKind::NONE) { \
+ times[ProfileKey::name] = SumPhase(phase, slice.phaseTimes); \
+ }
+ FOR_EACH_GC_PROFILE_TIME(GET_PROFILE_TIME)
+#undef GET_PROFILE_TIME
+
+ return times;
+}
+
+void Statistics::updateTotalProfileTimes(const ProfileDurations& times) {
+#define UPDATE_PROFILE_TIME(name, _, phase) \
+ totalTimes_[ProfileKey::name] += times[ProfileKey::name];
+ FOR_EACH_GC_PROFILE_TIME(UPDATE_PROFILE_TIME)
+#undef UPDATE_PROFILE_TIME
+}
+
+const char* Statistics::formatGCStates(const SliceData& slice) {
+ DebugOnly<int> r =
+ SprintfLiteral(formatBuffer_, "%1d -> %1d", int(slice.initialState),
+ int(slice.finalState));
+ MOZ_ASSERT(r > 0 && r < FormatBufferLength);
+ return formatBuffer_;
+}
+
+const char* Statistics::formatGCFlags(const SliceData& slice) {
+ bool fullGC = gc->fullGCRequested;
+ bool shrinkingGC = gcOptions == JS::GCOptions::Shrink;
+ bool nonIncrementalGC = nonincrementalReason_ != GCAbortReason::None;
+ bool wasReset = slice.resetReason != GCAbortReason::None;
+
+ MOZ_ASSERT(FormatBufferLength >= 5);
+ formatBuffer_[0] = fullGC ? 'F' : ' ';
+ formatBuffer_[1] = shrinkingGC ? 'S' : ' ';
+ formatBuffer_[2] = nonIncrementalGC ? 'N' : ' ';
+ formatBuffer_[3] = wasReset ? 'R' : ' ';
+ formatBuffer_[4] = '\0';
+
+ return formatBuffer_;
+}
+
+const char* Statistics::formatBudget(const SliceData& slice) {
+ if (nonincrementalReason_ != GCAbortReason::None ||
+ !slice.budget.isTimeBudget()) {
+ formatBuffer_[0] = '\0';
+ return formatBuffer_;
+ }
+
+ DebugOnly<int> r =
+ SprintfLiteral(formatBuffer_, " %6" PRIi64, slice.budget.timeBudget());
+ MOZ_ASSERT(r > 0 && r < FormatBufferLength);
+ return formatBuffer_;
+}
+
+/* static */
+bool Statistics::printProfileTimes(const ProfileDurations& times,
+ Sprinter& sprinter) {
+ for (auto time : times) {
+ int64_t millis = int64_t(time.ToMilliseconds());
+ if (!sprinter.jsprintf(" %6" PRIi64, millis)) {
+ return false;
+ }
+ }
+
+ return sprinter.put("\n");
+}
+
+constexpr size_t SliceMetadataFormatWidth() {
+ size_t fieldCount = 0;
+ size_t totalWidth = 0;
+
+#define UPDATE_COUNT_AND_WIDTH(_1, width, _2, _3) \
+ fieldCount++; \
+ totalWidth += width;
+ FOR_EACH_GC_PROFILE_SLICE_METADATA(UPDATE_COUNT_AND_WIDTH)
+#undef UPDATE_COUNT_AND_WIDTH
+
+ // Add padding between fields.
+ totalWidth += fieldCount - 1;
+
+ return totalWidth;
+}
+
+void Statistics::printTotalProfileTimes() {
+ if (!enableProfiling_) {
+ return;
+ }
+
+ Sprinter sprinter;
+ if (!sprinter.init() || !sprinter.put(MajorGCProfilePrefix)) {
+ return;
+ }
+
+ size_t pid = getpid();
+ JSRuntime* runtime = gc->rt;
+
+#define PRINT_FIELD_VALUE(_1, _2, format, value) \
+ if (!sprinter.jsprintf(" " format, value)) { \
+ return; \
+ }
+ FOR_EACH_GC_PROFILE_COMMON_METADATA(PRINT_FIELD_VALUE)
+#undef PRINT_FIELD_VALUE
+
+ // Use whole width of per-slice metadata to print total slices so the profile
+ // totals that follow line up.
+ size_t width = SliceMetadataFormatWidth();
+ if (!sprinter.jsprintf(" %-*s", int(width), formatTotalSlices())) {
+ return;
+ }
+
+ if (!printProfileTimes(totalTimes_, sprinter)) {
+ return;
+ }
+
+ fputs(sprinter.string(), profileFile());
+}
+
+const char* Statistics::formatTotalSlices() {
+ DebugOnly<int> r = SprintfLiteral(
+ formatBuffer_, "TOTALS: %7" PRIu64 " slices:", sliceCount_);
+ MOZ_ASSERT(r > 0 && r < FormatBufferLength);
+ return formatBuffer_;
+}
diff --git a/js/src/gc/Statistics.h b/js/src/gc/Statistics.h
new file mode 100644
index 0000000000..0e45cd651a
--- /dev/null
+++ b/js/src/gc/Statistics.h
@@ -0,0 +1,606 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Statistics_h
+#define gc_Statistics_h
+
+#include "mozilla/Array.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TimeStamp.h"
+
+#include "jspubtd.h"
+#include "NamespaceImports.h"
+
+#include "gc/GCEnum.h"
+#include "js/AllocPolicy.h"
+#include "js/GCAPI.h"
+#include "js/SliceBudget.h"
+#include "js/Vector.h"
+
+namespace js {
+
+class JS_PUBLIC_API Sprinter;
+class JSONPrinter;
+
+namespace gcstats {
+
+// Phase data is generated by a script. If you need to add phases, edit
+// js/src/gc/GenerateStatsPhases.py
+
+#include "gc/StatsPhasesGenerated.h"
+
+// Counts can be incremented with Statistics::count(). They're reset at the end
+// of a Major GC.
+enum Count {
+ COUNT_NEW_CHUNK,
+ COUNT_DESTROY_CHUNK,
+ COUNT_MINOR_GC,
+
+ // Number of times a 'put' into a storebuffer overflowed, triggering a
+ // compaction
+ COUNT_STOREBUFFER_OVERFLOW,
+
+ // Number of arenas relocated by compacting GC.
+ COUNT_ARENA_RELOCATED,
+
+ // Number of cells marked during the marking phase. Excludes atoms marked when
+ // not collecting the atoms zone.
+ COUNT_CELLS_MARKED,
+
+ // Number of times work was donated to a requesting thread during parallel
+ // marking.
+ COUNT_PARALLEL_MARK_INTERRUPTIONS,
+
+ COUNT_LIMIT
+};
+
+// Stats can be set with Statistics::setStat(). They're not reset automatically.
+enum Stat {
+ // Number of strings tenured.
+ STAT_STRINGS_TENURED,
+
+ // Number of strings deduplicated.
+ STAT_STRINGS_DEDUPLICATED,
+
+ // Number of BigInts tenured.
+ STAT_BIGINTS_TENURED,
+
+ STAT_LIMIT
+};
+
+struct ZoneGCStats {
+ /* Number of zones collected in this GC. */
+ int collectedZoneCount = 0;
+
+ /* Total number of zones in the Runtime at the start of this GC. */
+ int zoneCount = 0;
+
+ /* Number of zones swept in this GC. */
+ int sweptZoneCount = 0;
+
+ /* Total number of compartments in all zones collected. */
+ int collectedCompartmentCount = 0;
+
+ /* Total number of compartments in the Runtime at the start of this GC. */
+ int compartmentCount = 0;
+
+ /* Total number of compartments swept by this GC. */
+ int sweptCompartmentCount = 0;
+
+ ZoneGCStats() = default;
+};
+
+struct Trigger {
+ size_t amount = 0;
+ size_t threshold = 0;
+};
+
+#define FOR_EACH_GC_PROFILE_TIME(_) \
+ _(Total, "total", PhaseKind::NONE) \
+ _(Background, "bgwrk", PhaseKind::NONE) \
+ _(BeginCallback, "bgnCB", PhaseKind::GC_BEGIN) \
+ _(MinorForMajor, "evct4m", PhaseKind::EVICT_NURSERY_FOR_MAJOR_GC) \
+ _(WaitBgThread, "waitBG", PhaseKind::WAIT_BACKGROUND_THREAD) \
+ _(Prepare, "prep", PhaseKind::PREPARE) \
+ _(Mark, "mark", PhaseKind::MARK) \
+ _(Sweep, "sweep", PhaseKind::SWEEP) \
+ _(Compact, "cmpct", PhaseKind::COMPACT) \
+ _(EndCallback, "endCB", PhaseKind::GC_END) \
+ _(MinorGC, "minor", PhaseKind::MINOR_GC) \
+ _(EvictNursery, "evict", PhaseKind::EVICT_NURSERY)
+
+static const char* const MajorGCProfilePrefix = "MajorGC:";
+static const char* const MinorGCProfilePrefix = "MinorGC:";
+
+const char* ExplainAbortReason(GCAbortReason reason);
+
+/*
+ * Struct for collecting timing statistics on a "phase tree". The tree is
+ * specified as a limited DAG, but the timings are collected for the whole tree
+ * that you would get by expanding out the DAG by duplicating subtrees rooted
+ * at nodes with multiple parents.
+ *
+ * During execution, a child phase can be activated multiple times, and the
+ * total time will be accumulated. (So for example, you can start and end
+ * PhaseKind::MARK_ROOTS multiple times before completing the parent phase.)
+ *
+ * Incremental GC is represented by recording separate timing results for each
+ * slice within the overall GC.
+ */
+struct Statistics {
+ template <typename T, size_t Length>
+ using Array = mozilla::Array<T, Length>;
+
+ template <typename IndexType, IndexType SizeAsEnumValue, typename ValueType>
+ using EnumeratedArray =
+ mozilla::EnumeratedArray<IndexType, SizeAsEnumValue, ValueType>;
+
+ using TimeDuration = mozilla::TimeDuration;
+ using TimeStamp = mozilla::TimeStamp;
+
+ // Create types for tables of times, by phase and phase kind.
+ using PhaseTimes = EnumeratedArray<Phase, Phase::LIMIT, TimeDuration>;
+ using PhaseKindTimes =
+ EnumeratedArray<PhaseKind, PhaseKind::LIMIT, TimeDuration>;
+
+ using PhaseTimeStamps = EnumeratedArray<Phase, Phase::LIMIT, TimeStamp>;
+
+ [[nodiscard]] static bool initialize();
+
+ explicit Statistics(gc::GCRuntime* gc);
+ ~Statistics();
+
+ Statistics(const Statistics&) = delete;
+ Statistics& operator=(const Statistics&) = delete;
+
+ void beginPhase(PhaseKind phaseKind);
+ void endPhase(PhaseKind phaseKind);
+ void recordParallelPhase(PhaseKind phaseKind, TimeDuration duration);
+
+ // Occasionally, we may be in the middle of something that is tracked by
+ // this class, and we need to do something unusual (eg evict the nursery)
+ // that doesn't normally nest within the current phase. Suspend the
+ // currently tracked phase stack, at which time the caller is free to do
+ // other tracked operations.
+ //
+ // This also happens internally with the PhaseKind::MUTATOR "phase". While in
+ // this phase, any beginPhase will automatically suspend the non-GC phase,
+ // until that inner stack is complete, at which time it will automatically
+ // resume the non-GC phase. Explicit suspensions do not get auto-resumed.
+ void suspendPhases(PhaseKind suspension = PhaseKind::EXPLICIT_SUSPENSION);
+
+ // Resume a suspended stack of phases.
+ void resumePhases();
+
+ void beginSlice(const ZoneGCStats& zoneStats, JS::GCOptions options,
+ const SliceBudget& budget, JS::GCReason reason,
+ bool budgetWasIncreased);
+ void endSlice();
+
+ [[nodiscard]] bool startTimingMutator();
+ [[nodiscard]] bool stopTimingMutator(double& mutator_ms, double& gc_ms);
+
+ // Note when we sweep a zone or compartment.
+ void sweptZone() { ++zoneStats.sweptZoneCount; }
+ void sweptCompartment() { ++zoneStats.sweptCompartmentCount; }
+
+ void reset(GCAbortReason reason) {
+ MOZ_ASSERT(reason != GCAbortReason::None);
+ if (!aborted) {
+ slices_.back().resetReason = reason;
+ }
+ }
+
+ void measureInitialHeapSize();
+
+ void nonincremental(GCAbortReason reason) {
+ MOZ_ASSERT(reason != GCAbortReason::None);
+ nonincrementalReason_ = reason;
+ log("Non-incremental reason: %s", nonincrementalReason());
+ }
+
+ bool nonincremental() const {
+ return nonincrementalReason_ != GCAbortReason::None;
+ }
+
+ const char* nonincrementalReason() const {
+ return ExplainAbortReason(nonincrementalReason_);
+ }
+
+ void count(Count s) { counts[s]++; }
+ void addCount(Count s, uint32_t count) { counts[s] += count; }
+
+ uint32_t getCount(Count s) const { return uint32_t(counts[s]); }
+
+ void setStat(Stat s, uint32_t value) { stats[s] = value; }
+
+ uint32_t getStat(Stat s) const { return stats[s]; }
+
+ void recordTrigger(size_t amount, size_t threshold) {
+ recordedTrigger = mozilla::Some(Trigger{amount, threshold});
+ }
+ bool hasTrigger() const { return recordedTrigger.isSome(); }
+
+ // tenured allocs don't include nursery evictions.
+ void setAllocsSinceMinorGCTenured(uint32_t allocs) {
+ tenuredAllocsSinceMinorGC = allocs;
+ }
+
+ uint32_t allocsSinceMinorGCTenured() { return tenuredAllocsSinceMinorGC; }
+
+ void beginNurseryCollection(JS::GCReason reason);
+ void endNurseryCollection(JS::GCReason reason);
+
+ TimeStamp beginSCC();
+ void endSCC(unsigned scc, TimeStamp start);
+
+ UniqueChars formatCompactSliceMessage() const;
+ UniqueChars formatCompactSummaryMessage() const;
+ UniqueChars formatDetailedMessage() const;
+
+ JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback);
+ JS::GCNurseryCollectionCallback setNurseryCollectionCallback(
+ JS::GCNurseryCollectionCallback callback);
+
+ TimeDuration clearMaxGCPauseAccumulator();
+ TimeDuration getMaxGCPauseSinceClear();
+
+ PhaseKind currentPhaseKind() const;
+
+ static const size_t MAX_SUSPENDED_PHASES = MAX_PHASE_NESTING * 3;
+
+ struct SliceData {
+ SliceData(const SliceBudget& budget, mozilla::Maybe<Trigger> trigger,
+ JS::GCReason reason, TimeStamp start, size_t startFaults,
+ gc::State initialState);
+
+ SliceBudget budget;
+ JS::GCReason reason = JS::GCReason::NO_REASON;
+ mozilla::Maybe<Trigger> trigger;
+ gc::State initialState = gc::State::NotActive;
+ gc::State finalState = gc::State::NotActive;
+ GCAbortReason resetReason = GCAbortReason::None;
+ TimeStamp start;
+ TimeStamp end;
+ size_t startFaults = 0;
+ size_t endFaults = 0;
+ PhaseTimes phaseTimes;
+ PhaseKindTimes totalParallelTimes;
+ PhaseKindTimes maxParallelTimes;
+
+ TimeDuration duration() const { return end - start; }
+ bool wasReset() const { return resetReason != GCAbortReason::None; }
+ };
+
+ typedef Vector<SliceData, 8, SystemAllocPolicy> SliceDataVector;
+
+ const SliceDataVector& slices() const { return slices_; }
+
+ const SliceData* lastSlice() const {
+ if (slices_.length() == 0) {
+ return nullptr;
+ }
+
+ return &slices_.back();
+ }
+
+ TimeStamp start() const { return slices_[0].start; }
+
+ TimeStamp end() const { return slices_.back().end; }
+
+ TimeStamp creationTime() const { return creationTime_; }
+
+ TimeDuration totalGCTime() const { return totalGCTime_; }
+ size_t initialCollectedBytes() const { return preCollectedHeapBytes; }
+
+ // File to write profiling information to, either stderr or file specified
+ // with JS_GC_PROFILE_FILE.
+ FILE* profileFile() const { return gcProfileFile; }
+
+ // Occasionally print header lines for profiling information.
+ void maybePrintProfileHeaders();
+
+ // Print header line for profile times.
+ void printProfileHeader();
+
+ // Print total profile times on shutdown.
+ void printTotalProfileTimes();
+
+ // These JSON strings are used by the firefox profiler to display the GC
+ // markers.
+
+ // Return JSON for a whole major GC
+ UniqueChars renderJsonMessage() const;
+
+ // Return JSON for the timings of just the given slice.
+ UniqueChars renderJsonSlice(size_t sliceNum) const;
+
+ // Return JSON for the previous nursery collection.
+ UniqueChars renderNurseryJson() const;
+
+#ifdef DEBUG
+ // Print a logging message.
+ void log(const char* fmt, ...);
+#else
+ void log(const char* fmt, ...){};
+#endif
+
+ private:
+ gc::GCRuntime* const gc;
+
+ /* File used for MOZ_GCTIMER output. */
+ FILE* gcTimerFile;
+
+ /* File used for JS_GC_DEBUG output. */
+ FILE* gcDebugFile;
+
+ /* File used for JS_GC_PROFILE output. */
+ FILE* gcProfileFile;
+
+ ZoneGCStats zoneStats;
+
+ JS::GCOptions gcOptions;
+
+ GCAbortReason nonincrementalReason_;
+
+ SliceDataVector slices_;
+
+ /* Most recent time when the given phase started. */
+ PhaseTimeStamps phaseStartTimes;
+
+#ifdef DEBUG
+ /* Most recent time when the given phase ended. */
+ PhaseTimeStamps phaseEndTimes;
+#endif
+
+ TimeStamp creationTime_;
+
+ /* Bookkeeping for GC timings when timingMutator is true */
+ TimeStamp timedGCStart;
+ TimeDuration timedGCTime;
+
+ /* Total main thread time in a given phase for this GC. */
+ PhaseTimes phaseTimes;
+
+ /* Total main thread time for this GC. */
+ TimeDuration totalGCTime_;
+
+ /* Number of events of this type for this GC. */
+ EnumeratedArray<Count, COUNT_LIMIT,
+ mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire>>
+ counts;
+
+ /* Other GC statistics. */
+ EnumeratedArray<Stat, STAT_LIMIT, uint32_t> stats;
+
+ /*
+ * These events cannot be kept in the above array, we need to take their
+ * address.
+ */
+ uint32_t tenuredAllocsSinceMinorGC;
+
+ /* Total GC heap size before and after the GC ran. */
+ size_t preTotalHeapBytes;
+ size_t postTotalHeapBytes;
+
+ /* GC heap size for collected zones before GC ran. */
+ size_t preCollectedHeapBytes;
+
+ /*
+ * If a GC slice was triggered by exceeding some threshold, record the
+ * threshold and the value that exceeded it. This happens before the slice
+ * starts so this is recorded here first and then transferred to SliceData.
+ */
+ mozilla::Maybe<Trigger> recordedTrigger;
+
+ /* GC numbers as of the beginning of the collection. */
+ uint64_t startingMinorGCNumber;
+ uint64_t startingMajorGCNumber;
+ uint64_t startingSliceNumber;
+
+ /* Records the maximum GC pause in an API-controlled interval. */
+ mutable TimeDuration maxPauseInInterval;
+
+ /* Phases that are currently on stack. */
+ Vector<Phase, MAX_PHASE_NESTING, SystemAllocPolicy> phaseStack;
+
+ /*
+ * Certain phases can interrupt the phase stack, eg callback phases. When
+ * this happens, we move the suspended phases over to a sepearate list,
+ * terminated by a dummy PhaseKind::SUSPENSION phase (so that we can nest
+ * suspensions by suspending multiple stacks with a PhaseKind::SUSPENSION in
+ * between).
+ */
+ Vector<Phase, MAX_SUSPENDED_PHASES, SystemAllocPolicy> suspendedPhases;
+
+ /* Sweep times for SCCs of compartments. */
+ Vector<TimeDuration, 0, SystemAllocPolicy> sccTimes;
+
+ TimeDuration timeSinceLastGC;
+
+ JS::GCSliceCallback sliceCallback;
+ JS::GCNurseryCollectionCallback nurseryCollectionCallback;
+
+ /*
+ * True if we saw an OOM while allocating slices or we saw an impossible
+ * timestamp. The statistics for this GC will be invalid.
+ */
+ bool aborted;
+
+ /* Profiling data. */
+
+ enum class ProfileKey {
+#define DEFINE_PROFILE_KEY(name, _1, _2) name,
+ FOR_EACH_GC_PROFILE_TIME(DEFINE_PROFILE_KEY)
+#undef DEFINE_PROFILE_KEY
+ KeyCount
+ };
+
+ using ProfileDurations =
+ EnumeratedArray<ProfileKey, ProfileKey::KeyCount, TimeDuration>;
+
+ bool enableProfiling_;
+ bool profileWorkers_;
+ TimeDuration profileThreshold_;
+ ProfileDurations totalTimes_;
+ uint64_t sliceCount_;
+
+ char formatBuffer_[32];
+ static constexpr int FormatBufferLength = sizeof(formatBuffer_);
+
+ JSContext* context();
+
+ Phase currentPhase() const;
+ Phase lookupChildPhase(PhaseKind phaseKind) const;
+
+ void beginGC(JS::GCOptions options, const TimeStamp& currentTime);
+ void endGC();
+
+ void sendGCTelemetry();
+ void sendSliceTelemetry(const SliceData& slice);
+
+ TimeDuration sumTotalParallelTime(PhaseKind phaseKind) const;
+
+ void recordPhaseBegin(Phase phase);
+ void recordPhaseEnd(Phase phase);
+
+ void gcDuration(TimeDuration* total, TimeDuration* maxPause) const;
+ void sccDurations(TimeDuration* total, TimeDuration* maxPause) const;
+ void printStats();
+
+ template <typename Fn>
+ void reportLongestPhaseInMajorGC(PhaseKind longest, Fn reportFn);
+
+ UniqueChars formatCompactSlicePhaseTimes(const PhaseTimes& phaseTimes) const;
+
+ UniqueChars formatDetailedDescription() const;
+ UniqueChars formatDetailedSliceDescription(unsigned i,
+ const SliceData& slice) const;
+ UniqueChars formatDetailedPhaseTimes(const PhaseTimes& phaseTimes) const;
+ UniqueChars formatDetailedTotals() const;
+
+ void formatJsonDescription(JSONPrinter&) const;
+ void formatJsonSliceDescription(unsigned i, const SliceData& slice,
+ JSONPrinter&) const;
+ void formatJsonPhaseTimes(const PhaseTimes& phaseTimes, JSONPrinter&) const;
+ void formatJsonSlice(size_t sliceNum, JSONPrinter&) const;
+
+ double computeMMU(TimeDuration resolution) const;
+
+ void printSliceProfile();
+ ProfileDurations getProfileTimes(const SliceData& slice) const;
+ void updateTotalProfileTimes(const ProfileDurations& times);
+ const char* formatGCStates(const SliceData& slice);
+ const char* formatGCFlags(const SliceData& slice);
+ const char* formatBudget(const SliceData& slice);
+ const char* formatTotalSlices();
+ static bool printProfileTimes(const ProfileDurations& times,
+ Sprinter& sprinter);
+};
+
+struct MOZ_RAII AutoGCSlice {
+ AutoGCSlice(Statistics& stats, const ZoneGCStats& zoneStats,
+ JS::GCOptions options, const SliceBudget& budget,
+ JS::GCReason reason, bool budgetWasIncreased)
+ : stats(stats) {
+ stats.beginSlice(zoneStats, options, budget, reason, budgetWasIncreased);
+ }
+ ~AutoGCSlice() { stats.endSlice(); }
+
+ Statistics& stats;
+};
+
+struct MOZ_RAII AutoPhase {
+ AutoPhase(Statistics& stats, PhaseKind phaseKind)
+ : stats(stats), phaseKind(phaseKind), enabled(true) {
+ stats.beginPhase(phaseKind);
+ }
+
+ AutoPhase(Statistics& stats, bool condition, PhaseKind phaseKind)
+ : stats(stats), phaseKind(phaseKind), enabled(condition) {
+ if (enabled) {
+ stats.beginPhase(phaseKind);
+ }
+ }
+
+ ~AutoPhase() {
+ if (enabled) {
+ stats.endPhase(phaseKind);
+ }
+ }
+
+ Statistics& stats;
+ PhaseKind phaseKind;
+ bool enabled;
+};
+
+struct MOZ_RAII AutoSCC {
+ AutoSCC(Statistics& stats, unsigned scc) : stats(stats), scc(scc) {
+ start = stats.beginSCC();
+ }
+ ~AutoSCC() { stats.endSCC(scc, start); }
+
+ Statistics& stats;
+ unsigned scc;
+ mozilla::TimeStamp start;
+};
+
+void ReadProfileEnv(const char* envName, const char* helpText, bool* enableOut,
+ bool* workersOut, mozilla::TimeDuration* thresholdOut);
+
+} /* namespace gcstats */
+
+struct StringStats {
+ // number of strings that were deduplicated, and their sizes in characters
+ // and bytes
+ uint64_t deduplicatedStrings = 0;
+ uint64_t deduplicatedChars = 0;
+ uint64_t deduplicatedBytes = 0;
+
+ // number of live nursery strings at the start of a nursery collection
+ uint64_t liveNurseryStrings = 0;
+
+ // number of new strings added to the tenured heap
+ uint64_t tenuredStrings = 0;
+
+ // Currently, liveNurseryStrings = tenuredStrings + deduplicatedStrings (but
+ // in the future we may do more transformation during tenuring, eg
+ // atomizing.)
+
+ // number of malloced bytes associated with tenured strings (the actual
+ // malloc will have happened when the strings were allocated in the nursery;
+ // the ownership of the bytes will be transferred to the tenured strings)
+ uint64_t tenuredBytes = 0;
+
+ StringStats& operator+=(const StringStats& other) {
+ deduplicatedStrings += other.deduplicatedStrings;
+ deduplicatedChars += other.deduplicatedChars;
+ deduplicatedBytes += other.deduplicatedBytes;
+ liveNurseryStrings += other.liveNurseryStrings;
+ tenuredStrings += other.tenuredStrings;
+ tenuredBytes += other.tenuredBytes;
+ return *this;
+ }
+
+ void noteTenured(size_t mallocBytes) {
+ liveNurseryStrings++;
+ tenuredStrings++;
+ tenuredBytes += mallocBytes;
+ }
+
+ void noteDeduplicated(size_t numChars, size_t mallocBytes) {
+ liveNurseryStrings++;
+ deduplicatedStrings++;
+ deduplicatedChars += numChars;
+ deduplicatedBytes += mallocBytes;
+ }
+};
+
+} /* namespace js */
+
+#endif /* gc_Statistics_h */
diff --git a/js/src/gc/StoreBuffer-inl.h b/js/src/gc/StoreBuffer-inl.h
new file mode 100644
index 0000000000..1c97654761
--- /dev/null
+++ b/js/src/gc/StoreBuffer-inl.h
@@ -0,0 +1,97 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_StoreBuffer_inl_h
+#define gc_StoreBuffer_inl_h
+
+#include "gc/StoreBuffer.h"
+
+#include "gc/Cell.h"
+#include "gc/Heap.h"
+
+#include "gc/Heap-inl.h"
+
+namespace js {
+namespace gc {
+
+inline /* static */ size_t ArenaCellSet::getCellIndex(const TenuredCell* cell) {
+ uintptr_t cellOffset = uintptr_t(cell) & ArenaMask;
+ MOZ_ASSERT(cellOffset % ArenaCellIndexBytes == 0);
+ return cellOffset / ArenaCellIndexBytes;
+}
+
+inline /* static */ void ArenaCellSet::getWordIndexAndMask(size_t cellIndex,
+ size_t* wordp,
+ uint32_t* maskp) {
+ BitArray<MaxArenaCellIndex>::getIndexAndMask(cellIndex, wordp, maskp);
+}
+
+inline bool ArenaCellSet::hasCell(size_t cellIndex) const {
+ MOZ_ASSERT(cellIndex < MaxArenaCellIndex);
+ return bits.get(cellIndex);
+}
+
+inline void ArenaCellSet::putCell(size_t cellIndex) {
+ MOZ_ASSERT(cellIndex < MaxArenaCellIndex);
+ MOZ_ASSERT(arena);
+
+ bits.set(cellIndex);
+ check();
+}
+
+inline void ArenaCellSet::check() const {
+#ifdef DEBUG
+ bool bitsZero = bits.isAllClear();
+ MOZ_ASSERT(isEmpty() == bitsZero);
+ MOZ_ASSERT(isEmpty() == !arena);
+ if (!isEmpty()) {
+ MOZ_ASSERT(IsCellPointerValid(arena));
+ MOZ_ASSERT(arena->bufferedCells() == this);
+ JSRuntime* runtime = arena->zone->runtimeFromMainThread();
+ MOZ_ASSERT(runtime->gc.minorGCCount() == minorGCNumberAtCreation);
+ }
+#endif
+}
+
+inline void StoreBuffer::WholeCellBuffer::put(const Cell* cell) {
+ if (cell != last_) {
+ putDontCheckLast(cell);
+ }
+}
+
+inline void StoreBuffer::WholeCellBuffer::putDontCheckLast(const Cell* cell) {
+ // This can still be called when |cell == last_| if the caller didn't check
+ // and that's OK.
+
+ MOZ_ASSERT(cell->isTenured());
+
+ // BigInts don't have any children, so shouldn't show up here.
+ MOZ_ASSERT(cell->getTraceKind() != JS::TraceKind::BigInt);
+
+ Arena* arena = cell->asTenured().arena();
+ ArenaCellSet* cells = arena->bufferedCells();
+ if (cells->isEmpty()) {
+ cells = allocateCellSet(arena);
+ if (!cells) {
+ return;
+ }
+ }
+
+ cells->putCell(&cell->asTenured());
+ cells->check();
+
+ last_ = cell;
+}
+
+inline void StoreBuffer::putWholeCell(Cell* cell) { bufferWholeCell.put(cell); }
+inline void StoreBuffer::putWholeCellDontCheckLast(Cell* cell) {
+ bufferWholeCell.putDontCheckLast(cell);
+}
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_StoreBuffer_inl_h
diff --git a/js/src/gc/StoreBuffer.cpp b/js/src/gc/StoreBuffer.cpp
new file mode 100644
index 0000000000..4f6240d48b
--- /dev/null
+++ b/js/src/gc/StoreBuffer.cpp
@@ -0,0 +1,255 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/StoreBuffer-inl.h"
+
+#include "mozilla/Assertions.h"
+
+#include "gc/Statistics.h"
+#include "vm/MutexIDs.h"
+#include "vm/Runtime.h"
+
+using namespace js;
+using namespace js::gc;
+
+JS_PUBLIC_API void js::gc::LockStoreBuffer(StoreBuffer* sb) {
+ MOZ_ASSERT(sb);
+ sb->lock();
+}
+
+JS_PUBLIC_API void js::gc::UnlockStoreBuffer(StoreBuffer* sb) {
+ MOZ_ASSERT(sb);
+ sb->unlock();
+}
+
+#ifdef DEBUG
+void StoreBuffer::checkAccess() const {
+ // The GC runs tasks that may access the storebuffer in parallel and so must
+ // take a lock. The mutator may only access the storebuffer from the main
+ // thread.
+ if (runtime_->heapState() != JS::HeapState::Idle) {
+ MOZ_ASSERT(!CurrentThreadIsGCMarking());
+ lock_.assertOwnedByCurrentThread();
+ } else {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_));
+ }
+}
+#endif
+
+bool StoreBuffer::WholeCellBuffer::init() {
+ MOZ_ASSERT(!stringHead_);
+ MOZ_ASSERT(!nonStringHead_);
+ if (!storage_) {
+ storage_ = MakeUnique<LifoAlloc>(LifoAllocBlockSize);
+ // This prevents LifoAlloc::Enum from crashing with a release
+ // assertion if we ever allocate one entry larger than
+ // LifoAllocBlockSize.
+ if (storage_) {
+ storage_->disableOversize();
+ }
+ }
+ clear();
+ return bool(storage_);
+}
+
+bool StoreBuffer::GenericBuffer::init() {
+ if (!storage_) {
+ storage_ = MakeUnique<LifoAlloc>(LifoAllocBlockSize);
+ }
+ clear();
+ return bool(storage_);
+}
+
+void StoreBuffer::GenericBuffer::trace(JSTracer* trc) {
+ mozilla::ReentrancyGuard g(*owner_);
+ MOZ_ASSERT(owner_->isEnabled());
+ if (!storage_) {
+ return;
+ }
+
+ for (LifoAlloc::Enum e(*storage_); !e.empty();) {
+ unsigned size = *e.read<unsigned>();
+ BufferableRef* edge = e.read<BufferableRef>(size);
+ edge->trace(trc);
+ }
+}
+
+StoreBuffer::StoreBuffer(JSRuntime* rt, const Nursery& nursery)
+ : lock_(mutexid::StoreBuffer),
+ bufferVal(this, JS::GCReason::FULL_VALUE_BUFFER),
+ bufStrCell(this, JS::GCReason::FULL_CELL_PTR_STR_BUFFER),
+ bufBigIntCell(this, JS::GCReason::FULL_CELL_PTR_BIGINT_BUFFER),
+ bufObjCell(this, JS::GCReason::FULL_CELL_PTR_OBJ_BUFFER),
+ bufferSlot(this, JS::GCReason::FULL_SLOT_BUFFER),
+ bufferWholeCell(this),
+ bufferGeneric(this),
+ runtime_(rt),
+ nursery_(nursery),
+ aboutToOverflow_(false),
+ enabled_(false),
+ mayHavePointersToDeadCells_(false)
+#ifdef DEBUG
+ ,
+ mEntered(false),
+ markingNondeduplicatable(false)
+#endif
+{
+}
+
+void StoreBuffer::checkEmpty() const { MOZ_ASSERT(isEmpty()); }
+
+bool StoreBuffer::isEmpty() const {
+ return bufferVal.isEmpty() && bufStrCell.isEmpty() &&
+ bufBigIntCell.isEmpty() && bufObjCell.isEmpty() &&
+ bufferSlot.isEmpty() && bufferWholeCell.isEmpty() &&
+ bufferGeneric.isEmpty();
+}
+
+bool StoreBuffer::enable() {
+ if (enabled_) {
+ return true;
+ }
+
+ checkEmpty();
+
+ if (!bufferWholeCell.init() || !bufferGeneric.init()) {
+ return false;
+ }
+
+ enabled_ = true;
+ return true;
+}
+
+void StoreBuffer::disable() {
+ checkEmpty();
+
+ if (!enabled_) {
+ return;
+ }
+
+ aboutToOverflow_ = false;
+
+ enabled_ = false;
+}
+
+void StoreBuffer::clear() {
+ if (!enabled_) {
+ return;
+ }
+
+ aboutToOverflow_ = false;
+ mayHavePointersToDeadCells_ = false;
+
+ bufferVal.clear();
+ bufStrCell.clear();
+ bufBigIntCell.clear();
+ bufObjCell.clear();
+ bufferSlot.clear();
+ bufferWholeCell.clear();
+ bufferGeneric.clear();
+}
+
+void StoreBuffer::setAboutToOverflow(JS::GCReason reason) {
+ if (!aboutToOverflow_) {
+ aboutToOverflow_ = true;
+ runtime_->gc.stats().count(gcstats::COUNT_STOREBUFFER_OVERFLOW);
+ }
+ nursery_.requestMinorGC(reason);
+}
+
+void StoreBuffer::addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ JS::GCSizes* sizes) {
+ sizes->storeBufferVals += bufferVal.sizeOfExcludingThis(mallocSizeOf);
+ sizes->storeBufferCells += bufStrCell.sizeOfExcludingThis(mallocSizeOf) +
+ bufBigIntCell.sizeOfExcludingThis(mallocSizeOf) +
+ bufObjCell.sizeOfExcludingThis(mallocSizeOf);
+ sizes->storeBufferSlots += bufferSlot.sizeOfExcludingThis(mallocSizeOf);
+ sizes->storeBufferWholeCells +=
+ bufferWholeCell.sizeOfExcludingThis(mallocSizeOf);
+ sizes->storeBufferGenerics += bufferGeneric.sizeOfExcludingThis(mallocSizeOf);
+}
+
+ArenaCellSet ArenaCellSet::Empty;
+
+ArenaCellSet::ArenaCellSet(Arena* arena, ArenaCellSet* next)
+ : arena(arena),
+ next(next)
+#ifdef DEBUG
+ ,
+ minorGCNumberAtCreation(
+ arena->zone->runtimeFromMainThread()->gc.minorGCCount())
+#endif
+{
+ MOZ_ASSERT(arena);
+ MOZ_ASSERT(bits.isAllClear());
+}
+
+ArenaCellSet* StoreBuffer::WholeCellBuffer::allocateCellSet(Arena* arena) {
+ Zone* zone = arena->zone;
+ JSRuntime* rt = zone->runtimeFromMainThread();
+ if (!rt->gc.nursery().isEnabled()) {
+ return nullptr;
+ }
+
+ // Maintain separate lists for strings and non-strings, so that all buffered
+ // string whole cells will be processed before anything else (to prevent them
+ // from being deduplicated when their chars are used by a tenured string.)
+ bool isString =
+ MapAllocToTraceKind(arena->getAllocKind()) == JS::TraceKind::String;
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ ArenaCellSet*& head = isString ? stringHead_ : nonStringHead_;
+ auto cells = storage_->new_<ArenaCellSet>(arena, head);
+ if (!cells) {
+ oomUnsafe.crash("Failed to allocate ArenaCellSet");
+ }
+
+ arena->bufferedCells() = cells;
+ head = cells;
+
+ if (isAboutToOverflow()) {
+ rt->gc.storeBuffer().setAboutToOverflow(
+ JS::GCReason::FULL_WHOLE_CELL_BUFFER);
+ }
+
+ return cells;
+}
+
+void gc::CellHeaderPostWriteBarrier(JSObject** ptr, JSObject* prev,
+ JSObject* next) {
+ InternalBarrierMethods<JSObject*>::postBarrier(ptr, prev, next);
+}
+
+void StoreBuffer::WholeCellBuffer::clear() {
+ for (auto** headPtr : {&stringHead_, &nonStringHead_}) {
+ for (auto* set = *headPtr; set; set = set->next) {
+ set->arena->bufferedCells() = &ArenaCellSet::Empty;
+ }
+ *headPtr = nullptr;
+ }
+
+ if (storage_) {
+ storage_->used() ? storage_->releaseAll() : storage_->freeAll();
+ }
+
+ last_ = nullptr;
+}
+
+template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::ValueEdge>;
+template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::SlotsEdge>;
+
+void js::gc::PostWriteBarrierCell(Cell* cell, Cell* prev, Cell* next) {
+ if (!next || !cell->isTenured()) {
+ return;
+ }
+
+ StoreBuffer* buffer = next->storeBuffer();
+ if (!buffer || (prev && prev->storeBuffer())) {
+ return;
+ }
+
+ buffer->putWholeCell(cell);
+}
diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h
new file mode 100644
index 0000000000..4700392074
--- /dev/null
+++ b/js/src/gc/StoreBuffer.h
@@ -0,0 +1,662 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_StoreBuffer_h
+#define gc_StoreBuffer_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/HashFunctions.h"
+#include "mozilla/ReentrancyGuard.h"
+
+#include <algorithm>
+
+#include "ds/BitArray.h"
+#include "ds/LifoAlloc.h"
+#include "gc/Cell.h"
+#include "gc/Nursery.h"
+#include "gc/TraceKind.h"
+#include "js/AllocPolicy.h"
+#include "js/UniquePtr.h"
+#include "threading/Mutex.h"
+
+namespace JS {
+struct GCSizes;
+}
+
+namespace js {
+
+class NativeObject;
+
+#ifdef DEBUG
+extern bool CurrentThreadIsGCMarking();
+#endif
+
+namespace gc {
+
+class Arena;
+class ArenaCellSet;
+
+#ifdef DEBUG
+extern bool CurrentThreadHasLockedGC();
+#endif
+
+/*
+ * BufferableRef represents an abstract reference for use in the generational
+ * GC's remembered set. Entries in the store buffer that cannot be represented
+ * with the simple pointer-to-a-pointer scheme must derive from this class and
+ * use the generic store buffer interface.
+ *
+ * A single BufferableRef entry in the generic buffer can represent many entries
+ * in the remembered set. For example js::OrderedHashTableRef represents all
+ * the incoming edges corresponding to keys in an ordered hash table.
+ */
+class BufferableRef {
+ public:
+ virtual void trace(JSTracer* trc) = 0;
+ bool maybeInRememberedSet(const Nursery&) const { return true; }
+};
+
+typedef HashSet<void*, PointerHasher<void*>, SystemAllocPolicy> EdgeSet;
+
+/* The size of a single block of store buffer storage space. */
+static const size_t LifoAllocBlockSize = 8 * 1024;
+
+/*
+ * The StoreBuffer observes all writes that occur in the system and performs
+ * efficient filtering of them to derive a remembered set for nursery GC.
+ */
+class StoreBuffer {
+ friend class mozilla::ReentrancyGuard;
+
+ /* The size at which a block is about to overflow for the generic buffer. */
+ static const size_t GenericBufferLowAvailableThreshold =
+ LifoAllocBlockSize / 2;
+
+ /* The size at which other store buffers are about to overflow. */
+ static const size_t BufferOverflowThresholdBytes = 128 * 1024;
+
+ /*
+ * This buffer holds only a single type of edge. Using this buffer is more
+ * efficient than the generic buffer when many writes will be to the same
+ * type of edge: e.g. Value or Cell*.
+ */
+ template <typename T>
+ struct MonoTypeBuffer {
+ /* The canonical set of stores. */
+ typedef HashSet<T, typename T::Hasher, SystemAllocPolicy> StoreSet;
+ StoreSet stores_;
+
+ /*
+ * A one element cache in front of the canonical set to speed up
+ * temporary instances of HeapPtr.
+ */
+ T last_;
+
+ StoreBuffer* owner_;
+
+ JS::GCReason gcReason_;
+
+ /* Maximum number of entries before we request a minor GC. */
+ const static size_t MaxEntries = BufferOverflowThresholdBytes / sizeof(T);
+
+ explicit MonoTypeBuffer(StoreBuffer* owner, JS::GCReason reason)
+ : last_(T()), owner_(owner), gcReason_(reason) {}
+
+ void clear() {
+ last_ = T();
+ stores_.clear();
+ }
+
+ /* Add one item to the buffer. */
+ void put(const T& t) {
+ sinkStore();
+ last_ = t;
+ }
+
+ /* Remove an item from the store buffer. */
+ void unput(const T& v) {
+ // Fast, hashless remove of last put.
+ if (last_ == v) {
+ last_ = T();
+ return;
+ }
+ stores_.remove(v);
+ }
+
+ /* Move any buffered stores to the canonical store set. */
+ void sinkStore() {
+ if (last_) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!stores_.put(last_)) {
+ oomUnsafe.crash("Failed to allocate for MonoTypeBuffer::put.");
+ }
+ }
+ last_ = T();
+
+ if (MOZ_UNLIKELY(stores_.count() > MaxEntries)) {
+ owner_->setAboutToOverflow(gcReason_);
+ }
+ }
+
+ /* Trace the source of all edges in the store buffer. */
+ void trace(TenuringTracer& mover);
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return stores_.shallowSizeOfExcludingThis(mallocSizeOf);
+ }
+
+ bool isEmpty() const { return last_ == T() && stores_.empty(); }
+
+ private:
+ MonoTypeBuffer(const MonoTypeBuffer& other) = delete;
+ MonoTypeBuffer& operator=(const MonoTypeBuffer& other) = delete;
+ };
+
+ struct WholeCellBuffer {
+ UniquePtr<LifoAlloc> storage_;
+ ArenaCellSet* stringHead_ = nullptr;
+ ArenaCellSet* nonStringHead_ = nullptr;
+ const Cell* last_ = nullptr;
+ StoreBuffer* owner_;
+
+ explicit WholeCellBuffer(StoreBuffer* owner) : owner_(owner) {}
+
+ [[nodiscard]] bool init();
+
+ void clear();
+
+ bool isAboutToOverflow() const {
+ return !storage_->isEmpty() &&
+ storage_->used() > BufferOverflowThresholdBytes;
+ }
+
+ void trace(TenuringTracer& mover);
+
+ inline void put(const Cell* cell);
+ inline void putDontCheckLast(const Cell* cell);
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return storage_ ? storage_->sizeOfIncludingThis(mallocSizeOf) : 0;
+ }
+
+ bool isEmpty() const {
+ MOZ_ASSERT_IF(!stringHead_ && !nonStringHead_,
+ !storage_ || storage_->isEmpty());
+ return !stringHead_ && !nonStringHead_;
+ }
+
+ const Cell** lastBufferedPtr() { return &last_; }
+
+ private:
+ ArenaCellSet* allocateCellSet(Arena* arena);
+
+ WholeCellBuffer(const WholeCellBuffer& other) = delete;
+ WholeCellBuffer& operator=(const WholeCellBuffer& other) = delete;
+ };
+
+ struct GenericBuffer {
+ UniquePtr<LifoAlloc> storage_;
+ StoreBuffer* owner_;
+
+ explicit GenericBuffer(StoreBuffer* owner)
+ : storage_(nullptr), owner_(owner) {}
+
+ [[nodiscard]] bool init();
+
+ void clear() {
+ if (storage_) {
+ storage_->used() ? storage_->releaseAll() : storage_->freeAll();
+ }
+ }
+
+ bool isAboutToOverflow() const {
+ return !storage_->isEmpty() && storage_->availableInCurrentChunk() <
+ GenericBufferLowAvailableThreshold;
+ }
+
+ /* Trace all generic edges. */
+ void trace(JSTracer* trc);
+
+ template <typename T>
+ void put(const T& t) {
+ MOZ_ASSERT(storage_);
+
+ /* Ensure T is derived from BufferableRef. */
+ (void)static_cast<const BufferableRef*>(&t);
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ unsigned size = sizeof(T);
+ unsigned* sizep = storage_->pod_malloc<unsigned>();
+ if (!sizep) {
+ oomUnsafe.crash("Failed to allocate for GenericBuffer::put.");
+ }
+ *sizep = size;
+
+ T* tp = storage_->new_<T>(t);
+ if (!tp) {
+ oomUnsafe.crash("Failed to allocate for GenericBuffer::put.");
+ }
+
+ if (isAboutToOverflow()) {
+ owner_->setAboutToOverflow(JS::GCReason::FULL_GENERIC_BUFFER);
+ }
+ }
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return storage_ ? storage_->sizeOfIncludingThis(mallocSizeOf) : 0;
+ }
+
+ bool isEmpty() const { return !storage_ || storage_->isEmpty(); }
+
+ private:
+ GenericBuffer(const GenericBuffer& other) = delete;
+ GenericBuffer& operator=(const GenericBuffer& other) = delete;
+ };
+
+ template <typename Edge>
+ struct PointerEdgeHasher {
+ using Lookup = Edge;
+ static HashNumber hash(const Lookup& l) {
+ return mozilla::HashGeneric(l.edge);
+ }
+ static bool match(const Edge& k, const Lookup& l) { return k == l; }
+ };
+
+ template <typename T>
+ struct CellPtrEdge {
+ T** edge = nullptr;
+
+ CellPtrEdge() = default;
+ explicit CellPtrEdge(T** v) : edge(v) {}
+ bool operator==(const CellPtrEdge& other) const {
+ return edge == other.edge;
+ }
+ bool operator!=(const CellPtrEdge& other) const {
+ return edge != other.edge;
+ }
+
+ bool maybeInRememberedSet(const Nursery& nursery) const {
+ MOZ_ASSERT(IsInsideNursery(*edge));
+ return !nursery.isInside(edge);
+ }
+
+ void trace(TenuringTracer& mover) const;
+
+ explicit operator bool() const { return edge != nullptr; }
+
+ using Hasher = PointerEdgeHasher<CellPtrEdge<T>>;
+ };
+
+ using ObjectPtrEdge = CellPtrEdge<JSObject>;
+ using StringPtrEdge = CellPtrEdge<JSString>;
+ using BigIntPtrEdge = CellPtrEdge<JS::BigInt>;
+
+ struct ValueEdge {
+ JS::Value* edge;
+
+ ValueEdge() : edge(nullptr) {}
+ explicit ValueEdge(JS::Value* v) : edge(v) {}
+ bool operator==(const ValueEdge& other) const { return edge == other.edge; }
+ bool operator!=(const ValueEdge& other) const { return edge != other.edge; }
+
+ Cell* deref() const {
+ return edge->isGCThing() ? static_cast<Cell*>(edge->toGCThing())
+ : nullptr;
+ }
+
+ bool maybeInRememberedSet(const Nursery& nursery) const {
+ MOZ_ASSERT(IsInsideNursery(deref()));
+ return !nursery.isInside(edge);
+ }
+
+ void trace(TenuringTracer& mover) const;
+
+ explicit operator bool() const { return edge != nullptr; }
+
+ using Hasher = PointerEdgeHasher<ValueEdge>;
+ };
+
+ struct SlotsEdge {
+ // These definitions must match those in HeapSlot::Kind.
+ const static int SlotKind = 0;
+ const static int ElementKind = 1;
+
+ uintptr_t objectAndKind_; // NativeObject* | Kind
+ uint32_t start_;
+ uint32_t count_;
+
+ SlotsEdge() : objectAndKind_(0), start_(0), count_(0) {}
+ SlotsEdge(NativeObject* object, int kind, uint32_t start, uint32_t count)
+ : objectAndKind_(uintptr_t(object) | kind),
+ start_(start),
+ count_(count) {
+ MOZ_ASSERT((uintptr_t(object) & 1) == 0);
+ MOZ_ASSERT(kind <= 1);
+ MOZ_ASSERT(count > 0);
+ MOZ_ASSERT(start + count > start);
+ }
+
+ NativeObject* object() const {
+ return reinterpret_cast<NativeObject*>(objectAndKind_ & ~1);
+ }
+ int kind() const { return (int)(objectAndKind_ & 1); }
+
+ bool operator==(const SlotsEdge& other) const {
+ return objectAndKind_ == other.objectAndKind_ && start_ == other.start_ &&
+ count_ == other.count_;
+ }
+
+ bool operator!=(const SlotsEdge& other) const { return !(*this == other); }
+
+ // True if this SlotsEdge range overlaps with the other SlotsEdge range,
+ // false if they do not overlap.
+ bool overlaps(const SlotsEdge& other) const {
+ if (objectAndKind_ != other.objectAndKind_) {
+ return false;
+ }
+
+ // Widen our range by one on each side so that we consider
+ // adjacent-but-not-actually-overlapping ranges as overlapping. This
+ // is particularly useful for coalescing a series of increasing or
+ // decreasing single index writes 0, 1, 2, ..., N into a SlotsEdge
+ // range of elements [0, N].
+ uint32_t end = start_ + count_ + 1;
+ uint32_t start = start_ > 0 ? start_ - 1 : 0;
+ MOZ_ASSERT(start < end);
+
+ uint32_t otherEnd = other.start_ + other.count_;
+ MOZ_ASSERT(other.start_ <= otherEnd);
+ return (start <= other.start_ && other.start_ <= end) ||
+ (start <= otherEnd && otherEnd <= end);
+ }
+
+ // Destructively make this SlotsEdge range the union of the other
+ // SlotsEdge range and this one. A precondition is that the ranges must
+ // overlap.
+ void merge(const SlotsEdge& other) {
+ MOZ_ASSERT(overlaps(other));
+ uint32_t end = std::max(start_ + count_, other.start_ + other.count_);
+ start_ = std::min(start_, other.start_);
+ count_ = end - start_;
+ }
+
+ bool maybeInRememberedSet(const Nursery& n) const {
+ return !IsInsideNursery(reinterpret_cast<Cell*>(object()));
+ }
+
+ void trace(TenuringTracer& mover) const;
+
+ explicit operator bool() const { return objectAndKind_ != 0; }
+
+ typedef struct Hasher {
+ using Lookup = SlotsEdge;
+ static HashNumber hash(const Lookup& l) {
+ return mozilla::HashGeneric(l.objectAndKind_, l.start_, l.count_);
+ }
+ static bool match(const SlotsEdge& k, const Lookup& l) { return k == l; }
+ } Hasher;
+ };
+
+#ifdef DEBUG
+ void checkAccess() const;
+#else
+ void checkAccess() const {}
+#endif
+
+ template <typename Buffer, typename Edge>
+ void unput(Buffer& buffer, const Edge& edge) {
+ checkAccess();
+ if (!isEnabled()) {
+ return;
+ }
+ mozilla::ReentrancyGuard g(*this);
+ buffer.unput(edge);
+ }
+
+ template <typename Buffer, typename Edge>
+ void put(Buffer& buffer, const Edge& edge) {
+ checkAccess();
+ if (!isEnabled()) {
+ return;
+ }
+ mozilla::ReentrancyGuard g(*this);
+ if (edge.maybeInRememberedSet(nursery_)) {
+ buffer.put(edge);
+ }
+ }
+
+ Mutex lock_ MOZ_UNANNOTATED;
+
+ MonoTypeBuffer<ValueEdge> bufferVal;
+ MonoTypeBuffer<StringPtrEdge> bufStrCell;
+ MonoTypeBuffer<BigIntPtrEdge> bufBigIntCell;
+ MonoTypeBuffer<ObjectPtrEdge> bufObjCell;
+ MonoTypeBuffer<SlotsEdge> bufferSlot;
+ WholeCellBuffer bufferWholeCell;
+ GenericBuffer bufferGeneric;
+
+ JSRuntime* runtime_;
+ const Nursery& nursery_;
+
+ bool aboutToOverflow_;
+ bool enabled_;
+ bool mayHavePointersToDeadCells_;
+#ifdef DEBUG
+ bool mEntered; /* For ReentrancyGuard. */
+#endif
+
+ public:
+#ifdef DEBUG
+ bool markingNondeduplicatable;
+#endif
+
+ explicit StoreBuffer(JSRuntime* rt, const Nursery& nursery);
+ [[nodiscard]] bool enable();
+
+ void disable();
+ bool isEnabled() const { return enabled_; }
+
+ bool isEmpty() const;
+ void clear();
+
+ const Nursery& nursery() const { return nursery_; }
+
+ /* Get the overflowed status. */
+ bool isAboutToOverflow() const { return aboutToOverflow_; }
+
+ /*
+ * Brain transplants may add whole cell buffer entires for dead cells. We must
+ * evict the nursery prior to sweeping arenas if any such entries are present.
+ */
+ bool mayHavePointersToDeadCells() const {
+ return mayHavePointersToDeadCells_;
+ }
+
+ /* Insert a single edge into the buffer/remembered set. */
+ void putValue(JS::Value* vp) { put(bufferVal, ValueEdge(vp)); }
+ void unputValue(JS::Value* vp) { unput(bufferVal, ValueEdge(vp)); }
+
+ void putCell(JSString** strp) { put(bufStrCell, StringPtrEdge(strp)); }
+ void unputCell(JSString** strp) { unput(bufStrCell, StringPtrEdge(strp)); }
+
+ void putCell(JS::BigInt** bip) { put(bufBigIntCell, BigIntPtrEdge(bip)); }
+ void unputCell(JS::BigInt** bip) { unput(bufBigIntCell, BigIntPtrEdge(bip)); }
+
+ void putCell(JSObject** strp) { put(bufObjCell, ObjectPtrEdge(strp)); }
+ void unputCell(JSObject** strp) { unput(bufObjCell, ObjectPtrEdge(strp)); }
+
+ void putSlot(NativeObject* obj, int kind, uint32_t start, uint32_t count) {
+ SlotsEdge edge(obj, kind, start, count);
+ if (bufferSlot.last_.overlaps(edge)) {
+ bufferSlot.last_.merge(edge);
+ } else {
+ put(bufferSlot, edge);
+ }
+ }
+
+ inline void putWholeCell(Cell* cell);
+ inline void putWholeCellDontCheckLast(Cell* cell);
+ const void* addressOfLastBufferedWholeCell() {
+ return bufferWholeCell.lastBufferedPtr();
+ }
+
+ /* Insert an entry into the generic buffer. */
+ template <typename T>
+ void putGeneric(const T& t) {
+ put(bufferGeneric, t);
+ }
+
+ void setMayHavePointersToDeadCells() { mayHavePointersToDeadCells_ = true; }
+
+ /* Methods to trace the source of all edges in the store buffer. */
+ void traceValues(TenuringTracer& mover) { bufferVal.trace(mover); }
+ void traceCells(TenuringTracer& mover) {
+ bufStrCell.trace(mover);
+ bufBigIntCell.trace(mover);
+ bufObjCell.trace(mover);
+ }
+ void traceSlots(TenuringTracer& mover) { bufferSlot.trace(mover); }
+ void traceWholeCells(TenuringTracer& mover) { bufferWholeCell.trace(mover); }
+ void traceGenericEntries(JSTracer* trc) { bufferGeneric.trace(trc); }
+
+ /* For use by our owned buffers and for testing. */
+ void setAboutToOverflow(JS::GCReason);
+
+ void addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ JS::GCSizes* sizes);
+
+ void checkEmpty() const;
+
+ // For use by the GC only.
+ void lock() { lock_.lock(); }
+ void unlock() { lock_.unlock(); }
+};
+
+// A set of cells in an arena used to implement the whole cell store buffer.
+class ArenaCellSet {
+ friend class StoreBuffer;
+
+ using ArenaCellBits = BitArray<MaxArenaCellIndex>;
+
+ // The arena this relates to.
+ Arena* arena;
+
+ // Pointer to next set forming a linked list.
+ ArenaCellSet* next;
+
+ // Bit vector for each possible cell start position.
+ ArenaCellBits bits;
+
+#ifdef DEBUG
+ // The minor GC number when this was created. This object should not survive
+ // past the next minor collection.
+ const uint64_t minorGCNumberAtCreation;
+#endif
+
+ // Construct the empty sentinel object.
+ constexpr ArenaCellSet()
+ : arena(nullptr),
+ next(nullptr)
+#ifdef DEBUG
+ ,
+ minorGCNumberAtCreation(0)
+#endif
+ {
+ }
+
+ public:
+ using WordT = ArenaCellBits::WordT;
+ const size_t BitsPerWord = ArenaCellBits::bitsPerElement;
+ const size_t NumWords = ArenaCellBits::numSlots;
+
+ ArenaCellSet(Arena* arena, ArenaCellSet* next);
+
+ bool hasCell(const TenuredCell* cell) const {
+ return hasCell(getCellIndex(cell));
+ }
+
+ void putCell(const TenuredCell* cell) { putCell(getCellIndex(cell)); }
+
+ bool isEmpty() const { return this == &Empty; }
+
+ bool hasCell(size_t cellIndex) const;
+
+ void putCell(size_t cellIndex);
+
+ void check() const;
+
+ WordT getWord(size_t wordIndex) const { return bits.getWord(wordIndex); }
+
+ void trace(TenuringTracer& mover);
+
+ // Sentinel object used for all empty sets.
+ //
+ // We use a sentinel because it simplifies the JIT code slightly as we can
+ // assume all arenas have a cell set.
+ static ArenaCellSet Empty;
+
+ static size_t getCellIndex(const TenuredCell* cell);
+ static void getWordIndexAndMask(size_t cellIndex, size_t* wordp,
+ uint32_t* maskp);
+
+ // Attempt to trigger a minor GC if free space in the nursery (where these
+ // objects are allocated) falls below this threshold.
+ static const size_t NurseryFreeThresholdBytes = 64 * 1024;
+
+ static size_t offsetOfArena() { return offsetof(ArenaCellSet, arena); }
+ static size_t offsetOfBits() { return offsetof(ArenaCellSet, bits); }
+};
+
+// Post-write barrier implementation for GC cells.
+
+// Implement the post-write barrier for nursery allocateable cell type |T|. Call
+// this from |T::postWriteBarrier|.
+template <typename T>
+MOZ_ALWAYS_INLINE void PostWriteBarrierImpl(void* cellp, T* prev, T* next) {
+ MOZ_ASSERT(cellp);
+
+ // If the target needs an entry, add it.
+ StoreBuffer* buffer;
+ if (next && (buffer = next->storeBuffer())) {
+ // If we know that the prev has already inserted an entry, we can skip
+ // doing the lookup to add the new entry. Note that we cannot safely
+ // assert the presence of the entry because it may have been added
+ // via a different store buffer.
+ if (prev && prev->storeBuffer()) {
+ return;
+ }
+ buffer->putCell(static_cast<T**>(cellp));
+ return;
+ }
+
+ // Remove the prev entry if the new value does not need it. There will only
+ // be a prev entry if the prev value was in the nursery.
+ if (prev && (buffer = prev->storeBuffer())) {
+ buffer->unputCell(static_cast<T**>(cellp));
+ }
+}
+
+template <typename T>
+MOZ_ALWAYS_INLINE void PostWriteBarrier(T** vp, T* prev, T* next) {
+ static_assert(std::is_base_of_v<Cell, T>);
+ static_assert(!std::is_same_v<Cell, T> && !std::is_same_v<TenuredCell, T>);
+
+ if constexpr (!GCTypeIsTenured<T>()) {
+ using BaseT = typename BaseGCType<T>::type;
+ PostWriteBarrierImpl<BaseT>(vp, prev, next);
+ return;
+ }
+
+ MOZ_ASSERT_IF(next, !IsInsideNursery(next));
+}
+
+// Used when we don't have a specific edge to put in the store buffer.
+void PostWriteBarrierCell(Cell* cell, Cell* prev, Cell* next);
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_StoreBuffer_h */
diff --git a/js/src/gc/Sweeping.cpp b/js/src/gc/Sweeping.cpp
new file mode 100644
index 0000000000..80f4a20e74
--- /dev/null
+++ b/js/src/gc/Sweeping.cpp
@@ -0,0 +1,2383 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Implementation of GC sweeping.
+ *
+ * In the SpiderMonkey GC, 'sweeping' is used to mean two things:
+ * - updating data structures to remove pointers to dead GC things and updating
+ * pointers to moved GC things
+ * - finalizing dead GC things
+ *
+ * Furthermore, the GC carries out gray and weak marking after the start of the
+ * sweep phase. This is also implemented in this file.
+ */
+
+#include "mozilla/Maybe.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/TimeStamp.h"
+
+#include "builtin/FinalizationRegistryObject.h"
+#include "builtin/WeakRefObject.h"
+#include "debugger/DebugAPI.h"
+#include "gc/AllocKind.h"
+#include "gc/FinalizationObservers.h"
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/GCProbes.h"
+#include "gc/GCRuntime.h"
+#include "gc/ParallelWork.h"
+#include "gc/Statistics.h"
+#include "gc/TraceKind.h"
+#include "gc/WeakMap.h"
+#include "gc/Zone.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "proxy/DeadObjectProxy.h"
+#include "vm/BigIntType.h"
+#include "vm/HelperThreads.h"
+#include "vm/JSContext.h"
+#include "vm/Time.h"
+#include "vm/WrapperObject.h"
+
+#include "gc/PrivateIterators-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSObject-inl.h"
+#include "vm/PropMap-inl.h"
+#include "vm/Shape-inl.h"
+#include "vm/StringType-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::TimeStamp;
+
+struct js::gc::FinalizePhase {
+ gcstats::PhaseKind statsPhase;
+ AllocKinds kinds;
+};
+
+/*
+ * Finalization order for objects swept incrementally on the main thread.
+ */
+static constexpr FinalizePhase ForegroundObjectFinalizePhase = {
+ gcstats::PhaseKind::FINALIZE_OBJECT,
+ {AllocKind::OBJECT0, AllocKind::OBJECT2, AllocKind::OBJECT4,
+ AllocKind::OBJECT8, AllocKind::OBJECT12, AllocKind::OBJECT16}};
+
+/*
+ * Finalization order for GC things swept incrementally on the main thread.
+ */
+static constexpr FinalizePhase ForegroundNonObjectFinalizePhase = {
+ gcstats::PhaseKind::FINALIZE_NON_OBJECT,
+ {AllocKind::SCRIPT, AllocKind::JITCODE}};
+
+/*
+ * Finalization order for GC things swept on the background thread.
+ */
+static constexpr FinalizePhase BackgroundFinalizePhases[] = {
+ {gcstats::PhaseKind::FINALIZE_OBJECT,
+ {AllocKind::FUNCTION, AllocKind::FUNCTION_EXTENDED,
+ AllocKind::OBJECT0_BACKGROUND, AllocKind::OBJECT2_BACKGROUND,
+ AllocKind::ARRAYBUFFER4, AllocKind::OBJECT4_BACKGROUND,
+ AllocKind::ARRAYBUFFER8, AllocKind::OBJECT8_BACKGROUND,
+ AllocKind::ARRAYBUFFER12, AllocKind::OBJECT12_BACKGROUND,
+ AllocKind::ARRAYBUFFER16, AllocKind::OBJECT16_BACKGROUND}},
+ {gcstats::PhaseKind::FINALIZE_NON_OBJECT,
+ {AllocKind::SCOPE, AllocKind::REGEXP_SHARED, AllocKind::FAT_INLINE_STRING,
+ AllocKind::STRING, AllocKind::EXTERNAL_STRING, AllocKind::FAT_INLINE_ATOM,
+ AllocKind::ATOM, AllocKind::SYMBOL, AllocKind::BIGINT, AllocKind::SHAPE,
+ AllocKind::BASE_SHAPE, AllocKind::GETTER_SETTER,
+ AllocKind::COMPACT_PROP_MAP, AllocKind::NORMAL_PROP_MAP,
+ AllocKind::DICT_PROP_MAP}}};
+
+template <typename T>
+inline size_t Arena::finalize(JS::GCContext* gcx, AllocKind thingKind,
+ size_t thingSize) {
+ /* Enforce requirements on size of T. */
+ MOZ_ASSERT(thingSize % CellAlignBytes == 0);
+ MOZ_ASSERT(thingSize >= MinCellSize);
+ MOZ_ASSERT(thingSize <= 255);
+
+ MOZ_ASSERT(allocated());
+ MOZ_ASSERT(thingKind == getAllocKind());
+ MOZ_ASSERT(thingSize == getThingSize());
+ MOZ_ASSERT(!onDelayedMarkingList_);
+
+ uint_fast16_t firstThing = firstThingOffset(thingKind);
+ uint_fast16_t firstThingOrSuccessorOfLastMarkedThing = firstThing;
+ uint_fast16_t lastThing = ArenaSize - thingSize;
+
+ FreeSpan newListHead;
+ FreeSpan* newListTail = &newListHead;
+ size_t nmarked = 0, nfinalized = 0;
+
+ for (ArenaCellIterUnderFinalize cell(this); !cell.done(); cell.next()) {
+ T* t = cell.as<T>();
+ if (TenuredThingIsMarkedAny(t)) {
+ uint_fast16_t thing = uintptr_t(t) & ArenaMask;
+ if (thing != firstThingOrSuccessorOfLastMarkedThing) {
+ // We just finished passing over one or more free things,
+ // so record a new FreeSpan.
+ newListTail->initBounds(firstThingOrSuccessorOfLastMarkedThing,
+ thing - thingSize, this);
+ newListTail = newListTail->nextSpanUnchecked(this);
+ }
+ firstThingOrSuccessorOfLastMarkedThing = thing + thingSize;
+ nmarked++;
+ } else {
+ t->finalize(gcx);
+ AlwaysPoison(t, JS_SWEPT_TENURED_PATTERN, thingSize,
+ MemCheckKind::MakeUndefined);
+ gcprobes::TenuredFinalize(t);
+ nfinalized++;
+ }
+ }
+
+ if constexpr (std::is_same_v<T, JSObject>) {
+ if (isNewlyCreated_) {
+ zone->pretenuring.updateCellCountsInNewlyCreatedArenas(
+ nmarked + nfinalized, nmarked);
+ }
+ }
+ isNewlyCreated_ = 0;
+
+ if (thingKind == AllocKind::STRING ||
+ thingKind == AllocKind::FAT_INLINE_STRING) {
+ zone->markedStrings += nmarked;
+ zone->finalizedStrings += nfinalized;
+ }
+
+ if (nmarked == 0) {
+ // Do nothing. The caller will update the arena appropriately.
+ MOZ_ASSERT(newListTail == &newListHead);
+ DebugOnlyPoison(data, JS_SWEPT_TENURED_PATTERN, sizeof(data),
+ MemCheckKind::MakeUndefined);
+ return nmarked;
+ }
+
+ MOZ_ASSERT(firstThingOrSuccessorOfLastMarkedThing != firstThing);
+ uint_fast16_t lastMarkedThing =
+ firstThingOrSuccessorOfLastMarkedThing - thingSize;
+ if (lastThing == lastMarkedThing) {
+ // If the last thing was marked, we will have already set the bounds of
+ // the final span, and we just need to terminate the list.
+ newListTail->initAsEmpty();
+ } else {
+ // Otherwise, end the list with a span that covers the final stretch of free
+ // things.
+ newListTail->initFinal(firstThingOrSuccessorOfLastMarkedThing, lastThing,
+ this);
+ }
+
+ firstFreeSpan = newListHead;
+#ifdef DEBUG
+ size_t nfree = numFreeThings(thingSize);
+ MOZ_ASSERT(nfree + nmarked == thingsPerArena(thingKind));
+#endif
+ return nmarked;
+}
+
+// Finalize arenas from src list, releasing empty arenas if keepArenas wasn't
+// specified and inserting the others into the appropriate destination size
+// bins.
+template <typename T>
+static inline bool FinalizeTypedArenas(JS::GCContext* gcx, ArenaList& src,
+ SortedArenaList& dest,
+ AllocKind thingKind,
+ SliceBudget& budget) {
+ MOZ_ASSERT(gcx->isFinalizing());
+
+ size_t thingSize = Arena::thingSize(thingKind);
+ size_t thingsPerArena = Arena::thingsPerArena(thingKind);
+ size_t markCount = 0;
+
+ auto updateMarkCount = mozilla::MakeScopeExit([&] {
+ GCRuntime* gc = &gcx->runtimeFromAnyThread()->gc;
+ gc->stats().addCount(gcstats::COUNT_CELLS_MARKED, markCount);
+ });
+
+ while (Arena* arena = src.takeFirstArena()) {
+ size_t nmarked = arena->finalize<T>(gcx, thingKind, thingSize);
+ size_t nfree = thingsPerArena - nmarked;
+
+ markCount += nmarked;
+
+ if (nmarked) {
+ dest.insertAt(arena, nfree);
+ } else {
+ arena->chunk()->recycleArena(arena, dest, thingsPerArena);
+ }
+
+ budget.step(thingsPerArena);
+ if (budget.isOverBudget()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/*
+ * Finalize the list of areans.
+ */
+static bool FinalizeArenas(JS::GCContext* gcx, ArenaList& src,
+ SortedArenaList& dest, AllocKind thingKind,
+ SliceBudget& budget) {
+ switch (thingKind) {
+#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery, \
+ compact) \
+ case AllocKind::allocKind: \
+ return FinalizeTypedArenas<type>(gcx, src, dest, thingKind, budget);
+ FOR_EACH_ALLOCKIND(EXPAND_CASE)
+#undef EXPAND_CASE
+
+ default:
+ MOZ_CRASH("Invalid alloc kind");
+ }
+}
+
+void GCRuntime::initBackgroundSweep(Zone* zone, JS::GCContext* gcx,
+ const FinalizePhase& phase) {
+ gcstats::AutoPhase ap(stats(), phase.statsPhase);
+ for (auto kind : phase.kinds) {
+ zone->arenas.initBackgroundSweep(kind);
+ }
+}
+
+void ArenaLists::initBackgroundSweep(AllocKind thingKind) {
+ MOZ_ASSERT(IsBackgroundFinalized(thingKind));
+ MOZ_ASSERT(concurrentUse(thingKind) == ConcurrentUse::None);
+
+ if (!collectingArenaList(thingKind).isEmpty()) {
+ concurrentUse(thingKind) = ConcurrentUse::BackgroundFinalize;
+ }
+}
+
+void GCRuntime::backgroundFinalize(JS::GCContext* gcx, Zone* zone,
+ AllocKind kind, Arena** empty) {
+ MOZ_ASSERT(empty);
+
+ ArenaLists* lists = &zone->arenas;
+ ArenaList& arenas = lists->collectingArenaList(kind);
+ if (arenas.isEmpty()) {
+ MOZ_ASSERT(lists->concurrentUse(kind) == ArenaLists::ConcurrentUse::None);
+ return;
+ }
+
+ SortedArenaList finalizedSorted(Arena::thingsPerArena(kind));
+
+ auto unlimited = SliceBudget::unlimited();
+ FinalizeArenas(gcx, arenas, finalizedSorted, kind, unlimited);
+ MOZ_ASSERT(arenas.isEmpty());
+
+ finalizedSorted.extractEmpty(empty);
+
+ // When marking begins, all arenas are moved from arenaLists to
+ // collectingArenaLists. When the mutator runs, new arenas are allocated in
+ // arenaLists. Now that finalization is complete, we want to merge these lists
+ // back together.
+
+ // We must take the GC lock to be able to safely modify the ArenaList;
+ // however, this does not by itself make the changes visible to all threads,
+ // as not all threads take the GC lock to read the ArenaLists.
+ // That safety is provided by the ReleaseAcquire memory ordering of the
+ // background finalize state, which we explicitly set as the final step.
+ {
+ AutoLockGC lock(rt);
+ MOZ_ASSERT(lists->concurrentUse(kind) ==
+ ArenaLists::ConcurrentUse::BackgroundFinalize);
+ lists->mergeFinalizedArenas(kind, finalizedSorted);
+ }
+
+ lists->concurrentUse(kind) = ArenaLists::ConcurrentUse::None;
+}
+
+// After finalizing arenas, merge the following to get the final state of an
+// arena list:
+// - arenas allocated during marking
+// - arenas allocated during sweeping
+// - finalized arenas
+void ArenaLists::mergeFinalizedArenas(AllocKind kind,
+ SortedArenaList& finalizedArenas) {
+#ifdef DEBUG
+ // Updating arena lists off-thread requires taking the GC lock because the
+ // main thread uses these when allocating.
+ if (IsBackgroundFinalized(kind)) {
+ runtimeFromAnyThread()->gc.assertCurrentThreadHasLockedGC();
+ }
+#endif
+
+ ArenaList& arenas = arenaList(kind);
+
+ ArenaList allocatedDuringCollection = std::move(arenas);
+ arenas = finalizedArenas.toArenaList();
+ arenas.insertListWithCursorAtEnd(allocatedDuringCollection);
+
+ collectingArenaList(kind).clear();
+}
+
+void ArenaLists::queueForegroundThingsForSweep() {
+ gcCompactPropMapArenasToUpdate =
+ collectingArenaList(AllocKind::COMPACT_PROP_MAP).head();
+ gcNormalPropMapArenasToUpdate =
+ collectingArenaList(AllocKind::NORMAL_PROP_MAP).head();
+}
+
+void GCRuntime::sweepBackgroundThings(ZoneList& zones) {
+ if (zones.isEmpty()) {
+ return;
+ }
+
+ JS::GCContext* gcx = TlsGCContext.get();
+ MOZ_ASSERT(gcx->isFinalizing());
+
+ // Sweep zones in order. The atoms zone must be finalized last as other
+ // zones may have direct pointers into it.
+ while (!zones.isEmpty()) {
+ Zone* zone = zones.removeFront();
+ MOZ_ASSERT(zone->isGCFinished());
+
+ TimeStamp startTime = TimeStamp::Now();
+
+ Arena* emptyArenas = zone->arenas.takeSweptEmptyArenas();
+
+ // We must finalize thing kinds in the order specified by
+ // BackgroundFinalizePhases.
+ for (auto phase : BackgroundFinalizePhases) {
+ for (auto kind : phase.kinds) {
+ backgroundFinalize(gcx, zone, kind, &emptyArenas);
+ }
+ }
+
+ // Release any arenas that are now empty.
+ //
+ // Empty arenas are only released after everything has been finalized so
+ // that it's still possible to get a thing's zone after the thing has been
+ // finalized. The HeapPtr destructor depends on this, and this allows
+ // HeapPtrs between things of different alloc kind regardless of
+ // finalization order.
+ //
+ // Periodically drop and reaquire the GC lock every so often to avoid
+ // blocking the main thread from allocating chunks.
+ static const size_t LockReleasePeriod = 32;
+
+ while (emptyArenas) {
+ AutoLockGC lock(this);
+ for (size_t i = 0; i < LockReleasePeriod && emptyArenas; i++) {
+ Arena* arena = emptyArenas;
+ emptyArenas = emptyArenas->next;
+ releaseArena(arena, lock);
+ }
+ }
+
+ // Record time spent sweeping this zone.
+ TimeStamp endTime = TimeStamp::Now();
+ zone->perZoneGCTime += endTime - startTime;
+ }
+}
+
+void GCRuntime::assertBackgroundSweepingFinished() {
+#ifdef DEBUG
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(backgroundSweepZones.ref().isEmpty());
+ }
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ for (auto kind : AllAllocKinds()) {
+ MOZ_ASSERT_IF(state() != State::Prepare && state() != State::Mark &&
+ state() != State::Sweep,
+ zone->arenas.collectingArenaList(kind).isEmpty());
+ MOZ_ASSERT(zone->arenas.doneBackgroundFinalize(kind));
+ }
+ }
+#endif
+}
+
+void GCRuntime::queueZonesAndStartBackgroundSweep(ZoneList&& zones) {
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(!requestSliceAfterBackgroundTask);
+ backgroundSweepZones.ref().appendList(std::move(zones));
+ if (useBackgroundThreads) {
+ sweepTask.startOrRunIfIdle(lock);
+ }
+ }
+ if (!useBackgroundThreads) {
+ sweepTask.join();
+ sweepTask.runFromMainThread();
+ }
+}
+
+BackgroundSweepTask::BackgroundSweepTask(GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::SWEEP, GCUse::Finalizing) {}
+
+void BackgroundSweepTask::run(AutoLockHelperThreadState& lock) {
+ gc->sweepFromBackgroundThread(lock);
+}
+
+void GCRuntime::sweepFromBackgroundThread(AutoLockHelperThreadState& lock) {
+ do {
+ ZoneList zones;
+ zones.appendList(std::move(backgroundSweepZones.ref()));
+
+ AutoUnlockHelperThreadState unlock(lock);
+ sweepBackgroundThings(zones);
+
+ // The main thread may call queueZonesAndStartBackgroundSweep() while this
+ // is running so we must check there is no more work after releasing the
+ // lock.
+ } while (!backgroundSweepZones.ref().isEmpty());
+
+ maybeRequestGCAfterBackgroundTask(lock);
+}
+
+void GCRuntime::waitBackgroundSweepEnd() {
+ sweepTask.join();
+ if (state() != State::Sweep) {
+ assertBackgroundSweepingFinished();
+ }
+}
+
+void GCRuntime::startBackgroundFree() {
+ AutoLockHelperThreadState lock;
+ freeTask.startOrRunIfIdle(lock);
+}
+
+BackgroundFreeTask::BackgroundFreeTask(GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::NONE) {
+ // This can occur outside GCs so doesn't have a stats phase.
+}
+
+void BackgroundFreeTask::run(AutoLockHelperThreadState& lock) {
+ gc->freeFromBackgroundThread(lock);
+}
+
+void GCRuntime::freeFromBackgroundThread(AutoLockHelperThreadState& lock) {
+ do {
+ LifoAlloc lifoBlocks(JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE);
+ lifoBlocks.transferFrom(&lifoBlocksToFree.ref());
+
+ Nursery::BufferSet buffers;
+ std::swap(buffers, buffersToFreeAfterMinorGC.ref());
+
+ AutoUnlockHelperThreadState unlock(lock);
+
+ lifoBlocks.freeAll();
+
+ JS::GCContext* gcx = TlsGCContext.get();
+ for (Nursery::BufferSet::Range r = buffers.all(); !r.empty();
+ r.popFront()) {
+ // Malloc memory associated with nursery objects is not tracked as these
+ // are assumed to be short lived.
+ gcx->freeUntracked(r.front());
+ }
+ } while (!lifoBlocksToFree.ref().isEmpty() ||
+ !buffersToFreeAfterMinorGC.ref().empty());
+}
+
+void GCRuntime::waitBackgroundFreeEnd() { freeTask.join(); }
+
+template <class ZoneIterT>
+IncrementalProgress GCRuntime::markWeakReferences(
+ SliceBudget& incrementalBudget) {
+ MOZ_ASSERT(!marker().isWeakMarking());
+
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::MARK_WEAK);
+
+ auto unlimited = SliceBudget::unlimited();
+ SliceBudget& budget =
+ marker().incrementalWeakMapMarkingEnabled ? incrementalBudget : unlimited;
+
+ // Ensure we don't return to the mutator while we're still in weak marking
+ // mode.
+ auto leaveOnExit =
+ mozilla::MakeScopeExit([&] { marker().leaveWeakMarkingMode(); });
+
+ if (marker().enterWeakMarkingMode()) {
+ // If there was an 'enter-weak-marking-mode' token in the queue, then it
+ // and everything after it will still be in the queue so we can process
+ // them now.
+ while (processTestMarkQueue() == QueueYielded) {
+ };
+
+ // Do not rely on the information about not-yet-marked weak keys that have
+ // been collected by barriers. Clear out the gcEphemeronEdges entries and
+ // rebuild the full table. Note that this a cross-zone operation; delegate
+ // zone entries will be populated by map zone traversals, so everything
+ // needs to be cleared first, then populated.
+ if (!marker().incrementalWeakMapMarkingEnabled) {
+ for (ZoneIterT zone(this); !zone.done(); zone.next()) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing weak keys when entering weak marking mode");
+ }
+ }
+ }
+
+ for (ZoneIterT zone(this); !zone.done(); zone.next()) {
+ if (zone->enterWeakMarkingMode(&marker(), budget) == NotFinished) {
+ return NotFinished;
+ }
+ }
+ }
+
+ bool markedAny = true;
+ while (markedAny) {
+ if (!marker().markUntilBudgetExhausted(budget)) {
+ MOZ_ASSERT(marker().incrementalWeakMapMarkingEnabled);
+ return NotFinished;
+ }
+
+ markedAny = false;
+
+ if (!marker().isWeakMarking()) {
+ for (ZoneIterT zone(this); !zone.done(); zone.next()) {
+ markedAny |= WeakMapBase::markZoneIteratively(zone, &marker());
+ }
+ }
+
+ markedAny |= jit::JitRuntime::MarkJitcodeGlobalTableIteratively(&marker());
+ }
+
+ assertNoMarkingWork();
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::markWeakReferencesInCurrentGroup(
+ SliceBudget& budget) {
+ return markWeakReferences<SweepGroupZonesIter>(budget);
+}
+
+template <class ZoneIterT>
+IncrementalProgress GCRuntime::markGrayRoots(SliceBudget& budget,
+ gcstats::PhaseKind phase) {
+ MOZ_ASSERT(marker().markColor() == MarkColor::Gray);
+
+ gcstats::AutoPhase ap(stats(), phase);
+
+ AutoUpdateLiveCompartments updateLive(this);
+ marker().setRootMarkingMode(true);
+ auto guard =
+ mozilla::MakeScopeExit([this]() { marker().setRootMarkingMode(false); });
+
+ IncrementalProgress result =
+ traceEmbeddingGrayRoots(marker().tracer(), budget);
+ if (result == NotFinished) {
+ return NotFinished;
+ }
+
+ Compartment::traceIncomingCrossCompartmentEdgesForZoneGC(
+ marker().tracer(), Compartment::GrayEdges);
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::markAllWeakReferences() {
+ SliceBudget budget = SliceBudget::unlimited();
+ return markWeakReferences<GCZonesIter>(budget);
+}
+
+void GCRuntime::markAllGrayReferences(gcstats::PhaseKind phase) {
+ SliceBudget budget = SliceBudget::unlimited();
+ markGrayRoots<GCZonesIter>(budget, phase);
+ drainMarkStack();
+}
+
+void GCRuntime::dropStringWrappers() {
+ /*
+ * String "wrappers" are dropped on GC because their presence would require
+ * us to sweep the wrappers in all compartments every time we sweep a
+ * compartment group.
+ */
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->dropStringWrappersOnGC();
+ }
+}
+
+/*
+ * Group zones that must be swept at the same time.
+ *
+ * From the point of view of the mutator, groups of zones transition atomically
+ * from marking to sweeping. If compartment A has an edge to an unmarked object
+ * in compartment B, then we must not start sweeping A in a later slice than we
+ * start sweeping B. That's because a write barrier in A could lead to the
+ * unmarked object in B becoming marked. However, if we had already swept that
+ * object, we would be in trouble.
+ *
+ * If we consider these dependencies as a graph, then all the compartments in
+ * any strongly-connected component of this graph must start sweeping in the
+ * same slice.
+ *
+ * Tarjan's algorithm is used to calculate the components.
+ */
+
+bool Compartment::findSweepGroupEdges() {
+ Zone* source = zone();
+ for (WrappedObjectCompartmentEnum e(this); !e.empty(); e.popFront()) {
+ Compartment* targetComp = e.front();
+ Zone* target = targetComp->zone();
+
+ if (!target->isGCMarking() || source->hasSweepGroupEdgeTo(target)) {
+ continue;
+ }
+
+ for (ObjectWrapperEnum e(this, targetComp); !e.empty(); e.popFront()) {
+ JSObject* key = e.front().mutableKey();
+ MOZ_ASSERT(key->zone() == target);
+
+ // Add an edge to the wrapped object's zone to ensure that the wrapper
+ // zone is not still being marked when we start sweeping the wrapped zone.
+ // As an optimization, if the wrapped object is already marked black there
+ // is no danger of later marking and we can skip this.
+ if (key->isMarkedBlack()) {
+ continue;
+ }
+
+ if (!source->addSweepGroupEdgeTo(target)) {
+ return false;
+ }
+
+ // We don't need to consider any more wrappers for this target
+ // compartment since we already added an edge.
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool Zone::findSweepGroupEdges(Zone* atomsZone) {
+ MOZ_ASSERT_IF(this != atomsZone, !isAtomsZone());
+
+#ifdef DEBUG
+ if (FinalizationObservers* observers = finalizationObservers()) {
+ observers->checkTables();
+ }
+#endif
+
+ // Any zone may have a pointer to an atom in the atoms zone, and these aren't
+ // in the cross compartment map.
+ if (atomsZone->wasGCStarted() && !addSweepGroupEdgeTo(atomsZone)) {
+ return false;
+ }
+
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) {
+ if (!comp->findSweepGroupEdges()) {
+ return false;
+ }
+ }
+
+ return WeakMapBase::findSweepGroupEdgesForZone(this);
+}
+
+bool GCRuntime::addEdgesForMarkQueue() {
+#ifdef DEBUG
+ // For testing only.
+ //
+ // Add edges between all objects mentioned in the test mark queue, since
+ // otherwise they will get marked in a different order than their sweep
+ // groups. Note that this is only done at the beginning of an incremental
+ // collection, so it is possible for objects to be added later that do not
+ // follow the sweep group ordering. These objects will wait until their sweep
+ // group comes up, or will be skipped if their sweep group is already past.
+ JS::Zone* prevZone = nullptr;
+ for (size_t i = 0; i < testMarkQueue.length(); i++) {
+ Value val = testMarkQueue[i].get();
+ if (!val.isObject()) {
+ continue;
+ }
+ JSObject* obj = &val.toObject();
+ JS::Zone* zone = obj->zone();
+ if (!zone->isGCMarking()) {
+ continue;
+ }
+ if (prevZone && prevZone != zone) {
+ if (!prevZone->addSweepGroupEdgeTo(zone)) {
+ return false;
+ }
+ }
+ prevZone = zone;
+ }
+#endif
+ return true;
+}
+
+bool GCRuntime::findSweepGroupEdges() {
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ if (!zone->findSweepGroupEdges(atomsZone())) {
+ return false;
+ }
+ }
+
+ if (!addEdgesForMarkQueue()) {
+ return false;
+ }
+
+ return DebugAPI::findSweepGroupEdges(rt);
+}
+
+void GCRuntime::groupZonesForSweeping(JS::GCReason reason) {
+#ifdef DEBUG
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->gcSweepGroupEdges().empty());
+ }
+#endif
+
+ JSContext* cx = rt->mainContextFromOwnThread();
+ ZoneComponentFinder finder(cx);
+ if (!isIncremental || !findSweepGroupEdges()) {
+ finder.useOneComponent();
+ }
+
+ // Use one component for two-slice zeal modes.
+ if (useZeal && hasIncrementalTwoSliceZealMode()) {
+ finder.useOneComponent();
+ }
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarking());
+ finder.addNode(zone);
+ }
+ sweepGroups = finder.getResultsList();
+ currentSweepGroup = sweepGroups;
+ sweepGroupIndex = 1;
+
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->clearSweepGroupEdges();
+ }
+
+#ifdef DEBUG
+ unsigned idx = sweepGroupIndex;
+ for (Zone* head = currentSweepGroup; head; head = head->nextGroup()) {
+ for (Zone* zone = head; zone; zone = zone->nextNodeInGroup()) {
+ MOZ_ASSERT(zone->isGCMarking());
+ zone->gcSweepGroupIndex = idx;
+ }
+ idx++;
+ }
+
+ MOZ_ASSERT_IF(!isIncremental, !currentSweepGroup->nextGroup());
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->gcSweepGroupEdges().empty());
+ }
+#endif
+}
+
+void GCRuntime::getNextSweepGroup() {
+ currentSweepGroup = currentSweepGroup->nextGroup();
+ ++sweepGroupIndex;
+ if (!currentSweepGroup) {
+ abortSweepAfterCurrentGroup = false;
+ return;
+ }
+
+ MOZ_ASSERT_IF(abortSweepAfterCurrentGroup, !isIncremental);
+ if (!isIncremental) {
+ ZoneComponentFinder::mergeGroups(currentSweepGroup);
+ }
+
+ for (Zone* zone = currentSweepGroup; zone; zone = zone->nextNodeInGroup()) {
+ MOZ_ASSERT(zone->gcState() == zone->initialMarkingState());
+ MOZ_ASSERT(!zone->isQueuedForBackgroundSweep());
+ }
+
+ if (abortSweepAfterCurrentGroup) {
+ markTask.join();
+
+ // Abort collection of subsequent sweep groups.
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ MOZ_ASSERT(!zone->gcNextGraphComponent);
+ zone->changeGCState(zone->initialMarkingState(), Zone::NoGC);
+ zone->arenas.unmarkPreMarkedFreeCells();
+ zone->arenas.mergeArenasFromCollectingLists();
+ zone->clearGCSliceThresholds();
+ }
+
+ for (SweepGroupCompartmentsIter comp(rt); !comp.done(); comp.next()) {
+ resetGrayList(comp);
+ }
+
+ abortSweepAfterCurrentGroup = false;
+ currentSweepGroup = nullptr;
+ }
+}
+
+/*
+ * Gray marking:
+ *
+ * At the end of collection, anything reachable from a gray root that has not
+ * otherwise been marked black must be marked gray.
+ *
+ * This means that when marking things gray we must not allow marking to leave
+ * the current compartment group, as that could result in things being marked
+ * gray when they might subsequently be marked black. To achieve this, when we
+ * find a cross compartment pointer we don't mark the referent but add it to a
+ * singly-linked list of incoming gray pointers that is stored with each
+ * compartment.
+ *
+ * The list head is stored in Compartment::gcIncomingGrayPointers and contains
+ * cross compartment wrapper objects. The next pointer is stored in the second
+ * extra slot of the cross compartment wrapper.
+ *
+ * The list is created during gray marking when one of the
+ * MarkCrossCompartmentXXX functions is called for a pointer that leaves the
+ * current compartent group. This calls DelayCrossCompartmentGrayMarking to
+ * push the referring object onto the list.
+ *
+ * The list is traversed and then unlinked in
+ * GCRuntime::markIncomingGrayCrossCompartmentPointers.
+ */
+
+static bool IsGrayListObject(JSObject* obj) {
+ MOZ_ASSERT(obj);
+ return obj->is<CrossCompartmentWrapperObject>() && !IsDeadProxyObject(obj);
+}
+
+/* static */
+unsigned ProxyObject::grayLinkReservedSlot(JSObject* obj) {
+ MOZ_ASSERT(IsGrayListObject(obj));
+ return CrossCompartmentWrapperObject::GrayLinkReservedSlot;
+}
+
+#ifdef DEBUG
+static void AssertNotOnGrayList(JSObject* obj) {
+ MOZ_ASSERT_IF(
+ IsGrayListObject(obj),
+ GetProxyReservedSlot(obj, ProxyObject::grayLinkReservedSlot(obj))
+ .isUndefined());
+}
+#endif
+
+static void AssertNoWrappersInGrayList(JSRuntime* rt) {
+#ifdef DEBUG
+ for (CompartmentsIter c(rt); !c.done(); c.next()) {
+ MOZ_ASSERT(!c->gcIncomingGrayPointers);
+ for (Compartment::ObjectWrapperEnum e(c); !e.empty(); e.popFront()) {
+ AssertNotOnGrayList(e.front().value().unbarrieredGet());
+ }
+ }
+#endif
+}
+
+static JSObject* CrossCompartmentPointerReferent(JSObject* obj) {
+ MOZ_ASSERT(IsGrayListObject(obj));
+ return &obj->as<ProxyObject>().private_().toObject();
+}
+
+static JSObject* NextIncomingCrossCompartmentPointer(JSObject* prev,
+ bool unlink) {
+ unsigned slot = ProxyObject::grayLinkReservedSlot(prev);
+ JSObject* next = GetProxyReservedSlot(prev, slot).toObjectOrNull();
+ MOZ_ASSERT_IF(next, IsGrayListObject(next));
+
+ if (unlink) {
+ SetProxyReservedSlot(prev, slot, UndefinedValue());
+ }
+
+ return next;
+}
+
+void js::gc::DelayCrossCompartmentGrayMarking(GCMarker* maybeMarker,
+ JSObject* src) {
+ MOZ_ASSERT_IF(!maybeMarker, !JS::RuntimeHeapIsBusy());
+ MOZ_ASSERT(IsGrayListObject(src));
+ MOZ_ASSERT(src->isMarkedGray());
+
+ AutoTouchingGrayThings tgt;
+
+ mozilla::Maybe<AutoLockGC> lock;
+ if (maybeMarker && maybeMarker->isParallelMarking()) {
+ // Synchronize access to JSCompartment::gcIncomingGrayPointers.
+ //
+ // TODO: Instead of building this list we could scan all incoming CCWs and
+ // mark through gray ones when marking gray roots for a sweep group.
+ lock.emplace(maybeMarker->runtime());
+ }
+
+ /* Called from MarkCrossCompartmentXXX functions. */
+ unsigned slot = ProxyObject::grayLinkReservedSlot(src);
+ JSObject* dest = CrossCompartmentPointerReferent(src);
+ Compartment* comp = dest->compartment();
+
+ if (GetProxyReservedSlot(src, slot).isUndefined()) {
+ SetProxyReservedSlot(src, slot,
+ ObjectOrNullValue(comp->gcIncomingGrayPointers));
+ comp->gcIncomingGrayPointers = src;
+ } else {
+ MOZ_ASSERT(GetProxyReservedSlot(src, slot).isObjectOrNull());
+ }
+
+#ifdef DEBUG
+ /*
+ * Assert that the object is in our list, also walking the list to check its
+ * integrity.
+ */
+ JSObject* obj = comp->gcIncomingGrayPointers;
+ bool found = false;
+ while (obj) {
+ if (obj == src) {
+ found = true;
+ }
+ obj = NextIncomingCrossCompartmentPointer(obj, false);
+ }
+ MOZ_ASSERT(found);
+#endif
+}
+
+void GCRuntime::markIncomingGrayCrossCompartmentPointers() {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_INCOMING_GRAY);
+
+ for (SweepGroupCompartmentsIter c(rt); !c.done(); c.next()) {
+ MOZ_ASSERT(c->zone()->isGCMarkingBlackAndGray());
+ MOZ_ASSERT_IF(c->gcIncomingGrayPointers,
+ IsGrayListObject(c->gcIncomingGrayPointers));
+
+ for (JSObject* src = c->gcIncomingGrayPointers; src;
+ src = NextIncomingCrossCompartmentPointer(src, true)) {
+ JSObject* dst = CrossCompartmentPointerReferent(src);
+ MOZ_ASSERT(dst->compartment() == c);
+ MOZ_ASSERT_IF(src->asTenured().isMarkedBlack(),
+ dst->asTenured().isMarkedBlack());
+
+ if (src->asTenured().isMarkedGray()) {
+ TraceManuallyBarrieredEdge(marker().tracer(), &dst,
+ "cross-compartment gray pointer");
+ }
+ }
+
+ c->gcIncomingGrayPointers = nullptr;
+ }
+}
+
+static bool RemoveFromGrayList(JSObject* wrapper) {
+ AutoTouchingGrayThings tgt;
+
+ if (!IsGrayListObject(wrapper)) {
+ return false;
+ }
+
+ unsigned slot = ProxyObject::grayLinkReservedSlot(wrapper);
+ if (GetProxyReservedSlot(wrapper, slot).isUndefined()) {
+ return false; /* Not on our list. */
+ }
+
+ JSObject* tail = GetProxyReservedSlot(wrapper, slot).toObjectOrNull();
+ SetProxyReservedSlot(wrapper, slot, UndefinedValue());
+
+ Compartment* comp = CrossCompartmentPointerReferent(wrapper)->compartment();
+ JSObject* obj = comp->gcIncomingGrayPointers;
+ if (obj == wrapper) {
+ comp->gcIncomingGrayPointers = tail;
+ return true;
+ }
+
+ while (obj) {
+ unsigned slot = ProxyObject::grayLinkReservedSlot(obj);
+ JSObject* next = GetProxyReservedSlot(obj, slot).toObjectOrNull();
+ if (next == wrapper) {
+ js::detail::SetProxyReservedSlotUnchecked(obj, slot,
+ ObjectOrNullValue(tail));
+ return true;
+ }
+ obj = next;
+ }
+
+ MOZ_CRASH("object not found in gray link list");
+}
+
+void GCRuntime::resetGrayList(Compartment* comp) {
+ JSObject* src = comp->gcIncomingGrayPointers;
+ while (src) {
+ src = NextIncomingCrossCompartmentPointer(src, true);
+ }
+ comp->gcIncomingGrayPointers = nullptr;
+}
+
+#ifdef DEBUG
+static bool HasIncomingCrossCompartmentPointers(JSRuntime* rt) {
+ for (SweepGroupCompartmentsIter c(rt); !c.done(); c.next()) {
+ if (c->gcIncomingGrayPointers) {
+ return true;
+ }
+ }
+
+ return false;
+}
+#endif
+
+void js::NotifyGCNukeWrapper(JSContext* cx, JSObject* wrapper) {
+ MOZ_ASSERT(IsCrossCompartmentWrapper(wrapper));
+
+ /*
+ * References to target of wrapper are being removed, we no longer have to
+ * remember to mark it.
+ */
+ RemoveFromGrayList(wrapper);
+
+ /*
+ * Clean up WeakRef maps which might include this wrapper.
+ */
+ JSObject* target = UncheckedUnwrapWithoutExpose(wrapper);
+ if (target->is<WeakRefObject>()) {
+ WeakRefObject* weakRef = &target->as<WeakRefObject>();
+ if (weakRef->target()) {
+ cx->runtime()->gc.nukeWeakRefWrapper(wrapper, weakRef);
+ }
+ }
+
+ /*
+ * Clean up FinalizationRecord record objects which might be the target of
+ * this wrapper.
+ */
+ if (target->is<FinalizationRecordObject>()) {
+ auto* record = &target->as<FinalizationRecordObject>();
+ cx->runtime()->gc.nukeFinalizationRecordWrapper(wrapper, record);
+ }
+}
+
+enum {
+ JS_GC_SWAP_OBJECT_A_REMOVED = 1 << 0,
+ JS_GC_SWAP_OBJECT_B_REMOVED = 1 << 1
+};
+
+unsigned js::NotifyGCPreSwap(JSObject* a, JSObject* b) {
+ /*
+ * Two objects in the same compartment are about to have had their contents
+ * swapped. If either of them are in our gray pointer list, then we remove
+ * them from the lists, returning a bitset indicating what happened.
+ */
+ return (RemoveFromGrayList(a) ? JS_GC_SWAP_OBJECT_A_REMOVED : 0) |
+ (RemoveFromGrayList(b) ? JS_GC_SWAP_OBJECT_B_REMOVED : 0);
+}
+
+void js::NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned removedFlags) {
+ /*
+ * Two objects in the same compartment have had their contents swapped. If
+ * either of them were in our gray pointer list, we re-add them again.
+ */
+ if (removedFlags & JS_GC_SWAP_OBJECT_A_REMOVED) {
+ DelayCrossCompartmentGrayMarking(nullptr, b);
+ }
+ if (removedFlags & JS_GC_SWAP_OBJECT_B_REMOVED) {
+ DelayCrossCompartmentGrayMarking(nullptr, a);
+ }
+}
+
+static inline void MaybeCheckWeakMapMarking(GCRuntime* gc) {
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+
+ bool shouldCheck;
+# if defined(DEBUG)
+ shouldCheck = true;
+# else
+ shouldCheck = gc->hasZealMode(ZealMode::CheckWeakMapMarking);
+# endif
+
+ if (shouldCheck) {
+ for (SweepGroupZonesIter zone(gc); !zone.done(); zone.next()) {
+ MOZ_RELEASE_ASSERT(WeakMapBase::checkMarkingForZone(zone));
+ }
+ }
+
+#endif
+}
+
+IncrementalProgress GCRuntime::beginMarkingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget) {
+#ifdef DEBUG
+ MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping);
+ assertNoMarkingWork();
+ for (auto& marker : markers) {
+ MOZ_ASSERT(marker->markColor() == MarkColor::Black);
+ }
+ MOZ_ASSERT(cellsToAssertNotGray.ref().empty());
+#endif
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+
+ // Change state of current group to MarkBlackAndGray to restrict gray marking
+ // to this group. Note that there may be pointers to the atoms zone, and these
+ // will be marked through, as they are not marked with
+ // TraceCrossCompartmentEdge.
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->changeGCState(zone->initialMarkingState(), Zone::MarkBlackAndGray);
+ }
+
+ AutoSetMarkColor setColorGray(marker(), MarkColor::Gray);
+
+ // Mark incoming gray pointers from previously swept compartments.
+ markIncomingGrayCrossCompartmentPointers();
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::markGrayRootsInCurrentGroup(
+ JS::GCContext* gcx, SliceBudget& budget) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+
+ AutoSetMarkColor setColorGray(marker(), MarkColor::Gray);
+
+ return markGrayRoots<SweepGroupZonesIter>(budget,
+ gcstats::PhaseKind::MARK_GRAY);
+}
+
+IncrementalProgress GCRuntime::markGray(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+
+ if (markUntilBudgetExhausted(budget, AllowParallelMarking) == NotFinished) {
+ return NotFinished;
+ }
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::endMarkingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget) {
+#ifdef DEBUG
+ MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping);
+ assertNoMarkingWork();
+ for (auto& marker : markers) {
+ MOZ_ASSERT(marker->markColor() == MarkColor::Black);
+ }
+ MOZ_ASSERT(!HasIncomingCrossCompartmentPointers(rt));
+#endif
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+
+ if (markWeakReferencesInCurrentGroup(budget) == NotFinished) {
+ return NotFinished;
+ }
+
+ AutoSetMarkColor setColorGray(marker(), MarkColor::Gray);
+
+ // Mark transitively inside the current compartment group.
+ if (markWeakReferencesInCurrentGroup(budget) == NotFinished) {
+ return NotFinished;
+ }
+
+ MOZ_ASSERT(marker().isDrained());
+
+ // We must not yield after this point before we start sweeping the group.
+ safeToYield = false;
+
+ MaybeCheckWeakMapMarking(this);
+
+ return Finished;
+}
+
+// Causes the given WeakCache to be swept when run.
+class ImmediateSweepWeakCacheTask : public GCParallelTask {
+ Zone* zone;
+ JS::detail::WeakCacheBase& cache;
+
+ ImmediateSweepWeakCacheTask(const ImmediateSweepWeakCacheTask&) = delete;
+
+ public:
+ ImmediateSweepWeakCacheTask(GCRuntime* gc, Zone* zone,
+ JS::detail::WeakCacheBase& wc)
+ : GCParallelTask(gc, gcstats::PhaseKind::SWEEP_WEAK_CACHES),
+ zone(zone),
+ cache(wc) {}
+
+ ImmediateSweepWeakCacheTask(ImmediateSweepWeakCacheTask&& other)
+ : GCParallelTask(std::move(other)),
+ zone(other.zone),
+ cache(other.cache) {}
+
+ void run(AutoLockHelperThreadState& lock) override {
+ AutoUnlockHelperThreadState unlock(lock);
+ AutoSetThreadIsSweeping threadIsSweeping(zone);
+ SweepingTracer trc(gc->rt);
+ cache.traceWeak(&trc, &gc->storeBuffer());
+ }
+};
+
+void GCRuntime::updateAtomsBitmap() {
+ DenseBitmap marked;
+ if (atomMarking.computeBitmapFromChunkMarkBits(rt, marked)) {
+ for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+ atomMarking.refineZoneBitmapForCollectedZone(zone, marked);
+ }
+ } else {
+ // Ignore OOM in computeBitmapFromChunkMarkBits. The
+ // refineZoneBitmapForCollectedZone call can only remove atoms from the
+ // zone bitmap, so it is conservative to just not call it.
+ }
+
+ atomMarking.markAtomsUsedByUncollectedZones(rt);
+
+ // For convenience sweep these tables non-incrementally as part of bitmap
+ // sweeping; they are likely to be much smaller than the main atoms table.
+ SweepingTracer trc(rt);
+ rt->symbolRegistry().traceWeak(&trc);
+}
+
+void GCRuntime::sweepCCWrappers() {
+ SweepingTracer trc(rt);
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->traceWeakCCWEdges(&trc);
+ }
+}
+
+void GCRuntime::sweepRealmGlobals() {
+ SweepingTracer trc(rt);
+ for (SweepGroupRealmsIter r(this); !r.done(); r.next()) {
+ AutoSetThreadIsSweeping threadIsSweeping(r->zone());
+ r->traceWeakGlobalEdge(&trc);
+ }
+}
+
+void GCRuntime::sweepMisc() {
+ SweepingTracer trc(rt);
+ for (SweepGroupRealmsIter r(this); !r.done(); r.next()) {
+ AutoSetThreadIsSweeping threadIsSweeping(r->zone());
+ r->traceWeakSavedStacks(&trc);
+ r->traceWeakRegExps(&trc);
+ }
+ for (SweepGroupCompartmentsIter c(this); !c.done(); c.next()) {
+ AutoSetThreadIsSweeping threadIsSweeping(c->zone());
+ c->traceWeakNativeIterators(&trc);
+ }
+}
+
+void GCRuntime::sweepCompressionTasks() {
+ JSRuntime* runtime = rt;
+
+ // Attach finished compression tasks.
+ AutoLockHelperThreadState lock;
+ AttachFinishedCompressions(runtime, lock);
+ SweepPendingCompressions(lock);
+}
+
+void GCRuntime::sweepWeakMaps() {
+ SweepingTracer trc(rt);
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ /* No need to look up any more weakmap keys from this sweep group. */
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing weak keys in beginSweepingSweepGroup()");
+ }
+
+ // Lock the storebuffer since this may access it when rehashing or resizing
+ // the tables.
+ AutoLockStoreBuffer lock(&storeBuffer());
+ zone->sweepWeakMaps(&trc);
+ }
+}
+
+void GCRuntime::sweepUniqueIds() {
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ AutoSetThreadIsSweeping threadIsSweeping(zone);
+ zone->sweepUniqueIds();
+ }
+}
+
+void JS::Zone::sweepUniqueIds() {
+ SweepingTracer trc(runtimeFromAnyThread());
+ uniqueIds().traceWeak(&trc);
+}
+
+/* static */
+bool UniqueIdGCPolicy::traceWeak(JSTracer* trc, Cell** keyp, uint64_t* valuep) {
+ // Since this is only ever used for sweeping, we can optimize it for that
+ // case. (Compacting GC updates this table manually when it moves a cell.)
+ MOZ_ASSERT(trc->kind() == JS::TracerKind::Sweeping);
+ return (*keyp)->isMarkedAny();
+}
+
+void GCRuntime::sweepFinalizationObserversOnMainThread() {
+ // This calls back into the browser which expects to be called from the main
+ // thread.
+ gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS);
+ gcstats::AutoPhase ap2(stats(),
+ gcstats::PhaseKind::SWEEP_FINALIZATION_OBSERVERS);
+ SweepingTracer trc(rt);
+ AutoLockStoreBuffer lock(&storeBuffer());
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ traceWeakFinalizationObserverEdges(&trc, zone);
+ }
+}
+
+void GCRuntime::startTask(GCParallelTask& task,
+ AutoLockHelperThreadState& lock) {
+ if (!CanUseExtraThreads()) {
+ AutoUnlockHelperThreadState unlock(lock);
+ task.runFromMainThread();
+ stats().recordParallelPhase(task.phaseKind, task.duration());
+ return;
+ }
+
+ task.startWithLockHeld(lock);
+}
+
+void GCRuntime::joinTask(GCParallelTask& task,
+ AutoLockHelperThreadState& lock) {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::JOIN_PARALLEL_TASKS);
+ task.joinWithLockHeld(lock);
+}
+
+void GCRuntime::sweepDebuggerOnMainThread(JS::GCContext* gcx) {
+ SweepingTracer trc(rt);
+ AutoLockStoreBuffer lock(&storeBuffer());
+
+ // Detach unreachable debuggers and global objects from each other.
+ // This can modify weakmaps and so must happen before weakmap sweeping.
+ DebugAPI::sweepAll(gcx);
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS);
+
+ // Sweep debug environment information. This performs lookups in the Zone's
+ // unique IDs table and so must not happen in parallel with sweeping that
+ // table.
+ {
+ gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::SWEEP_MISC);
+ for (SweepGroupRealmsIter r(rt); !r.done(); r.next()) {
+ r->traceWeakDebugEnvironmentEdges(&trc);
+ }
+ }
+}
+
+void GCRuntime::sweepJitDataOnMainThread(JS::GCContext* gcx) {
+ SweepingTracer trc(rt);
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_JIT_DATA);
+
+ if (initialState != State::NotActive) {
+ // Cancel any active or pending off thread compilations. We also did
+ // this before marking (in DiscardJITCodeForGC) so this is a no-op
+ // for non-incremental GCs.
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Sweep);
+ }
+
+ // Bug 1071218: the following method has not yet been refactored to
+ // work on a single zone-group at once.
+
+ // Sweep entries containing about-to-be-finalized JitCode and
+ // update relocated TypeSet::Types inside the JitcodeGlobalTable.
+ jit::JitRuntime::TraceWeakJitcodeGlobalTable(rt, &trc);
+ }
+
+ if (initialState != State::NotActive) {
+ gcstats::AutoPhase apdc(stats(), gcstats::PhaseKind::SWEEP_DISCARD_CODE);
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ zone->discardJitCode(gcx);
+ }
+ }
+
+ // JitZone/JitRealm must be swept *after* discarding JIT code, because
+ // Zone::discardJitCode might access CacheIRStubInfos deleted here.
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_JIT_DATA);
+
+ for (SweepGroupRealmsIter r(rt); !r.done(); r.next()) {
+ r->traceWeakEdgesInJitRealm(&trc);
+ }
+
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ if (jit::JitZone* jitZone = zone->jitZone()) {
+ jitZone->traceWeak(&trc);
+ }
+ }
+ }
+}
+
+using WeakCacheTaskVector =
+ mozilla::Vector<ImmediateSweepWeakCacheTask, 0, SystemAllocPolicy>;
+
+// Call a functor for all weak caches that need to be swept in the current
+// sweep group.
+template <typename Functor>
+static inline bool IterateWeakCaches(JSRuntime* rt, Functor f) {
+ for (SweepGroupZonesIter zone(rt); !zone.done(); zone.next()) {
+ for (JS::detail::WeakCacheBase* cache : zone->weakCaches()) {
+ if (!f(cache, zone.get())) {
+ return false;
+ }
+ }
+ }
+
+ for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) {
+ if (!f(cache, nullptr)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool PrepareWeakCacheTasks(JSRuntime* rt,
+ WeakCacheTaskVector* immediateTasks) {
+ // Start incremental sweeping for caches that support it or add to a vector
+ // of sweep tasks to run on a helper thread.
+
+ MOZ_ASSERT(immediateTasks->empty());
+
+ GCRuntime* gc = &rt->gc;
+ bool ok =
+ IterateWeakCaches(rt, [&](JS::detail::WeakCacheBase* cache, Zone* zone) {
+ if (cache->empty()) {
+ return true;
+ }
+
+ // Caches that support incremental sweeping will be swept later.
+ if (zone && cache->setIncrementalBarrierTracer(&gc->sweepingTracer)) {
+ return true;
+ }
+
+ return immediateTasks->emplaceBack(gc, zone, *cache);
+ });
+
+ if (!ok) {
+ immediateTasks->clearAndFree();
+ }
+
+ return ok;
+}
+
+static void SweepAllWeakCachesOnMainThread(JSRuntime* rt) {
+ // If we ran out of memory, do all the work on the main thread.
+ gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::SWEEP_WEAK_CACHES);
+ SweepingTracer trc(rt);
+ IterateWeakCaches(rt, [&](JS::detail::WeakCacheBase* cache, Zone* zone) {
+ if (cache->needsIncrementalBarrier()) {
+ cache->setIncrementalBarrierTracer(nullptr);
+ }
+ cache->traceWeak(&trc, &rt->gc.storeBuffer());
+ return true;
+ });
+}
+
+void GCRuntime::sweepEmbeddingWeakPointers(JS::GCContext* gcx) {
+ using namespace gcstats;
+
+ AutoLockStoreBuffer lock(&storeBuffer());
+
+ AutoPhase ap(stats(), PhaseKind::FINALIZE_START);
+ callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_PREPARE);
+ {
+ AutoPhase ap2(stats(), PhaseKind::WEAK_ZONES_CALLBACK);
+ callWeakPointerZonesCallbacks(&sweepingTracer);
+ }
+ {
+ AutoPhase ap2(stats(), PhaseKind::WEAK_COMPARTMENT_CALLBACK);
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ callWeakPointerCompartmentCallbacks(&sweepingTracer, comp);
+ }
+ }
+ }
+ callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_START);
+}
+
+IncrementalProgress GCRuntime::beginSweepingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ /*
+ * Begin sweeping the group of zones in currentSweepGroup, performing
+ * actions that must be done before yielding to caller.
+ */
+
+ using namespace gcstats;
+
+ AutoSCC scc(stats(), sweepGroupIndex);
+
+ bool sweepingAtoms = false;
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ /* Set the GC state to sweeping. */
+ zone->changeGCState(Zone::MarkBlackAndGray, Zone::Sweep);
+
+ /* Purge the ArenaLists before sweeping. */
+ zone->arenas.checkSweepStateNotInUse();
+ zone->arenas.unmarkPreMarkedFreeCells();
+ zone->arenas.clearFreeLists();
+
+ if (zone->isAtomsZone()) {
+ sweepingAtoms = true;
+ }
+ }
+
+#ifdef DEBUG
+ for (auto cell : cellsToAssertNotGray.ref()) {
+ JS::AssertCellIsNotGray(cell);
+ }
+ cellsToAssertNotGray.ref().clearAndFree();
+#endif
+
+ // Updating the atom marking bitmaps. This marks atoms referenced by
+ // uncollected zones so cannot be done in parallel with the other sweeping
+ // work below.
+ if (sweepingAtoms) {
+ AutoPhase ap(stats(), PhaseKind::UPDATE_ATOMS_BITMAP);
+ updateAtomsBitmap();
+ }
+
+#ifdef JS_GC_ZEAL
+ validateIncrementalMarking();
+#endif
+
+ AutoSetThreadIsSweeping threadIsSweeping;
+
+ // This must happen before sweeping realm globals.
+ sweepDebuggerOnMainThread(gcx);
+
+ // FinalizationRegistry sweeping touches weak maps and so must not run in
+ // parallel with that. This triggers a read barrier and can add marking work
+ // for zones that are still marking. Must happen before sweeping realm
+ // globals.
+ sweepFinalizationObserversOnMainThread();
+
+ // This must happen before updating embedding weak pointers.
+ sweepRealmGlobals();
+
+ sweepEmbeddingWeakPointers(gcx);
+
+ {
+ AutoLockHelperThreadState lock;
+
+ AutoPhase ap(stats(), PhaseKind::SWEEP_COMPARTMENTS);
+
+ AutoRunParallelTask sweepCCWrappers(this, &GCRuntime::sweepCCWrappers,
+ PhaseKind::SWEEP_CC_WRAPPER,
+ GCUse::Sweeping, lock);
+ AutoRunParallelTask sweepMisc(this, &GCRuntime::sweepMisc,
+ PhaseKind::SWEEP_MISC, GCUse::Sweeping, lock);
+ AutoRunParallelTask sweepCompTasks(this, &GCRuntime::sweepCompressionTasks,
+ PhaseKind::SWEEP_COMPRESSION,
+ GCUse::Sweeping, lock);
+ AutoRunParallelTask sweepWeakMaps(this, &GCRuntime::sweepWeakMaps,
+ PhaseKind::SWEEP_WEAKMAPS,
+ GCUse::Sweeping, lock);
+ AutoRunParallelTask sweepUniqueIds(this, &GCRuntime::sweepUniqueIds,
+ PhaseKind::SWEEP_UNIQUEIDS,
+ GCUse::Sweeping, lock);
+
+ WeakCacheTaskVector sweepCacheTasks;
+ bool canSweepWeakCachesOffThread =
+ PrepareWeakCacheTasks(rt, &sweepCacheTasks);
+ if (canSweepWeakCachesOffThread) {
+ weakCachesToSweep.ref().emplace(currentSweepGroup);
+ for (auto& task : sweepCacheTasks) {
+ startTask(task, lock);
+ }
+ }
+
+ {
+ AutoUnlockHelperThreadState unlock(lock);
+ sweepJitDataOnMainThread(gcx);
+
+ if (!canSweepWeakCachesOffThread) {
+ MOZ_ASSERT(sweepCacheTasks.empty());
+ SweepAllWeakCachesOnMainThread(rt);
+ }
+ }
+
+ for (auto& task : sweepCacheTasks) {
+ joinTask(task, lock);
+ }
+ }
+
+ if (sweepingAtoms) {
+ startSweepingAtomsTable();
+ }
+
+ // Queue all GC things in all zones for sweeping, either on the foreground
+ // or on the background thread.
+
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ for (const auto& phase : BackgroundFinalizePhases) {
+ initBackgroundSweep(zone, gcx, phase);
+ }
+
+ zone->arenas.queueForegroundThingsForSweep();
+ }
+
+ MOZ_ASSERT(!sweepZone);
+
+ safeToYield = true;
+ markOnBackgroundThreadDuringSweeping = CanUseExtraThreads();
+
+ return Finished;
+}
+
+#ifdef JS_GC_ZEAL
+bool GCRuntime::shouldYieldForZeal(ZealMode mode) {
+ bool yield = useZeal && hasZealMode(mode);
+
+ // Only yield on the first sweep slice for this mode.
+ bool firstSweepSlice = initialState != State::Sweep;
+ if (mode == ZealMode::IncrementalMultipleSlices && !firstSweepSlice) {
+ yield = false;
+ }
+
+ return yield;
+}
+#endif
+
+IncrementalProgress GCRuntime::endSweepingSweepGroup(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ // This is to prevent a race between markTask checking the zone state and
+ // us changing it below.
+ if (joinBackgroundMarkTask() == NotFinished) {
+ return NotFinished;
+ }
+
+ assertNoMarkingWork();
+
+ // Disable background marking during sweeping until we start sweeping the next
+ // zone group.
+ markOnBackgroundThreadDuringSweeping = false;
+
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END);
+ AutoLockStoreBuffer lock(&storeBuffer());
+ callFinalizeCallbacks(gcx, JSFINALIZE_GROUP_END);
+ }
+
+ /* Free LIFO blocks on a background thread if possible. */
+ startBackgroundFree();
+
+ /* Update the GC state for zones we have swept. */
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ if (jit::JitZone* jitZone = zone->jitZone()) {
+ // Clear out any small pools that we're hanging on to.
+ jitZone->execAlloc().purge();
+ }
+ AutoLockGC lock(this);
+ zone->changeGCState(Zone::Sweep, Zone::Finished);
+ zone->arenas.unmarkPreMarkedFreeCells();
+ zone->arenas.checkNoArenasToUpdate();
+ zone->pretenuring.clearCellCountsInNewlyCreatedArenas();
+ }
+
+ /*
+ * Start background thread to sweep zones if required, sweeping any atoms
+ * zones last if present.
+ */
+ ZoneList zones;
+ for (SweepGroupZonesIter zone(this); !zone.done(); zone.next()) {
+ if (zone->isAtomsZone()) {
+ zones.append(zone);
+ } else {
+ zones.prepend(zone);
+ }
+ }
+
+ queueZonesAndStartBackgroundSweep(std::move(zones));
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::markDuringSweeping(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ MOZ_ASSERT(markTask.isIdle());
+
+ if (markOnBackgroundThreadDuringSweeping) {
+ if (!marker().isDrained() || hasDelayedMarking()) {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(markTask.isIdle(lock));
+ markTask.setBudget(budget);
+ markTask.startOrRunIfIdle(lock);
+ }
+ return Finished; // This means don't yield to the mutator here.
+ }
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK);
+ return markUntilBudgetExhausted(budget, AllowParallelMarking);
+}
+
+void GCRuntime::beginSweepPhase(JS::GCReason reason, AutoGCSession& session) {
+ /*
+ * Sweep phase.
+ *
+ * Finalize as we sweep, outside of lock but with RuntimeHeapIsBusy()
+ * true so that any attempt to allocate a GC-thing from a finalizer will
+ * fail, rather than nest badly and leave the unmarked newborn to be swept.
+ */
+
+ MOZ_ASSERT(!abortSweepAfterCurrentGroup);
+ MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping);
+
+#ifdef DEBUG
+ releaseHeldRelocatedArenas();
+ verifyAllChunks();
+#endif
+
+#ifdef JS_GC_ZEAL
+ computeNonIncrementalMarkingForValidation(session);
+#endif
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
+
+ AssertNoWrappersInGrayList(rt);
+ dropStringWrappers();
+
+ groupZonesForSweeping(reason);
+
+ sweepActions->assertFinished();
+}
+
+bool GCRuntime::foregroundFinalize(JS::GCContext* gcx, Zone* zone,
+ AllocKind thingKind,
+ SliceBudget& sliceBudget,
+ SortedArenaList& sweepList) {
+ ArenaLists& lists = zone->arenas;
+ lists.checkNoArenasToUpdateForKind(thingKind);
+
+ // Non-empty arenas are reused for use for new allocations as soon as the
+ // finalizers for that allocation kind have run. Empty arenas are only
+ // released when everything in the zone has been swept (see
+ // GCRuntime::sweepBackgroundThings for more details).
+ if (!FinalizeArenas(gcx, lists.collectingArenaList(thingKind), sweepList,
+ thingKind, sliceBudget)) {
+ // Copy the current contents of sweepList so that ArenaIter can find them.
+ lists.setIncrementalSweptArenas(thingKind, sweepList);
+ return false;
+ }
+
+ sweepList.extractEmpty(&lists.savedEmptyArenas.ref());
+ lists.mergeFinalizedArenas(thingKind, sweepList);
+ lists.clearIncrementalSweptArenas();
+
+ return true;
+}
+
+BackgroundMarkTask::BackgroundMarkTask(GCRuntime* gc)
+ : GCParallelTask(gc, gcstats::PhaseKind::MARK, GCUse::Marking),
+ budget(SliceBudget::unlimited()) {}
+
+void js::gc::BackgroundMarkTask::run(AutoLockHelperThreadState& lock) {
+ AutoUnlockHelperThreadState unlock(lock);
+
+ // Time reporting is handled separately for parallel tasks.
+ gc->sweepMarkResult = gc->markUntilBudgetExhausted(
+ this->budget, GCRuntime::SingleThreadedMarking, DontReportMarkTime);
+}
+
+IncrementalProgress GCRuntime::joinBackgroundMarkTask() {
+ AutoLockHelperThreadState lock;
+ if (markTask.isIdle(lock)) {
+ return Finished;
+ }
+
+ joinTask(markTask, lock);
+
+ IncrementalProgress result = sweepMarkResult;
+ sweepMarkResult = Finished;
+ return result;
+}
+
+template <typename T>
+static void SweepThing(JS::GCContext* gcx, T* thing) {
+ if (!TenuredThingIsMarkedAny(thing)) {
+ thing->sweep(gcx);
+ }
+}
+
+template <typename T>
+static bool SweepArenaList(JS::GCContext* gcx, Arena** arenasToSweep,
+ SliceBudget& sliceBudget) {
+ while (Arena* arena = *arenasToSweep) {
+ MOZ_ASSERT(arena->zone->isGCSweeping());
+
+ for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+ SweepThing(gcx, cell.as<T>());
+ }
+
+ Arena* next = arena->next;
+ MOZ_ASSERT_IF(next, next->zone == arena->zone);
+ *arenasToSweep = next;
+
+ AllocKind kind = MapTypeToAllocKind<T>::kind;
+ sliceBudget.step(Arena::thingsPerArena(kind));
+ if (sliceBudget.isOverBudget()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void GCRuntime::startSweepingAtomsTable() {
+ auto& maybeAtoms = maybeAtomsToSweep.ref();
+ MOZ_ASSERT(maybeAtoms.isNothing());
+
+ AtomsTable* atomsTable = rt->atomsForSweeping();
+ if (!atomsTable) {
+ return;
+ }
+
+ // Create secondary tables to hold new atoms added while we're sweeping the
+ // main tables incrementally.
+ if (!atomsTable->startIncrementalSweep(maybeAtoms)) {
+ SweepingTracer trc(rt);
+ atomsTable->traceWeak(&trc);
+ }
+}
+
+IncrementalProgress GCRuntime::sweepAtomsTable(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ if (!atomsZone()->isGCSweeping()) {
+ return Finished;
+ }
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_ATOMS_TABLE);
+
+ auto& maybeAtoms = maybeAtomsToSweep.ref();
+ if (!maybeAtoms) {
+ return Finished;
+ }
+
+ if (!rt->atomsForSweeping()->sweepIncrementally(maybeAtoms.ref(), budget)) {
+ return NotFinished;
+ }
+
+ maybeAtoms.reset();
+
+ return Finished;
+}
+
+static size_t IncrementalSweepWeakCache(GCRuntime* gc,
+ const WeakCacheToSweep& item) {
+ AutoSetThreadIsSweeping threadIsSweeping(item.zone);
+
+ JS::detail::WeakCacheBase* cache = item.cache;
+ MOZ_ASSERT(cache->needsIncrementalBarrier());
+
+ SweepingTracer trc(gc->rt);
+ size_t steps = cache->traceWeak(&trc, &gc->storeBuffer());
+ cache->setIncrementalBarrierTracer(nullptr);
+
+ return steps;
+}
+
+WeakCacheSweepIterator::WeakCacheSweepIterator(JS::Zone* sweepGroup)
+ : sweepZone(sweepGroup), sweepCache(sweepZone->weakCaches().getFirst()) {
+ settle();
+}
+
+bool WeakCacheSweepIterator::done() const { return !sweepZone; }
+
+WeakCacheToSweep WeakCacheSweepIterator::get() const {
+ MOZ_ASSERT(!done());
+
+ return {sweepCache, sweepZone};
+}
+
+void WeakCacheSweepIterator::next() {
+ MOZ_ASSERT(!done());
+
+ sweepCache = sweepCache->getNext();
+ settle();
+}
+
+void WeakCacheSweepIterator::settle() {
+ while (sweepZone) {
+ while (sweepCache && !sweepCache->needsIncrementalBarrier()) {
+ sweepCache = sweepCache->getNext();
+ }
+
+ if (sweepCache) {
+ break;
+ }
+
+ sweepZone = sweepZone->nextNodeInGroup();
+ if (sweepZone) {
+ sweepCache = sweepZone->weakCaches().getFirst();
+ }
+ }
+
+ MOZ_ASSERT((!sweepZone && !sweepCache) ||
+ (sweepCache && sweepCache->needsIncrementalBarrier()));
+}
+
+IncrementalProgress GCRuntime::sweepWeakCaches(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ if (weakCachesToSweep.ref().isNothing()) {
+ return Finished;
+ }
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS);
+
+ WeakCacheSweepIterator& work = weakCachesToSweep.ref().ref();
+
+ AutoLockHelperThreadState lock;
+
+ {
+ AutoRunParallelWork runWork(this, IncrementalSweepWeakCache,
+ gcstats::PhaseKind::SWEEP_WEAK_CACHES,
+ GCUse::Sweeping, work, budget, lock);
+ AutoUnlockHelperThreadState unlock(lock);
+ }
+
+ if (work.done()) {
+ weakCachesToSweep.ref().reset();
+ return Finished;
+ }
+
+ return NotFinished;
+}
+
+IncrementalProgress GCRuntime::finalizeAllocKind(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ MOZ_ASSERT(sweepZone->isGCSweeping());
+
+ // Set the number of things per arena for this AllocKind.
+ size_t thingsPerArena = Arena::thingsPerArena(sweepAllocKind);
+ auto& sweepList = incrementalSweepList.ref();
+ sweepList.setThingsPerArena(thingsPerArena);
+
+ AutoSetThreadIsFinalizing threadIsFinalizing(gcx);
+
+ if (!foregroundFinalize(gcx, sweepZone, sweepAllocKind, budget, sweepList)) {
+ return NotFinished;
+ }
+
+ // Reset the slots of the sweep list that we used.
+ sweepList.reset(thingsPerArena);
+
+ return Finished;
+}
+
+IncrementalProgress GCRuntime::sweepPropMapTree(JS::GCContext* gcx,
+ SliceBudget& budget) {
+ // Remove dead SharedPropMaps from the tree. This happens incrementally on the
+ // main thread. PropMaps are finalized later on the a background thread.
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_PROP_MAP);
+
+ ArenaLists& al = sweepZone->arenas;
+
+ if (!SweepArenaList<CompactPropMap>(
+ gcx, &al.gcCompactPropMapArenasToUpdate.ref(), budget)) {
+ return NotFinished;
+ }
+ if (!SweepArenaList<NormalPropMap>(
+ gcx, &al.gcNormalPropMapArenasToUpdate.ref(), budget)) {
+ return NotFinished;
+ }
+
+ return Finished;
+}
+
+// An iterator for a standard container that provides an STL-like begin()/end()
+// interface. This iterator provides a done()/get()/next() style interface.
+template <typename Container>
+class ContainerIter {
+ using Iter = decltype(std::declval<const Container>().begin());
+ using Elem = decltype(*std::declval<Iter>());
+
+ Iter iter;
+ const Iter end;
+
+ public:
+ explicit ContainerIter(const Container& container)
+ : iter(container.begin()), end(container.end()) {}
+
+ bool done() const { return iter == end; }
+
+ Elem get() const { return *iter; }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ ++iter;
+ }
+};
+
+// IncrementalIter is a template class that makes a normal iterator into one
+// that can be used to perform incremental work by using external state that
+// persists between instantiations. The state is only initialised on the first
+// use and subsequent uses carry on from the previous state.
+template <typename Iter>
+struct IncrementalIter {
+ using State = mozilla::Maybe<Iter>;
+ using Elem = decltype(std::declval<Iter>().get());
+
+ private:
+ State& maybeIter;
+
+ public:
+ template <typename... Args>
+ explicit IncrementalIter(State& maybeIter, Args&&... args)
+ : maybeIter(maybeIter) {
+ if (maybeIter.isNothing()) {
+ maybeIter.emplace(std::forward<Args>(args)...);
+ }
+ }
+
+ ~IncrementalIter() {
+ if (done()) {
+ maybeIter.reset();
+ }
+ }
+
+ bool done() const { return maybeIter.ref().done(); }
+
+ Elem get() const { return maybeIter.ref().get(); }
+
+ void next() { maybeIter.ref().next(); }
+};
+
+// Iterate through the sweep groups created by
+// GCRuntime::groupZonesForSweeping().
+class js::gc::SweepGroupsIter {
+ GCRuntime* gc;
+
+ public:
+ explicit SweepGroupsIter(JSRuntime* rt) : gc(&rt->gc) {
+ MOZ_ASSERT(gc->currentSweepGroup);
+ }
+
+ bool done() const { return !gc->currentSweepGroup; }
+
+ Zone* get() const { return gc->currentSweepGroup; }
+
+ void next() {
+ MOZ_ASSERT(!done());
+ gc->getNextSweepGroup();
+ }
+};
+
+namespace sweepaction {
+
+// Implementation of the SweepAction interface that calls a method on GCRuntime.
+class SweepActionCall final : public SweepAction {
+ using Method = IncrementalProgress (GCRuntime::*)(JS::GCContext* gcx,
+ SliceBudget& budget);
+
+ Method method;
+
+ public:
+ explicit SweepActionCall(Method m) : method(m) {}
+ IncrementalProgress run(Args& args) override {
+ return (args.gc->*method)(args.gcx, args.budget);
+ }
+ void assertFinished() const override {}
+};
+
+// Implementation of the SweepAction interface that yields in a specified zeal
+// mode.
+class SweepActionMaybeYield final : public SweepAction {
+#ifdef JS_GC_ZEAL
+ ZealMode mode;
+ bool isYielding;
+#endif
+
+ public:
+ explicit SweepActionMaybeYield(ZealMode mode)
+#ifdef JS_GC_ZEAL
+ : mode(mode),
+ isYielding(false)
+#endif
+ {
+ }
+
+ IncrementalProgress run(Args& args) override {
+#ifdef JS_GC_ZEAL
+ if (!isYielding && args.gc->shouldYieldForZeal(mode)) {
+ isYielding = true;
+ return NotFinished;
+ }
+
+ isYielding = false;
+#endif
+ return Finished;
+ }
+
+ void assertFinished() const override { MOZ_ASSERT(!isYielding); }
+
+ // These actions should be skipped if GC zeal is not configured.
+#ifndef JS_GC_ZEAL
+ bool shouldSkip() override { return true; }
+#endif
+};
+
+// Implementation of the SweepAction interface that calls a list of actions in
+// sequence.
+class SweepActionSequence final : public SweepAction {
+ using ActionVector = Vector<UniquePtr<SweepAction>, 0, SystemAllocPolicy>;
+ using Iter = IncrementalIter<ContainerIter<ActionVector>>;
+
+ ActionVector actions;
+ typename Iter::State iterState;
+
+ public:
+ bool init(UniquePtr<SweepAction>* acts, size_t count) {
+ for (size_t i = 0; i < count; i++) {
+ auto& action = acts[i];
+ if (!action) {
+ return false;
+ }
+ if (action->shouldSkip()) {
+ continue;
+ }
+ if (!actions.emplaceBack(std::move(action))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ IncrementalProgress run(Args& args) override {
+ for (Iter iter(iterState, actions); !iter.done(); iter.next()) {
+ if (iter.get()->run(args) == NotFinished) {
+ return NotFinished;
+ }
+ }
+ return Finished;
+ }
+
+ void assertFinished() const override {
+ MOZ_ASSERT(iterState.isNothing());
+ for (const auto& action : actions) {
+ action->assertFinished();
+ }
+ }
+};
+
+template <typename Iter, typename Init>
+class SweepActionForEach final : public SweepAction {
+ using Elem = decltype(std::declval<Iter>().get());
+ using IncrIter = IncrementalIter<Iter>;
+
+ Init iterInit;
+ Elem* elemOut;
+ UniquePtr<SweepAction> action;
+ typename IncrIter::State iterState;
+
+ public:
+ SweepActionForEach(const Init& init, Elem* maybeElemOut,
+ UniquePtr<SweepAction> action)
+ : iterInit(init), elemOut(maybeElemOut), action(std::move(action)) {}
+
+ IncrementalProgress run(Args& args) override {
+ MOZ_ASSERT_IF(elemOut, *elemOut == Elem());
+ auto clearElem = mozilla::MakeScopeExit([&] { setElem(Elem()); });
+ for (IncrIter iter(iterState, iterInit); !iter.done(); iter.next()) {
+ setElem(iter.get());
+ if (action->run(args) == NotFinished) {
+ return NotFinished;
+ }
+ }
+ return Finished;
+ }
+
+ void assertFinished() const override {
+ MOZ_ASSERT(iterState.isNothing());
+ MOZ_ASSERT_IF(elemOut, *elemOut == Elem());
+ action->assertFinished();
+ }
+
+ private:
+ void setElem(const Elem& value) {
+ if (elemOut) {
+ *elemOut = value;
+ }
+ }
+};
+
+static UniquePtr<SweepAction> Call(IncrementalProgress (GCRuntime::*method)(
+ JS::GCContext* gcx, SliceBudget& budget)) {
+ return MakeUnique<SweepActionCall>(method);
+}
+
+static UniquePtr<SweepAction> MaybeYield(ZealMode zealMode) {
+ return MakeUnique<SweepActionMaybeYield>(zealMode);
+}
+
+template <typename... Rest>
+static UniquePtr<SweepAction> Sequence(UniquePtr<SweepAction> first,
+ Rest... rest) {
+ UniquePtr<SweepAction> actions[] = {std::move(first), std::move(rest)...};
+ auto seq = MakeUnique<SweepActionSequence>();
+ if (!seq || !seq->init(actions, std::size(actions))) {
+ return nullptr;
+ }
+
+ return UniquePtr<SweepAction>(std::move(seq));
+}
+
+static UniquePtr<SweepAction> RepeatForSweepGroup(
+ JSRuntime* rt, UniquePtr<SweepAction> action) {
+ if (!action) {
+ return nullptr;
+ }
+
+ using Action = SweepActionForEach<SweepGroupsIter, JSRuntime*>;
+ return js::MakeUnique<Action>(rt, nullptr, std::move(action));
+}
+
+static UniquePtr<SweepAction> ForEachZoneInSweepGroup(
+ JSRuntime* rt, Zone** zoneOut, UniquePtr<SweepAction> action) {
+ if (!action) {
+ return nullptr;
+ }
+
+ using Action = SweepActionForEach<SweepGroupZonesIter, JSRuntime*>;
+ return js::MakeUnique<Action>(rt, zoneOut, std::move(action));
+}
+
+static UniquePtr<SweepAction> ForEachAllocKind(AllocKinds kinds,
+ AllocKind* kindOut,
+ UniquePtr<SweepAction> action) {
+ if (!action) {
+ return nullptr;
+ }
+
+ using Action = SweepActionForEach<ContainerIter<AllocKinds>, AllocKinds>;
+ return js::MakeUnique<Action>(kinds, kindOut, std::move(action));
+}
+
+} // namespace sweepaction
+
+bool GCRuntime::initSweepActions() {
+ using namespace sweepaction;
+ using sweepaction::Call;
+
+ sweepActions.ref() = RepeatForSweepGroup(
+ rt,
+ Sequence(
+ Call(&GCRuntime::beginMarkingSweepGroup),
+ Call(&GCRuntime::markGrayRootsInCurrentGroup),
+ MaybeYield(ZealMode::YieldWhileGrayMarking),
+ Call(&GCRuntime::markGray), Call(&GCRuntime::endMarkingSweepGroup),
+ Call(&GCRuntime::beginSweepingSweepGroup),
+ MaybeYield(ZealMode::IncrementalMultipleSlices),
+ MaybeYield(ZealMode::YieldBeforeSweepingAtoms),
+ Call(&GCRuntime::sweepAtomsTable),
+ MaybeYield(ZealMode::YieldBeforeSweepingCaches),
+ Call(&GCRuntime::sweepWeakCaches),
+ ForEachZoneInSweepGroup(
+ rt, &sweepZone.ref(),
+ Sequence(MaybeYield(ZealMode::YieldBeforeSweepingObjects),
+ ForEachAllocKind(ForegroundObjectFinalizePhase.kinds,
+ &sweepAllocKind.ref(),
+ Call(&GCRuntime::finalizeAllocKind)),
+ MaybeYield(ZealMode::YieldBeforeSweepingNonObjects),
+ ForEachAllocKind(ForegroundNonObjectFinalizePhase.kinds,
+ &sweepAllocKind.ref(),
+ Call(&GCRuntime::finalizeAllocKind)),
+ MaybeYield(ZealMode::YieldBeforeSweepingPropMapTrees),
+ Call(&GCRuntime::sweepPropMapTree))),
+ Call(&GCRuntime::endSweepingSweepGroup)));
+
+ return sweepActions != nullptr;
+}
+
+IncrementalProgress GCRuntime::performSweepActions(SliceBudget& budget) {
+ AutoMajorGCProfilerEntry s(this);
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
+
+ JS::GCContext* gcx = rt->gcContext();
+ AutoSetThreadIsSweeping threadIsSweeping(gcx);
+ AutoPoisonFreedJitCode pjc(gcx);
+
+ // Don't trigger pre-barriers when finalizing.
+ AutoDisableBarriers disableBarriers(this);
+
+ // Drain the mark stack, possibly in a parallel task if we're in a part of
+ // sweeping that allows it.
+ //
+ // In the first sweep slice where we must not yield to the mutator until we've
+ // starting sweeping a sweep group but in that case the stack must be empty
+ // already.
+
+#ifdef DEBUG
+ MOZ_ASSERT(initialState <= State::Sweep);
+ if (initialState != State::Sweep) {
+ assertNoMarkingWork();
+ }
+#endif
+
+ if (initialState == State::Sweep &&
+ markDuringSweeping(gcx, budget) == NotFinished) {
+ return NotFinished;
+ }
+
+ // Then continue running sweep actions.
+
+ SweepAction::Args args{this, gcx, budget};
+ IncrementalProgress sweepProgress = sweepActions->run(args);
+ IncrementalProgress markProgress = joinBackgroundMarkTask();
+
+ if (sweepProgress == Finished && markProgress == Finished) {
+ return Finished;
+ }
+
+ MOZ_ASSERT(isIncremental);
+ return NotFinished;
+}
+
+bool GCRuntime::allCCVisibleZonesWereCollected() {
+ // Calculate whether the gray marking state is now valid.
+ //
+ // The gray bits change from invalid to valid if we finished a full GC from
+ // the point of view of the cycle collector. We ignore the following:
+ //
+ // - Helper thread zones, as these are not reachable from the main heap.
+ // - The atoms zone, since strings and symbols are never marked gray.
+ // - Empty zones.
+ //
+ // These exceptions ensure that when the CC requests a full GC the gray mark
+ // state ends up valid even it we don't collect all of the zones.
+
+ for (ZonesIter zone(this, SkipAtoms); !zone.done(); zone.next()) {
+ if (!zone->isCollecting() && !zone->arenas.arenaListsAreEmpty()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void GCRuntime::endSweepPhase(bool destroyingRuntime) {
+ MOZ_ASSERT(!markOnBackgroundThreadDuringSweeping);
+
+ sweepActions->assertFinished();
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
+
+ MOZ_ASSERT_IF(destroyingRuntime, !useBackgroundThreads);
+
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::DESTROY);
+
+ // Sweep shared script bytecode now all zones have been swept and finalizers
+ // for BaseScripts have released their references.
+ SweepScriptData(rt);
+ }
+
+ {
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END);
+ AutoLockStoreBuffer lock(&storeBuffer());
+ callFinalizeCallbacks(rt->gcContext(), JSFINALIZE_COLLECTION_END);
+
+ if (allCCVisibleZonesWereCollected()) {
+ grayBitsValid = true;
+ }
+ }
+
+ if (isIncremental) {
+ findDeadCompartments();
+ }
+
+#ifdef JS_GC_ZEAL
+ finishMarkingValidation();
+#endif
+
+#ifdef DEBUG
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT_IF(!IsBackgroundFinalized(i) || !useBackgroundThreads,
+ zone->arenas.collectingArenaList(i).isEmpty());
+ }
+ }
+#endif
+
+ AssertNoWrappersInGrayList(rt);
+}
diff --git a/js/src/gc/Tenuring.cpp b/js/src/gc/Tenuring.cpp
new file mode 100644
index 0000000000..457aa4876c
--- /dev/null
+++ b/js/src/gc/Tenuring.cpp
@@ -0,0 +1,1016 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Implementation of nursery eviction (tenuring).
+ */
+
+#include "gc/Tenuring.h"
+
+#include "mozilla/PodOperations.h"
+
+#include "gc/Cell.h"
+#include "gc/GCInternals.h"
+#include "gc/GCProbes.h"
+#include "gc/Pretenuring.h"
+#include "gc/Zone.h"
+#include "jit/JitCode.h"
+#include "proxy/Proxy.h"
+#include "vm/BigIntType.h"
+#include "vm/JSScript.h"
+#include "vm/NativeObject.h"
+#include "vm/Runtime.h"
+#include "vm/TypedArrayObject.h"
+
+#include "gc/Heap-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/ObjectKind-inl.h"
+#include "gc/StoreBuffer-inl.h"
+#include "gc/TraceMethods-inl.h"
+#include "vm/JSObject-inl.h"
+#include "vm/PlainObject-inl.h"
+#ifdef ENABLE_RECORD_TUPLE
+# include "vm/TupleType.h"
+#endif
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::PodCopy;
+
+constexpr size_t MAX_DEDUPLICATABLE_STRING_LENGTH = 500;
+
+TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery)
+ : JSTracer(rt, JS::TracerKind::Tenuring,
+ JS::WeakMapTraceAction::TraceKeysAndValues),
+ nursery_(*nursery) {
+ stringDeDupSet.emplace();
+}
+
+size_t TenuringTracer::getTenuredSize() const {
+ return tenuredSize + tenuredCells * sizeof(NurseryCellHeader);
+}
+
+size_t TenuringTracer::getTenuredCells() const { return tenuredCells; }
+
+static inline void UpdateAllocSiteOnTenure(Cell* cell) {
+ AllocSite* site = NurseryCellHeader::from(cell)->allocSite();
+ site->incTenuredCount();
+}
+
+void TenuringTracer::onObjectEdge(JSObject** objp, const char* name) {
+ JSObject* obj = *objp;
+ if (!IsInsideNursery(obj)) {
+ return;
+ }
+
+ if (obj->isForwarded()) {
+ const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(obj);
+ *objp = static_cast<JSObject*>(overlay->forwardingAddress());
+ return;
+ }
+
+ UpdateAllocSiteOnTenure(obj);
+
+ // Take a fast path for tenuring a plain object which is by far the most
+ // common case.
+ if (obj->is<PlainObject>()) {
+ *objp = movePlainObjectToTenured(&obj->as<PlainObject>());
+ return;
+ }
+
+ *objp = moveToTenuredSlow(obj);
+}
+
+void TenuringTracer::onStringEdge(JSString** strp, const char* name) {
+ JSString* str = *strp;
+ if (!IsInsideNursery(str)) {
+ return;
+ }
+
+ if (str->isForwarded()) {
+ const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(str);
+ *strp = static_cast<JSString*>(overlay->forwardingAddress());
+ return;
+ }
+
+ UpdateAllocSiteOnTenure(str);
+
+ *strp = moveToTenured(str);
+}
+
+void TenuringTracer::onBigIntEdge(JS::BigInt** bip, const char* name) {
+ JS::BigInt* bi = *bip;
+ if (!IsInsideNursery(bi)) {
+ return;
+ }
+
+ if (bi->isForwarded()) {
+ const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(bi);
+ *bip = static_cast<JS::BigInt*>(overlay->forwardingAddress());
+ return;
+ }
+
+ UpdateAllocSiteOnTenure(bi);
+
+ *bip = moveToTenured(bi);
+}
+
+void TenuringTracer::onSymbolEdge(JS::Symbol** symp, const char* name) {}
+void TenuringTracer::onScriptEdge(BaseScript** scriptp, const char* name) {}
+void TenuringTracer::onShapeEdge(Shape** shapep, const char* name) {}
+void TenuringTracer::onRegExpSharedEdge(RegExpShared** sharedp,
+ const char* name) {}
+void TenuringTracer::onBaseShapeEdge(BaseShape** basep, const char* name) {}
+void TenuringTracer::onGetterSetterEdge(GetterSetter** gsp, const char* name) {}
+void TenuringTracer::onPropMapEdge(PropMap** mapp, const char* name) {}
+void TenuringTracer::onJitCodeEdge(jit::JitCode** codep, const char* name) {}
+void TenuringTracer::onScopeEdge(Scope** scopep, const char* name) {}
+
+void TenuringTracer::traverse(JS::Value* thingp) {
+ MOZ_ASSERT(!nursery().isInside(thingp));
+
+ Value value = *thingp;
+ CheckTracedThing(this, value);
+
+ // We only care about a few kinds of GC thing here and this generates much
+ // tighter code than using MapGCThingTyped.
+ Value post;
+ if (value.isObject()) {
+ JSObject* obj = &value.toObject();
+ onObjectEdge(&obj, "value");
+ post = JS::ObjectValue(*obj);
+ }
+#ifdef ENABLE_RECORD_TUPLE
+ else if (value.isExtendedPrimitive()) {
+ JSObject* obj = &value.toExtendedPrimitive();
+ onObjectEdge(&obj, "value");
+ post = JS::ExtendedPrimitiveValue(*obj);
+ }
+#endif
+ else if (value.isString()) {
+ JSString* str = value.toString();
+ onStringEdge(&str, "value");
+ post = JS::StringValue(str);
+ } else if (value.isBigInt()) {
+ JS::BigInt* bi = value.toBigInt();
+ onBigIntEdge(&bi, "value");
+ post = JS::BigIntValue(bi);
+ } else {
+ MOZ_ASSERT_IF(value.isGCThing(), !IsInsideNursery(value.toGCThing()));
+ return;
+ }
+
+ if (post != value) {
+ *thingp = post;
+ }
+}
+
+template <typename T>
+void js::gc::StoreBuffer::MonoTypeBuffer<T>::trace(TenuringTracer& mover) {
+ mozilla::ReentrancyGuard g(*owner_);
+ MOZ_ASSERT(owner_->isEnabled());
+ if (last_) {
+ last_.trace(mover);
+ }
+ for (typename StoreSet::Range r = stores_.all(); !r.empty(); r.popFront()) {
+ r.front().trace(mover);
+ }
+}
+
+namespace js {
+namespace gc {
+template void StoreBuffer::MonoTypeBuffer<StoreBuffer::ValueEdge>::trace(
+ TenuringTracer&);
+template void StoreBuffer::MonoTypeBuffer<StoreBuffer::SlotsEdge>::trace(
+ TenuringTracer&);
+template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::StringPtrEdge>;
+template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::BigIntPtrEdge>;
+template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::ObjectPtrEdge>;
+} // namespace gc
+} // namespace js
+
+void js::gc::StoreBuffer::SlotsEdge::trace(TenuringTracer& mover) const {
+ NativeObject* obj = object();
+ MOZ_ASSERT(IsCellPointerValid(obj));
+
+ // Beware JSObject::swap exchanging a native object for a non-native one.
+ if (!obj->is<NativeObject>()) {
+ return;
+ }
+
+ MOZ_ASSERT(!IsInsideNursery(obj), "obj shouldn't live in nursery.");
+
+ if (kind() == ElementKind) {
+ uint32_t initLen = obj->getDenseInitializedLength();
+ uint32_t numShifted = obj->getElementsHeader()->numShiftedElements();
+ uint32_t clampedStart = start_;
+ clampedStart = numShifted < clampedStart ? clampedStart - numShifted : 0;
+ clampedStart = std::min(clampedStart, initLen);
+ uint32_t clampedEnd = start_ + count_;
+ clampedEnd = numShifted < clampedEnd ? clampedEnd - numShifted : 0;
+ clampedEnd = std::min(clampedEnd, initLen);
+ MOZ_ASSERT(clampedStart <= clampedEnd);
+ mover.traceSlots(
+ static_cast<HeapSlot*>(obj->getDenseElements() + clampedStart)
+ ->unbarrieredAddress(),
+ clampedEnd - clampedStart);
+ } else {
+ uint32_t start = std::min(start_, obj->slotSpan());
+ uint32_t end = std::min(start_ + count_, obj->slotSpan());
+ MOZ_ASSERT(start <= end);
+ mover.traceObjectSlots(obj, start, end);
+ }
+}
+
+static inline void TraceWholeCell(TenuringTracer& mover, JSObject* object) {
+ MOZ_ASSERT_IF(object->storeBuffer(),
+ !object->storeBuffer()->markingNondeduplicatable);
+ mover.traceObject(object);
+}
+
+// Non-deduplicatable marking is necessary because of the following 2 reasons:
+//
+// 1. Tenured string chars cannot be updated:
+//
+// If any of the tenured string's bases were deduplicated during tenuring,
+// the tenured string's chars pointer would need to be adjusted. This would
+// then require updating any other tenured strings that are dependent on the
+// first tenured string, and we have no way to find them without scanning
+// the entire tenured heap.
+//
+// 2. Tenured string cannot store its nursery base or base's chars:
+//
+// Tenured strings have no place to stash a pointer to their nursery base or
+// its chars. You need to be able to traverse any dependent string's chain
+// of bases up to a nursery "root base" that points to the malloced chars
+// that the dependent strings started out pointing to, so that you can
+// calculate the offset of any dependent string and update the ptr+offset if
+// the root base gets deduplicated to a different allocation. Tenured
+// strings in this base chain will stop you from reaching the nursery
+// version of the root base; you can only get to the tenured version, and it
+// has no place to store the original chars pointer.
+static inline void PreventDeduplicationOfReachableStrings(JSString* str) {
+ MOZ_ASSERT(str->isTenured());
+ MOZ_ASSERT(!str->isForwarded());
+
+ JSLinearString* baseOrRelocOverlay = str->nurseryBaseOrRelocOverlay();
+
+ // Walk along the chain of dependent strings' base string pointers
+ // to mark them all non-deduplicatable.
+ while (true) {
+ // baseOrRelocOverlay can be one of the three cases:
+ // 1. forwarded nursery string:
+ // The forwarded string still retains the flag that can tell whether
+ // this string is a dependent string with a base. Its
+ // StringRelocationOverlay holds a saved pointer to its base in the
+ // nursery.
+ // 2. not yet forwarded nursery string:
+ // Retrieve the base field directly from the string.
+ // 3. tenured string:
+ // The nursery base chain ends here, so stop traversing.
+ if (baseOrRelocOverlay->isForwarded()) {
+ JSLinearString* tenuredBase = Forwarded(baseOrRelocOverlay);
+ if (!tenuredBase->hasBase()) {
+ break;
+ }
+ baseOrRelocOverlay = StringRelocationOverlay::fromCell(baseOrRelocOverlay)
+ ->savedNurseryBaseOrRelocOverlay();
+ } else {
+ JSLinearString* base = baseOrRelocOverlay;
+ if (base->isTenured()) {
+ break;
+ }
+ if (base->isDeduplicatable()) {
+ base->setNonDeduplicatable();
+ }
+ if (!base->hasBase()) {
+ break;
+ }
+ baseOrRelocOverlay = base->nurseryBaseOrRelocOverlay();
+ }
+ }
+}
+
+static inline void TraceWholeCell(TenuringTracer& mover, JSString* str) {
+ MOZ_ASSERT_IF(str->storeBuffer(),
+ str->storeBuffer()->markingNondeduplicatable);
+
+ // Mark all strings reachable from the tenured string `str` as
+ // non-deduplicatable. These strings are the bases of the tenured dependent
+ // string.
+ if (str->hasBase()) {
+ PreventDeduplicationOfReachableStrings(str);
+ }
+
+ str->traceChildren(&mover);
+}
+
+static inline void TraceWholeCell(TenuringTracer& mover, BaseScript* script) {
+ script->traceChildren(&mover);
+}
+
+static inline void TraceWholeCell(TenuringTracer& mover,
+ jit::JitCode* jitcode) {
+ jitcode->traceChildren(&mover);
+}
+
+template <typename T>
+static void TraceBufferedCells(TenuringTracer& mover, Arena* arena,
+ ArenaCellSet* cells) {
+ for (size_t i = 0; i < MaxArenaCellIndex; i += cells->BitsPerWord) {
+ ArenaCellSet::WordT bitset = cells->getWord(i / cells->BitsPerWord);
+ while (bitset) {
+ size_t bit = i + js::detail::CountTrailingZeroes(bitset);
+ auto cell =
+ reinterpret_cast<T*>(uintptr_t(arena) + ArenaCellIndexBytes * bit);
+ TraceWholeCell(mover, cell);
+ bitset &= bitset - 1; // Clear the low bit.
+ }
+ }
+}
+
+void ArenaCellSet::trace(TenuringTracer& mover) {
+ for (ArenaCellSet* cells = this; cells; cells = cells->next) {
+ cells->check();
+
+ Arena* arena = cells->arena;
+ arena->bufferedCells() = &ArenaCellSet::Empty;
+
+ JS::TraceKind kind = MapAllocToTraceKind(arena->getAllocKind());
+ switch (kind) {
+ case JS::TraceKind::Object:
+ TraceBufferedCells<JSObject>(mover, arena, cells);
+ break;
+ case JS::TraceKind::String:
+ TraceBufferedCells<JSString>(mover, arena, cells);
+ break;
+ case JS::TraceKind::Script:
+ TraceBufferedCells<BaseScript>(mover, arena, cells);
+ break;
+ case JS::TraceKind::JitCode:
+ TraceBufferedCells<jit::JitCode>(mover, arena, cells);
+ break;
+ default:
+ MOZ_CRASH("Unexpected trace kind");
+ }
+ }
+}
+
+void js::gc::StoreBuffer::WholeCellBuffer::trace(TenuringTracer& mover) {
+ MOZ_ASSERT(owner_->isEnabled());
+
+#ifdef DEBUG
+ // Verify that all string whole cells are traced first before any other
+ // strings are visited for any reason.
+ MOZ_ASSERT(!owner_->markingNondeduplicatable);
+ owner_->markingNondeduplicatable = true;
+#endif
+ // Trace all of the strings to mark the non-deduplicatable bits, then trace
+ // all other whole cells.
+ if (stringHead_) {
+ stringHead_->trace(mover);
+ }
+#ifdef DEBUG
+ owner_->markingNondeduplicatable = false;
+#endif
+ if (nonStringHead_) {
+ nonStringHead_->trace(mover);
+ }
+
+ stringHead_ = nonStringHead_ = nullptr;
+}
+
+template <typename T>
+void js::gc::StoreBuffer::CellPtrEdge<T>::trace(TenuringTracer& mover) const {
+ static_assert(std::is_base_of_v<Cell, T>, "T must be a Cell type");
+ static_assert(!GCTypeIsTenured<T>(), "T must not be a tenured Cell type");
+
+ T* thing = *edge;
+ if (!thing) {
+ return;
+ }
+
+ MOZ_ASSERT(IsCellPointerValid(thing));
+ MOZ_ASSERT(thing->getTraceKind() == JS::MapTypeToTraceKind<T>::kind);
+
+ if (std::is_same_v<JSString, T>) {
+ // Nursery string deduplication requires all tenured string -> nursery
+ // string edges to be registered with the whole cell buffer in order to
+ // correctly set the non-deduplicatable bit.
+ MOZ_ASSERT(!mover.runtime()->gc.isPointerWithinTenuredCell(
+ edge, JS::TraceKind::String));
+ }
+
+ DispatchToOnEdge(&mover, edge, "CellPtrEdge");
+}
+
+void js::gc::StoreBuffer::ValueEdge::trace(TenuringTracer& mover) const {
+ if (deref()) {
+ mover.traverse(edge);
+ }
+}
+
+// Visit all object children of the object and trace them.
+void js::gc::TenuringTracer::traceObject(JSObject* obj) {
+ const JSClass* clasp = obj->getClass();
+ MOZ_ASSERT(clasp);
+
+ if (clasp->hasTrace()) {
+ clasp->doTrace(this, obj);
+ }
+
+ if (!obj->is<NativeObject>()) {
+ return;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!nobj->hasEmptyElements()) {
+ HeapSlotArray elements = nobj->getDenseElements();
+ Value* elems = elements.begin()->unbarrieredAddress();
+ traceSlots(elems, elems + nobj->getDenseInitializedLength());
+ }
+
+ traceObjectSlots(nobj, 0, nobj->slotSpan());
+}
+
+void js::gc::TenuringTracer::traceObjectSlots(NativeObject* nobj,
+ uint32_t start, uint32_t end) {
+ auto traceRange = [this](HeapSlot* slotStart, HeapSlot* slotEnd) {
+ traceSlots(slotStart->unbarrieredAddress(), slotEnd->unbarrieredAddress());
+ };
+ nobj->forEachSlotRange(start, end, traceRange);
+}
+
+void js::gc::TenuringTracer::traceSlots(Value* vp, Value* end) {
+ for (; vp != end; ++vp) {
+ traverse(vp);
+ }
+}
+
+inline void js::gc::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots) {
+ traceSlots(vp, vp + nslots);
+}
+
+void js::gc::TenuringTracer::traceString(JSString* str) {
+ str->traceChildren(this);
+}
+
+void js::gc::TenuringTracer::traceBigInt(JS::BigInt* bi) {
+ bi->traceChildren(this);
+}
+
+#ifdef DEBUG
+static inline uintptr_t OffsetFromChunkStart(void* p) {
+ return uintptr_t(p) & gc::ChunkMask;
+}
+static inline ptrdiff_t OffsetToChunkEnd(void* p) {
+ return ChunkSize - (uintptr_t(p) & gc::ChunkMask);
+}
+#endif
+
+/* Insert the given relocation entry into the list of things to visit. */
+inline void js::gc::TenuringTracer::insertIntoObjectFixupList(
+ RelocationOverlay* entry) {
+ entry->setNext(objHead);
+ objHead = entry;
+}
+
+template <typename T>
+inline T* js::gc::TenuringTracer::allocTenured(Zone* zone, AllocKind kind) {
+ return static_cast<T*>(static_cast<Cell*>(AllocateCellInGC(zone, kind)));
+}
+
+JSString* js::gc::TenuringTracer::allocTenuredString(JSString* src, Zone* zone,
+ AllocKind dstKind) {
+ JSString* dst = allocTenured<JSString>(zone, dstKind);
+ tenuredSize += moveStringToTenured(dst, src, dstKind);
+ tenuredCells++;
+
+ return dst;
+}
+
+JSObject* js::gc::TenuringTracer::moveToTenuredSlow(JSObject* src) {
+ MOZ_ASSERT(IsInsideNursery(src));
+ MOZ_ASSERT(!src->is<PlainObject>());
+
+ AllocKind dstKind = src->allocKindForTenure(nursery());
+ auto dst = allocTenured<JSObject>(src->nurseryZone(), dstKind);
+
+ size_t srcSize = Arena::thingSize(dstKind);
+
+ // Arrays and Tuples do not necessarily have the same AllocKind between src
+ // and dst. We deal with this by copying elements manually, possibly
+ // re-inlining them if there is adequate room inline in dst.
+ //
+ // For Arrays and Tuples we're reducing tenuredSize to the smaller srcSize
+ // because moveElementsToTenured() accounts for all Array or Tuple elements,
+ // even if they are inlined.
+ if (src->is<TypedArrayObject>()) {
+ TypedArrayObject* tarray = &src->as<TypedArrayObject>();
+ // Typed arrays with inline data do not necessarily have the same
+ // AllocKind between src and dst. The nursery does not allocate an
+ // inline data buffer that has the same size as the slow path will do.
+ // In the slow path, the Typed Array Object stores the inline data
+ // in the allocated space that fits the AllocKind. In the fast path,
+ // the nursery will allocate another buffer that is directly behind the
+ // minimal JSObject. That buffer size plus the JSObject size is not
+ // necessarily as large as the slow path's AllocKind size.
+ if (tarray->hasInlineElements()) {
+ AllocKind srcKind = GetGCObjectKind(TypedArrayObject::FIXED_DATA_START);
+ size_t headerSize = Arena::thingSize(srcKind);
+ srcSize = headerSize + tarray->byteLength();
+ }
+ } else if (src->canHaveFixedElements()) {
+ srcSize = sizeof(NativeObject);
+ }
+
+ tenuredSize += srcSize;
+ tenuredCells++;
+
+ // Copy the Cell contents.
+ MOZ_ASSERT(OffsetFromChunkStart(src) >= sizeof(ChunkBase));
+ MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize));
+ js_memcpy(dst, src, srcSize);
+
+ // Move the slots and elements, if we need to.
+ if (src->is<NativeObject>()) {
+ NativeObject* ndst = &dst->as<NativeObject>();
+ NativeObject* nsrc = &src->as<NativeObject>();
+ tenuredSize += moveSlotsToTenured(ndst, nsrc);
+ tenuredSize += moveElementsToTenured(ndst, nsrc, dstKind);
+ }
+
+ JSObjectMovedOp op = dst->getClass()->extObjectMovedOp();
+ MOZ_ASSERT_IF(src->is<ProxyObject>(), op == proxy_ObjectMoved);
+ if (op) {
+ // Tell the hazard analysis that the object moved hook can't GC.
+ JS::AutoSuppressGCAnalysis nogc;
+ tenuredSize += op(dst, src);
+ } else {
+ MOZ_ASSERT_IF(src->getClass()->hasFinalize(),
+ CanNurseryAllocateFinalizedClass(src->getClass()));
+ }
+
+ RelocationOverlay* overlay = RelocationOverlay::forwardCell(src, dst);
+ insertIntoObjectFixupList(overlay);
+
+ gcprobes::PromoteToTenured(src, dst);
+ return dst;
+}
+
+inline JSObject* js::gc::TenuringTracer::movePlainObjectToTenured(
+ PlainObject* src) {
+ // Fast path version of moveToTenuredSlow() for specialized for PlainObject.
+
+ MOZ_ASSERT(IsInsideNursery(src));
+
+ AllocKind dstKind = src->allocKindForTenure();
+ auto dst = allocTenured<PlainObject>(src->nurseryZone(), dstKind);
+
+ size_t srcSize = Arena::thingSize(dstKind);
+ tenuredSize += srcSize;
+ tenuredCells++;
+
+ // Copy the Cell contents.
+ MOZ_ASSERT(OffsetFromChunkStart(src) >= sizeof(ChunkBase));
+ MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize));
+ js_memcpy(dst, src, srcSize);
+
+ // Move the slots and elements.
+ tenuredSize += moveSlotsToTenured(dst, src);
+ tenuredSize += moveElementsToTenured(dst, src, dstKind);
+
+ MOZ_ASSERT(!dst->getClass()->extObjectMovedOp());
+
+ RelocationOverlay* overlay = RelocationOverlay::forwardCell(src, dst);
+ insertIntoObjectFixupList(overlay);
+
+ gcprobes::PromoteToTenured(src, dst);
+ return dst;
+}
+
+size_t js::gc::TenuringTracer::moveSlotsToTenured(NativeObject* dst,
+ NativeObject* src) {
+ /* Fixed slots have already been copied over. */
+ if (!src->hasDynamicSlots()) {
+ return 0;
+ }
+
+ Zone* zone = src->nurseryZone();
+ size_t count = src->numDynamicSlots();
+
+ uint64_t uid = src->maybeUniqueId();
+
+ size_t allocSize = ObjectSlots::allocSize(count);
+
+ ObjectSlots* srcHeader = src->getSlotsHeader();
+ if (!nursery().isInside(srcHeader)) {
+ AddCellMemory(dst, allocSize, MemoryUse::ObjectSlots);
+ nursery().removeMallocedBufferDuringMinorGC(srcHeader);
+ return 0;
+ }
+
+ {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ HeapSlot* allocation =
+ zone->pod_malloc<HeapSlot>(ObjectSlots::allocCount(count));
+ if (!allocation) {
+ oomUnsafe.crash(allocSize, "Failed to allocate slots while tenuring.");
+ }
+
+ ObjectSlots* slotsHeader = new (allocation)
+ ObjectSlots(count, srcHeader->dictionarySlotSpan(), uid);
+ dst->slots_ = slotsHeader->slots();
+ }
+
+ AddCellMemory(dst, allocSize, MemoryUse::ObjectSlots);
+
+ PodCopy(dst->slots_, src->slots_, count);
+ if (count) {
+ nursery().setSlotsForwardingPointer(src->slots_, dst->slots_, count);
+ }
+
+ return allocSize;
+}
+
+size_t js::gc::TenuringTracer::moveElementsToTenured(NativeObject* dst,
+ NativeObject* src,
+ AllocKind dstKind) {
+ if (src->hasEmptyElements()) {
+ return 0;
+ }
+
+ Zone* zone = src->nurseryZone();
+
+ ObjectElements* srcHeader = src->getElementsHeader();
+ size_t nslots = srcHeader->numAllocatedElements();
+ size_t allocSize = nslots * sizeof(HeapSlot);
+
+ void* srcAllocatedHeader = src->getUnshiftedElementsHeader();
+
+ /* TODO Bug 874151: Prefer to put element data inline if we have space. */
+ if (!nursery().isInside(srcAllocatedHeader)) {
+ MOZ_ASSERT(src->elements_ == dst->elements_);
+ nursery().removeMallocedBufferDuringMinorGC(srcAllocatedHeader);
+
+ AddCellMemory(dst, allocSize, MemoryUse::ObjectElements);
+
+ return 0;
+ }
+
+ // Shifted elements are copied too.
+ uint32_t numShifted = srcHeader->numShiftedElements();
+
+ /* Unlike other objects, Arrays and Tuples can have fixed elements. */
+ if (src->canHaveFixedElements() && nslots <= GetGCKindSlots(dstKind)) {
+ dst->as<NativeObject>().setFixedElements();
+ js_memcpy(dst->getElementsHeader(), srcAllocatedHeader, allocSize);
+ dst->elements_ += numShifted;
+ dst->getElementsHeader()->flags |= ObjectElements::FIXED;
+ nursery().setElementsForwardingPointer(srcHeader, dst->getElementsHeader(),
+ srcHeader->capacity);
+ return allocSize;
+ }
+
+ MOZ_ASSERT(nslots >= 2);
+
+ ObjectElements* dstHeader;
+ {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ dstHeader =
+ reinterpret_cast<ObjectElements*>(zone->pod_malloc<HeapSlot>(nslots));
+ if (!dstHeader) {
+ oomUnsafe.crash(allocSize, "Failed to allocate elements while tenuring.");
+ }
+ }
+
+ AddCellMemory(dst, allocSize, MemoryUse::ObjectElements);
+
+ js_memcpy(dstHeader, srcAllocatedHeader, allocSize);
+ dst->elements_ = dstHeader->elements() + numShifted;
+ dst->getElementsHeader()->flags &= ~ObjectElements::FIXED;
+ nursery().setElementsForwardingPointer(srcHeader, dst->getElementsHeader(),
+ srcHeader->capacity);
+ return allocSize;
+}
+
+inline void js::gc::TenuringTracer::insertIntoStringFixupList(
+ StringRelocationOverlay* entry) {
+ entry->setNext(stringHead);
+ stringHead = entry;
+}
+
+JSString* js::gc::TenuringTracer::moveToTenured(JSString* src) {
+ MOZ_ASSERT(IsInsideNursery(src));
+ MOZ_ASSERT(!src->isExternal());
+
+ AllocKind dstKind = src->getAllocKind();
+ Zone* zone = src->nurseryZone();
+
+ // If this string is in the StringToAtomCache, try to deduplicate it by using
+ // the atom. Don't do this for dependent strings because they're more
+ // complicated. See StringRelocationOverlay and DeduplicationStringHasher
+ // comments.
+ if (src->isLinear() && src->inStringToAtomCache() &&
+ src->isDeduplicatable() && !src->hasBase()) {
+ JSLinearString* linear = &src->asLinear();
+ JSAtom* atom = runtime()->caches().stringToAtomCache.lookupInMap(linear);
+ MOZ_ASSERT(atom, "Why was the cache purged before minor GC?");
+
+ // Only deduplicate if both strings have the same encoding, to not confuse
+ // dependent strings.
+ if (src->hasTwoByteChars() == atom->hasTwoByteChars()) {
+ // The StringToAtomCache isn't used for inline strings (due to the minimum
+ // length) so canOwnDependentChars must be true for both src and atom.
+ // This means if there are dependent strings floating around using str's
+ // chars, they will be able to use the chars from the atom.
+ static_assert(StringToAtomCache::MinStringLength >
+ JSFatInlineString::MAX_LENGTH_LATIN1);
+ static_assert(StringToAtomCache::MinStringLength >
+ JSFatInlineString::MAX_LENGTH_TWO_BYTE);
+ MOZ_ASSERT(src->canOwnDependentChars());
+ MOZ_ASSERT(atom->canOwnDependentChars());
+
+ StringRelocationOverlay::forwardCell(src, atom);
+ gcprobes::PromoteToTenured(src, atom);
+ return atom;
+ }
+ }
+
+ JSString* dst;
+
+ // A live nursery string can only get deduplicated when:
+ // 1. Its length is smaller than MAX_DEDUPLICATABLE_STRING_LENGTH:
+ // Hashing a long string can affect performance.
+ // 2. It is linear:
+ // Deduplicating every node in it would end up doing O(n^2) hashing work.
+ // 3. It is deduplicatable:
+ // The JSString NON_DEDUP_BIT flag is unset.
+ // 4. It matches an entry in stringDeDupSet.
+
+ if (src->length() < MAX_DEDUPLICATABLE_STRING_LENGTH && src->isLinear() &&
+ src->isDeduplicatable() && stringDeDupSet.isSome()) {
+ if (auto p = stringDeDupSet->lookup(src)) {
+ // Deduplicate to the looked-up string!
+ dst = *p;
+ zone->stringStats.ref().noteDeduplicated(src->length(), src->allocSize());
+ StringRelocationOverlay::forwardCell(src, dst);
+ gcprobes::PromoteToTenured(src, dst);
+ return dst;
+ }
+
+ dst = allocTenuredString(src, zone, dstKind);
+
+ if (!stringDeDupSet->putNew(dst)) {
+ // When there is oom caused by the stringDeDupSet, stop deduplicating
+ // strings.
+ stringDeDupSet.reset();
+ }
+ } else {
+ dst = allocTenuredString(src, zone, dstKind);
+ dst->clearNonDeduplicatable();
+ }
+
+ zone->stringStats.ref().noteTenured(src->allocSize());
+
+ auto* overlay = StringRelocationOverlay::forwardCell(src, dst);
+ MOZ_ASSERT(dst->isDeduplicatable());
+
+ if (dst->hasBase() || dst->isRope()) {
+ // dst or one of its leaves might have a base that will be deduplicated.
+ // Insert the overlay into the fixup list to relocate it later.
+ insertIntoStringFixupList(overlay);
+ }
+
+ gcprobes::PromoteToTenured(src, dst);
+ return dst;
+}
+
+template <typename CharT>
+void js::gc::TenuringTracer::relocateDependentStringChars(
+ JSDependentString* tenuredDependentStr, JSLinearString* baseOrRelocOverlay,
+ size_t* offset, bool* rootBaseNotYetForwarded, JSLinearString** rootBase) {
+ MOZ_ASSERT(*offset == 0);
+ MOZ_ASSERT(*rootBaseNotYetForwarded == false);
+ MOZ_ASSERT(*rootBase == nullptr);
+
+ JS::AutoCheckCannotGC nogc;
+
+ const CharT* dependentStrChars =
+ tenuredDependentStr->nonInlineChars<CharT>(nogc);
+
+ // Traverse the dependent string nursery base chain to find the base that
+ // it's using chars from.
+ while (true) {
+ if (baseOrRelocOverlay->isForwarded()) {
+ JSLinearString* tenuredBase = Forwarded(baseOrRelocOverlay);
+ StringRelocationOverlay* relocOverlay =
+ StringRelocationOverlay::fromCell(baseOrRelocOverlay);
+
+ if (!tenuredBase->hasBase()) {
+ // The nursery root base is relocOverlay, it is tenured to tenuredBase.
+ // Relocate tenuredDependentStr chars and reassign the tenured root base
+ // as its base.
+ JSLinearString* tenuredRootBase = tenuredBase;
+ const CharT* rootBaseChars = relocOverlay->savedNurseryChars<CharT>();
+ *offset = dependentStrChars - rootBaseChars;
+ MOZ_ASSERT(*offset < tenuredRootBase->length());
+ tenuredDependentStr->relocateNonInlineChars<const CharT*>(
+ tenuredRootBase->nonInlineChars<CharT>(nogc), *offset);
+ tenuredDependentStr->setBase(tenuredRootBase);
+ return;
+ }
+
+ baseOrRelocOverlay = relocOverlay->savedNurseryBaseOrRelocOverlay();
+
+ } else {
+ JSLinearString* base = baseOrRelocOverlay;
+
+ if (!base->hasBase()) {
+ // The root base is not forwarded yet, it is simply base.
+ *rootBase = base;
+
+ // The root base can be in either the nursery or the tenured heap.
+ // dependentStr chars needs to be relocated after traceString if the
+ // root base is in the nursery.
+ if (!(*rootBase)->isTenured()) {
+ *rootBaseNotYetForwarded = true;
+ const CharT* rootBaseChars = (*rootBase)->nonInlineChars<CharT>(nogc);
+ *offset = dependentStrChars - rootBaseChars;
+ MOZ_ASSERT(*offset < base->length(), "Tenured root base");
+ }
+
+ tenuredDependentStr->setBase(*rootBase);
+
+ return;
+ }
+
+ baseOrRelocOverlay = base->nurseryBaseOrRelocOverlay();
+ }
+ }
+}
+
+JS::BigInt* js::gc::TenuringTracer::moveToTenured(JS::BigInt* src) {
+ MOZ_ASSERT(IsInsideNursery(src));
+
+ AllocKind dstKind = src->getAllocKind();
+ Zone* zone = src->nurseryZone();
+ zone->tenuredBigInts++;
+
+ JS::BigInt* dst = allocTenured<JS::BigInt>(zone, dstKind);
+ tenuredSize += moveBigIntToTenured(dst, src, dstKind);
+ tenuredCells++;
+
+ RelocationOverlay::forwardCell(src, dst);
+
+ gcprobes::PromoteToTenured(src, dst);
+ return dst;
+}
+
+void js::gc::TenuringTracer::collectToObjectFixedPoint() {
+ while (RelocationOverlay* p = objHead) {
+ objHead = objHead->next();
+ auto* obj = static_cast<JSObject*>(p->forwardingAddress());
+ traceObject(obj);
+ }
+}
+
+void js::gc::TenuringTracer::collectToStringFixedPoint() {
+ while (StringRelocationOverlay* p = stringHead) {
+ stringHead = stringHead->next();
+
+ auto* tenuredStr = static_cast<JSString*>(p->forwardingAddress());
+ // To ensure the NON_DEDUP_BIT was reset properly.
+ MOZ_ASSERT(tenuredStr->isDeduplicatable());
+
+ // The nursery root base might not be forwarded before
+ // traceString(tenuredStr). traceString(tenuredStr) will forward the root
+ // base if that's the case. Dependent string chars needs to be relocated
+ // after traceString if root base was not forwarded.
+ size_t offset = 0;
+ bool rootBaseNotYetForwarded = false;
+ JSLinearString* rootBase = nullptr;
+
+ if (tenuredStr->isDependent()) {
+ if (tenuredStr->hasTwoByteChars()) {
+ relocateDependentStringChars<char16_t>(
+ &tenuredStr->asDependent(), p->savedNurseryBaseOrRelocOverlay(),
+ &offset, &rootBaseNotYetForwarded, &rootBase);
+ } else {
+ relocateDependentStringChars<JS::Latin1Char>(
+ &tenuredStr->asDependent(), p->savedNurseryBaseOrRelocOverlay(),
+ &offset, &rootBaseNotYetForwarded, &rootBase);
+ }
+ }
+
+ traceString(tenuredStr);
+
+ if (rootBaseNotYetForwarded) {
+ MOZ_ASSERT(rootBase->isForwarded(),
+ "traceString() should make it forwarded");
+ JS::AutoCheckCannotGC nogc;
+
+ JSLinearString* tenuredRootBase = Forwarded(rootBase);
+ MOZ_ASSERT(offset < tenuredRootBase->length());
+
+ if (tenuredStr->hasTwoByteChars()) {
+ tenuredStr->asDependent().relocateNonInlineChars<const char16_t*>(
+ tenuredRootBase->twoByteChars(nogc), offset);
+ } else {
+ tenuredStr->asDependent().relocateNonInlineChars<const JS::Latin1Char*>(
+ tenuredRootBase->latin1Chars(nogc), offset);
+ }
+ tenuredStr->setBase(tenuredRootBase);
+ }
+ }
+}
+
+size_t js::gc::TenuringTracer::moveStringToTenured(JSString* dst, JSString* src,
+ AllocKind dstKind) {
+ size_t size = Arena::thingSize(dstKind);
+
+ // At the moment, strings always have the same AllocKind between src and
+ // dst. This may change in the future.
+ MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind());
+
+ // Copy the Cell contents.
+ MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size));
+ js_memcpy(dst, src, size);
+
+ if (src->ownsMallocedChars()) {
+ void* chars = src->asLinear().nonInlineCharsRaw();
+ nursery().removeMallocedBufferDuringMinorGC(chars);
+ AddCellMemory(dst, dst->asLinear().allocSize(), MemoryUse::StringContents);
+ }
+
+ return size;
+}
+
+size_t js::gc::TenuringTracer::moveBigIntToTenured(JS::BigInt* dst,
+ JS::BigInt* src,
+ AllocKind dstKind) {
+ size_t size = Arena::thingSize(dstKind);
+
+ // At the moment, BigInts always have the same AllocKind between src and
+ // dst. This may change in the future.
+ MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind());
+
+ // Copy the Cell contents.
+ MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size));
+ js_memcpy(dst, src, size);
+
+ MOZ_ASSERT(dst->zone() == src->nurseryZone());
+
+ if (src->hasHeapDigits()) {
+ size_t length = dst->digitLength();
+ if (!nursery().isInside(src->heapDigits_)) {
+ nursery().removeMallocedBufferDuringMinorGC(src->heapDigits_);
+ } else {
+ Zone* zone = src->nurseryZone();
+ {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ dst->heapDigits_ = zone->pod_malloc<JS::BigInt::Digit>(length);
+ if (!dst->heapDigits_) {
+ oomUnsafe.crash(sizeof(JS::BigInt::Digit) * length,
+ "Failed to allocate digits while tenuring.");
+ }
+ }
+
+ PodCopy(dst->heapDigits_, src->heapDigits_, length);
+ nursery().setDirectForwardingPointer(src->heapDigits_, dst->heapDigits_);
+
+ size += length * sizeof(JS::BigInt::Digit);
+ }
+
+ AddCellMemory(dst, length * sizeof(JS::BigInt::Digit),
+ MemoryUse::BigIntDigits);
+ }
+
+ return size;
+}
+
+MinorSweepingTracer::MinorSweepingTracer(JSRuntime* rt)
+ : GenericTracerImpl(rt, JS::TracerKind::MinorSweeping,
+ JS::WeakMapTraceAction::TraceKeysAndValues) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+ MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+}
+
+template <typename T>
+inline void MinorSweepingTracer::onEdge(T** thingp, const char* name) {
+ T* thing = *thingp;
+ if (thing->isTenured()) {
+ return;
+ }
+
+ if (IsForwarded(thing)) {
+ *thingp = Forwarded(thing);
+ return;
+ }
+
+ *thingp = nullptr;
+}
diff --git a/js/src/gc/Tenuring.h b/js/src/gc/Tenuring.h
new file mode 100644
index 0000000000..34ef950e1d
--- /dev/null
+++ b/js/src/gc/Tenuring.h
@@ -0,0 +1,169 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Tenuring_h
+#define gc_Tenuring_h
+
+#include "mozilla/Maybe.h"
+
+#include "gc/AllocKind.h"
+#include "js/GCAPI.h"
+#include "js/TracingAPI.h"
+#include "util/Text.h"
+
+namespace js {
+
+class NativeObject;
+class Nursery;
+class PlainObject;
+
+namespace gc {
+
+class RelocationOverlay;
+class StringRelocationOverlay;
+
+template <typename Key>
+struct DeduplicationStringHasher {
+ using Lookup = Key;
+
+ static inline HashNumber hash(const Lookup& lookup) {
+ JS::AutoCheckCannotGC nogc;
+ HashNumber strHash;
+
+ // Include flags in the hash. A string relocation overlay stores either
+ // the nursery root base chars or the dependent string nursery base, but
+ // does not indicate which one. If strings with different string types
+ // were deduplicated, for example, a dependent string gets deduplicated
+ // into an extensible string, the base chain would be broken and the root
+ // base would be unreachable.
+
+ if (lookup->asLinear().hasLatin1Chars()) {
+ strHash = mozilla::HashString(lookup->asLinear().latin1Chars(nogc),
+ lookup->length());
+ } else {
+ MOZ_ASSERT(lookup->asLinear().hasTwoByteChars());
+ strHash = mozilla::HashString(lookup->asLinear().twoByteChars(nogc),
+ lookup->length());
+ }
+
+ return mozilla::HashGeneric(strHash, lookup->zone(), lookup->flags());
+ }
+
+ static MOZ_ALWAYS_INLINE bool match(const Key& key, const Lookup& lookup) {
+ if (!key->sameLengthAndFlags(*lookup) ||
+ key->asTenured().zone() != lookup->zone() ||
+ key->asTenured().getAllocKind() != lookup->getAllocKind()) {
+ return false;
+ }
+
+ JS::AutoCheckCannotGC nogc;
+
+ if (key->asLinear().hasLatin1Chars()) {
+ MOZ_ASSERT(lookup->asLinear().hasLatin1Chars());
+ return EqualChars(key->asLinear().latin1Chars(nogc),
+ lookup->asLinear().latin1Chars(nogc), lookup->length());
+ } else {
+ MOZ_ASSERT(key->asLinear().hasTwoByteChars());
+ MOZ_ASSERT(lookup->asLinear().hasTwoByteChars());
+ return EqualChars(key->asLinear().twoByteChars(nogc),
+ lookup->asLinear().twoByteChars(nogc),
+ lookup->length());
+ }
+ }
+};
+
+class TenuringTracer final : public JSTracer {
+ Nursery& nursery_;
+
+ // Amount of data moved to the tenured generation during collection.
+ size_t tenuredSize = 0;
+ // Number of cells moved to the tenured generation.
+ size_t tenuredCells = 0;
+
+ // These lists are threaded through the Nursery using the space from
+ // already moved things. The lists are used to fix up the moved things and
+ // to find things held live by intra-Nursery pointers.
+ gc::RelocationOverlay* objHead = nullptr;
+ gc::StringRelocationOverlay* stringHead = nullptr;
+
+ using StringDeDupSet =
+ HashSet<JSString*, DeduplicationStringHasher<JSString*>,
+ SystemAllocPolicy>;
+
+ // deDupSet is emplaced at the beginning of the nursery collection and reset
+ // at the end of the nursery collection. It can also be reset during nursery
+ // collection when out of memory to insert new entries.
+ mozilla::Maybe<StringDeDupSet> stringDeDupSet;
+
+#define DEFINE_ON_EDGE_METHOD(name, type, _1, _2) \
+ void on##name##Edge(type** thingp, const char* name) override;
+ JS_FOR_EACH_TRACEKIND(DEFINE_ON_EDGE_METHOD)
+#undef DEFINE_ON_EDGE_METHOD
+
+ public:
+ TenuringTracer(JSRuntime* rt, Nursery* nursery);
+
+ Nursery& nursery() { return nursery_; }
+
+ // Move all objects and everything they can reach to the tenured heap. Called
+ // after all roots have been traced.
+ void collectToObjectFixedPoint();
+
+ // Move all strings and all strings they can reach to the tenured heap, and
+ // additionally do any fixups for when strings are pointing into memory that
+ // was deduplicated. Called after collectToObjectFixedPoint().
+ void collectToStringFixedPoint();
+
+ size_t getTenuredSize() const;
+ size_t getTenuredCells() const;
+
+ void traverse(JS::Value* thingp);
+
+ // The store buffers need to be able to call these directly.
+ void traceObject(JSObject* src);
+ void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t end);
+ void traceSlots(JS::Value* vp, uint32_t nslots);
+ void traceString(JSString* src);
+ void traceBigInt(JS::BigInt* src);
+
+ private:
+ // The dependent string chars needs to be relocated if the base which it's
+ // using chars from has been deduplicated.
+ template <typename CharT>
+ void relocateDependentStringChars(JSDependentString* tenuredDependentStr,
+ JSLinearString* baseOrRelocOverlay,
+ size_t* offset,
+ bool* rootBaseNotYetForwarded,
+ JSLinearString** rootBase);
+
+ inline void insertIntoObjectFixupList(gc::RelocationOverlay* entry);
+ inline void insertIntoStringFixupList(gc::StringRelocationOverlay* entry);
+
+ template <typename T>
+ inline T* allocTenured(JS::Zone* zone, gc::AllocKind kind);
+ JSString* allocTenuredString(JSString* src, JS::Zone* zone,
+ gc::AllocKind dstKind);
+
+ inline JSObject* movePlainObjectToTenured(PlainObject* src);
+ JSObject* moveToTenuredSlow(JSObject* src);
+ JSString* moveToTenured(JSString* src);
+ JS::BigInt* moveToTenured(JS::BigInt* src);
+
+ size_t moveElementsToTenured(NativeObject* dst, NativeObject* src,
+ gc::AllocKind dstKind);
+ size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src);
+ size_t moveStringToTenured(JSString* dst, JSString* src,
+ gc::AllocKind dstKind);
+ size_t moveBigIntToTenured(JS::BigInt* dst, JS::BigInt* src,
+ gc::AllocKind dstKind);
+
+ void traceSlots(JS::Value* vp, JS::Value* end);
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_Tenuring_h
diff --git a/js/src/gc/TraceKind.h b/js/src/gc/TraceKind.h
new file mode 100644
index 0000000000..fdcb162239
--- /dev/null
+++ b/js/src/gc/TraceKind.h
@@ -0,0 +1,59 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_TraceKind_h
+#define gc_TraceKind_h
+
+#include "js/TraceKind.h"
+
+namespace js {
+namespace gc {
+
+// Map from all trace kinds to the base GC type.
+template <JS::TraceKind kind>
+struct MapTraceKindToType {};
+
+#define DEFINE_TRACE_KIND_MAP(name, type, _, _1) \
+ template <> \
+ struct MapTraceKindToType<JS::TraceKind::name> { \
+ using Type = type; \
+ };
+JS_FOR_EACH_TRACEKIND(DEFINE_TRACE_KIND_MAP);
+#undef DEFINE_TRACE_KIND_MAP
+
+// Map from a possibly-derived type to the base GC type.
+template <typename T>
+struct BaseGCType {
+ using type =
+ typename MapTraceKindToType<JS::MapTypeToTraceKind<T>::kind>::Type;
+ static_assert(std::is_base_of_v<type, T>, "Failed to find base type");
+};
+
+template <typename T>
+struct TraceKindCanBeGray {};
+#define EXPAND_TRACEKIND_DEF(_, type, canBeGray, _1) \
+ template <> \
+ struct TraceKindCanBeGray<type> { \
+ static constexpr bool value = canBeGray; \
+ };
+JS_FOR_EACH_TRACEKIND(EXPAND_TRACEKIND_DEF)
+#undef EXPAND_TRACEKIND_DEF
+
+struct TraceKindCanBeGrayFunctor {
+ template <typename T>
+ bool operator()() {
+ return TraceKindCanBeGray<T>::value;
+ }
+};
+
+static inline bool TraceKindCanBeMarkedGray(JS::TraceKind kind) {
+ return DispatchTraceKindTyped(TraceKindCanBeGrayFunctor(), kind);
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_TraceKind_h */
diff --git a/js/src/gc/TraceMethods-inl.h b/js/src/gc/TraceMethods-inl.h
new file mode 100644
index 0000000000..67d4142e0a
--- /dev/null
+++ b/js/src/gc/TraceMethods-inl.h
@@ -0,0 +1,384 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Trace methods for all GC things, defined in a separate header to allow
+ * inlining.
+ *
+ * This also includes eager inline marking versions. Both paths must end up
+ * traversing equivalent subgraphs.
+ */
+
+#ifndef gc_TraceMethods_inl_h
+#define gc_TraceMethods_inl_h
+
+#include "gc/GCMarker.h"
+#include "gc/Tracer.h"
+#include "jit/JitCode.h"
+#include "vm/BigIntType.h"
+#include "vm/GetterSetter.h"
+#include "vm/GlobalObject.h"
+#include "vm/JSScript.h"
+#include "vm/PropMap.h"
+#include "vm/Realm.h"
+#include "vm/Scope.h"
+#include "vm/Shape.h"
+#include "vm/StringType.h"
+#include "vm/SymbolType.h"
+#include "wasm/WasmJS.h"
+
+inline void js::BaseScript::traceChildren(JSTracer* trc) {
+ TraceNullableEdge(trc, &function_, "function");
+ TraceEdge(trc, &sourceObject_, "sourceObject");
+
+ warmUpData_.trace(trc);
+
+ if (data_) {
+ data_->trace(trc);
+ }
+}
+
+inline void js::Shape::traceChildren(JSTracer* trc) {
+ TraceCellHeaderEdge(trc, this, "base");
+ if (isNative()) {
+ asNative().traceChildren(trc);
+ }
+}
+
+inline void js::NativeShape::traceChildren(JSTracer* trc) {
+ TraceNullableEdge(trc, &propMap_, "propertymap");
+}
+
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(Shape* shape) {
+ MOZ_ASSERT(shape->isMarked(markColor()));
+
+ BaseShape* base = shape->base();
+ checkTraversedEdge(shape, base);
+ if (mark<opts>(base)) {
+ base->traceChildren(tracer());
+ }
+
+ if (shape->isNative()) {
+ if (PropMap* map = shape->asNative().propMap()) {
+ markAndTraverseEdge<opts>(shape, map);
+ }
+ }
+}
+
+inline void JSString::traceChildren(JSTracer* trc) {
+ if (hasBase()) {
+ traceBase(trc);
+ } else if (isRope()) {
+ asRope().traceChildren(trc);
+ }
+}
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(JSString* str) {
+ if (str->isLinear()) {
+ eagerlyMarkChildren<opts>(&str->asLinear());
+ } else {
+ eagerlyMarkChildren<opts>(&str->asRope());
+ }
+}
+
+inline void JSString::traceBase(JSTracer* trc) {
+ MOZ_ASSERT(hasBase());
+ js::TraceManuallyBarrieredEdge(trc, &d.s.u3.base, "base");
+}
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(JSLinearString* linearStr) {
+ gc::AssertShouldMarkInZone(this, linearStr);
+ MOZ_ASSERT(linearStr->isMarkedAny());
+ MOZ_ASSERT(linearStr->JSString::isLinear());
+
+ // Use iterative marking to avoid blowing out the stack.
+ while (linearStr->hasBase()) {
+ linearStr = linearStr->base();
+
+ // It's possible to observe a rope as the base of a linear string if we
+ // process barriers during rope flattening. See the assignment of base in
+ // JSRope::flattenInternal's finish_node section.
+ if (static_cast<JSString*>(linearStr)->isRope()) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsMajorCollecting());
+ break;
+ }
+
+ MOZ_ASSERT(linearStr->JSString::isLinear());
+ MOZ_ASSERT(!linearStr->isPermanentAtom());
+ gc::AssertShouldMarkInZone(this, linearStr);
+ if (!mark<opts>(static_cast<JSString*>(linearStr))) {
+ break;
+ }
+ }
+}
+
+inline void JSRope::traceChildren(JSTracer* trc) {
+ js::TraceManuallyBarrieredEdge(trc, &d.s.u2.left, "left child");
+ js::TraceManuallyBarrieredEdge(trc, &d.s.u3.right, "right child");
+}
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(JSRope* rope) {
+ // This function tries to scan the whole rope tree using the marking stack
+ // as temporary storage. If that becomes full, the unscanned ropes are
+ // added to the delayed marking list. When the function returns, the
+ // marking stack is at the same depth as it was on entry. This way we avoid
+ // using tags when pushing ropes to the stack as ropes never leak to other
+ // users of the stack. This also assumes that a rope can only point to
+ // other ropes or linear strings, it cannot refer to GC things of other
+ // types.
+ size_t savedPos = stack.position();
+ MOZ_DIAGNOSTIC_ASSERT(rope->getTraceKind() == JS::TraceKind::String);
+ while (true) {
+ MOZ_DIAGNOSTIC_ASSERT(rope->getTraceKind() == JS::TraceKind::String);
+ MOZ_DIAGNOSTIC_ASSERT(rope->JSString::isRope());
+ gc::AssertShouldMarkInZone(this, rope);
+ MOZ_ASSERT(rope->isMarkedAny());
+ JSRope* next = nullptr;
+
+ JSString* right = rope->rightChild();
+ if (mark<opts>(right)) {
+ MOZ_ASSERT(!right->isPermanentAtom());
+ if (right->isLinear()) {
+ eagerlyMarkChildren<opts>(&right->asLinear());
+ } else {
+ next = &right->asRope();
+ }
+ }
+
+ JSString* left = rope->leftChild();
+ if (mark<opts>(left)) {
+ MOZ_ASSERT(!left->isPermanentAtom());
+ if (left->isLinear()) {
+ eagerlyMarkChildren<opts>(&left->asLinear());
+ } else {
+ // When both children are ropes, set aside the right one to
+ // scan it later.
+ if (next && !stack.pushTempRope(next)) {
+ delayMarkingChildrenOnOOM(next);
+ }
+ next = &left->asRope();
+ }
+ }
+ if (next) {
+ rope = next;
+ } else if (savedPos != stack.position()) {
+ MOZ_ASSERT(savedPos < stack.position());
+ rope = stack.popPtr().asTempRope();
+ } else {
+ break;
+ }
+ }
+ MOZ_ASSERT(savedPos == stack.position());
+}
+
+inline void JS::Symbol::traceChildren(JSTracer* trc) {
+ js::TraceNullableCellHeaderEdge(trc, this, "symbol description");
+}
+
+template <typename SlotInfo>
+void js::RuntimeScopeData<SlotInfo>::trace(JSTracer* trc) {
+ TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length);
+}
+
+inline void js::FunctionScope::RuntimeData::trace(JSTracer* trc) {
+ TraceNullableEdge(trc, &canonicalFunction, "scope canonical function");
+ TraceNullableBindingNames(trc, GetScopeDataTrailingNamesPointer(this),
+ length);
+}
+inline void js::ModuleScope::RuntimeData::trace(JSTracer* trc) {
+ TraceNullableEdge(trc, &module, "scope module");
+ TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length);
+}
+inline void js::WasmInstanceScope::RuntimeData::trace(JSTracer* trc) {
+ TraceNullableEdge(trc, &instance, "wasm instance");
+ TraceBindingNames(trc, GetScopeDataTrailingNamesPointer(this), length);
+}
+
+inline void js::Scope::traceChildren(JSTracer* trc) {
+ TraceNullableEdge(trc, &environmentShape_, "scope env shape");
+ TraceNullableEdge(trc, &enclosingScope_, "scope enclosing");
+ applyScopeDataTyped([trc](auto data) { data->trace(trc); });
+}
+
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(Scope* scope) {
+ do {
+ if (Shape* shape = scope->environmentShape()) {
+ markAndTraverseEdge<opts>(scope, shape);
+ }
+ mozilla::Span<AbstractBindingName<JSAtom>> names;
+ switch (scope->kind()) {
+ case ScopeKind::Function: {
+ FunctionScope::RuntimeData& data = scope->as<FunctionScope>().data();
+ if (data.canonicalFunction) {
+ markAndTraverseObjectEdge<opts>(scope, data.canonicalFunction);
+ }
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::FunctionBodyVar: {
+ VarScope::RuntimeData& data = scope->as<VarScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::Lexical:
+ case ScopeKind::SimpleCatch:
+ case ScopeKind::Catch:
+ case ScopeKind::NamedLambda:
+ case ScopeKind::StrictNamedLambda:
+ case ScopeKind::FunctionLexical: {
+ LexicalScope::RuntimeData& data = scope->as<LexicalScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::ClassBody: {
+ ClassBodyScope::RuntimeData& data = scope->as<ClassBodyScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::Global:
+ case ScopeKind::NonSyntactic: {
+ GlobalScope::RuntimeData& data = scope->as<GlobalScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::Eval:
+ case ScopeKind::StrictEval: {
+ EvalScope::RuntimeData& data = scope->as<EvalScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::Module: {
+ ModuleScope::RuntimeData& data = scope->as<ModuleScope>().data();
+ if (data.module) {
+ markAndTraverseObjectEdge<opts>(scope, data.module);
+ }
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::With:
+ break;
+
+ case ScopeKind::WasmInstance: {
+ WasmInstanceScope::RuntimeData& data =
+ scope->as<WasmInstanceScope>().data();
+ markAndTraverseObjectEdge<opts>(scope, data.instance);
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+
+ case ScopeKind::WasmFunction: {
+ WasmFunctionScope::RuntimeData& data =
+ scope->as<WasmFunctionScope>().data();
+ names = GetScopeDataTrailingNames(&data);
+ break;
+ }
+ }
+ if (scope->kind_ == ScopeKind::Function) {
+ for (auto& binding : names) {
+ if (JSAtom* name = binding.name()) {
+ markAndTraverseStringEdge<opts>(scope, name);
+ }
+ }
+ } else {
+ for (auto& binding : names) {
+ markAndTraverseStringEdge<opts>(scope, binding.name());
+ }
+ }
+ scope = scope->enclosing();
+ } while (scope && mark<opts>(scope));
+}
+
+inline void js::BaseShape::traceChildren(JSTracer* trc) {
+ // Note: the realm's global can be nullptr if we GC while creating the global.
+ if (JSObject* global = realm()->unsafeUnbarrieredMaybeGlobal()) {
+ TraceManuallyBarrieredEdge(trc, &global, "baseshape_global");
+ }
+
+ if (proto_.isObject()) {
+ TraceEdge(trc, &proto_, "baseshape_proto");
+ }
+}
+
+inline void js::GetterSetter::traceChildren(JSTracer* trc) {
+ if (getter()) {
+ TraceCellHeaderEdge(trc, this, "gettersetter_getter");
+ }
+ if (setter()) {
+ TraceEdge(trc, &setter_, "gettersetter_setter");
+ }
+}
+
+inline void js::PropMap::traceChildren(JSTracer* trc) {
+ if (hasPrevious()) {
+ TraceEdge(trc, &asLinked()->data_.previous, "propmap_previous");
+ }
+
+ if (isShared()) {
+ SharedPropMap::TreeData& treeData = asShared()->treeDataRef();
+ if (SharedPropMap* parent = treeData.parent.maybeMap()) {
+ TraceManuallyBarrieredEdge(trc, &parent, "propmap_parent");
+ if (parent != treeData.parent.map()) {
+ treeData.setParent(parent, treeData.parent.index());
+ }
+ }
+ }
+
+ for (uint32_t i = 0; i < PropMap::Capacity; i++) {
+ if (hasKey(i)) {
+ TraceEdge(trc, &keys_[i], "propmap_key");
+ }
+ }
+
+ if (canHaveTable() && asLinked()->hasTable()) {
+ asLinked()->data_.table->trace(trc);
+ }
+}
+
+template <uint32_t opts>
+void js::GCMarker::eagerlyMarkChildren(PropMap* map) {
+ MOZ_ASSERT(map->isMarkedAny());
+ do {
+ for (uint32_t i = 0; i < PropMap::Capacity; i++) {
+ if (map->hasKey(i)) {
+ markAndTraverseEdge<opts>(map, map->getKey(i));
+ }
+ }
+
+ if (map->canHaveTable()) {
+ // Special case: if a map has a table then all its pointers must point to
+ // this map or an ancestor. Since these pointers will be traced by this
+ // loop they do not need to be traced here as well.
+ MOZ_ASSERT(map->asLinked()->canSkipMarkingTable());
+ }
+
+ if (map->isDictionary()) {
+ map = map->asDictionary()->previous();
+ } else {
+ // For shared maps follow the |parent| link and not the |previous| link.
+ // They're different when a map had a branch that wasn't at the end of the
+ // map, but in this case they must have the same |previous| map. This is
+ // asserted in SharedPropMap::addChild. In other words, marking all
+ // |parent| maps will also mark all |previous| maps.
+ map = map->asShared()->treeDataRef().parent.maybeMap();
+ }
+ } while (map && mark<opts>(map));
+}
+
+inline void JS::BigInt::traceChildren(JSTracer* trc) {}
+
+// JitCode::traceChildren is not defined inline due to its dependence on
+// MacroAssembler.
+
+#endif // gc_TraceMethods_inl_h
diff --git a/js/src/gc/Tracer.cpp b/js/src/gc/Tracer.cpp
new file mode 100644
index 0000000000..d150e4d2b7
--- /dev/null
+++ b/js/src/gc/Tracer.cpp
@@ -0,0 +1,297 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Tracer.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include "NamespaceImports.h"
+
+#include "gc/PublicIterators.h"
+#include "jit/JitCode.h"
+#include "util/Memory.h"
+#include "util/Text.h"
+#include "vm/BigIntType.h"
+#include "vm/JSContext.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+#include "vm/RegExpShared.h"
+#include "vm/Scope.h"
+#include "vm/Shape.h"
+#include "vm/StringType.h"
+#include "vm/SymbolType.h"
+
+#include "gc/TraceMethods-inl.h"
+#include "vm/Shape-inl.h"
+
+using namespace js;
+using namespace js::gc;
+using mozilla::DebugOnly;
+
+template void RuntimeScopeData<LexicalScope::SlotInfo>::trace(JSTracer* trc);
+template void RuntimeScopeData<ClassBodyScope::SlotInfo>::trace(JSTracer* trc);
+template void RuntimeScopeData<VarScope::SlotInfo>::trace(JSTracer* trc);
+template void RuntimeScopeData<GlobalScope::SlotInfo>::trace(JSTracer* trc);
+template void RuntimeScopeData<EvalScope::SlotInfo>::trace(JSTracer* trc);
+template void RuntimeScopeData<WasmFunctionScope::SlotInfo>::trace(
+ JSTracer* trc);
+
+void JS::TracingContext::getEdgeName(const char* name, char* buffer,
+ size_t bufferSize) {
+ MOZ_ASSERT(bufferSize > 0);
+ if (functor_) {
+ (*functor_)(this, buffer, bufferSize);
+ return;
+ }
+ if (index_ != InvalidIndex) {
+ snprintf(buffer, bufferSize, "%s[%zu]", name, index_);
+ return;
+ }
+ snprintf(buffer, bufferSize, "%s", name);
+}
+
+/*** Public Tracing API *****************************************************/
+
+JS_PUBLIC_API void JS::TraceChildren(JSTracer* trc, GCCellPtr thing) {
+ ApplyGCThingTyped(thing.asCell(), thing.kind(), [trc](auto t) {
+ MOZ_ASSERT_IF(t->runtimeFromAnyThread() != trc->runtime(),
+ t->isPermanentAndMayBeShared());
+ t->traceChildren(trc);
+ });
+}
+
+void js::gc::TraceIncomingCCWs(JSTracer* trc,
+ const JS::CompartmentSet& compartments) {
+ for (CompartmentsIter source(trc->runtime()); !source.done(); source.next()) {
+ if (compartments.has(source)) {
+ continue;
+ }
+ // Iterate over all compartments that |source| has wrappers for.
+ for (Compartment::WrappedObjectCompartmentEnum dest(source); !dest.empty();
+ dest.popFront()) {
+ if (!compartments.has(dest)) {
+ continue;
+ }
+ // Iterate over all wrappers from |source| to |dest| compartments.
+ for (Compartment::ObjectWrapperEnum e(source, dest); !e.empty();
+ e.popFront()) {
+ JSObject* obj = e.front().key();
+ MOZ_ASSERT(compartments.has(obj->compartment()));
+ mozilla::DebugOnly<JSObject*> prior = obj;
+ TraceManuallyBarrieredEdge(trc, &obj,
+ "cross-compartment wrapper target");
+ MOZ_ASSERT(obj == prior);
+ }
+ }
+ }
+}
+
+/*** Cycle Collector Helpers ************************************************/
+
+// This function is used by the Cycle Collector (CC) to trace through -- or in
+// CC parlance, traverse -- a Shape. The CC does not care about Shapes,
+// BaseShapes or PropMaps, only the JSObjects held live by them. Thus, we only
+// report non-Shape things.
+void gc::TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape) {
+ shape->base()->traceChildren(trc);
+ // Don't trace the PropMap because the CC doesn't care about PropertyKey.
+}
+
+/*** Traced Edge Printer ****************************************************/
+
+static size_t CountDecimalDigits(size_t num) {
+ size_t numDigits = 0;
+ do {
+ num /= 10;
+ numDigits++;
+ } while (num > 0);
+
+ return numDigits;
+}
+
+static const char* StringKindHeader(JSString* str) {
+ MOZ_ASSERT(str->isLinear());
+
+ if (str->isAtom()) {
+ if (str->isPermanentAtom()) {
+ return "permanent atom: ";
+ }
+ return "atom: ";
+ }
+
+ if (str->isExtensible()) {
+ return "extensible: ";
+ }
+
+ if (str->isInline()) {
+ if (str->isFatInline()) {
+ return "fat inline: ";
+ }
+ return "inline: ";
+ }
+
+ if (str->isDependent()) {
+ return "dependent: ";
+ }
+
+ if (str->isExternal()) {
+ return "external: ";
+ }
+
+ return "linear: ";
+}
+
+void js::gc::GetTraceThingInfo(char* buf, size_t bufsize, void* thing,
+ JS::TraceKind kind, bool details) {
+ const char* name = nullptr; /* silence uninitialized warning */
+ size_t n;
+
+ if (bufsize == 0) {
+ return;
+ }
+
+ switch (kind) {
+ case JS::TraceKind::BaseShape:
+ name = "base_shape";
+ break;
+
+ case JS::TraceKind::GetterSetter:
+ name = "getter_setter";
+ break;
+
+ case JS::TraceKind::PropMap:
+ name = "prop_map";
+ break;
+
+ case JS::TraceKind::JitCode:
+ name = "jitcode";
+ break;
+
+ case JS::TraceKind::Null:
+ name = "null_pointer";
+ break;
+
+ case JS::TraceKind::Object: {
+ name = static_cast<JSObject*>(thing)->getClass()->name;
+ break;
+ }
+
+ case JS::TraceKind::RegExpShared:
+ name = "reg_exp_shared";
+ break;
+
+ case JS::TraceKind::Scope:
+ name = "scope";
+ break;
+
+ case JS::TraceKind::Script:
+ name = "script";
+ break;
+
+ case JS::TraceKind::Shape:
+ name = "shape";
+ break;
+
+ case JS::TraceKind::String:
+ name = ((JSString*)thing)->isDependent() ? "substring" : "string";
+ break;
+
+ case JS::TraceKind::Symbol:
+ name = "symbol";
+ break;
+
+ case JS::TraceKind::BigInt:
+ name = "BigInt";
+ break;
+
+ default:
+ name = "INVALID";
+ break;
+ }
+
+ n = strlen(name);
+ if (n > bufsize - 1) {
+ n = bufsize - 1;
+ }
+ js_memcpy(buf, name, n + 1);
+ buf += n;
+ bufsize -= n;
+ *buf = '\0';
+
+ if (details && bufsize > 2) {
+ switch (kind) {
+ case JS::TraceKind::Object: {
+ JSObject* obj = (JSObject*)thing;
+ if (obj->is<JSFunction>()) {
+ JSFunction* fun = &obj->as<JSFunction>();
+ if (fun->displayAtom()) {
+ *buf++ = ' ';
+ bufsize--;
+ PutEscapedString(buf, bufsize, fun->displayAtom(), 0);
+ }
+ } else {
+ snprintf(buf, bufsize, " <unknown object>");
+ }
+ break;
+ }
+
+ case JS::TraceKind::Script: {
+ auto* script = static_cast<js::BaseScript*>(thing);
+ snprintf(buf, bufsize, " %s:%u", script->filename(), script->lineno());
+ break;
+ }
+
+ case JS::TraceKind::String: {
+ *buf++ = ' ';
+ bufsize--;
+ JSString* str = (JSString*)thing;
+
+ if (str->isLinear()) {
+ const char* header = StringKindHeader(str);
+ bool willFit = str->length() + strlen("<length > ") + strlen(header) +
+ CountDecimalDigits(str->length()) <
+ bufsize;
+
+ n = snprintf(buf, bufsize, "<%slength %zu%s> ", header, str->length(),
+ willFit ? "" : " (truncated)");
+ buf += n;
+ bufsize -= n;
+
+ PutEscapedString(buf, bufsize, &str->asLinear(), 0);
+ } else {
+ snprintf(buf, bufsize, "<rope: length %zu>", str->length());
+ }
+ break;
+ }
+
+ case JS::TraceKind::Symbol: {
+ *buf++ = ' ';
+ bufsize--;
+ auto* sym = static_cast<JS::Symbol*>(thing);
+ if (JSAtom* desc = sym->description()) {
+ PutEscapedString(buf, bufsize, desc, 0);
+ } else {
+ snprintf(buf, bufsize, "<null>");
+ }
+ break;
+ }
+
+ case JS::TraceKind::Scope: {
+ auto* scope = static_cast<js::Scope*>(thing);
+ snprintf(buf, bufsize, " %s", js::ScopeKindString(scope->kind()));
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+ buf[bufsize - 1] = '\0';
+}
+
+JS::CallbackTracer::CallbackTracer(JSContext* cx, JS::TracerKind kind,
+ JS::TraceOptions options)
+ : CallbackTracer(cx->runtime(), kind, options) {}
diff --git a/js/src/gc/Tracer.h b/js/src/gc/Tracer.h
new file mode 100644
index 0000000000..e1f10980de
--- /dev/null
+++ b/js/src/gc/Tracer.h
@@ -0,0 +1,401 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef js_Tracer_h
+#define js_Tracer_h
+
+#include "gc/Barrier.h"
+#include "gc/TraceKind.h"
+#include "js/HashTable.h"
+#include "js/TracingAPI.h"
+
+namespace JS {
+using CompartmentSet =
+ js::HashSet<Compartment*, js::DefaultHasher<Compartment*>,
+ js::SystemAllocPolicy>;
+} // namespace JS
+
+namespace js {
+
+class TaggedProto;
+
+// Internal Tracing API
+//
+// Tracing is an abstract visitation of each edge in a JS heap graph.[1] The
+// most common (and performance sensitive) use of this infrastructure is for GC
+// "marking" as part of the mark-and-sweep collector; however, this
+// infrastructure is much more general than that and is used for many other
+// purposes as well.
+//
+// One commonly misunderstood subtlety of the tracing architecture is the role
+// of graph vertices versus graph edges. Graph vertices are the heap
+// allocations -- GC things -- that are returned by Allocate. Graph edges are
+// pointers -- including tagged pointers like Value and jsid -- that link the
+// allocations into a complex heap. The tracing API deals *only* with edges.
+// Any action taken on the target of a graph edge is independent of the tracing
+// itself.
+//
+// Another common misunderstanding relates to the role of the JSTracer. The
+// JSTracer instance determines what tracing does when visiting an edge; it
+// does not itself participate in the tracing process, other than to be passed
+// through as opaque data. It works like a closure in that respect.
+//
+// Tracing implementations internal to SpiderMonkey should use these interfaces
+// instead of the public interfaces in js/TracingAPI.h. Unlike the public
+// tracing methods, these work on internal types and avoid an external call.
+//
+// Note that the implementations for these methods are, surprisingly, in
+// js/src/gc/Marking.cpp. This is so that the compiler can inline as much as
+// possible in the common, marking pathways. Conceptually, however, they remain
+// as part of the generic "tracing" architecture, rather than the more specific
+// marking implementation of tracing.
+//
+// 1 - In SpiderMonkey, we call this concept tracing rather than visiting
+// because "visiting" is already used by the compiler. Also, it's been
+// called "tracing" forever and changing it would be extremely difficult at
+// this point.
+
+class GCMarker;
+
+// Debugging functions to check tracing invariants.
+#ifdef DEBUG
+template <typename T>
+void CheckTracedThing(JSTracer* trc, T* thing);
+template <typename T>
+void CheckTracedThing(JSTracer* trc, const T& thing);
+#else
+template <typename T>
+inline void CheckTracedThing(JSTracer* trc, T* thing) {}
+template <typename T>
+inline void CheckTracedThing(JSTracer* trc, const T& thing) {}
+#endif
+
+namespace gc {
+
+// Our barrier templates are parameterized on the pointer types so that we can
+// share the definitions with Value and jsid. Thus, we need to strip the
+// pointer before sending the type to BaseGCType and re-add it on the other
+// side. As such:
+template <typename T>
+struct PtrBaseGCType {
+ using type = T;
+};
+template <typename T>
+struct PtrBaseGCType<T*> {
+ using type = typename BaseGCType<T>::type*;
+};
+
+// Cast a possibly-derived T** pointer to a base class pointer.
+template <typename T>
+typename PtrBaseGCType<T>::type* ConvertToBase(T* thingp) {
+ return reinterpret_cast<typename PtrBaseGCType<T>::type*>(thingp);
+}
+
+// Internal methods to trace edges.
+
+#define DEFINE_TRACE_FUNCTION(name, type, _1, _2) \
+ MOZ_ALWAYS_INLINE bool TraceEdgeInternal(JSTracer* trc, type** thingp, \
+ const char* name) { \
+ CheckTracedThing(trc, *thingp); \
+ trc->on##name##Edge(thingp, name); \
+ return *thingp; \
+ }
+JS_FOR_EACH_TRACEKIND(DEFINE_TRACE_FUNCTION)
+#undef DEFINE_TRACE_FUNCTION
+
+bool TraceEdgeInternal(JSTracer* trc, Value* thingp, const char* name);
+bool TraceEdgeInternal(JSTracer* trc, jsid* thingp, const char* name);
+bool TraceEdgeInternal(JSTracer* trc, TaggedProto* thingp, const char* name);
+
+template <typename T>
+void TraceRangeInternal(JSTracer* trc, size_t len, T* vec, const char* name);
+template <typename T>
+bool TraceWeakMapKeyInternal(JSTracer* trc, Zone* zone, T* thingp,
+ const char* name);
+
+#ifdef DEBUG
+void AssertRootMarkingPhase(JSTracer* trc);
+void AssertShouldMarkInZone(GCMarker* marker, gc::Cell* thing);
+#else
+inline void AssertRootMarkingPhase(JSTracer* trc) {}
+inline void AssertShouldMarkInZone(GCMarker* marker, gc::Cell* thing) {}
+#endif
+
+} // namespace gc
+
+// Trace through a strong edge in the live object graph on behalf of
+// tracing. The effect of tracing the edge depends on the JSTracer being
+// used. For pointer types, |*thingp| must not be null.
+//
+// Note that weak edges are handled separately. GC things with weak edges must
+// not trace those edges during marking tracing (which would keep the referent
+// alive) but instead arrange for the edge to be swept by calling
+// js::gc::IsAboutToBeFinalized or TraceWeakEdge during sweeping.
+//
+// GC things that are weakly held in containers can use WeakMap or a container
+// wrapped in the WeakCache<> template to perform the appropriate sweeping.
+
+template <typename T>
+inline void TraceEdge(JSTracer* trc, const WriteBarriered<T>* thingp,
+ const char* name) {
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unbarrieredAddress()),
+ name);
+}
+
+template <typename T>
+inline void TraceEdge(JSTracer* trc, WeakHeapPtr<T>* thingp, const char* name) {
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unbarrieredAddress()),
+ name);
+}
+
+template <class BC, class T>
+inline void TraceCellHeaderEdge(JSTracer* trc,
+ gc::CellWithTenuredGCPointer<BC, T>* thingp,
+ const char* name) {
+ T* thing = thingp->headerPtr();
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name);
+ if (thing != thingp->headerPtr()) {
+ thingp->unbarrieredSetHeaderPtr(thing);
+ }
+}
+
+template <class T>
+inline void TraceCellHeaderEdge(JSTracer* trc,
+ gc::TenuredCellWithGCPointer<T>* thingp,
+ const char* name) {
+ T* thing = thingp->headerPtr();
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name);
+ if (thing != thingp->headerPtr()) {
+ thingp->unbarrieredSetHeaderPtr(thing);
+ }
+}
+
+// Trace through a possibly-null edge in the live object graph on behalf of
+// tracing.
+
+template <typename T>
+inline void TraceNullableEdge(JSTracer* trc, const WriteBarriered<T>* thingp,
+ const char* name) {
+ if (InternalBarrierMethods<T>::isMarkable(thingp->get())) {
+ TraceEdge(trc, thingp, name);
+ }
+}
+
+template <typename T>
+inline void TraceNullableEdge(JSTracer* trc, WeakHeapPtr<T>* thingp,
+ const char* name) {
+ if (InternalBarrierMethods<T>::isMarkable(thingp->unbarrieredGet())) {
+ TraceEdge(trc, thingp, name);
+ }
+}
+
+template <class BC, class T>
+inline void TraceNullableCellHeaderEdge(
+ JSTracer* trc, gc::CellWithTenuredGCPointer<BC, T>* thingp,
+ const char* name) {
+ T* thing = thingp->headerPtr();
+ if (thing) {
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(&thing), name);
+ if (thing != thingp->headerPtr()) {
+ thingp->unbarrieredSetHeaderPtr(thing);
+ }
+ }
+}
+
+// Trace through a "root" edge. These edges are the initial edges in the object
+// graph traversal. Root edges are asserted to only be traversed in the initial
+// phase of a GC.
+
+template <typename T>
+inline void TraceRoot(JSTracer* trc, T* thingp, const char* name) {
+ gc::AssertRootMarkingPhase(trc);
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
+
+template <typename T>
+inline void TraceRoot(JSTracer* trc, const HeapPtr<T>* thingp,
+ const char* name) {
+ TraceRoot(trc, thingp->unbarrieredAddress(), name);
+}
+
+// Idential to TraceRoot, except that this variant will not crash if |*thingp|
+// is null.
+
+template <typename T>
+inline void TraceNullableRoot(JSTracer* trc, T* thingp, const char* name) {
+ gc::AssertRootMarkingPhase(trc);
+ if (InternalBarrierMethods<T>::isMarkable(*thingp)) {
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+ }
+}
+
+template <typename T>
+inline void TraceNullableRoot(JSTracer* trc, WeakHeapPtr<T>* thingp,
+ const char* name) {
+ TraceNullableRoot(trc, thingp->unbarrieredAddress(), name);
+}
+
+// Like TraceEdge, but for edges that do not use one of the automatic barrier
+// classes and, thus, must be treated specially for moving GC. This method is
+// separate from TraceEdge to make accidental use of such edges more obvious.
+
+template <typename T>
+inline void TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp,
+ const char* name) {
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
+
+// Trace through a weak edge. If *thingp is not marked at the end of marking,
+// it is replaced by nullptr, and this method will return false to indicate that
+// the edge no longer exists.
+template <typename T>
+inline bool TraceManuallyBarrieredWeakEdge(JSTracer* trc, T* thingp,
+ const char* name) {
+ return gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
+
+// The result of tracing a weak edge, which can be either:
+//
+// - the target is dead (and the edge has been cleared), or
+// - the target is alive (and the edge may have been updated)
+//
+// This includes the initial and final values of the edge to allow cleanup if
+// the target is dead or access to the referent if it is alive.
+template <typename T>
+struct TraceWeakResult {
+ const bool live_;
+ const T initial_;
+ const T final_;
+
+ bool isLive() const { return live_; }
+ bool isDead() const { return !live_; }
+
+ MOZ_IMPLICIT operator bool() const { return isLive(); }
+
+ T initialTarget() const {
+ MOZ_ASSERT(isDead());
+ return initial_;
+ }
+
+ T finalTarget() const {
+ MOZ_ASSERT(isLive());
+ return final_;
+ }
+};
+
+template <typename T>
+inline TraceWeakResult<T> TraceWeakEdge(JSTracer* trc, BarrieredBase<T>* thingp,
+ const char* name) {
+ T* addr = thingp->unbarrieredAddress();
+ T initial = *addr;
+ bool live = !InternalBarrierMethods<T>::isMarkable(initial) ||
+ gc::TraceEdgeInternal(trc, gc::ConvertToBase(addr), name);
+ return TraceWeakResult<T>{live, initial, *addr};
+}
+
+// Trace all edges contained in the given array.
+
+template <typename T>
+void TraceRange(JSTracer* trc, size_t len, BarrieredBase<T>* vec,
+ const char* name) {
+ gc::TraceRangeInternal(trc, len,
+ gc::ConvertToBase(vec[0].unbarrieredAddress()), name);
+}
+
+// Trace all root edges in the given array.
+
+template <typename T>
+void TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name) {
+ gc::AssertRootMarkingPhase(trc);
+ gc::TraceRangeInternal(trc, len, gc::ConvertToBase(vec), name);
+}
+
+// As below but with manual barriers.
+template <typename T>
+void TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src,
+ T* dst, const char* name);
+
+// Trace an edge that crosses compartment boundaries. If the compartment of the
+// destination thing is not being GC'd, then the edge will not be traced.
+template <typename T>
+void TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src,
+ const WriteBarriered<T>* dst, const char* name) {
+ TraceManuallyBarrieredCrossCompartmentEdge(
+ trc, src, gc::ConvertToBase(dst->unbarrieredAddress()), name);
+}
+
+// Trace an edge that's guaranteed to be same-zone but may cross a compartment
+// boundary. This should NOT be used for object => object edges, as those have
+// to be in the cross-compartment wrapper map.
+//
+// WARNING: because this turns off certain compartment checks, you most likely
+// don't want to use this! If you still think you need this function, talk to a
+// GC peer first.
+template <typename T>
+void TraceSameZoneCrossCompartmentEdge(JSTracer* trc,
+ const WriteBarriered<T>* dst,
+ const char* name);
+
+// Trace a weak map key. For debugger weak maps these may be cross compartment,
+// but the compartment must always be within the current sweep group.
+template <typename T>
+void TraceWeakMapKeyEdgeInternal(JSTracer* trc, Zone* weakMapZone, T** thingp,
+ const char* name);
+
+template <typename T>
+inline void TraceWeakMapKeyEdge(JSTracer* trc, Zone* weakMapZone,
+ const WriteBarriered<T>* thingp,
+ const char* name) {
+ TraceWeakMapKeyEdgeInternal(
+ trc, weakMapZone, gc::ConvertToBase(thingp->unbarrieredAddress()), name);
+}
+
+// Trace a root edge that uses the base GC thing type, instead of a more
+// specific type.
+void TraceGenericPointerRoot(JSTracer* trc, gc::Cell** thingp,
+ const char* name);
+
+// Trace a non-root edge that uses the base GC thing type, instead of a more
+// specific type.
+void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp,
+ const char* name);
+
+void TraceGCCellPtrRoot(JSTracer* trc, JS::GCCellPtr* thingp, const char* name);
+
+void TraceManuallyBarrieredGCCellPtr(JSTracer* trc, JS::GCCellPtr* thingp,
+ const char* name);
+
+namespace gc {
+
+// Trace through a shape or group iteratively during cycle collection to avoid
+// deep or infinite recursion.
+void TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape);
+
+/**
+ * Trace every value within |compartments| that is wrapped by a
+ * cross-compartment wrapper from a compartment that is not an element of
+ * |compartments|.
+ */
+void TraceIncomingCCWs(JSTracer* trc, const JS::CompartmentSet& compartments);
+
+/* Get information about a GC thing. Used when dumping the heap. */
+void GetTraceThingInfo(char* buf, size_t bufsize, void* thing,
+ JS::TraceKind kind, bool includeDetails);
+
+// Overloaded function to call the correct GenericTracer method based on the
+// argument type.
+#define DEFINE_DISPATCH_FUNCTION(name, type, _1, _2) \
+ inline void DispatchToOnEdge(JSTracer* trc, type** thingp, \
+ const char* name) { \
+ trc->on##name##Edge(thingp, name); \
+ }
+JS_FOR_EACH_TRACEKIND(DEFINE_DISPATCH_FUNCTION)
+#undef DEFINE_DISPATCH_FUNCTION
+
+} // namespace gc
+} // namespace js
+
+#endif /* js_Tracer_h */
diff --git a/js/src/gc/Verifier.cpp b/js/src/gc/Verifier.cpp
new file mode 100644
index 0000000000..299d73c51f
--- /dev/null
+++ b/js/src/gc/Verifier.cpp
@@ -0,0 +1,1135 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Maybe.h"
+#include "mozilla/Sprintf.h"
+
+#include <algorithm>
+#include <utility>
+
+#ifdef MOZ_VALGRIND
+# include <valgrind/memcheck.h>
+#endif
+
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/PublicIterators.h"
+#include "gc/WeakMap.h"
+#include "gc/Zone.h"
+#include "js/friend/DumpFunctions.h" // js::DumpObject
+#include "js/HashTable.h"
+#include "vm/JSContext.h"
+
+#include "gc/ArenaList-inl.h"
+#include "gc/GC-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/PrivateIterators-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::DebugOnly;
+
+#ifdef JS_GC_ZEAL
+
+/*
+ * Write barrier verification
+ *
+ * The next few functions are for write barrier verification.
+ *
+ * The VerifyBarriers function is a shorthand. It checks if a verification phase
+ * is currently running. If not, it starts one. Otherwise, it ends the current
+ * phase and starts a new one.
+ *
+ * The user can adjust the frequency of verifications, which causes
+ * VerifyBarriers to be a no-op all but one out of N calls. However, if the
+ * |always| parameter is true, it starts a new phase no matter what.
+ *
+ * Pre-Barrier Verifier:
+ * When StartVerifyBarriers is called, a snapshot is taken of all objects in
+ * the GC heap and saved in an explicit graph data structure. Later,
+ * EndVerifyBarriers traverses the heap again. Any pointer values that were in
+ * the snapshot and are no longer found must be marked; otherwise an assertion
+ * triggers. Note that we must not GC in between starting and finishing a
+ * verification phase.
+ */
+
+struct EdgeValue {
+ JS::GCCellPtr thing;
+ const char* label;
+};
+
+struct VerifyNode {
+ JS::GCCellPtr thing;
+ uint32_t count;
+ EdgeValue edges[1];
+};
+
+typedef HashMap<Cell*, VerifyNode*, DefaultHasher<Cell*>, SystemAllocPolicy>
+ NodeMap;
+
+/*
+ * The verifier data structures are simple. The entire graph is stored in a
+ * single block of memory. At the beginning is a VerifyNode for the root
+ * node. It is followed by a sequence of EdgeValues--the exact number is given
+ * in the node. After the edges come more nodes and their edges.
+ *
+ * The edgeptr and term fields are used to allocate out of the block of memory
+ * for the graph. If we run out of memory (i.e., if edgeptr goes beyond term),
+ * we just abandon the verification.
+ *
+ * The nodemap field is a hashtable that maps from the address of the GC thing
+ * to the VerifyNode that represents it.
+ */
+class js::VerifyPreTracer final : public JS::CallbackTracer {
+ JS::AutoDisableGenerationalGC noggc;
+
+ void onChild(JS::GCCellPtr thing, const char* name) override;
+
+ public:
+ /* The gcNumber when the verification began. */
+ uint64_t number;
+
+ /* This counts up to gcZealFrequency to decide whether to verify. */
+ int count;
+
+ /* This graph represents the initial GC "snapshot". */
+ VerifyNode* curnode;
+ VerifyNode* root;
+ char* edgeptr;
+ char* term;
+ NodeMap nodemap;
+
+ explicit VerifyPreTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt, JS::TracerKind::Callback,
+ JS::WeakEdgeTraceAction::Skip),
+ noggc(rt->mainContextFromOwnThread()),
+ number(rt->gc.gcNumber()),
+ count(0),
+ curnode(nullptr),
+ root(nullptr),
+ edgeptr(nullptr),
+ term(nullptr) {
+ // We don't care about weak edges here. Since they are not marked they
+ // cannot cause the problem that the pre-write barrier protects against.
+ }
+
+ ~VerifyPreTracer() { js_free(root); }
+};
+
+/*
+ * This function builds up the heap snapshot by adding edges to the current
+ * node.
+ */
+void VerifyPreTracer::onChild(JS::GCCellPtr thing, const char* name) {
+ MOZ_ASSERT(!IsInsideNursery(thing.asCell()));
+
+ // Skip things in other runtimes.
+ if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) {
+ return;
+ }
+
+ edgeptr += sizeof(EdgeValue);
+ if (edgeptr >= term) {
+ edgeptr = term;
+ return;
+ }
+
+ VerifyNode* node = curnode;
+ uint32_t i = node->count;
+
+ node->edges[i].thing = thing;
+ node->edges[i].label = name;
+ node->count++;
+}
+
+static VerifyNode* MakeNode(VerifyPreTracer* trc, JS::GCCellPtr thing) {
+ NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing.asCell());
+ if (!p) {
+ VerifyNode* node = (VerifyNode*)trc->edgeptr;
+ trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue);
+ if (trc->edgeptr >= trc->term) {
+ trc->edgeptr = trc->term;
+ return nullptr;
+ }
+
+ node->thing = thing;
+ node->count = 0;
+ if (!trc->nodemap.add(p, thing.asCell(), node)) {
+ trc->edgeptr = trc->term;
+ return nullptr;
+ }
+
+ return node;
+ }
+ return nullptr;
+}
+
+static VerifyNode* NextNode(VerifyNode* node) {
+ if (node->count == 0) {
+ return (VerifyNode*)((char*)node + sizeof(VerifyNode) - sizeof(EdgeValue));
+ } else {
+ return (VerifyNode*)((char*)node + sizeof(VerifyNode) +
+ sizeof(EdgeValue) * (node->count - 1));
+ }
+}
+
+template <typename ZonesIterT>
+static void ClearMarkBits(GCRuntime* gc) {
+ // This does not clear the mark bits for permanent atoms, whose arenas are
+ // removed from the arena lists by GCRuntime::freezePermanentAtoms.
+
+ for (ZonesIterT zone(gc); !zone.done(); zone.next()) {
+ for (auto kind : AllAllocKinds()) {
+ for (ArenaIter arena(zone, kind); !arena.done(); arena.next()) {
+ arena->unmarkAll();
+ }
+ }
+ }
+}
+
+void gc::GCRuntime::startVerifyPreBarriers() {
+ if (verifyPreData || isIncrementalGCInProgress()) {
+ return;
+ }
+
+ JSContext* cx = rt->mainContextFromOwnThread();
+
+ if (IsIncrementalGCUnsafe(rt) != GCAbortReason::None) {
+ return;
+ }
+
+ number++;
+
+ VerifyPreTracer* trc = js_new<VerifyPreTracer>(rt);
+ if (!trc) {
+ return;
+ }
+
+ AutoPrepareForTracing prep(cx);
+
+ ClearMarkBits<AllZonesIter>(this);
+
+ gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::TRACE_HEAP);
+
+ const size_t size = 64 * 1024 * 1024;
+ trc->root = (VerifyNode*)js_malloc(size);
+ if (!trc->root) {
+ goto oom;
+ }
+ trc->edgeptr = (char*)trc->root;
+ trc->term = trc->edgeptr + size;
+
+ /* Create the root node. */
+ trc->curnode = MakeNode(trc, JS::GCCellPtr());
+
+ MOZ_ASSERT(incrementalState == State::NotActive);
+ incrementalState = State::MarkRoots;
+
+ /* Make all the roots be edges emanating from the root node. */
+ traceRuntime(trc, prep);
+
+ VerifyNode* node;
+ node = trc->curnode;
+ if (trc->edgeptr == trc->term) {
+ goto oom;
+ }
+
+ /* For each edge, make a node for it if one doesn't already exist. */
+ while ((char*)node < trc->edgeptr) {
+ for (uint32_t i = 0; i < node->count; i++) {
+ EdgeValue& e = node->edges[i];
+ VerifyNode* child = MakeNode(trc, e.thing);
+ if (child) {
+ trc->curnode = child;
+ JS::TraceChildren(trc, e.thing);
+ }
+ if (trc->edgeptr == trc->term) {
+ goto oom;
+ }
+ }
+
+ node = NextNode(node);
+ }
+
+ verifyPreData = trc;
+ incrementalState = State::Mark;
+ marker().start();
+
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ zone->changeGCState(Zone::NoGC, Zone::VerifyPreBarriers);
+ zone->setNeedsIncrementalBarrier(true);
+ zone->arenas.clearFreeLists();
+ }
+
+ return;
+
+oom:
+ incrementalState = State::NotActive;
+ js_delete(trc);
+ verifyPreData = nullptr;
+}
+
+static bool IsMarkedOrAllocated(TenuredCell* cell) {
+ return cell->isMarkedAny();
+}
+
+struct CheckEdgeTracer final : public JS::CallbackTracer {
+ VerifyNode* node;
+ explicit CheckEdgeTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt), node(nullptr) {}
+ void onChild(JS::GCCellPtr thing, const char* name) override;
+};
+
+static const uint32_t MAX_VERIFIER_EDGES = 1000;
+
+/*
+ * This function is called by EndVerifyBarriers for every heap edge. If the edge
+ * already existed in the original snapshot, we "cancel it out" by overwriting
+ * it with nullptr. EndVerifyBarriers later asserts that the remaining
+ * non-nullptr edges (i.e., the ones from the original snapshot that must have
+ * been modified) must point to marked objects.
+ */
+void CheckEdgeTracer::onChild(JS::GCCellPtr thing, const char* name) {
+ // Skip things in other runtimes.
+ if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) {
+ return;
+ }
+
+ /* Avoid n^2 behavior. */
+ if (node->count > MAX_VERIFIER_EDGES) {
+ return;
+ }
+
+ for (uint32_t i = 0; i < node->count; i++) {
+ if (node->edges[i].thing == thing) {
+ node->edges[i].thing = JS::GCCellPtr();
+ return;
+ }
+ }
+}
+
+static bool IsMarkedOrAllocated(const EdgeValue& edge) {
+ if (!edge.thing || IsMarkedOrAllocated(&edge.thing.asCell()->asTenured())) {
+ return true;
+ }
+
+ // Permanent atoms and well-known symbols aren't marked during graph
+ // traversal.
+ if (edge.thing.is<JSString>() &&
+ edge.thing.as<JSString>().isPermanentAtom()) {
+ return true;
+ }
+ if (edge.thing.is<JS::Symbol>() &&
+ edge.thing.as<JS::Symbol>().isWellKnownSymbol()) {
+ return true;
+ }
+
+ return false;
+}
+
+void gc::GCRuntime::endVerifyPreBarriers() {
+ VerifyPreTracer* trc = verifyPreData;
+
+ if (!trc) {
+ return;
+ }
+
+ MOZ_ASSERT(!JS::IsGenerationalGCEnabled(rt));
+
+ // Now that barrier marking has finished, prepare the heap to allow this
+ // method to trace cells and discover their outgoing edges.
+ AutoPrepareForTracing prep(rt->mainContextFromOwnThread());
+
+ bool compartmentCreated = false;
+
+ /* We need to disable barriers before tracing, which may invoke barriers. */
+ for (ZonesIter zone(this, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isVerifyingPreBarriers()) {
+ zone->changeGCState(Zone::VerifyPreBarriers, Zone::NoGC);
+ } else {
+ compartmentCreated = true;
+ }
+
+ MOZ_ASSERT(!zone->wasGCStarted());
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ }
+
+ verifyPreData = nullptr;
+ MOZ_ASSERT(incrementalState == State::Mark);
+ incrementalState = State::NotActive;
+
+ if (!compartmentCreated && IsIncrementalGCUnsafe(rt) == GCAbortReason::None) {
+ CheckEdgeTracer cetrc(rt);
+
+ /* Start after the roots. */
+ VerifyNode* node = NextNode(trc->root);
+ while ((char*)node < trc->edgeptr) {
+ cetrc.node = node;
+ JS::TraceChildren(&cetrc, node->thing);
+
+ if (node->count <= MAX_VERIFIER_EDGES) {
+ for (uint32_t i = 0; i < node->count; i++) {
+ EdgeValue& edge = node->edges[i];
+ if (!IsMarkedOrAllocated(edge)) {
+ char msgbuf[1024];
+ SprintfLiteral(
+ msgbuf,
+ "[barrier verifier] Unmarked edge: %s %p '%s' edge to %s %p",
+ JS::GCTraceKindToAscii(node->thing.kind()),
+ node->thing.asCell(), edge.label,
+ JS::GCTraceKindToAscii(edge.thing.kind()), edge.thing.asCell());
+ MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__);
+ MOZ_CRASH();
+ }
+ }
+ }
+
+ node = NextNode(node);
+ }
+ }
+
+ marker().reset();
+ marker().stop();
+ resetDelayedMarking();
+
+ js_delete(trc);
+}
+
+/*** Barrier Verifier Scheduling ***/
+
+void gc::GCRuntime::verifyPreBarriers() {
+ if (verifyPreData) {
+ endVerifyPreBarriers();
+ } else {
+ startVerifyPreBarriers();
+ }
+}
+
+void gc::VerifyBarriers(JSRuntime* rt, VerifierType type) {
+ if (type == PreBarrierVerifier) {
+ rt->gc.verifyPreBarriers();
+ }
+}
+
+void gc::GCRuntime::maybeVerifyPreBarriers(bool always) {
+ if (!hasZealMode(ZealMode::VerifierPre)) {
+ return;
+ }
+
+ if (rt->mainContextFromOwnThread()->suppressGC) {
+ return;
+ }
+
+ if (verifyPreData) {
+ if (++verifyPreData->count < zealFrequency && !always) {
+ return;
+ }
+
+ endVerifyPreBarriers();
+ }
+
+ startVerifyPreBarriers();
+}
+
+void js::gc::MaybeVerifyBarriers(JSContext* cx, bool always) {
+ GCRuntime* gc = &cx->runtime()->gc;
+ gc->maybeVerifyPreBarriers(always);
+}
+
+void js::gc::GCRuntime::finishVerifier() {
+ if (verifyPreData) {
+ js_delete(verifyPreData.ref());
+ verifyPreData = nullptr;
+ }
+}
+
+struct GCChunkHasher {
+ typedef gc::TenuredChunk* Lookup;
+
+ /*
+ * Strip zeros for better distribution after multiplying by the golden
+ * ratio.
+ */
+ static HashNumber hash(gc::TenuredChunk* chunk) {
+ MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
+ return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
+ }
+
+ static bool match(gc::TenuredChunk* k, gc::TenuredChunk* l) {
+ MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
+ MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
+ return k == l;
+ }
+};
+
+class js::gc::MarkingValidator {
+ public:
+ explicit MarkingValidator(GCRuntime* gc);
+ void nonIncrementalMark(AutoGCSession& session);
+ void validate();
+
+ private:
+ GCRuntime* gc;
+ bool initialized;
+
+ using BitmapMap = HashMap<TenuredChunk*, UniquePtr<MarkBitmap>, GCChunkHasher,
+ SystemAllocPolicy>;
+ BitmapMap map;
+};
+
+js::gc::MarkingValidator::MarkingValidator(GCRuntime* gc)
+ : gc(gc), initialized(false) {}
+
+void js::gc::MarkingValidator::nonIncrementalMark(AutoGCSession& session) {
+ /*
+ * Perform a non-incremental mark for all collecting zones and record
+ * the results for later comparison.
+ *
+ * Currently this does not validate gray marking.
+ */
+
+ JSRuntime* runtime = gc->rt;
+ GCMarker* gcmarker = &gc->marker();
+
+ MOZ_ASSERT(!gcmarker->isWeakMarking());
+
+ /* Wait for off-thread parsing which can allocate. */
+ WaitForAllHelperThreads();
+
+ gc->waitBackgroundAllocEnd();
+ gc->waitBackgroundSweepEnd();
+
+ /* Save existing mark bits. */
+ {
+ AutoLockGC lock(gc);
+ for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done();
+ chunk.next()) {
+ MarkBitmap* bitmap = &chunk->markBits;
+ auto entry = MakeUnique<MarkBitmap>();
+ if (!entry) {
+ return;
+ }
+
+ memcpy((void*)entry->bitmap, (void*)bitmap->bitmap,
+ sizeof(bitmap->bitmap));
+
+ if (!map.putNew(chunk, std::move(entry))) {
+ return;
+ }
+ }
+ }
+
+ /*
+ * Temporarily clear the weakmaps' mark flags for the compartments we are
+ * collecting.
+ */
+
+ WeakMapColors markedWeakMaps;
+
+ /*
+ * For saving, smush all of the keys into one big table and split them back
+ * up into per-zone tables when restoring.
+ */
+ gc::EphemeronEdgeTable savedEphemeronEdges(
+ SystemAllocPolicy(), runtime->randomHashCodeScrambler());
+ if (!savedEphemeronEdges.init()) {
+ return;
+ }
+
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ if (!WeakMapBase::saveZoneMarkedWeakMaps(zone, markedWeakMaps)) {
+ return;
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ for (auto r = zone->gcEphemeronEdges().mutableAll(); !r.empty();
+ r.popFront()) {
+ MOZ_ASSERT(r.front().key->asTenured().zone() == zone);
+ if (!savedEphemeronEdges.put(r.front().key, std::move(r.front().value))) {
+ oomUnsafe.crash("saving weak keys table for validator");
+ }
+ }
+
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing weak keys table for validator");
+ }
+ }
+
+ /* Save and restore test mark queue state. */
+# ifdef DEBUG
+ size_t savedQueuePos = gc->queuePos;
+ mozilla::Maybe<MarkColor> savedQueueColor = gc->queueMarkColor;
+# endif
+
+ /*
+ * After this point, the function should run to completion, so we shouldn't
+ * do anything fallible.
+ */
+ initialized = true;
+
+ /* Re-do all the marking, but non-incrementally. */
+ js::gc::State state = gc->incrementalState;
+ gc->incrementalState = State::MarkRoots;
+
+ {
+ gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::PREPARE);
+
+ {
+ gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::UNMARK);
+
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ WeakMapBase::unmarkZone(zone);
+ }
+
+ MOZ_ASSERT(gcmarker->isDrained());
+ gcmarker->reset();
+
+ ClearMarkBits<GCZonesIter>(gc);
+ }
+ }
+
+ {
+ gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::MARK);
+
+ gc->traceRuntimeForMajorGC(gcmarker->tracer(), session);
+
+ gc->incrementalState = State::Mark;
+ gc->drainMarkStack();
+ }
+
+ gc->incrementalState = State::Sweep;
+ {
+ gcstats::AutoPhase ap1(gc->stats(), gcstats::PhaseKind::SWEEP);
+ gcstats::AutoPhase ap2(gc->stats(), gcstats::PhaseKind::MARK);
+
+ gc->markAllWeakReferences();
+
+ /* Update zone state for gray marking. */
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ zone->changeGCState(zone->initialMarkingState(), Zone::MarkBlackAndGray);
+ }
+
+ AutoSetMarkColor setColorGray(*gcmarker, MarkColor::Gray);
+
+ gc->markAllGrayReferences(gcstats::PhaseKind::MARK_GRAY);
+ gc->markAllWeakReferences();
+
+ /* Restore zone state. */
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ zone->changeGCState(Zone::MarkBlackAndGray, zone->initialMarkingState());
+ }
+ MOZ_ASSERT(gc->marker().isDrained());
+ }
+
+ /* Take a copy of the non-incremental mark state and restore the original. */
+ {
+ AutoLockGC lock(gc);
+ for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done();
+ chunk.next()) {
+ MarkBitmap* bitmap = &chunk->markBits;
+ auto ptr = map.lookup(chunk);
+ MOZ_RELEASE_ASSERT(ptr, "Chunk not found in map");
+ MarkBitmap* entry = ptr->value().get();
+ for (size_t i = 0; i < MarkBitmap::WordCount; i++) {
+ uintptr_t v = entry->bitmap[i];
+ entry->bitmap[i] = uintptr_t(bitmap->bitmap[i]);
+ bitmap->bitmap[i] = v;
+ }
+ }
+ }
+
+ for (GCZonesIter zone(gc); !zone.done(); zone.next()) {
+ WeakMapBase::unmarkZone(zone);
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing weak keys table for validator");
+ }
+ }
+
+ WeakMapBase::restoreMarkedWeakMaps(markedWeakMaps);
+
+ for (auto r = savedEphemeronEdges.mutableAll(); !r.empty(); r.popFront()) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Zone* zone = r.front().key->asTenured().zone();
+ if (!zone->gcEphemeronEdges().put(r.front().key,
+ std::move(r.front().value))) {
+ oomUnsafe.crash("restoring weak keys table for validator");
+ }
+ }
+
+# ifdef DEBUG
+ gc->queuePos = savedQueuePos;
+ gc->queueMarkColor = savedQueueColor;
+# endif
+
+ gc->incrementalState = state;
+}
+
+void js::gc::MarkingValidator::validate() {
+ /*
+ * Validates the incremental marking for a single compartment by comparing
+ * the mark bits to those previously recorded for a non-incremental mark.
+ */
+
+ if (!initialized) {
+ return;
+ }
+
+ MOZ_ASSERT(!gc->marker().isWeakMarking());
+
+ gc->waitBackgroundSweepEnd();
+
+ bool ok = true;
+ AutoLockGC lock(gc->rt);
+ for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) {
+ BitmapMap::Ptr ptr = map.lookup(chunk);
+ if (!ptr) {
+ continue; /* Allocated after we did the non-incremental mark. */
+ }
+
+ MarkBitmap* bitmap = ptr->value().get();
+ MarkBitmap* incBitmap = &chunk->markBits;
+
+ for (size_t i = 0; i < ArenasPerChunk; i++) {
+ if (chunk->decommittedPages[chunk->pageIndex(i)]) {
+ continue;
+ }
+ Arena* arena = &chunk->arenas[i];
+ if (!arena->allocated()) {
+ continue;
+ }
+ if (!arena->zone->isGCSweeping()) {
+ continue;
+ }
+
+ AllocKind kind = arena->getAllocKind();
+ uintptr_t thing = arena->thingsStart();
+ uintptr_t end = arena->thingsEnd();
+ while (thing < end) {
+ auto* cell = reinterpret_cast<TenuredCell*>(thing);
+
+ /*
+ * If a non-incremental GC wouldn't have collected a cell, then
+ * an incremental GC won't collect it.
+ */
+ if (bitmap->isMarkedAny(cell)) {
+ if (!incBitmap->isMarkedAny(cell)) {
+ ok = false;
+ const char* color = TenuredCell::getColor(bitmap, cell).name();
+ fprintf(stderr,
+ "%p: cell not marked, but would be marked %s by "
+ "non-incremental marking\n",
+ cell, color);
+# ifdef DEBUG
+ cell->dump();
+ fprintf(stderr, "\n");
+# endif
+ }
+ }
+
+ /*
+ * If the cycle collector isn't allowed to collect an object
+ * after a non-incremental GC has run, then it isn't allowed to
+ * collected it after an incremental GC.
+ */
+ if (!bitmap->isMarkedGray(cell)) {
+ if (incBitmap->isMarkedGray(cell)) {
+ ok = false;
+ const char* color = TenuredCell::getColor(bitmap, cell).name();
+ fprintf(stderr,
+ "%p: cell marked gray, but would be marked %s by "
+ "non-incremental marking\n",
+ cell, color);
+# ifdef DEBUG
+ cell->dump();
+ fprintf(stderr, "\n");
+# endif
+ }
+ }
+
+ thing += Arena::thingSize(kind);
+ }
+ }
+ }
+
+ MOZ_RELEASE_ASSERT(ok, "Incremental marking verification failed");
+}
+
+void GCRuntime::computeNonIncrementalMarkingForValidation(
+ AutoGCSession& session) {
+ MOZ_ASSERT(!markingValidator);
+ if (isIncremental && hasZealMode(ZealMode::IncrementalMarkingValidator)) {
+ markingValidator = js_new<MarkingValidator>(this);
+ }
+ if (markingValidator) {
+ markingValidator->nonIncrementalMark(session);
+ }
+}
+
+void GCRuntime::validateIncrementalMarking() {
+ if (markingValidator) {
+ markingValidator->validate();
+ }
+}
+
+void GCRuntime::finishMarkingValidation() {
+ js_delete(markingValidator.ref());
+ markingValidator = nullptr;
+}
+
+#endif /* JS_GC_ZEAL */
+
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+
+class HeapCheckTracerBase : public JS::CallbackTracer {
+ public:
+ explicit HeapCheckTracerBase(JSRuntime* rt, JS::TraceOptions options);
+ bool traceHeap(AutoTraceSession& session);
+ virtual void checkCell(Cell* cell, const char* name) = 0;
+
+ protected:
+ void dumpCellInfo(Cell* cell);
+ void dumpCellPath(const char* name);
+
+ Cell* parentCell() {
+ return parentIndex == -1 ? nullptr : stack[parentIndex].thing.asCell();
+ }
+
+ size_t failures;
+
+ private:
+ void onChild(JS::GCCellPtr thing, const char* name) override;
+
+ struct WorkItem {
+ WorkItem(JS::GCCellPtr thing, const char* name, int parentIndex)
+ : thing(thing),
+ name(name),
+ parentIndex(parentIndex),
+ processed(false) {}
+
+ JS::GCCellPtr thing;
+ const char* name;
+ int parentIndex;
+ bool processed;
+ };
+
+ JSRuntime* rt;
+ bool oom;
+ HashSet<Cell*, DefaultHasher<Cell*>, SystemAllocPolicy> visited;
+ Vector<WorkItem, 0, SystemAllocPolicy> stack;
+ int parentIndex;
+};
+
+HeapCheckTracerBase::HeapCheckTracerBase(JSRuntime* rt,
+ JS::TraceOptions options)
+ : CallbackTracer(rt, JS::TracerKind::Callback, options),
+ failures(0),
+ rt(rt),
+ oom(false),
+ parentIndex(-1) {}
+
+void HeapCheckTracerBase::onChild(JS::GCCellPtr thing, const char* name) {
+ Cell* cell = thing.asCell();
+ checkCell(cell, name);
+
+ if (visited.lookup(cell)) {
+ return;
+ }
+
+ if (!visited.put(cell)) {
+ oom = true;
+ return;
+ }
+
+ // Don't trace into GC things owned by another runtime.
+ if (cell->runtimeFromAnyThread() != rt) {
+ return;
+ }
+
+ WorkItem item(thing, name, parentIndex);
+ if (!stack.append(item)) {
+ oom = true;
+ }
+}
+
+bool HeapCheckTracerBase::traceHeap(AutoTraceSession& session) {
+ // The analysis thinks that traceRuntime might GC by calling a GC callback.
+ JS::AutoSuppressGCAnalysis nogc;
+ if (!rt->isBeingDestroyed()) {
+ rt->gc.traceRuntime(this, session);
+ }
+
+ while (!stack.empty() && !oom) {
+ WorkItem item = stack.back();
+ if (item.processed) {
+ stack.popBack();
+ } else {
+ parentIndex = stack.length() - 1;
+ stack.back().processed = true;
+ TraceChildren(this, item.thing);
+ }
+ }
+
+ return !oom;
+}
+
+void HeapCheckTracerBase::dumpCellInfo(Cell* cell) {
+ auto kind = cell->getTraceKind();
+ JSObject* obj =
+ kind == JS::TraceKind::Object ? static_cast<JSObject*>(cell) : nullptr;
+
+ fprintf(stderr, "%s %s", cell->color().name(), GCTraceKindToAscii(kind));
+ if (obj) {
+ fprintf(stderr, " %s", obj->getClass()->name);
+ }
+ fprintf(stderr, " %p", cell);
+ if (obj) {
+ fprintf(stderr, " (compartment %p)", obj->compartment());
+ }
+}
+
+void HeapCheckTracerBase::dumpCellPath(const char* name) {
+ for (int index = parentIndex; index != -1; index = stack[index].parentIndex) {
+ const WorkItem& parent = stack[index];
+ Cell* cell = parent.thing.asCell();
+ fprintf(stderr, " from ");
+ dumpCellInfo(cell);
+ fprintf(stderr, " %s edge\n", name);
+ name = parent.name;
+ }
+ fprintf(stderr, " from root %s\n", name);
+}
+
+class CheckHeapTracer final : public HeapCheckTracerBase {
+ public:
+ enum GCType { Moving, NonMoving };
+
+ explicit CheckHeapTracer(JSRuntime* rt, GCType type);
+ void check(AutoTraceSession& session);
+
+ private:
+ void checkCell(Cell* cell, const char* name) override;
+ GCType gcType;
+};
+
+CheckHeapTracer::CheckHeapTracer(JSRuntime* rt, GCType type)
+ : HeapCheckTracerBase(rt, JS::WeakMapTraceAction::TraceKeysAndValues),
+ gcType(type) {}
+
+inline static bool IsValidGCThingPointer(Cell* cell) {
+ return (uintptr_t(cell) & CellAlignMask) == 0;
+}
+
+void CheckHeapTracer::checkCell(Cell* cell, const char* name) {
+ // Moving
+ if (!IsValidGCThingPointer(cell) ||
+ ((gcType == GCType::Moving) && !IsGCThingValidAfterMovingGC(cell)) ||
+ ((gcType == GCType::NonMoving) && cell->isForwarded())) {
+ failures++;
+ fprintf(stderr, "Bad pointer %p\n", cell);
+ dumpCellPath(name);
+ }
+}
+
+void CheckHeapTracer::check(AutoTraceSession& session) {
+ if (!traceHeap(session)) {
+ return;
+ }
+
+ if (failures) {
+ fprintf(stderr, "Heap check: %zu failure(s)\n", failures);
+ }
+ MOZ_RELEASE_ASSERT(failures == 0);
+}
+
+void js::gc::CheckHeapAfterGC(JSRuntime* rt) {
+ AutoTraceSession session(rt);
+ CheckHeapTracer::GCType gcType;
+
+ if (rt->gc.nursery().isEmpty()) {
+ gcType = CheckHeapTracer::GCType::Moving;
+ } else {
+ gcType = CheckHeapTracer::GCType::NonMoving;
+ }
+
+ CheckHeapTracer tracer(rt, gcType);
+ tracer.check(session);
+}
+
+class CheckGrayMarkingTracer final : public HeapCheckTracerBase {
+ public:
+ explicit CheckGrayMarkingTracer(JSRuntime* rt);
+ bool check(AutoTraceSession& session);
+
+ private:
+ void checkCell(Cell* cell, const char* name) override;
+};
+
+CheckGrayMarkingTracer::CheckGrayMarkingTracer(JSRuntime* rt)
+ : HeapCheckTracerBase(rt, JS::TraceOptions(JS::WeakMapTraceAction::Skip,
+ JS::WeakEdgeTraceAction::Skip)) {
+ // Weak gray->black edges are allowed.
+}
+
+void CheckGrayMarkingTracer::checkCell(Cell* cell, const char* name) {
+ Cell* parent = parentCell();
+ if (!parent) {
+ return;
+ }
+
+ if (parent->isMarkedBlack() && cell->isMarkedGray()) {
+ failures++;
+
+ fprintf(stderr, "Found black to gray edge to ");
+ dumpCellInfo(cell);
+ fprintf(stderr, "\n");
+ dumpCellPath(name);
+
+# ifdef DEBUG
+ if (parent->is<JSObject>()) {
+ fprintf(stderr, "\nSource: ");
+ DumpObject(parent->as<JSObject>(), stderr);
+ }
+ if (cell->is<JSObject>()) {
+ fprintf(stderr, "\nTarget: ");
+ DumpObject(cell->as<JSObject>(), stderr);
+ }
+# endif
+ }
+}
+
+bool CheckGrayMarkingTracer::check(AutoTraceSession& session) {
+ if (!traceHeap(session)) {
+ return true; // Ignore failure.
+ }
+
+ return failures == 0;
+}
+
+JS_PUBLIC_API bool js::CheckGrayMarkingState(JSRuntime* rt) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+ MOZ_ASSERT(!rt->gc.isIncrementalGCInProgress());
+ if (!rt->gc.areGrayBitsValid()) {
+ return true;
+ }
+
+ gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PhaseKind::TRACE_HEAP);
+ AutoTraceSession session(rt);
+ CheckGrayMarkingTracer tracer(rt);
+
+ return tracer.check(session);
+}
+
+static JSObject* MaybeGetDelegate(Cell* cell) {
+ if (!cell->is<JSObject>()) {
+ return nullptr;
+ }
+
+ JSObject* object = cell->as<JSObject>();
+ return js::UncheckedUnwrapWithoutExpose(object);
+}
+
+bool js::gc::CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key,
+ Cell* value) {
+ bool ok = true;
+
+ Zone* zone = map->zone();
+ MOZ_ASSERT(CurrentThreadCanAccessZone(zone));
+ MOZ_ASSERT(zone->isGCMarking());
+
+ JSObject* object = map->memberOf;
+ MOZ_ASSERT_IF(object, object->zone() == zone);
+
+ // Debugger weak maps can have keys in different zones.
+ Zone* keyZone = key->zoneFromAnyThread();
+ MOZ_ASSERT_IF(!map->allowKeysInOtherZones(),
+ keyZone == zone || keyZone->isAtomsZone());
+
+ Zone* valueZone = value->zoneFromAnyThread();
+ MOZ_ASSERT(valueZone == zone || valueZone->isAtomsZone());
+
+ if (object && object->color() != map->mapColor) {
+ fprintf(stderr, "WeakMap object is marked differently to the map\n");
+ fprintf(stderr, "(map %p is %s, object %p is %s)\n", map,
+ map->mapColor.name(), object, object->color().name());
+ ok = false;
+ }
+
+ // Values belonging to other runtimes or in uncollected zones are treated as
+ // black.
+ JSRuntime* mapRuntime = zone->runtimeFromAnyThread();
+ auto effectiveColor = [=](Cell* cell, Zone* cellZone) -> CellColor {
+ if (cell->runtimeFromAnyThread() != mapRuntime) {
+ return CellColor::Black;
+ }
+ if (cellZone->isGCMarkingOrSweeping()) {
+ return cell->color();
+ }
+ return CellColor::Black;
+ };
+
+ CellColor valueColor = effectiveColor(value, valueZone);
+ CellColor keyColor = effectiveColor(key, keyZone);
+
+ if (valueColor < std::min(map->mapColor, keyColor)) {
+ fprintf(stderr, "WeakMap value is less marked than map and key\n");
+ fprintf(stderr, "(map %p is %s, key %p is %s, value %p is %s)\n", map,
+ map->mapColor.name(), key, keyColor.name(), value,
+ valueColor.name());
+# ifdef DEBUG
+ fprintf(stderr, "Key:\n");
+ key->dump();
+ if (auto delegate = MaybeGetDelegate(key); delegate) {
+ fprintf(stderr, "Delegate:\n");
+ delegate->dump();
+ }
+ fprintf(stderr, "Value:\n");
+ value->dump();
+# endif
+
+ ok = false;
+ }
+
+ JSObject* delegate = MaybeGetDelegate(key);
+ if (!delegate) {
+ return ok;
+ }
+
+ CellColor delegateColor = effectiveColor(delegate, delegate->zone());
+ if (keyColor < std::min(map->mapColor, delegateColor)) {
+ fprintf(stderr, "WeakMap key is less marked than map or delegate\n");
+ fprintf(stderr, "(map %p is %s, delegate %p is %s, key %p is %s)\n", map,
+ map->mapColor.name(), delegate, delegateColor.name(), key,
+ keyColor.name());
+ ok = false;
+ }
+
+ return ok;
+}
+
+#endif // defined(JS_GC_ZEAL) || defined(DEBUG)
+
+#ifdef DEBUG
+// Return whether an arbitrary pointer is within a cell with the given
+// traceKind. Only for assertions.
+bool GCRuntime::isPointerWithinTenuredCell(void* ptr, JS::TraceKind traceKind) {
+ AutoLockGC lock(this);
+ for (auto chunk = allNonEmptyChunks(lock); !chunk.done(); chunk.next()) {
+ MOZ_ASSERT(!chunk->isNurseryChunk());
+ if (ptr >= &chunk->arenas[0] && ptr < &chunk->arenas[ArenasPerChunk]) {
+ auto* arena = reinterpret_cast<Arena*>(uintptr_t(ptr) & ~ArenaMask);
+ if (!arena->allocated()) {
+ return false;
+ }
+
+ return MapAllocToTraceKind(arena->getAllocKind()) == traceKind;
+ }
+ }
+
+ return false;
+}
+#endif // DEBUG
diff --git a/js/src/gc/WeakMap-inl.h b/js/src/gc/WeakMap-inl.h
new file mode 100644
index 0000000000..1b5626211f
--- /dev/null
+++ b/js/src/gc/WeakMap-inl.h
@@ -0,0 +1,413 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_WeakMap_inl_h
+#define gc_WeakMap_inl_h
+
+#include "gc/WeakMap.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include <algorithm>
+#include <type_traits>
+
+#include "gc/GCLock.h"
+#include "gc/Marking.h"
+#include "gc/Zone.h"
+#include "js/TraceKind.h"
+#include "vm/JSContext.h"
+
+#include "gc/StableCellHasher-inl.h"
+
+namespace js {
+namespace gc {
+
+namespace detail {
+
+// Return the effective cell color given the current marking state.
+// This must be kept in sync with ShouldMark in Marking.cpp.
+template <typename T>
+static CellColor GetEffectiveColor(GCMarker* marker, const T& item) {
+ Cell* cell = ToMarkable(item);
+ if (!cell->isTenured()) {
+ return CellColor::Black;
+ }
+ const TenuredCell& t = cell->asTenured();
+ if (!t.zoneFromAnyThread()->shouldMarkInZone(marker->markColor())) {
+ return CellColor::Black;
+ }
+ MOZ_ASSERT(t.runtimeFromAnyThread() == marker->runtime());
+ return t.color();
+}
+
+// Only objects have delegates, so default to returning nullptr. Note that some
+// compilation units will only ever use the object version.
+static MOZ_MAYBE_UNUSED JSObject* GetDelegateInternal(gc::Cell* key) {
+ return nullptr;
+}
+
+static MOZ_MAYBE_UNUSED JSObject* GetDelegateInternal(JSObject* key) {
+ JSObject* delegate = UncheckedUnwrapWithoutExpose(key);
+ return (key == delegate) ? nullptr : delegate;
+}
+
+// Use a helper function to do overload resolution to handle cases like
+// Heap<ObjectSubclass*>: find everything that is convertible to JSObject* (and
+// avoid calling barriers).
+template <typename T>
+static inline JSObject* GetDelegate(const T& key) {
+ return GetDelegateInternal(key);
+}
+
+template <>
+inline JSObject* GetDelegate(gc::Cell* const&) = delete;
+
+} /* namespace detail */
+} /* namespace gc */
+
+// Weakmap entry -> value edges are only visible if the map is traced, which
+// only happens if the map zone is being collected. If the map and the value
+// were in different zones, then we could have a case where the map zone is not
+// collecting but the value zone is, and incorrectly free a value that is
+// reachable solely through weakmaps.
+template <class K, class V>
+void WeakMap<K, V>::assertMapIsSameZoneWithValue(const V& v) {
+#ifdef DEBUG
+ gc::Cell* cell = gc::ToMarkable(v);
+ if (cell) {
+ Zone* cellZone = cell->zoneFromAnyThread();
+ MOZ_ASSERT(zone() == cellZone || cellZone->isAtomsZone());
+ }
+#endif
+}
+
+template <class K, class V>
+WeakMap<K, V>::WeakMap(JSContext* cx, JSObject* memOf)
+ : WeakMap(cx->zone(), memOf) {}
+
+template <class K, class V>
+WeakMap<K, V>::WeakMap(JS::Zone* zone, JSObject* memOf)
+ : Base(zone), WeakMapBase(memOf, zone) {
+ using ElemType = typename K::ElementType;
+ using NonPtrType = std::remove_pointer_t<ElemType>;
+
+ // The object's TraceKind needs to be added to CC graph if this object is
+ // used as a WeakMap key, otherwise the key is considered to be pointed from
+ // somewhere unknown, and results in leaking the subgraph which contains the
+ // key. See the comments in NoteWeakMapsTracer::trace for more details.
+ static_assert(JS::IsCCTraceKind(NonPtrType::TraceKind),
+ "Object's TraceKind should be added to CC graph.");
+
+ zone->gcWeakMapList().insertFront(this);
+ if (zone->gcState() > Zone::Prepare) {
+ mapColor = CellColor::Black;
+ }
+}
+
+// If the entry is live, ensure its key and value are marked. Also make sure the
+// key is at least as marked as min(map, delegate), so it cannot get discarded
+// and then recreated by rewrapping the delegate.
+//
+// Optionally adds edges to the ephemeron edges table for any keys (or
+// delegates) where future changes to their mark color would require marking the
+// value (or the key).
+template <class K, class V>
+bool WeakMap<K, V>::markEntry(GCMarker* marker, K& key, V& value,
+ bool populateWeakKeysTable) {
+#ifdef DEBUG
+ MOZ_ASSERT(mapColor);
+ if (marker->isParallelMarking()) {
+ marker->runtime()->gc.assertCurrentThreadHasLockedGC();
+ }
+#endif
+
+ bool marked = false;
+ CellColor markColor = marker->markColor();
+ CellColor keyColor = gc::detail::GetEffectiveColor(marker, key);
+ JSObject* delegate = gc::detail::GetDelegate(key);
+ JSTracer* trc = marker->tracer();
+
+ if (delegate) {
+ CellColor delegateColor = gc::detail::GetEffectiveColor(marker, delegate);
+ // The key needs to stay alive while both the delegate and map are live.
+ CellColor proxyPreserveColor = std::min(delegateColor, mapColor);
+ if (keyColor < proxyPreserveColor) {
+ MOZ_ASSERT(markColor >= proxyPreserveColor);
+ if (markColor == proxyPreserveColor) {
+ TraceWeakMapKeyEdge(trc, zone(), &key,
+ "proxy-preserved WeakMap entry key");
+ MOZ_ASSERT(key->color() >= proxyPreserveColor);
+ marked = true;
+ keyColor = proxyPreserveColor;
+ }
+ }
+ }
+
+ gc::Cell* cellValue = gc::ToMarkable(value);
+ if (keyColor) {
+ if (cellValue) {
+ CellColor targetColor = std::min(mapColor, keyColor);
+ CellColor valueColor = gc::detail::GetEffectiveColor(marker, cellValue);
+ if (valueColor < targetColor) {
+ MOZ_ASSERT(markColor >= targetColor);
+ if (markColor == targetColor) {
+ TraceEdge(trc, &value, "WeakMap entry value");
+ MOZ_ASSERT(cellValue->color() >= targetColor);
+ marked = true;
+ }
+ }
+ }
+ }
+
+ if (populateWeakKeysTable) {
+ // Note that delegateColor >= keyColor because marking a key marks its
+ // delegate, so we only need to check whether keyColor < mapColor to tell
+ // this.
+
+ if (keyColor < mapColor) {
+ MOZ_ASSERT(trc->weakMapAction() == JS::WeakMapTraceAction::Expand);
+ // The final color of the key is not yet known. Record this weakmap and
+ // the lookup key in the list of weak keys. If the key has a delegate,
+ // then the lookup key is the delegate (because marking the key will end
+ // up marking the delegate and thereby mark the entry.)
+ gc::TenuredCell* tenuredValue = nullptr;
+ if (cellValue && cellValue->isTenured()) {
+ tenuredValue = &cellValue->asTenured();
+ }
+
+ if (!this->addImplicitEdges(key, delegate, tenuredValue)) {
+ marker->abortLinearWeakMarking();
+ }
+ }
+ }
+
+ return marked;
+}
+
+template <class K, class V>
+void WeakMap<K, V>::trace(JSTracer* trc) {
+ MOZ_ASSERT(isInList());
+
+ TraceNullableEdge(trc, &memberOf, "WeakMap owner");
+
+ if (trc->isMarkingTracer()) {
+ MOZ_ASSERT(trc->weakMapAction() == JS::WeakMapTraceAction::Expand);
+ auto marker = GCMarker::fromTracer(trc);
+
+ // Lock if we are marking in parallel to synchronize updates to:
+ // - the weak map's color
+ // - the ephemeron edges table
+ mozilla::Maybe<AutoLockGC> lock;
+ if (marker->isParallelMarking()) {
+ lock.emplace(marker->runtime());
+ }
+
+ // Don't downgrade the map color from black to gray. This can happen when a
+ // barrier pushes the map object onto the black mark stack when it's
+ // already present on the gray mark stack, which is marked later.
+ if (mapColor < marker->markColor()) {
+ mapColor = marker->markColor();
+ (void)markEntries(marker);
+ }
+ return;
+ }
+
+ if (trc->weakMapAction() == JS::WeakMapTraceAction::Skip) {
+ return;
+ }
+
+ // Trace keys only if weakMapAction() says to.
+ if (trc->weakMapAction() == JS::WeakMapTraceAction::TraceKeysAndValues) {
+ for (Enum e(*this); !e.empty(); e.popFront()) {
+ TraceWeakMapKeyEdge(trc, zone(), &e.front().mutableKey(),
+ "WeakMap entry key");
+ }
+ }
+
+ // Always trace all values (unless weakMapAction() is Skip).
+ for (Range r = Base::all(); !r.empty(); r.popFront()) {
+ TraceEdge(trc, &r.front().value(), "WeakMap entry value");
+ }
+}
+
+bool WeakMapBase::addImplicitEdges(gc::Cell* key, gc::Cell* delegate,
+ gc::TenuredCell* value) {
+ if (delegate) {
+ auto& edgeTable = delegate->zone()->gcEphemeronEdges(delegate);
+ auto* p = edgeTable.get(delegate);
+
+ gc::EphemeronEdgeVector newVector;
+ gc::EphemeronEdgeVector& edges = p ? p->value : newVector;
+
+ // Add a <weakmap, delegate> -> key edge: the key must be preserved for
+ // future lookups until either the weakmap or the delegate dies.
+ gc::EphemeronEdge keyEdge{mapColor, key};
+ if (!edges.append(keyEdge)) {
+ return false;
+ }
+
+ if (value) {
+ gc::EphemeronEdge valueEdge{mapColor, value};
+ if (!edges.append(valueEdge)) {
+ return false;
+ }
+ }
+
+ if (!p) {
+ return edgeTable.put(delegate, std::move(newVector));
+ }
+
+ return true;
+ }
+
+ // No delegate. Insert just the key -> value edge.
+
+ if (!value) {
+ return true;
+ }
+
+ auto& edgeTable = key->zone()->gcEphemeronEdges(key);
+ auto* p = edgeTable.get(key);
+ gc::EphemeronEdge valueEdge{mapColor, value};
+ if (p) {
+ return p->value.append(valueEdge);
+ } else {
+ gc::EphemeronEdgeVector edges;
+ MOZ_ALWAYS_TRUE(edges.append(valueEdge));
+ return edgeTable.put(key, std::move(edges));
+ }
+}
+
+template <class K, class V>
+bool WeakMap<K, V>::markEntries(GCMarker* marker) {
+ // This method is called whenever the map's mark color changes. Mark values
+ // (and keys with delegates) as required for the new color and populate the
+ // ephemeron edges if we're in incremental marking mode.
+
+#ifdef DEBUG
+ if (marker->isParallelMarking()) {
+ marker->runtime()->gc.assertCurrentThreadHasLockedGC();
+ }
+#endif
+
+ MOZ_ASSERT(mapColor);
+ bool markedAny = false;
+
+ // If we don't populate the weak keys table now then we do it when we enter
+ // weak marking mode.
+ bool populateWeakKeysTable =
+ marker->incrementalWeakMapMarkingEnabled || marker->isWeakMarking();
+
+ for (Enum e(*this); !e.empty(); e.popFront()) {
+ if (markEntry(marker, e.front().mutableKey(), e.front().value(),
+ populateWeakKeysTable)) {
+ markedAny = true;
+ }
+ }
+
+ return markedAny;
+}
+
+template <class K, class V>
+void WeakMap<K, V>::traceWeakEdges(JSTracer* trc) {
+ // Remove all entries whose keys remain unmarked.
+ for (Enum e(*this); !e.empty(); e.popFront()) {
+ if (!TraceWeakEdge(trc, &e.front().mutableKey(), "WeakMap key")) {
+ e.removeFront();
+ }
+ }
+
+#if DEBUG
+ // Once we've swept, all remaining edges should stay within the known-live
+ // part of the graph.
+ assertEntriesNotAboutToBeFinalized();
+#endif
+}
+
+// memberOf can be nullptr, which means that the map is not part of a JSObject.
+template <class K, class V>
+void WeakMap<K, V>::traceMappings(WeakMapTracer* tracer) {
+ for (Range r = Base::all(); !r.empty(); r.popFront()) {
+ gc::Cell* key = gc::ToMarkable(r.front().key());
+ gc::Cell* value = gc::ToMarkable(r.front().value());
+ if (key && value) {
+ tracer->trace(memberOf, JS::GCCellPtr(r.front().key().get()),
+ JS::GCCellPtr(r.front().value().get()));
+ }
+ }
+}
+
+template <class K, class V>
+bool WeakMap<K, V>::findSweepGroupEdges() {
+ // For weakmap keys with delegates in a different zone, add a zone edge to
+ // ensure that the delegate zone finishes marking before the key zone.
+ JS::AutoSuppressGCAnalysis nogc;
+ for (Range r = all(); !r.empty(); r.popFront()) {
+ const K& key = r.front().key();
+
+ // If the key type doesn't have delegates, then this will always return
+ // nullptr and the optimizer can remove the entire body of this function.
+ JSObject* delegate = gc::detail::GetDelegate(key);
+ if (!delegate) {
+ continue;
+ }
+
+ // Marking a WeakMap key's delegate will mark the key, so process the
+ // delegate zone no later than the key zone.
+ Zone* delegateZone = delegate->zone();
+ Zone* keyZone = key->zone();
+ if (delegateZone != keyZone && delegateZone->isGCMarking() &&
+ keyZone->isGCMarking()) {
+ if (!delegateZone->addSweepGroupEdgeTo(keyZone)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+template <class K, class V>
+size_t WeakMap<K, V>::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return mallocSizeOf(this) + shallowSizeOfExcludingThis(mallocSizeOf);
+}
+
+#if DEBUG
+template <class K, class V>
+void WeakMap<K, V>::assertEntriesNotAboutToBeFinalized() {
+ for (Range r = Base::all(); !r.empty(); r.popFront()) {
+ UnbarrieredKey k = r.front().key();
+ MOZ_ASSERT(!gc::IsAboutToBeFinalizedUnbarriered(k));
+ JSObject* delegate = gc::detail::GetDelegate(k);
+ if (delegate) {
+ MOZ_ASSERT(!gc::IsAboutToBeFinalizedUnbarriered(delegate),
+ "weakmap marking depends on a key tracing its delegate");
+ }
+ MOZ_ASSERT(!gc::IsAboutToBeFinalized(r.front().value()));
+ }
+}
+#endif
+
+#ifdef JS_GC_ZEAL
+template <class K, class V>
+bool WeakMap<K, V>::checkMarking() const {
+ bool ok = true;
+ for (Range r = Base::all(); !r.empty(); r.popFront()) {
+ gc::Cell* key = gc::ToMarkable(r.front().key());
+ gc::Cell* value = gc::ToMarkable(r.front().value());
+ if (key && value) {
+ if (!gc::CheckWeakMapEntryMarking(this, key, value)) {
+ ok = false;
+ }
+ }
+ }
+ return ok;
+}
+#endif
+
+} /* namespace js */
+
+#endif /* gc_WeakMap_inl_h */
diff --git a/js/src/gc/WeakMap.cpp b/js/src/gc/WeakMap.cpp
new file mode 100644
index 0000000000..82684c7c9c
--- /dev/null
+++ b/js/src/gc/WeakMap.cpp
@@ -0,0 +1,175 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/WeakMap-inl.h"
+
+#include <string.h>
+
+#include "gc/PublicIterators.h"
+#include "vm/JSObject.h"
+
+#include "gc/Marking-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+WeakMapBase::WeakMapBase(JSObject* memOf, Zone* zone)
+ : memberOf(memOf), zone_(zone), mapColor(CellColor::White) {
+ MOZ_ASSERT_IF(memberOf, memberOf->compartment()->zone() == zone);
+}
+
+WeakMapBase::~WeakMapBase() {
+ MOZ_ASSERT(CurrentThreadIsGCFinalizing() ||
+ CurrentThreadCanAccessZone(zone_));
+}
+
+void WeakMapBase::unmarkZone(JS::Zone* zone) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcEphemeronEdges().clear()) {
+ oomUnsafe.crash("clearing ephemeron edges table");
+ }
+ MOZ_ASSERT(zone->gcNurseryEphemeronEdges().count() == 0);
+
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ m->mapColor = CellColor::White;
+ }
+}
+
+void Zone::traceWeakMaps(JSTracer* trc) {
+ MOZ_ASSERT(trc->weakMapAction() != JS::WeakMapTraceAction::Skip);
+ for (WeakMapBase* m : gcWeakMapList()) {
+ m->trace(trc);
+ TraceNullableEdge(trc, &m->memberOf, "memberOf");
+ }
+}
+
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+bool WeakMapBase::checkMarkingForZone(JS::Zone* zone) {
+ // This is called at the end of marking.
+ MOZ_ASSERT(zone->isGCMarking());
+
+ bool ok = true;
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ if (m->mapColor && !m->checkMarking()) {
+ ok = false;
+ }
+ }
+
+ return ok;
+}
+#endif
+
+bool WeakMapBase::markZoneIteratively(JS::Zone* zone, GCMarker* marker) {
+ bool markedAny = false;
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ if (m->mapColor && m->markEntries(marker)) {
+ markedAny = true;
+ }
+ }
+ return markedAny;
+}
+
+bool WeakMapBase::findSweepGroupEdgesForZone(JS::Zone* zone) {
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ if (!m->findSweepGroupEdges()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void Zone::sweepWeakMaps(JSTracer* trc) {
+ for (WeakMapBase* m = gcWeakMapList().getFirst(); m;) {
+ WeakMapBase* next = m->getNext();
+ if (m->mapColor) {
+ m->traceWeakEdges(trc);
+ } else {
+ m->clearAndCompact();
+ m->removeFrom(gcWeakMapList());
+ }
+ m = next;
+ }
+
+#ifdef DEBUG
+ for (WeakMapBase* m : gcWeakMapList()) {
+ MOZ_ASSERT(m->isInList() && m->mapColor);
+ }
+#endif
+}
+
+void WeakMapBase::traceAllMappings(WeakMapTracer* tracer) {
+ JSRuntime* rt = tracer->runtime;
+ for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ // The WeakMapTracer callback is not allowed to GC.
+ JS::AutoSuppressGCAnalysis nogc;
+ m->traceMappings(tracer);
+ }
+ }
+}
+
+bool WeakMapBase::saveZoneMarkedWeakMaps(JS::Zone* zone,
+ WeakMapColors& markedWeakMaps) {
+ for (WeakMapBase* m : zone->gcWeakMapList()) {
+ if (m->mapColor && !markedWeakMaps.put(m, m->mapColor)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void WeakMapBase::restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps) {
+ for (WeakMapColors::Range r = markedWeakMaps.all(); !r.empty();
+ r.popFront()) {
+ WeakMapBase* map = r.front().key();
+ MOZ_ASSERT(map->zone()->isGCMarking());
+ MOZ_ASSERT(map->mapColor == CellColor::White);
+ map->mapColor = r.front().value();
+ }
+}
+
+ObjectWeakMap::ObjectWeakMap(JSContext* cx) : map(cx, nullptr) {}
+
+JSObject* ObjectWeakMap::lookup(const JSObject* obj) {
+ if (ObjectValueWeakMap::Ptr p = map.lookup(const_cast<JSObject*>(obj))) {
+ return &p->value().toObject();
+ }
+ return nullptr;
+}
+
+bool ObjectWeakMap::add(JSContext* cx, JSObject* obj, JSObject* target) {
+ MOZ_ASSERT(obj && target);
+
+ Value targetVal(ObjectValue(*target));
+ if (!map.putNew(obj, targetVal)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+void ObjectWeakMap::remove(JSObject* key) {
+ MOZ_ASSERT(key);
+ map.remove(key);
+}
+
+void ObjectWeakMap::clear() { map.clear(); }
+
+void ObjectWeakMap::trace(JSTracer* trc) { map.trace(trc); }
+
+size_t ObjectWeakMap::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return map.shallowSizeOfExcludingThis(mallocSizeOf);
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void ObjectWeakMap::checkAfterMovingGC() {
+ for (ObjectValueWeakMap::Range r = map.all(); !r.empty(); r.popFront()) {
+ CheckGCThingAfterMovingGC(r.front().key().get());
+ CheckGCThingAfterMovingGC(&r.front().value().toObject());
+ }
+}
+#endif // JSGC_HASH_TABLE_CHECKS
diff --git a/js/src/gc/WeakMap.h b/js/src/gc/WeakMap.h
new file mode 100644
index 0000000000..4693bd838c
--- /dev/null
+++ b/js/src/gc/WeakMap.h
@@ -0,0 +1,354 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_WeakMap_h
+#define gc_WeakMap_h
+
+#include "mozilla/LinkedList.h"
+
+#include "gc/Barrier.h"
+#include "gc/Tracer.h"
+#include "gc/ZoneAllocator.h"
+#include "js/HashTable.h"
+#include "js/HeapAPI.h"
+
+namespace JS {
+class Zone;
+}
+
+namespace js {
+
+class GCMarker;
+class WeakMapBase;
+struct WeakMapTracer;
+
+extern void DumpWeakMapLog(JSRuntime* rt);
+
+namespace gc {
+
+struct WeakMarkable;
+
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+// Check whether a weak map entry is marked correctly.
+bool CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key, Cell* value);
+#endif
+
+} // namespace gc
+
+// A subclass template of js::HashMap whose keys and values may be
+// garbage-collected. When a key is collected, the table entry disappears,
+// dropping its reference to the value.
+//
+// More precisely:
+//
+// A WeakMap entry is live if and only if both the WeakMap and the entry's
+// key are live. An entry holds a strong reference to its value.
+//
+// You must call this table's 'trace' method when its owning object is reached
+// by the garbage collection tracer. Once a table is known to be live, the
+// implementation takes care of the special weak marking (ie, marking through
+// the implicit edges stored in the map) and of removing (sweeping) table
+// entries when collection is complete.
+
+// WeakMaps are marked with an incremental linear-time algorithm that handles
+// all orderings of map and key marking. The basic algorithm is:
+//
+// At first while marking, do nothing special when marking WeakMap keys (there
+// is no straightforward way to know whether a particular object is being used
+// as a key in some weakmap.) When a WeakMap is marked, scan through it to mark
+// all entries with live keys, and collect all unmarked keys into a "weak keys"
+// table.
+//
+// At some point, everything reachable has been marked. At this point, enter
+// "weak marking mode". In this mode, whenever any object is marked, look it up
+// in the weak keys table to see if it is the key for any WeakMap entry and if
+// so, mark the value. When entering weak marking mode, scan the weak key table
+// to find all keys that have been marked since we added them to the table, and
+// mark those entries.
+//
+// In addition, we want weakmap marking to work incrementally. So WeakMap
+// mutations are barriered to keep the weak keys table up to date: entries are
+// removed if their key is removed from the table, etc.
+//
+// You can break down various ways that WeakMap values get marked based on the
+// order that the map and key are marked. All of these assume the map and key
+// get marked at some point:
+//
+// key marked, then map marked:
+// - value was marked with map in `markEntries()`
+// map marked, key already in map, key marked before weak marking mode:
+// - key added to gcEphemeronEdges when map marked in `markEntries()`
+// - value marked during `enterWeakMarkingMode`
+// map marked, key already in map, key marked after weak marking mode:
+// - when key is marked, gcEphemeronEdges[key] triggers marking of value in
+// `markImplicitEdges()`
+// map marked, key inserted into map, key marked:
+// - value was live when inserted and must get marked at some point
+//
+
+using WeakMapColors = HashMap<WeakMapBase*, js::gc::CellColor,
+ DefaultHasher<WeakMapBase*>, SystemAllocPolicy>;
+
+// Common base class for all WeakMap specializations, used for calling
+// subclasses' GC-related methods.
+class WeakMapBase : public mozilla::LinkedListElement<WeakMapBase> {
+ friend class js::GCMarker;
+
+ public:
+ using CellColor = js::gc::CellColor;
+
+ WeakMapBase(JSObject* memOf, JS::Zone* zone);
+ virtual ~WeakMapBase();
+
+ JS::Zone* zone() const { return zone_; }
+
+ // Garbage collector entry points.
+
+ // Unmark all weak maps in a zone.
+ static void unmarkZone(JS::Zone* zone);
+
+ // Check all weak maps in a zone that have been marked as live in this garbage
+ // collection, and mark the values of all entries that have become strong
+ // references to them. Return true if we marked any new values, indicating
+ // that we need to make another pass. In other words, mark my marked maps'
+ // marked members' mid-collection.
+ static bool markZoneIteratively(JS::Zone* zone, GCMarker* marker);
+
+ // Add zone edges for weakmaps with key delegates in a different zone.
+ [[nodiscard]] static bool findSweepGroupEdgesForZone(JS::Zone* zone);
+
+ // Sweep the marked weak maps in a zone, updating moved keys.
+ static void sweepZoneAfterMinorGC(JS::Zone* zone);
+
+ // Trace all weak map bindings. Used by the cycle collector.
+ static void traceAllMappings(WeakMapTracer* tracer);
+
+ // Save information about which weak maps are marked for a zone.
+ static bool saveZoneMarkedWeakMaps(JS::Zone* zone,
+ WeakMapColors& markedWeakMaps);
+
+ // Restore information about which weak maps are marked for many zones.
+ static void restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps);
+
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
+ static bool checkMarkingForZone(JS::Zone* zone);
+#endif
+
+ protected:
+ // Instance member functions called by the above. Instantiations of WeakMap
+ // override these with definitions appropriate for their Key and Value types.
+ virtual void trace(JSTracer* tracer) = 0;
+ virtual bool findSweepGroupEdges() = 0;
+ virtual void traceWeakEdges(JSTracer* trc) = 0;
+ virtual void traceMappings(WeakMapTracer* tracer) = 0;
+ virtual void clearAndCompact() = 0;
+
+ // We have a key that, if it or its delegate is marked, may lead to a WeakMap
+ // value getting marked. Insert it or its delegate (if any) into the
+ // appropriate zone's gcEphemeronEdges or gcNurseryEphemeronEdges.
+ inline bool addImplicitEdges(gc::Cell* key, gc::Cell* delegate,
+ gc::TenuredCell* value);
+
+ virtual bool markEntries(GCMarker* marker) = 0;
+
+#ifdef JS_GC_ZEAL
+ virtual bool checkMarking() const = 0;
+ virtual bool allowKeysInOtherZones() const { return false; }
+ friend bool gc::CheckWeakMapEntryMarking(const WeakMapBase*, gc::Cell*,
+ gc::Cell*);
+#endif
+
+ // Object that this weak map is part of, if any.
+ HeapPtr<JSObject*> memberOf;
+
+ // Zone containing this weak map.
+ JS::Zone* zone_;
+
+ // Whether this object has been marked during garbage collection and which
+ // color it was marked.
+ gc::CellColor mapColor;
+
+ friend class JS::Zone;
+};
+
+template <class Key, class Value>
+class WeakMap
+ : private HashMap<Key, Value, StableCellHasher<Key>, ZoneAllocPolicy>,
+ public WeakMapBase {
+ public:
+ using Base = HashMap<Key, Value, StableCellHasher<Key>, ZoneAllocPolicy>;
+
+ using Lookup = typename Base::Lookup;
+ using Entry = typename Base::Entry;
+ using Range = typename Base::Range;
+ using Ptr = typename Base::Ptr;
+ using AddPtr = typename Base::AddPtr;
+
+ struct Enum : public Base::Enum {
+ explicit Enum(WeakMap& map) : Base::Enum(static_cast<Base&>(map)) {}
+ };
+
+ using Base::all;
+ using Base::clear;
+ using Base::count;
+ using Base::empty;
+ using Base::has;
+ using Base::shallowSizeOfExcludingThis;
+
+ // Resolve ambiguity with LinkedListElement<>::remove.
+ using Base::remove;
+
+ using UnbarrieredKey = typename RemoveBarrier<Key>::Type;
+
+ explicit WeakMap(JSContext* cx, JSObject* memOf = nullptr);
+ explicit WeakMap(JS::Zone* zone, JSObject* memOf = nullptr);
+
+ // Add a read barrier to prevent an incorrectly gray value from escaping the
+ // weak map. See the UnmarkGrayTracer::onChild comment in gc/Marking.cpp.
+ Ptr lookup(const Lookup& l) const {
+ Ptr p = Base::lookup(l);
+ if (p) {
+ exposeGCThingToActiveJS(p->value());
+ }
+ return p;
+ }
+
+ Ptr lookupUnbarriered(const Lookup& l) const { return Base::lookup(l); }
+
+ AddPtr lookupForAdd(const Lookup& l) {
+ AddPtr p = Base::lookupForAdd(l);
+ if (p) {
+ exposeGCThingToActiveJS(p->value());
+ }
+ return p;
+ }
+
+ template <typename KeyInput, typename ValueInput>
+ [[nodiscard]] bool add(AddPtr& p, KeyInput&& k, ValueInput&& v) {
+ MOZ_ASSERT(k);
+ return Base::add(p, std::forward<KeyInput>(k), std::forward<ValueInput>(v));
+ }
+
+ template <typename KeyInput, typename ValueInput>
+ [[nodiscard]] bool relookupOrAdd(AddPtr& p, KeyInput&& k, ValueInput&& v) {
+ MOZ_ASSERT(k);
+ return Base::relookupOrAdd(p, std::forward<KeyInput>(k),
+ std::forward<ValueInput>(v));
+ }
+
+ template <typename KeyInput, typename ValueInput>
+ [[nodiscard]] bool put(KeyInput&& k, ValueInput&& v) {
+ MOZ_ASSERT(k);
+ return Base::put(std::forward<KeyInput>(k), std::forward<ValueInput>(v));
+ }
+
+ template <typename KeyInput, typename ValueInput>
+ [[nodiscard]] bool putNew(KeyInput&& k, ValueInput&& v) {
+ MOZ_ASSERT(k);
+ return Base::putNew(std::forward<KeyInput>(k), std::forward<ValueInput>(v));
+ }
+
+ template <typename KeyInput, typename ValueInput>
+ void putNewInfallible(KeyInput&& k, ValueInput&& v) {
+ MOZ_ASSERT(k);
+ Base::putNewInfallible(std::forward(k), std::forward<KeyInput>(k));
+ }
+
+#ifdef DEBUG
+ template <typename KeyInput, typename ValueInput>
+ bool hasEntry(KeyInput&& key, ValueInput&& value) {
+ Ptr p = Base::lookup(std::forward<KeyInput>(key));
+ return p && p->value() == value;
+ }
+#endif
+
+ bool markEntry(GCMarker* marker, Key& key, Value& value,
+ bool populateWeakKeysTable);
+
+ void trace(JSTracer* trc) override;
+
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
+
+ protected:
+ inline void assertMapIsSameZoneWithValue(const Value& v);
+
+ bool markEntries(GCMarker* marker) override;
+
+ protected:
+ // Find sweep group edges for delegates, if the key type has delegates. (If
+ // not, the optimizer should make this a nop.)
+ bool findSweepGroupEdges() override;
+
+ /**
+ * If a wrapper is used as a key in a weakmap, the garbage collector should
+ * keep that object around longer than it otherwise would. We want to avoid
+ * collecting the wrapper (and removing the weakmap entry) as long as the
+ * wrapped object is alive (because the object can be rewrapped and looked up
+ * again). As long as the wrapper is used as a weakmap key, it will not be
+ * collected (and remain in the weakmap) until the wrapped object is
+ * collected.
+ */
+ private:
+ void exposeGCThingToActiveJS(const JS::Value& v) const {
+ JS::ExposeValueToActiveJS(v);
+ }
+ void exposeGCThingToActiveJS(JSObject* obj) const {
+ JS::ExposeObjectToActiveJS(obj);
+ }
+
+ void traceWeakEdges(JSTracer* trc) override;
+
+ void clearAndCompact() override {
+ Base::clear();
+ Base::compact();
+ }
+
+ // memberOf can be nullptr, which means that the map is not part of a
+ // JSObject.
+ void traceMappings(WeakMapTracer* tracer) override;
+
+ protected:
+#if DEBUG
+ void assertEntriesNotAboutToBeFinalized();
+#endif
+
+#ifdef JS_GC_ZEAL
+ bool checkMarking() const override;
+#endif
+};
+
+using ObjectValueWeakMap = WeakMap<HeapPtr<JSObject*>, HeapPtr<Value>>;
+
+// Generic weak map for mapping objects to other objects.
+class ObjectWeakMap {
+ ObjectValueWeakMap map;
+
+ public:
+ explicit ObjectWeakMap(JSContext* cx);
+
+ JS::Zone* zone() const { return map.zone(); }
+
+ JSObject* lookup(const JSObject* obj);
+ bool add(JSContext* cx, JSObject* obj, JSObject* target);
+ void remove(JSObject* key);
+ void clear();
+
+ void trace(JSTracer* trc);
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ return mallocSizeOf(this) + sizeOfExcludingThis(mallocSizeOf);
+ }
+
+ ObjectValueWeakMap& valueMap() { return map; }
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+ void checkAfterMovingGC();
+#endif
+};
+
+} /* namespace js */
+
+#endif /* gc_WeakMap_h */
diff --git a/js/src/gc/WeakMapPtr.cpp b/js/src/gc/WeakMapPtr.cpp
new file mode 100644
index 0000000000..f58fc52372
--- /dev/null
+++ b/js/src/gc/WeakMapPtr.cpp
@@ -0,0 +1,114 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "js/WeakMapPtr.h"
+
+#include "gc/WeakMap-inl.h"
+
+//
+// Machinery for the externally-linkable JS::WeakMapPtr, which wraps js::WeakMap
+// for a few public data types.
+//
+
+using namespace js;
+
+namespace WeakMapDetails {
+
+template <typename T>
+struct DataType {};
+
+template <>
+struct DataType<JSObject*> {
+ using BarrieredType = HeapPtr<JSObject*>;
+ using HasherType = StableCellHasher<BarrieredType>;
+ static JSObject* NullValue() { return nullptr; }
+};
+
+template <>
+struct DataType<JS::Value> {
+ using BarrieredType = HeapPtr<Value>;
+ static JS::Value NullValue() { return JS::UndefinedValue(); }
+};
+
+template <typename K, typename V>
+struct Utils {
+ using KeyType = typename DataType<K>::BarrieredType;
+ using ValueType = typename DataType<V>::BarrieredType;
+ typedef WeakMap<KeyType, ValueType> Type;
+ using PtrType = Type*;
+ static PtrType cast(void* ptr) { return static_cast<PtrType>(ptr); }
+};
+
+} // namespace WeakMapDetails
+
+template <typename K, typename V>
+void JS::WeakMapPtr<K, V>::destroy() {
+ MOZ_ASSERT(initialized());
+ js_delete(WeakMapDetails::Utils<K, V>::cast(ptr));
+ ptr = nullptr;
+}
+
+template <typename K, typename V>
+bool JS::WeakMapPtr<K, V>::init(JSContext* cx) {
+ MOZ_ASSERT(!initialized());
+ typename WeakMapDetails::Utils<K, V>::PtrType map =
+ cx->new_<typename WeakMapDetails::Utils<K, V>::Type>(cx);
+ if (!map) {
+ return false;
+ }
+ ptr = map;
+ return true;
+}
+
+template <typename K, typename V>
+void JS::WeakMapPtr<K, V>::trace(JSTracer* trc) {
+ MOZ_ASSERT(initialized());
+ return WeakMapDetails::Utils<K, V>::cast(ptr)->trace(trc);
+}
+
+template <typename K, typename V>
+V JS::WeakMapPtr<K, V>::lookup(const K& key) {
+ MOZ_ASSERT(initialized());
+ typename WeakMapDetails::Utils<K, V>::Type::Ptr result =
+ WeakMapDetails::Utils<K, V>::cast(ptr)->lookup(key);
+ if (!result) {
+ return WeakMapDetails::DataType<V>::NullValue();
+ }
+ return result->value();
+}
+
+template <typename K, typename V>
+bool JS::WeakMapPtr<K, V>::put(JSContext* cx, const K& key, const V& value) {
+ MOZ_ASSERT(initialized());
+ return WeakMapDetails::Utils<K, V>::cast(ptr)->put(key, value);
+}
+
+template <typename K, typename V>
+V JS::WeakMapPtr<K, V>::removeValue(const K& key) {
+ typedef typename WeakMapDetails::Utils<K, V>::Type Map;
+ using Ptr = typename Map::Ptr;
+
+ MOZ_ASSERT(initialized());
+
+ Map* map = WeakMapDetails::Utils<K, V>::cast(ptr);
+ if (Ptr result = map->lookup(key)) {
+ V value = result->value();
+ map->remove(result);
+ return value;
+ }
+ return WeakMapDetails::DataType<V>::NullValue();
+}
+
+//
+// Supported specializations of JS::WeakMap:
+//
+
+template class JS_PUBLIC_API JS::WeakMapPtr<JSObject*, JSObject*>;
+
+#ifdef DEBUG
+// Nobody's using this at the moment, but we want to make sure it compiles.
+template class JS_PUBLIC_API JS::WeakMapPtr<JSObject*, JS::Value>;
+#endif
diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp
new file mode 100644
index 0000000000..982b12f17f
--- /dev/null
+++ b/js/src/gc/Zone.cpp
@@ -0,0 +1,979 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Zone.h"
+#include "js/shadow/Zone.h" // JS::shadow::Zone
+
+#include "mozilla/Sprintf.h"
+#include "mozilla/TimeStamp.h"
+
+#include <type_traits>
+
+#include "gc/FinalizationObservers.h"
+#include "gc/GCContext.h"
+#include "gc/PublicIterators.h"
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/Invalidation.h"
+#include "jit/JitZone.h"
+#include "vm/Runtime.h"
+#include "vm/Time.h"
+
+#include "debugger/DebugAPI-inl.h"
+#include "gc/GC-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/Nursery-inl.h"
+#include "gc/WeakMap-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/Realm-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+Zone* const Zone::NotOnList = reinterpret_cast<Zone*>(1);
+
+ZoneAllocator::ZoneAllocator(JSRuntime* rt, Kind kind)
+ : JS::shadow::Zone(rt, rt->gc.marker().tracer(), kind),
+ jitHeapThreshold(jit::MaxCodeBytesPerProcess * 0.8) {}
+
+ZoneAllocator::~ZoneAllocator() {
+#ifdef DEBUG
+ mallocTracker.checkEmptyOnDestroy();
+ MOZ_ASSERT(gcHeapSize.bytes() == 0);
+ MOZ_ASSERT(mallocHeapSize.bytes() == 0);
+ MOZ_ASSERT(jitHeapSize.bytes() == 0);
+#endif
+}
+
+void ZoneAllocator::fixupAfterMovingGC() {
+#ifdef DEBUG
+ mallocTracker.fixupAfterMovingGC();
+#endif
+}
+
+void js::ZoneAllocator::updateSchedulingStateOnGCStart() {
+ gcHeapSize.updateOnGCStart();
+ mallocHeapSize.updateOnGCStart();
+ jitHeapSize.updateOnGCStart();
+ perZoneGCTime = mozilla::TimeDuration();
+}
+
+void js::ZoneAllocator::updateGCStartThresholds(GCRuntime& gc) {
+ bool isAtomsZone = JS::Zone::from(this)->isAtomsZone();
+ gcHeapThreshold.updateStartThreshold(
+ gcHeapSize.retainedBytes(), smoothedAllocationRate.ref(),
+ smoothedCollectionRate.ref(), gc.tunables, gc.schedulingState,
+ isAtomsZone);
+
+ mallocHeapThreshold.updateStartThreshold(mallocHeapSize.retainedBytes(),
+ gc.tunables, gc.schedulingState);
+}
+
+void js::ZoneAllocator::setGCSliceThresholds(GCRuntime& gc,
+ bool waitingOnBGTask) {
+ gcHeapThreshold.setSliceThreshold(this, gcHeapSize, gc.tunables,
+ waitingOnBGTask);
+ mallocHeapThreshold.setSliceThreshold(this, mallocHeapSize, gc.tunables,
+ waitingOnBGTask);
+ jitHeapThreshold.setSliceThreshold(this, jitHeapSize, gc.tunables,
+ waitingOnBGTask);
+}
+
+void js::ZoneAllocator::clearGCSliceThresholds() {
+ gcHeapThreshold.clearSliceThreshold();
+ mallocHeapThreshold.clearSliceThreshold();
+ jitHeapThreshold.clearSliceThreshold();
+}
+
+bool ZoneAllocator::addSharedMemory(void* mem, size_t nbytes, MemoryUse use) {
+ // nbytes can be zero here for SharedArrayBuffers.
+
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_));
+
+ auto ptr = sharedMemoryUseCounts.lookupForAdd(mem);
+ MOZ_ASSERT_IF(ptr, ptr->value().use == use);
+
+ if (!ptr && !sharedMemoryUseCounts.add(ptr, mem, gc::SharedMemoryUse(use))) {
+ return false;
+ }
+
+ ptr->value().count++;
+
+ // Allocations can grow, so add any increase over the previous size and record
+ // the new size.
+ if (nbytes > ptr->value().nbytes) {
+ mallocHeapSize.addBytes(nbytes - ptr->value().nbytes);
+ ptr->value().nbytes = nbytes;
+ }
+
+ maybeTriggerGCOnMalloc();
+
+ return true;
+}
+
+void ZoneAllocator::removeSharedMemory(void* mem, size_t nbytes,
+ MemoryUse use) {
+ // nbytes can be zero here for SharedArrayBuffers.
+
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_));
+ MOZ_ASSERT(CurrentThreadIsGCFinalizing());
+
+ auto ptr = sharedMemoryUseCounts.lookup(mem);
+
+ MOZ_ASSERT(ptr);
+ MOZ_ASSERT(ptr->value().count != 0);
+ MOZ_ASSERT(ptr->value().use == use);
+ MOZ_ASSERT(ptr->value().nbytes >= nbytes);
+
+ ptr->value().count--;
+ if (ptr->value().count == 0) {
+ mallocHeapSize.removeBytes(ptr->value().nbytes, true);
+ sharedMemoryUseCounts.remove(ptr);
+ }
+}
+
+template <TrackingKind kind>
+void js::TrackedAllocPolicy<kind>::decMemory(size_t nbytes) {
+ bool updateRetainedSize = false;
+ if constexpr (kind == TrackingKind::Cell) {
+ // Only subtract freed cell memory from retained size for cell associations
+ // during sweeping.
+ JS::GCContext* gcx = TlsGCContext.get();
+ updateRetainedSize = gcx->isFinalizing();
+ }
+
+ zone_->decNonGCMemory(this, nbytes, MemoryUse::TrackedAllocPolicy,
+ updateRetainedSize);
+}
+
+namespace js {
+template class TrackedAllocPolicy<TrackingKind::Zone>;
+template class TrackedAllocPolicy<TrackingKind::Cell>;
+} // namespace js
+
+JS::Zone::Zone(JSRuntime* rt, Kind kind)
+ : ZoneAllocator(rt, kind),
+ arenas(this),
+ data(nullptr),
+ tenuredBigInts(0),
+ markedStrings(0),
+ finalizedStrings(0),
+ suppressAllocationMetadataBuilder(false),
+ allocNurseryObjects_(true),
+ allocNurseryStrings_(true),
+ allocNurseryBigInts_(true),
+ pretenuring(this),
+ compartments_(),
+ crossZoneStringWrappers_(this),
+ gcEphemeronEdges_(SystemAllocPolicy(), rt->randomHashCodeScrambler()),
+ gcNurseryEphemeronEdges_(SystemAllocPolicy(),
+ rt->randomHashCodeScrambler()),
+ shapeZone_(this),
+ gcScheduled_(false),
+ gcScheduledSaved_(false),
+ gcPreserveCode_(false),
+ keepPropMapTables_(false),
+ wasCollected_(false),
+ listNext_(NotOnList),
+ keptObjects(this) {
+ /* Ensure that there are no vtables to mess us up here. */
+ MOZ_ASSERT(reinterpret_cast<JS::shadow::Zone*>(this) ==
+ static_cast<JS::shadow::Zone*>(this));
+ MOZ_ASSERT_IF(isAtomsZone(), rt->gc.zones().empty());
+
+ updateGCStartThresholds(rt->gc);
+ rt->gc.nursery().setAllocFlagsForZone(this);
+}
+
+Zone::~Zone() {
+ MOZ_ASSERT_IF(regExps_.ref(), regExps().empty());
+
+ DebugAPI::deleteDebugScriptMap(debugScriptMap);
+ js_delete(finalizationObservers_.ref().release());
+
+ MOZ_ASSERT(gcWeakMapList().isEmpty());
+
+ JSRuntime* rt = runtimeFromAnyThread();
+ if (this == rt->gc.systemZone) {
+ MOZ_ASSERT(isSystemZone());
+ rt->gc.systemZone = nullptr;
+ }
+
+ js_delete(jitZone_.ref());
+}
+
+bool Zone::init() {
+ regExps_.ref() = make_unique<RegExpZone>(this);
+ return regExps_.ref() && gcEphemeronEdges().init() &&
+ gcNurseryEphemeronEdges().init();
+}
+
+void Zone::setNeedsIncrementalBarrier(bool needs) {
+ needsIncrementalBarrier_ = needs;
+}
+
+void Zone::changeGCState(GCState prev, GCState next) {
+ MOZ_ASSERT(RuntimeHeapIsBusy());
+ MOZ_ASSERT(gcState() == prev);
+
+ // This can be called when barriers have been temporarily disabled by
+ // AutoDisableBarriers. In that case, don't update needsIncrementalBarrier_
+ // and barriers will be re-enabled by ~AutoDisableBarriers() if necessary.
+ bool barriersDisabled = isGCMarking() && !needsIncrementalBarrier();
+
+ gcState_ = next;
+
+ // Update the barriers state when we transition between marking and
+ // non-marking states, unless barriers have been disabled.
+ if (!barriersDisabled) {
+ needsIncrementalBarrier_ = isGCMarking();
+ }
+}
+
+template <class Pred>
+static void EraseIf(js::gc::EphemeronEdgeVector& entries, Pred pred) {
+ auto* begin = entries.begin();
+ auto* const end = entries.end();
+
+ auto* newEnd = begin;
+ for (auto* p = begin; p != end; p++) {
+ if (!pred(*p)) {
+ *newEnd++ = *p;
+ }
+ }
+
+ size_t removed = end - newEnd;
+ entries.shrinkBy(removed);
+}
+
+static void SweepEphemeronEdgesWhileMinorSweeping(
+ js::gc::EphemeronEdgeVector& entries) {
+ EraseIf(entries, [](js::gc::EphemeronEdge& edge) -> bool {
+ return IsAboutToBeFinalizedDuringMinorSweep(&edge.target);
+ });
+}
+
+void Zone::sweepAfterMinorGC(JSTracer* trc) {
+ sweepEphemeronTablesAfterMinorGC();
+ crossZoneStringWrappers().sweepAfterMinorGC(trc);
+
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) {
+ comp->sweepAfterMinorGC(trc);
+ }
+}
+
+void Zone::sweepEphemeronTablesAfterMinorGC() {
+ for (auto r = gcNurseryEphemeronEdges().mutableAll(); !r.empty();
+ r.popFront()) {
+ // Sweep gcNurseryEphemeronEdges to move live (forwarded) keys to
+ // gcEphemeronEdges, scanning through all the entries for such keys to
+ // update them.
+ //
+ // Forwarded and dead keys may also appear in their delegates' entries,
+ // so sweep those too (see below.)
+
+ // The tricky case is when the key has a delegate that was already
+ // tenured. Then it will be in its compartment's gcEphemeronEdges, but we
+ // still need to update the key (which will be in the entries
+ // associated with it.)
+ gc::Cell* key = r.front().key;
+ MOZ_ASSERT(!key->isTenured());
+ if (!Nursery::getForwardedPointer(&key)) {
+ // Dead nursery cell => discard.
+ continue;
+ }
+
+ // Key been moved. The value is an array of <color,cell> pairs; update all
+ // cells in that array.
+ EphemeronEdgeVector& entries = r.front().value;
+ SweepEphemeronEdgesWhileMinorSweeping(entries);
+
+ // Live (moved) nursery cell. Append entries to gcEphemeronEdges.
+ EphemeronEdgeTable& tenuredEdges = gcEphemeronEdges();
+ auto* entry = tenuredEdges.get(key);
+ if (!entry) {
+ if (!tenuredEdges.put(key, gc::EphemeronEdgeVector())) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ oomUnsafe.crash("Failed to tenure weak keys entry");
+ }
+ entry = tenuredEdges.get(key);
+ }
+
+ if (!entry->value.appendAll(entries)) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ oomUnsafe.crash("Failed to tenure weak keys entry");
+ }
+
+ // If the key has a delegate, then it will map to a WeakKeyEntryVector
+ // containing the key that needs to be updated.
+
+ JSObject* delegate = gc::detail::GetDelegate(key->as<JSObject>());
+ if (!delegate) {
+ continue;
+ }
+ MOZ_ASSERT(delegate->isTenured());
+
+ // If delegate was formerly nursery-allocated, we will sweep its entries
+ // when we visit its gcNurseryEphemeronEdges (if we haven't already). Note
+ // that we don't know the nursery address of the delegate, since the
+ // location it was stored in has already been updated.
+ //
+ // Otherwise, it will be in gcEphemeronEdges and we sweep it here.
+ auto* p = delegate->zone()->gcEphemeronEdges().get(delegate);
+ if (p) {
+ SweepEphemeronEdgesWhileMinorSweeping(p->value);
+ }
+ }
+
+ if (!gcNurseryEphemeronEdges().clear()) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ oomUnsafe.crash("OOM while clearing gcNurseryEphemeronEdges.");
+ }
+}
+
+void Zone::traceWeakCCWEdges(JSTracer* trc) {
+ crossZoneStringWrappers().traceWeak(trc);
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) {
+ comp->traceCrossCompartmentObjectWrapperEdges(trc);
+ }
+}
+
+/* static */
+void Zone::fixupAllCrossCompartmentWrappersAfterMovingGC(JSTracer* trc) {
+ MOZ_ASSERT(trc->runtime()->gc.isHeapCompacting());
+
+ for (ZonesIter zone(trc->runtime(), WithAtoms); !zone.done(); zone.next()) {
+ // Trace the wrapper map to update keys (wrapped values) in other
+ // compartments that may have been moved.
+ zone->crossZoneStringWrappers().traceWeak(trc);
+
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ comp->fixupCrossCompartmentObjectWrappersAfterMovingGC(trc);
+ }
+ }
+}
+
+void Zone::dropStringWrappersOnGC() {
+ MOZ_ASSERT(JS::RuntimeHeapIsCollecting());
+ crossZoneStringWrappers().clear();
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+void Zone::checkAllCrossCompartmentWrappersAfterMovingGC() {
+ checkStringWrappersAfterMovingGC();
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) {
+ comp->checkObjectWrappersAfterMovingGC();
+ }
+}
+
+void Zone::checkStringWrappersAfterMovingGC() {
+ for (StringWrapperMap::Enum e(crossZoneStringWrappers()); !e.empty();
+ e.popFront()) {
+ // Assert that the postbarriers have worked and that nothing is left in the
+ // wrapper map that points into the nursery, and that the hash table entries
+ // are discoverable.
+ auto key = e.front().key();
+ CheckGCThingAfterMovingGC(key.get());
+
+ auto ptr = crossZoneStringWrappers().lookup(key);
+ MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &e.front());
+ }
+}
+#endif
+
+void Zone::discardJitCode(JS::GCContext* gcx, const DiscardOptions& options) {
+ if (!isPreservingCode()) {
+ forceDiscardJitCode(gcx, options);
+ }
+}
+
+void Zone::forceDiscardJitCode(JS::GCContext* gcx,
+ const DiscardOptions& options) {
+ if (!jitZone()) {
+ return;
+ }
+
+ if (options.discardJitScripts && options.discardBaselineCode) {
+ lastDiscardedCodeTime_ = mozilla::TimeStamp::Now();
+ }
+
+ if (options.discardBaselineCode || options.discardJitScripts) {
+#ifdef DEBUG
+ // Assert no JitScripts are marked as active.
+ for (auto iter = cellIter<BaseScript>(); !iter.done(); iter.next()) {
+ BaseScript* base = iter.unbarrieredGet();
+ if (jit::JitScript* jitScript = base->maybeJitScript()) {
+ MOZ_ASSERT(!jitScript->active());
+ }
+ }
+#endif
+
+ // Mark JitScripts on the stack as active.
+ jit::MarkActiveJitScripts(this);
+ }
+
+ // Invalidate all Ion code in this zone.
+ jit::InvalidateAll(gcx, this);
+
+ for (auto base = cellIterUnsafe<BaseScript>(); !base.done(); base.next()) {
+ jit::JitScript* jitScript = base->maybeJitScript();
+ if (!jitScript) {
+ continue;
+ }
+
+ JSScript* script = base->asJSScript();
+ jit::FinishInvalidation(gcx, script);
+
+ // Discard baseline script if it's not marked as active.
+ if (options.discardBaselineCode) {
+ if (jitScript->hasBaselineScript() && !jitScript->active()) {
+ jit::FinishDiscardBaselineScript(gcx, script);
+ }
+ }
+
+#ifdef JS_CACHEIR_SPEW
+ maybeUpdateWarmUpCount(script);
+#endif
+
+ // Warm-up counter for scripts are reset on GC. After discarding code we
+ // need to let it warm back up to get information such as which
+ // opcodes are setting array holes or accessing getter properties.
+ script->resetWarmUpCounterForGC();
+
+ // Try to release the script's JitScript. This should happen after
+ // releasing JIT code because we can't do this when the script still has
+ // JIT code.
+ if (options.discardJitScripts) {
+ script->maybeReleaseJitScript(gcx);
+ jitScript = script->maybeJitScript();
+ if (!jitScript) {
+ // Try to discard the ScriptCounts too.
+ if (!script->realm()->collectCoverageForDebug() &&
+ !gcx->runtime()->profilingScripts) {
+ script->destroyScriptCounts();
+ }
+ continue;
+ }
+ }
+
+ // If we did not release the JitScript, we need to purge optimized IC
+ // stubs because the optimizedStubSpace will be purged below.
+ if (options.discardBaselineCode) {
+ jitScript->purgeOptimizedStubs(script);
+ }
+
+ if (options.resetNurseryAllocSites || options.resetPretenuredAllocSites) {
+ jitScript->resetAllocSites(options.resetNurseryAllocSites,
+ options.resetPretenuredAllocSites);
+ }
+
+ // Finally, reset the active flag.
+ jitScript->resetActive();
+ }
+
+ // Also clear references to jit code from RegExpShared cells at this point.
+ // This avoid holding onto ExecutablePools.
+ for (auto regExp = cellIterUnsafe<RegExpShared>(); !regExp.done();
+ regExp.next()) {
+ regExp->discardJitCode();
+ }
+
+ /*
+ * When scripts contains pointers to nursery things, the store buffer
+ * can contain entries that point into the optimized stub space. Since
+ * this method can be called outside the context of a GC, this situation
+ * could result in us trying to mark invalid store buffer entries.
+ *
+ * Defer freeing any allocated blocks until after the next minor GC.
+ */
+ if (options.discardBaselineCode) {
+ jitZone()->optimizedStubSpace()->freeAllAfterMinorGC(this);
+ jitZone()->purgeIonCacheIRStubInfo();
+ }
+
+ // Generate a profile marker
+ if (gcx->runtime()->geckoProfiler().enabled()) {
+ char discardingJitScript = options.discardJitScripts ? 'Y' : 'N';
+ char discardingBaseline = options.discardBaselineCode ? 'Y' : 'N';
+ char discardingIon = 'Y';
+
+ char discardingRegExp = 'Y';
+ char discardingNurserySites = options.resetNurseryAllocSites ? 'Y' : 'N';
+ char discardingPretenuredSites =
+ options.resetPretenuredAllocSites ? 'Y' : 'N';
+
+ char buf[100];
+ SprintfLiteral(buf,
+ "JitScript:%c Baseline:%c Ion:%c "
+ "RegExp:%c NurserySites:%c PretenuredSites:%c",
+ discardingJitScript, discardingBaseline, discardingIon,
+ discardingRegExp, discardingNurserySites,
+ discardingPretenuredSites);
+ gcx->runtime()->geckoProfiler().markEvent("DiscardJit", buf);
+ }
+}
+
+void JS::Zone::resetAllocSitesAndInvalidate(bool resetNurserySites,
+ bool resetPretenuredSites) {
+ MOZ_ASSERT(resetNurserySites || resetPretenuredSites);
+
+ if (!jitZone()) {
+ return;
+ }
+
+ JSContext* cx = runtime_->mainContextFromOwnThread();
+ for (auto base = cellIterUnsafe<BaseScript>(); !base.done(); base.next()) {
+ jit::JitScript* jitScript = base->maybeJitScript();
+ if (!jitScript) {
+ continue;
+ }
+
+ if (!jitScript->resetAllocSites(resetNurserySites, resetPretenuredSites)) {
+ continue;
+ }
+
+ JSScript* script = base->asJSScript();
+ CancelOffThreadIonCompile(script);
+
+ if (!script->hasIonScript()) {
+ continue;
+ }
+
+ jit::Invalidate(cx, script,
+ /* resetUses = */ true,
+ /* cancelOffThread = */ true);
+ }
+}
+
+void JS::Zone::beforeClearDelegateInternal(JSObject* wrapper,
+ JSObject* delegate) {
+ MOZ_ASSERT(js::gc::detail::GetDelegate(wrapper) == delegate);
+ MOZ_ASSERT(needsIncrementalBarrier());
+ MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(this));
+ runtimeFromMainThread()->gc.marker().severWeakDelegate(wrapper, delegate);
+}
+
+void JS::Zone::afterAddDelegateInternal(JSObject* wrapper) {
+ MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(this));
+ JSObject* delegate = js::gc::detail::GetDelegate(wrapper);
+ if (delegate) {
+ runtimeFromMainThread()->gc.marker().restoreWeakDelegate(wrapper, delegate);
+ }
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void JS::Zone::checkUniqueIdTableAfterMovingGC() {
+ for (auto r = uniqueIds().all(); !r.empty(); r.popFront()) {
+ js::gc::CheckGCThingAfterMovingGC(r.front().key());
+ }
+}
+#endif
+
+js::jit::JitZone* Zone::createJitZone(JSContext* cx) {
+ MOZ_ASSERT(!jitZone_);
+ MOZ_ASSERT(cx->runtime()->hasJitRuntime());
+
+ UniquePtr<jit::JitZone> jitZone(cx->new_<js::jit::JitZone>());
+ if (!jitZone) {
+ return nullptr;
+ }
+
+ jitZone_ = jitZone.release();
+ return jitZone_;
+}
+
+bool Zone::hasMarkedRealms() {
+ for (RealmsInZoneIter realm(this); !realm.done(); realm.next()) {
+ if (realm->marked()) {
+ return true;
+ }
+ }
+ return false;
+}
+
+void Zone::notifyObservingDebuggers() {
+ AutoAssertNoGC nogc;
+ MOZ_ASSERT(JS::RuntimeHeapIsCollecting(),
+ "This method should be called during GC.");
+
+ JSRuntime* rt = runtimeFromMainThread();
+
+ for (RealmsInZoneIter realms(this); !realms.done(); realms.next()) {
+ GlobalObject* global = realms->unsafeUnbarrieredMaybeGlobal();
+ if (!global) {
+ continue;
+ }
+
+ DebugAPI::notifyParticipatesInGC(global, rt->gc.majorGCCount());
+ }
+}
+
+bool Zone::isOnList() const { return listNext_ != NotOnList; }
+
+Zone* Zone::nextZone() const {
+ MOZ_ASSERT(isOnList());
+ return listNext_;
+}
+
+void Zone::fixupAfterMovingGC() {
+ ZoneAllocator::fixupAfterMovingGC();
+ shapeZone().fixupPropMapShapeTableAfterMovingGC();
+}
+
+void Zone::purgeAtomCache() {
+ atomCache().clearAndCompact();
+
+ // Also purge the dtoa caches so that subsequent lookups populate atom
+ // cache too.
+ for (RealmsInZoneIter r(this); !r.done(); r.next()) {
+ r->dtoaCache.purge();
+ }
+}
+
+void Zone::addSizeOfIncludingThis(
+ mozilla::MallocSizeOf mallocSizeOf, JS::CodeSizes* code, size_t* regexpZone,
+ size_t* jitZone, size_t* baselineStubsOptimized, size_t* uniqueIdMap,
+ size_t* initialPropMapTable, size_t* shapeTables, size_t* atomsMarkBitmaps,
+ size_t* compartmentObjects, size_t* crossCompartmentWrappersTables,
+ size_t* compartmentsPrivateData, size_t* scriptCountsMapArg) {
+ *regexpZone += regExps().sizeOfIncludingThis(mallocSizeOf);
+ if (jitZone_) {
+ jitZone_->addSizeOfIncludingThis(mallocSizeOf, code, jitZone,
+ baselineStubsOptimized);
+ }
+ *uniqueIdMap += uniqueIds().shallowSizeOfExcludingThis(mallocSizeOf);
+ shapeZone().addSizeOfExcludingThis(mallocSizeOf, initialPropMapTable,
+ shapeTables);
+ *atomsMarkBitmaps += markedAtoms().sizeOfExcludingThis(mallocSizeOf);
+ *crossCompartmentWrappersTables +=
+ crossZoneStringWrappers().sizeOfExcludingThis(mallocSizeOf);
+
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) {
+ comp->addSizeOfIncludingThis(mallocSizeOf, compartmentObjects,
+ crossCompartmentWrappersTables,
+ compartmentsPrivateData);
+ }
+
+ if (scriptCountsMap) {
+ *scriptCountsMapArg +=
+ scriptCountsMap->shallowSizeOfIncludingThis(mallocSizeOf);
+ for (auto r = scriptCountsMap->all(); !r.empty(); r.popFront()) {
+ *scriptCountsMapArg +=
+ r.front().value()->sizeOfIncludingThis(mallocSizeOf);
+ }
+ }
+}
+
+void* ZoneAllocator::onOutOfMemory(js::AllocFunction allocFunc,
+ arena_id_t arena, size_t nbytes,
+ void* reallocPtr) {
+ if (!js::CurrentThreadCanAccessRuntime(runtime_)) {
+ return nullptr;
+ }
+ // The analysis sees that JSRuntime::onOutOfMemory could report an error,
+ // which with a JSErrorInterceptor could GC. But we're passing a null cx (to
+ // a default parameter) so the error will not be reported.
+ JS::AutoSuppressGCAnalysis suppress;
+ return runtimeFromMainThread()->onOutOfMemory(allocFunc, arena, nbytes,
+ reallocPtr);
+}
+
+void ZoneAllocator::reportAllocationOverflow() const {
+ js::ReportAllocationOverflow(static_cast<JSContext*>(nullptr));
+}
+
+ZoneList::ZoneList() : head(nullptr), tail(nullptr) {}
+
+ZoneList::ZoneList(Zone* zone) : head(zone), tail(zone) {
+ MOZ_RELEASE_ASSERT(!zone->isOnList());
+ zone->listNext_ = nullptr;
+}
+
+ZoneList::~ZoneList() { MOZ_ASSERT(isEmpty()); }
+
+void ZoneList::check() const {
+#ifdef DEBUG
+ MOZ_ASSERT((head == nullptr) == (tail == nullptr));
+ if (!head) {
+ return;
+ }
+
+ Zone* zone = head;
+ for (;;) {
+ MOZ_ASSERT(zone && zone->isOnList());
+ if (zone == tail) break;
+ zone = zone->listNext_;
+ }
+ MOZ_ASSERT(!zone->listNext_);
+#endif
+}
+
+bool ZoneList::isEmpty() const { return head == nullptr; }
+
+Zone* ZoneList::front() const {
+ MOZ_ASSERT(!isEmpty());
+ MOZ_ASSERT(head->isOnList());
+ return head;
+}
+
+void ZoneList::prepend(Zone* zone) { prependList(ZoneList(zone)); }
+
+void ZoneList::append(Zone* zone) { appendList(ZoneList(zone)); }
+
+void ZoneList::prependList(ZoneList&& other) {
+ check();
+ other.check();
+
+ if (other.isEmpty()) {
+ return;
+ }
+
+ MOZ_ASSERT(tail != other.tail);
+
+ if (!isEmpty()) {
+ other.tail->listNext_ = head;
+ } else {
+ tail = other.tail;
+ }
+ head = other.head;
+
+ other.head = nullptr;
+ other.tail = nullptr;
+}
+
+void ZoneList::appendList(ZoneList&& other) {
+ check();
+ other.check();
+
+ if (other.isEmpty()) {
+ return;
+ }
+
+ MOZ_ASSERT(tail != other.tail);
+
+ if (!isEmpty()) {
+ tail->listNext_ = other.head;
+ } else {
+ head = other.head;
+ }
+ tail = other.tail;
+
+ other.head = nullptr;
+ other.tail = nullptr;
+}
+
+Zone* ZoneList::removeFront() {
+ MOZ_ASSERT(!isEmpty());
+ check();
+
+ Zone* front = head;
+ head = head->listNext_;
+ if (!head) {
+ tail = nullptr;
+ }
+
+ front->listNext_ = Zone::NotOnList;
+
+ return front;
+}
+
+void ZoneList::clear() {
+ while (!isEmpty()) {
+ removeFront();
+ }
+}
+
+JS_PUBLIC_API void JS::shadow::RegisterWeakCache(
+ JS::Zone* zone, detail::WeakCacheBase* cachep) {
+ zone->registerWeakCache(cachep);
+}
+
+void Zone::traceRootsInMajorGC(JSTracer* trc) {
+ if (trc->isMarkingTracer() && !isGCMarking()) {
+ return;
+ }
+
+ // Trace zone script-table roots. See comment below for justification re:
+ // calling this only during major (non-nursery) collections.
+ traceScriptTableRoots(trc);
+
+ if (FinalizationObservers* observers = finalizationObservers()) {
+ observers->traceRoots(trc);
+ }
+}
+
+void Zone::traceScriptTableRoots(JSTracer* trc) {
+ static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>,
+ "BaseScript must not be nursery-allocated for script-table "
+ "tracing to work");
+
+ // Performance optimization: the script-table keys are JSScripts, which
+ // cannot be in the nursery, so we can skip this tracing if we are only in a
+ // minor collection. We static-assert this fact above.
+ MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting());
+
+ // N.B.: the script-table keys are weak *except* in an exceptional case: when
+ // then --dump-bytecode command line option or the PCCount JSFriend API is
+ // used, then the scripts for all counts must remain alive. We only trace
+ // when the `trc->runtime()->profilingScripts` flag is set. This flag is
+ // cleared in JSRuntime::destroyRuntime() during shutdown to ensure that
+ // scripts are collected before the runtime goes away completely.
+ if (scriptCountsMap && trc->runtime()->profilingScripts) {
+ for (ScriptCountsMap::Range r = scriptCountsMap->all(); !r.empty();
+ r.popFront()) {
+ BaseScript* script = r.front().key();
+ MOZ_ASSERT(script->hasScriptCounts());
+ TraceRoot(trc, &script, "profilingScripts");
+ }
+ }
+
+ // Trace the debugger's DebugScript weak map.
+ if (debugScriptMap) {
+ DebugAPI::traceDebugScriptMap(trc, debugScriptMap);
+ }
+}
+
+void Zone::fixupScriptMapsAfterMovingGC(JSTracer* trc) {
+ // Map entries are removed by BaseScript::finalize, but we need to update the
+ // script pointers here in case they are moved by the GC.
+
+ if (scriptCountsMap) {
+ scriptCountsMap->traceWeak(trc);
+ }
+
+ if (scriptLCovMap) {
+ scriptLCovMap->traceWeak(trc);
+ }
+
+#ifdef MOZ_VTUNE
+ if (scriptVTuneIdMap) {
+ scriptVTuneIdMap->traceWeak(trc);
+ }
+#endif
+
+#ifdef JS_CACHEIR_SPEW
+ if (scriptFinalWarmUpCountMap) {
+ scriptFinalWarmUpCountMap->traceWeak(trc);
+ }
+#endif
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void Zone::checkScriptMapsAfterMovingGC() {
+ if (scriptCountsMap) {
+ for (auto r = scriptCountsMap->all(); !r.empty(); r.popFront()) {
+ BaseScript* script = r.front().key();
+ MOZ_ASSERT(script->zone() == this);
+ CheckGCThingAfterMovingGC(script);
+ auto ptr = scriptCountsMap->lookup(script);
+ MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front());
+ }
+ }
+
+ if (scriptLCovMap) {
+ for (auto r = scriptLCovMap->all(); !r.empty(); r.popFront()) {
+ BaseScript* script = r.front().key();
+ MOZ_ASSERT(script->zone() == this);
+ CheckGCThingAfterMovingGC(script);
+ auto ptr = scriptLCovMap->lookup(script);
+ MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front());
+ }
+ }
+
+# ifdef MOZ_VTUNE
+ if (scriptVTuneIdMap) {
+ for (auto r = scriptVTuneIdMap->all(); !r.empty(); r.popFront()) {
+ BaseScript* script = r.front().key();
+ MOZ_ASSERT(script->zone() == this);
+ CheckGCThingAfterMovingGC(script);
+ auto ptr = scriptVTuneIdMap->lookup(script);
+ MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front());
+ }
+ }
+# endif // MOZ_VTUNE
+
+# ifdef JS_CACHEIR_SPEW
+ if (scriptFinalWarmUpCountMap) {
+ for (auto r = scriptFinalWarmUpCountMap->all(); !r.empty(); r.popFront()) {
+ BaseScript* script = r.front().key();
+ MOZ_ASSERT(script->zone() == this);
+ CheckGCThingAfterMovingGC(script);
+ auto ptr = scriptFinalWarmUpCountMap->lookup(script);
+ MOZ_RELEASE_ASSERT(ptr.found() && &*ptr == &r.front());
+ }
+ }
+# endif // JS_CACHEIR_SPEW
+}
+#endif
+
+void Zone::clearScriptCounts(Realm* realm) {
+ if (!scriptCountsMap) {
+ return;
+ }
+
+ // Clear all hasScriptCounts_ flags of BaseScript, in order to release all
+ // ScriptCounts entries of the given realm.
+ for (auto i = scriptCountsMap->modIter(); !i.done(); i.next()) {
+ BaseScript* script = i.get().key();
+ if (script->realm() != realm) {
+ continue;
+ }
+ // We can't destroy the ScriptCounts yet if the script has Baseline code,
+ // because Baseline code bakes in pointers to the counters. The ScriptCounts
+ // will be destroyed in Zone::discardJitCode when discarding the JitScript.
+ if (script->hasBaselineScript()) {
+ continue;
+ }
+ script->clearHasScriptCounts();
+ i.remove();
+ }
+}
+
+void Zone::clearScriptLCov(Realm* realm) {
+ if (!scriptLCovMap) {
+ return;
+ }
+
+ for (auto i = scriptLCovMap->modIter(); !i.done(); i.next()) {
+ BaseScript* script = i.get().key();
+ if (script->realm() == realm) {
+ i.remove();
+ }
+ }
+}
+
+void Zone::clearRootsForShutdownGC() {
+ // Finalization callbacks are not called if we're shutting down.
+ if (finalizationObservers()) {
+ finalizationObservers()->clearRecords();
+ }
+
+ clearKeptObjects();
+}
+
+void Zone::finishRoots() {
+ for (RealmsInZoneIter r(this); !r.done(); r.next()) {
+ r->finishRoots();
+ }
+}
+
+void Zone::traceKeptObjects(JSTracer* trc) { keptObjects.ref().trace(trc); }
+
+bool Zone::keepDuringJob(HandleObject target) {
+ return keptObjects.ref().put(target);
+}
+
+void Zone::clearKeptObjects() { keptObjects.ref().clear(); }
+
+bool Zone::ensureFinalizationObservers() {
+ if (finalizationObservers_.ref()) {
+ return true;
+ }
+
+ finalizationObservers_ = js::MakeUnique<FinalizationObservers>(this);
+ return bool(finalizationObservers_.ref());
+}
diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h
new file mode 100644
index 0000000000..ba3de7ec3e
--- /dev/null
+++ b/js/src/gc/Zone.h
@@ -0,0 +1,653 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Zone_h
+#define gc_Zone_h
+
+#include "mozilla/Array.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/LinkedList.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/TimeStamp.h"
+
+#include "jstypes.h"
+
+#include "ds/Bitmap.h"
+#include "gc/ArenaList.h"
+#include "gc/Barrier.h"
+#include "gc/FindSCCs.h"
+#include "gc/GCMarker.h"
+#include "gc/NurseryAwareHashMap.h"
+#include "gc/Pretenuring.h"
+#include "gc/Statistics.h"
+#include "gc/ZoneAllocator.h"
+#include "js/GCHashTable.h"
+#include "js/Vector.h"
+#include "vm/AtomsTable.h"
+#include "vm/JSObject.h"
+#include "vm/JSScript.h"
+#include "vm/ShapeZone.h"
+
+namespace js {
+
+class DebugScriptMap;
+class RegExpZone;
+class WeakRefObject;
+
+namespace jit {
+class JitZone;
+} // namespace jit
+
+namespace gc {
+
+class FinalizationObservers;
+class ZoneList;
+
+using ZoneComponentFinder = ComponentFinder<JS::Zone>;
+
+struct UniqueIdGCPolicy {
+ static bool traceWeak(JSTracer* trc, Cell** keyp, uint64_t* valuep);
+};
+
+// Maps a Cell* to a unique, 64bit id.
+using UniqueIdMap = GCHashMap<Cell*, uint64_t, PointerHasher<Cell*>,
+ SystemAllocPolicy, UniqueIdGCPolicy>;
+
+template <typename T>
+class ZoneAllCellIter;
+
+template <typename T>
+class ZoneCellIter;
+
+} // namespace gc
+
+// If two different nursery strings are wrapped into the same zone, and have
+// the same contents, then deduplication may make them duplicates.
+// `DuplicatesPossible` will allow this and map both wrappers to the same (now
+// tenured) source string.
+using StringWrapperMap =
+ NurseryAwareHashMap<JSString*, JSString*, ZoneAllocPolicy,
+ DuplicatesPossible>;
+
+class MOZ_NON_TEMPORARY_CLASS ExternalStringCache {
+ static const size_t NumEntries = 4;
+ mozilla::Array<JSString*, NumEntries> entries_;
+
+ ExternalStringCache(const ExternalStringCache&) = delete;
+ void operator=(const ExternalStringCache&) = delete;
+
+ public:
+ ExternalStringCache() { purge(); }
+ void purge() { mozilla::PodArrayZero(entries_); }
+
+ MOZ_ALWAYS_INLINE JSString* lookup(const char16_t* chars, size_t len) const;
+ MOZ_ALWAYS_INLINE void put(JSString* s);
+};
+
+class MOZ_NON_TEMPORARY_CLASS FunctionToStringCache {
+ struct Entry {
+ BaseScript* script;
+ JSString* string;
+
+ void set(BaseScript* scriptArg, JSString* stringArg) {
+ script = scriptArg;
+ string = stringArg;
+ }
+ };
+ static const size_t NumEntries = 2;
+ mozilla::Array<Entry, NumEntries> entries_;
+
+ FunctionToStringCache(const FunctionToStringCache&) = delete;
+ void operator=(const FunctionToStringCache&) = delete;
+
+ public:
+ FunctionToStringCache() { purge(); }
+ void purge() { mozilla::PodArrayZero(entries_); }
+
+ MOZ_ALWAYS_INLINE JSString* lookup(BaseScript* script) const;
+ MOZ_ALWAYS_INLINE void put(BaseScript* script, JSString* string);
+};
+
+} // namespace js
+
+namespace JS {
+
+// [SMDOC] GC Zones
+//
+// A zone is a collection of compartments. Every compartment belongs to exactly
+// one zone. In Firefox, there is roughly one zone per tab along with a system
+// zone for everything else. Zones mainly serve as boundaries for garbage
+// collection. Unlike compartments, they have no special security properties.
+//
+// Every GC thing belongs to exactly one zone. GC things from the same zone but
+// different compartments can share an arena (4k page). GC things from different
+// zones cannot be stored in the same arena. The garbage collector is capable of
+// collecting one zone at a time; it cannot collect at the granularity of
+// compartments.
+//
+// GC things are tied to zones and compartments as follows:
+//
+// - JSObjects belong to a compartment and cannot be shared between
+// compartments. If an object needs to point to a JSObject in a different
+// compartment, regardless of zone, it must go through a cross-compartment
+// wrapper. Each compartment keeps track of its outgoing wrappers in a table.
+// JSObjects find their compartment via their ObjectGroup.
+//
+// - JSStrings do not belong to any particular compartment, but they do belong
+// to a zone. Thus, two different compartments in the same zone can point to a
+// JSString. When a string needs to be wrapped, we copy it if it's in a
+// different zone and do nothing if it's in the same zone. Thus, transferring
+// strings within a zone is very efficient.
+//
+// - Shapes and base shapes belong to a zone and are shared between compartments
+// in that zone where possible. Accessor shapes store getter and setter
+// JSObjects which belong to a single compartment, so these shapes and all
+// their descendants can't be shared with other compartments.
+//
+// - Scripts are also compartment-local and cannot be shared. A script points to
+// its compartment.
+//
+// - ObjectGroup and JitCode objects belong to a compartment and cannot be
+// shared. There is no mechanism to obtain the compartment from a JitCode
+// object.
+//
+// A zone remains alive as long as any GC things in the zone are alive. A
+// compartment remains alive as long as any JSObjects, scripts, shapes, or base
+// shapes within it are alive.
+//
+// We always guarantee that a zone has at least one live compartment by refusing
+// to delete the last compartment in a live zone.
+class Zone : public js::ZoneAllocator, public js::gc::GraphNodeBase<JS::Zone> {
+ public:
+ js::gc::ArenaLists arenas;
+
+ // Per-zone data for use by an embedder.
+ js::MainThreadData<void*> data;
+
+ js::MainThreadData<uint32_t> tenuredBigInts;
+
+ // Number of marked/finalized JSStrings/JSFatInlineStrings during major GC.
+ js::MainThreadOrGCTaskData<size_t> markedStrings;
+ js::MainThreadOrGCTaskData<size_t> finalizedStrings;
+
+ // When true, skip calling the metadata callback. We use this:
+ // - to avoid invoking the callback recursively;
+ // - to avoid observing lazy prototype setup (which confuses callbacks that
+ // want to use the types being set up!);
+ // - to avoid attaching allocation stacks to allocation stack nodes, which
+ // is silly
+ // And so on.
+ js::MainThreadData<bool> suppressAllocationMetadataBuilder;
+
+ // Flags permanently set when nursery allocation is disabled for this zone.
+ js::MainThreadData<bool> nurseryStringsDisabled;
+ js::MainThreadData<bool> nurseryBigIntsDisabled;
+
+ private:
+ // Flags dynamically updated based on more than one condition, including the
+ // flags above.
+ js::MainThreadOrIonCompileData<bool> allocNurseryObjects_;
+ js::MainThreadOrIonCompileData<bool> allocNurseryStrings_;
+ js::MainThreadOrIonCompileData<bool> allocNurseryBigInts_;
+
+ // Minimum Heap value which results in tenured allocation.
+ js::MainThreadData<js::gc::Heap> minObjectHeapToTenure_;
+ js::MainThreadData<js::gc::Heap> minStringHeapToTenure_;
+ js::MainThreadData<js::gc::Heap> minBigintHeapToTenure_;
+
+ public:
+ // Script side-tables. These used to be held by Realm, but are now placed
+ // here in order to allow JSScript to access them during finalize (see bug
+ // 1568245; this change in 1575350). The tables are initialized lazily by
+ // JSScript.
+ js::UniquePtr<js::ScriptCountsMap> scriptCountsMap;
+ js::UniquePtr<js::ScriptLCovMap> scriptLCovMap;
+ js::MainThreadData<js::DebugScriptMap*> debugScriptMap;
+#ifdef MOZ_VTUNE
+ js::UniquePtr<js::ScriptVTuneIdMap> scriptVTuneIdMap;
+#endif
+#ifdef JS_CACHEIR_SPEW
+ js::UniquePtr<js::ScriptFinalWarmUpCountMap> scriptFinalWarmUpCountMap;
+#endif
+
+ js::MainThreadData<js::StringStats> previousGCStringStats;
+ js::MainThreadData<js::StringStats> stringStats;
+
+#ifdef DEBUG
+ js::MainThreadData<unsigned> gcSweepGroupIndex;
+#endif
+
+ js::gc::PretenuringZone pretenuring;
+
+ private:
+ // Side map for storing unique ids for cells, independent of address.
+ js::MainThreadOrGCTaskData<js::gc::UniqueIdMap> uniqueIds_;
+
+ // Number of allocations since the most recent minor GC for this thread.
+ uint32_t tenuredAllocsSinceMinorGC_ = 0;
+
+ // Live weakmaps in this zone.
+ js::MainThreadOrGCTaskData<mozilla::LinkedList<js::WeakMapBase>>
+ gcWeakMapList_;
+
+ // The set of compartments in this zone.
+ using CompartmentVector =
+ js::Vector<JS::Compartment*, 1, js::SystemAllocPolicy>;
+ js::MainThreadOrGCTaskData<CompartmentVector> compartments_;
+
+ // All cross-zone string wrappers in the zone.
+ js::MainThreadOrGCTaskData<js::StringWrapperMap> crossZoneStringWrappers_;
+
+ // List of non-ephemeron weak containers to sweep during
+ // beginSweepingSweepGroup.
+ js::MainThreadOrGCTaskData<mozilla::LinkedList<detail::WeakCacheBase>>
+ weakCaches_;
+
+ // Mapping from not yet marked keys to a vector of all values that the key
+ // maps to in any live weak map. Separate tables for nursery and tenured
+ // keys.
+ js::MainThreadOrGCTaskData<js::gc::EphemeronEdgeTable> gcEphemeronEdges_;
+ js::MainThreadOrGCTaskData<js::gc::EphemeronEdgeTable>
+ gcNurseryEphemeronEdges_;
+
+ js::MainThreadData<js::UniquePtr<js::RegExpZone>> regExps_;
+
+ // Bitmap of atoms marked by this zone.
+ js::MainThreadOrGCTaskData<js::SparseBitmap> markedAtoms_;
+
+ // Set of atoms recently used by this Zone. Purged on GC.
+ js::MainThreadOrGCTaskData<js::AtomSet> atomCache_;
+
+ // Cache storing allocated external strings. Purged on GC.
+ js::MainThreadOrGCTaskData<js::ExternalStringCache> externalStringCache_;
+
+ // Cache for Function.prototype.toString. Purged on GC.
+ js::MainThreadOrGCTaskData<js::FunctionToStringCache> functionToStringCache_;
+
+ // Cache for Function.prototype.bind mapping an atom `name` to atom
+ // `"bound " + name`. Purged on GC.
+ using BoundPrefixCache =
+ js::HashMap<JSAtom*, JSAtom*, js::PointerHasher<JSAtom*>,
+ js::SystemAllocPolicy>;
+ js::MainThreadData<BoundPrefixCache> boundPrefixCache_;
+
+ // Information about Shapes and BaseShapes.
+ js::MainThreadData<js::ShapeZone> shapeZone_;
+
+ // Information about finalization registries, created on demand.
+ js::MainThreadOrGCTaskData<js::UniquePtr<js::gc::FinalizationObservers>>
+ finalizationObservers_;
+
+ js::MainThreadOrGCTaskData<js::jit::JitZone*> jitZone_;
+
+ // Last time at which JIT code was discarded for this zone. This is only set
+ // when JitScripts and Baseline code are discarded as well.
+ js::MainThreadData<mozilla::TimeStamp> lastDiscardedCodeTime_;
+
+ js::MainThreadData<bool> gcScheduled_;
+ js::MainThreadData<bool> gcScheduledSaved_;
+ js::MainThreadData<bool> gcPreserveCode_;
+ js::MainThreadData<bool> keepPropMapTables_;
+ js::MainThreadData<bool> wasCollected_;
+
+ // Allow zones to be linked into a list
+ js::MainThreadOrGCTaskData<Zone*> listNext_;
+ static Zone* const NotOnList;
+ friend class js::gc::ZoneList;
+
+ using KeptAliveSet =
+ JS::GCHashSet<js::HeapPtr<JSObject*>,
+ js::StableCellHasher<js::HeapPtr<JSObject*>>,
+ js::ZoneAllocPolicy>;
+ friend class js::WeakRefObject;
+ js::MainThreadOrGCTaskData<KeptAliveSet> keptObjects;
+
+ public:
+ static JS::Zone* from(ZoneAllocator* zoneAlloc) {
+ return static_cast<Zone*>(zoneAlloc);
+ }
+
+ explicit Zone(JSRuntime* rt, Kind kind = NormalZone);
+ ~Zone();
+
+ [[nodiscard]] bool init();
+
+ void destroy(JS::GCContext* gcx);
+
+ [[nodiscard]] bool findSweepGroupEdges(Zone* atomsZone);
+
+ struct DiscardOptions {
+ DiscardOptions() {}
+ bool discardBaselineCode = true;
+ bool discardJitScripts = false;
+ bool resetNurseryAllocSites = false;
+ bool resetPretenuredAllocSites = false;
+ };
+
+ void discardJitCode(JS::GCContext* gcx,
+ const DiscardOptions& options = DiscardOptions());
+
+ // Discard JIT code regardless of isPreservingCode().
+ void forceDiscardJitCode(JS::GCContext* gcx,
+ const DiscardOptions& options = DiscardOptions());
+
+ void resetAllocSitesAndInvalidate(bool resetNurserySites,
+ bool resetPretenuredSites);
+
+ void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ JS::CodeSizes* code, size_t* regexpZone,
+ size_t* jitZone, size_t* baselineStubsOptimized,
+ size_t* uniqueIdMap, size_t* initialPropMapTable,
+ size_t* shapeTables, size_t* atomsMarkBitmaps,
+ size_t* compartmentObjects,
+ size_t* crossCompartmentWrappersTables,
+ size_t* compartmentsPrivateData,
+ size_t* scriptCountsMapArg);
+
+ // Iterate over all cells in the zone. See the definition of ZoneCellIter
+ // in gc/GC-inl.h for the possible arguments and documentation.
+ template <typename T, typename... Args>
+ js::gc::ZoneCellIter<T> cellIter(Args&&... args) {
+ return js::gc::ZoneCellIter<T>(const_cast<Zone*>(this),
+ std::forward<Args>(args)...);
+ }
+
+ // As above, but can return about-to-be-finalised things.
+ template <typename T, typename... Args>
+ js::gc::ZoneAllCellIter<T> cellIterUnsafe(Args&&... args) {
+ return js::gc::ZoneAllCellIter<T>(const_cast<Zone*>(this),
+ std::forward<Args>(args)...);
+ }
+
+ bool hasMarkedRealms();
+
+ void scheduleGC() {
+ MOZ_ASSERT(!RuntimeHeapIsBusy());
+ gcScheduled_ = true;
+ }
+ void unscheduleGC() { gcScheduled_ = false; }
+ bool isGCScheduled() { return gcScheduled_; }
+
+ void setPreservingCode(bool preserving) { gcPreserveCode_ = preserving; }
+ bool isPreservingCode() const { return gcPreserveCode_; }
+
+ mozilla::TimeStamp lastDiscardedCodeTime() const {
+ return lastDiscardedCodeTime_;
+ }
+
+ void changeGCState(GCState prev, GCState next);
+
+ bool isCollecting() const {
+ MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(runtimeFromMainThread()));
+ return isCollectingFromAnyThread();
+ }
+
+ inline bool isCollectingFromAnyThread() const {
+ return needsIncrementalBarrier() || wasGCStarted();
+ }
+
+ GCState initialMarkingState() const;
+
+ bool shouldMarkInZone(js::gc::MarkColor color) const {
+ // Check whether the zone is in one or both of the MarkBlackOnly and
+ // MarkBlackAndGray states, depending on the mark color. Also check for
+ // VerifyPreBarriers when the mark color is black (we don't do any gray
+ // marking when verifying pre-barriers).
+ if (color == js::gc::MarkColor::Black) {
+ return isGCMarkingOrVerifyingPreBarriers();
+ }
+
+ return isGCMarkingBlackAndGray();
+ }
+
+ // Was this zone collected in the last GC.
+ bool wasCollected() const { return wasCollected_; }
+ void setWasCollected(bool v) { wasCollected_ = v; }
+
+ void setNeedsIncrementalBarrier(bool needs);
+ const BarrierState* addressOfNeedsIncrementalBarrier() const {
+ return &needsIncrementalBarrier_;
+ }
+
+ static constexpr size_t offsetOfNeedsIncrementalBarrier() {
+ return offsetof(Zone, needsIncrementalBarrier_);
+ }
+
+ js::jit::JitZone* getJitZone(JSContext* cx) {
+ return jitZone_ ? jitZone_ : createJitZone(cx);
+ }
+ js::jit::JitZone* jitZone() { return jitZone_; }
+
+ void prepareForCompacting();
+
+ void traceRootsInMajorGC(JSTracer* trc);
+
+ void sweepAfterMinorGC(JSTracer* trc);
+ void sweepUniqueIds();
+ void sweepCompartments(JS::GCContext* gcx, bool keepAtleastOne, bool lastGC);
+
+ // Remove dead weak maps from gcWeakMapList_ and remove entries from the
+ // remaining weak maps whose keys are dead.
+ void sweepWeakMaps(JSTracer* trc);
+
+ // Trace all weak maps in this zone. Used to update edges after a moving GC.
+ void traceWeakMaps(JSTracer* trc);
+
+ js::gc::UniqueIdMap& uniqueIds() { return uniqueIds_.ref(); }
+
+ void notifyObservingDebuggers();
+
+ void noteTenuredAlloc() { tenuredAllocsSinceMinorGC_++; }
+
+ uint32_t* addressOfTenuredAllocCount() { return &tenuredAllocsSinceMinorGC_; }
+
+ uint32_t getAndResetTenuredAllocsSinceMinorGC() {
+ uint32_t res = tenuredAllocsSinceMinorGC_;
+ tenuredAllocsSinceMinorGC_ = 0;
+ return res;
+ }
+
+ mozilla::LinkedList<js::WeakMapBase>& gcWeakMapList() {
+ return gcWeakMapList_.ref();
+ }
+
+ CompartmentVector& compartments() { return compartments_.ref(); }
+
+ js::StringWrapperMap& crossZoneStringWrappers() {
+ return crossZoneStringWrappers_.ref();
+ }
+ const js::StringWrapperMap& crossZoneStringWrappers() const {
+ return crossZoneStringWrappers_.ref();
+ }
+
+ void dropStringWrappersOnGC();
+
+ void traceWeakCCWEdges(JSTracer* trc);
+ static void fixupAllCrossCompartmentWrappersAfterMovingGC(JSTracer* trc);
+
+ void fixupAfterMovingGC();
+ void fixupScriptMapsAfterMovingGC(JSTracer* trc);
+
+ void setNurseryAllocFlags(bool allocObjects, bool allocStrings,
+ bool allocBigInts);
+
+ bool allocKindInNursery(JS::TraceKind kind) const {
+ switch (kind) {
+ case JS::TraceKind::Object:
+ return allocNurseryObjects_;
+ case JS::TraceKind::String:
+ return allocNurseryStrings_;
+ case JS::TraceKind::BigInt:
+ return allocNurseryBigInts_;
+ default:
+ MOZ_CRASH("Unsupported kind for nursery allocation");
+ }
+ }
+ bool allocNurseryObjects() const { return allocNurseryObjects_; }
+ bool allocNurseryStrings() const { return allocNurseryStrings_; }
+ bool allocNurseryBigInts() const { return allocNurseryBigInts_; }
+
+ js::gc::Heap minHeapToTenure(JS::TraceKind kind) const {
+ switch (kind) {
+ case JS::TraceKind::Object:
+ return minObjectHeapToTenure_;
+ case JS::TraceKind::String:
+ return minStringHeapToTenure_;
+ case JS::TraceKind::BigInt:
+ return minBigintHeapToTenure_;
+ default:
+ MOZ_CRASH("Unsupported kind for nursery allocation");
+ }
+ }
+
+ mozilla::LinkedList<detail::WeakCacheBase>& weakCaches() {
+ return weakCaches_.ref();
+ }
+ void registerWeakCache(detail::WeakCacheBase* cachep) {
+ weakCaches().insertBack(cachep);
+ }
+
+ void beforeClearDelegate(JSObject* wrapper, JSObject* delegate) {
+ if (needsIncrementalBarrier()) {
+ beforeClearDelegateInternal(wrapper, delegate);
+ }
+ }
+
+ void afterAddDelegate(JSObject* wrapper) {
+ if (needsIncrementalBarrier()) {
+ afterAddDelegateInternal(wrapper);
+ }
+ }
+
+ void beforeClearDelegateInternal(JSObject* wrapper, JSObject* delegate);
+ void afterAddDelegateInternal(JSObject* wrapper);
+ js::gc::EphemeronEdgeTable& gcEphemeronEdges() {
+ return gcEphemeronEdges_.ref();
+ }
+ js::gc::EphemeronEdgeTable& gcNurseryEphemeronEdges() {
+ return gcNurseryEphemeronEdges_.ref();
+ }
+
+ js::gc::EphemeronEdgeTable& gcEphemeronEdges(const js::gc::Cell* cell) {
+ return cell->isTenured() ? gcEphemeronEdges() : gcNurseryEphemeronEdges();
+ }
+
+ // Perform all pending weakmap entry marking for this zone after
+ // transitioning to weak marking mode.
+ js::gc::IncrementalProgress enterWeakMarkingMode(js::GCMarker* marker,
+ js::SliceBudget& budget);
+
+ // A set of edges from this zone to other zones used during GC to calculate
+ // sweep groups.
+ NodeSet& gcSweepGroupEdges() {
+ return gcGraphEdges; // Defined in GraphNodeBase base class.
+ }
+ bool hasSweepGroupEdgeTo(Zone* otherZone) const {
+ return gcGraphEdges.has(otherZone);
+ }
+ [[nodiscard]] bool addSweepGroupEdgeTo(Zone* otherZone) {
+ MOZ_ASSERT(otherZone->isGCMarking());
+ return gcSweepGroupEdges().put(otherZone);
+ }
+ void clearSweepGroupEdges() { gcSweepGroupEdges().clear(); }
+
+ js::RegExpZone& regExps() { return *regExps_.ref(); }
+
+ js::SparseBitmap& markedAtoms() { return markedAtoms_.ref(); }
+
+ js::AtomSet& atomCache() { return atomCache_.ref(); }
+
+ void purgeAtomCache();
+
+ js::ExternalStringCache& externalStringCache() {
+ return externalStringCache_.ref();
+ };
+
+ js::FunctionToStringCache& functionToStringCache() {
+ return functionToStringCache_.ref();
+ }
+
+ BoundPrefixCache& boundPrefixCache() { return boundPrefixCache_.ref(); }
+
+ js::ShapeZone& shapeZone() { return shapeZone_.ref(); }
+
+ bool keepPropMapTables() const { return keepPropMapTables_; }
+ void setKeepPropMapTables(bool b) { keepPropMapTables_ = b; }
+
+ void clearRootsForShutdownGC();
+ void finishRoots();
+
+ void traceScriptTableRoots(JSTracer* trc);
+
+ void clearScriptCounts(Realm* realm);
+ void clearScriptLCov(Realm* realm);
+
+ // Add the target of JS WeakRef to a kept-alive set maintained by GC.
+ // See: https://tc39.es/proposal-weakrefs/#sec-keepduringjob
+ bool keepDuringJob(HandleObject target);
+
+ void traceKeptObjects(JSTracer* trc);
+
+ // Clear the kept-alive set.
+ // See: https://tc39.es/proposal-weakrefs/#sec-clear-kept-objects
+ void clearKeptObjects();
+
+ js::gc::AllocSite* unknownAllocSite(JS::TraceKind kind) {
+ return &pretenuring.unknownAllocSite(kind);
+ }
+ js::gc::AllocSite* optimizedAllocSite() {
+ return &pretenuring.optimizedAllocSite;
+ }
+ uint32_t nurseryAllocCount(JS::TraceKind kind) const {
+ return pretenuring.nurseryAllocCount(kind);
+ }
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+ void checkAllCrossCompartmentWrappersAfterMovingGC();
+ void checkStringWrappersAfterMovingGC();
+
+ // Assert that the UniqueId table has been redirected successfully.
+ void checkUniqueIdTableAfterMovingGC();
+
+ void checkScriptMapsAfterMovingGC();
+#endif
+
+#ifdef DEBUG
+ // For testing purposes, return the index of the sweep group which this zone
+ // was swept in in the last GC.
+ unsigned lastSweepGroupIndex() { return gcSweepGroupIndex; }
+#endif
+
+ private:
+ js::jit::JitZone* createJitZone(JSContext* cx);
+
+ bool isQueuedForBackgroundSweep() { return isOnList(); }
+
+ void sweepEphemeronTablesAfterMinorGC();
+
+ js::gc::FinalizationObservers* finalizationObservers() {
+ return finalizationObservers_.ref().get();
+ }
+ bool ensureFinalizationObservers();
+
+ bool isOnList() const;
+ Zone* nextZone() const;
+
+ friend bool js::CurrentThreadCanAccessZone(Zone* zone);
+ friend class js::gc::GCRuntime;
+};
+
+} // namespace JS
+
+namespace js {
+namespace gc {
+const char* StateName(JS::Zone::GCState state);
+} // namespace gc
+} // namespace js
+
+#endif // gc_Zone_h
diff --git a/js/src/gc/ZoneAllocator.h b/js/src/gc/ZoneAllocator.h
new file mode 100644
index 0000000000..de2dd7da28
--- /dev/null
+++ b/js/src/gc/ZoneAllocator.h
@@ -0,0 +1,354 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Public header for allocating memory associated with GC things.
+ */
+
+#ifndef gc_ZoneAllocator_h
+#define gc_ZoneAllocator_h
+
+#include "mozilla/Maybe.h"
+
+#include "jsfriendapi.h"
+#include "jstypes.h"
+#include "gc/Cell.h"
+#include "gc/Scheduling.h"
+#include "js/GCAPI.h"
+#include "js/shadow/Zone.h" // JS::shadow::Zone
+#include "vm/MallocProvider.h"
+
+namespace JS {
+class JS_PUBLIC_API Zone;
+} // namespace JS
+
+namespace js {
+
+class ZoneAllocator;
+
+#ifdef DEBUG
+bool CurrentThreadIsGCFinalizing();
+#endif
+
+namespace gc {
+void MaybeMallocTriggerZoneGC(JSRuntime* rt, ZoneAllocator* zoneAlloc,
+ const HeapSize& heap,
+ const HeapThreshold& threshold,
+ JS::GCReason reason);
+}
+
+// Base class of JS::Zone that provides malloc memory allocation and accounting.
+class ZoneAllocator : public JS::shadow::Zone,
+ public js::MallocProvider<JS::Zone> {
+ protected:
+ explicit ZoneAllocator(JSRuntime* rt, Kind kind);
+ ~ZoneAllocator();
+ void fixupAfterMovingGC();
+
+ public:
+ static ZoneAllocator* from(JS::Zone* zone) {
+ // This is a safe upcast, but the compiler hasn't seen the definition yet.
+ return reinterpret_cast<ZoneAllocator*>(zone);
+ }
+
+ [[nodiscard]] void* onOutOfMemory(js::AllocFunction allocFunc,
+ arena_id_t arena, size_t nbytes,
+ void* reallocPtr = nullptr);
+ void reportAllocationOverflow() const;
+
+ void updateSchedulingStateOnGCStart();
+ void updateGCStartThresholds(gc::GCRuntime& gc);
+ void setGCSliceThresholds(gc::GCRuntime& gc, bool waitingOnBGTask);
+ void clearGCSliceThresholds();
+
+ // Memory accounting APIs for malloc memory owned by GC cells.
+
+ void addCellMemory(js::gc::Cell* cell, size_t nbytes, js::MemoryUse use) {
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(nbytes);
+
+ mallocHeapSize.addBytes(nbytes);
+
+#ifdef DEBUG
+ mallocTracker.trackGCMemory(cell, nbytes, use);
+#endif
+
+ maybeTriggerGCOnMalloc();
+ }
+
+ void removeCellMemory(js::gc::Cell* cell, size_t nbytes, js::MemoryUse use,
+ bool updateRetainedSize = false) {
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(nbytes);
+ MOZ_ASSERT_IF(CurrentThreadIsGCFinalizing(), updateRetainedSize);
+
+ mallocHeapSize.removeBytes(nbytes, updateRetainedSize);
+
+#ifdef DEBUG
+ mallocTracker.untrackGCMemory(cell, nbytes, use);
+#endif
+ }
+
+ void swapCellMemory(js::gc::Cell* a, js::gc::Cell* b, js::MemoryUse use) {
+#ifdef DEBUG
+ mallocTracker.swapGCMemory(a, b, use);
+#endif
+ }
+
+ void registerNonGCMemory(void* mem, MemoryUse use) {
+#ifdef DEBUG
+ return mallocTracker.registerNonGCMemory(mem, use);
+#endif
+ }
+ void unregisterNonGCMemory(void* mem, MemoryUse use) {
+#ifdef DEBUG
+ return mallocTracker.unregisterNonGCMemory(mem, use);
+#endif
+ }
+ void moveOtherMemory(void* dst, void* src, MemoryUse use) {
+#ifdef DEBUG
+ return mallocTracker.moveNonGCMemory(dst, src, use);
+#endif
+ }
+
+ void incNonGCMemory(void* mem, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(nbytes);
+ mallocHeapSize.addBytes(nbytes);
+
+#ifdef DEBUG
+ mallocTracker.incNonGCMemory(mem, nbytes, use);
+#endif
+
+ maybeTriggerGCOnMalloc();
+ }
+ void decNonGCMemory(void* mem, size_t nbytes, MemoryUse use,
+ bool updateRetainedSize) {
+ MOZ_ASSERT(nbytes);
+
+ mallocHeapSize.removeBytes(nbytes, updateRetainedSize);
+
+#ifdef DEBUG
+ mallocTracker.decNonGCMemory(mem, nbytes, use);
+#endif
+ }
+
+ // Account for allocations that may be referenced by more than one GC thing.
+ bool addSharedMemory(void* mem, size_t nbytes, MemoryUse use);
+ void removeSharedMemory(void* mem, size_t nbytes, MemoryUse use);
+
+ void incJitMemory(size_t nbytes) {
+ MOZ_ASSERT(nbytes);
+ jitHeapSize.addBytes(nbytes);
+ maybeTriggerZoneGC(jitHeapSize, jitHeapThreshold,
+ JS::GCReason::TOO_MUCH_JIT_CODE);
+ }
+ void decJitMemory(size_t nbytes) {
+ MOZ_ASSERT(nbytes);
+ jitHeapSize.removeBytes(nbytes, true);
+ }
+
+ // Check malloc allocation threshold and trigger a zone GC if necessary.
+ void maybeTriggerGCOnMalloc() {
+ maybeTriggerZoneGC(mallocHeapSize, mallocHeapThreshold,
+ JS::GCReason::TOO_MUCH_MALLOC);
+ }
+
+ private:
+ void maybeTriggerZoneGC(const js::gc::HeapSize& heap,
+ const js::gc::HeapThreshold& threshold,
+ JS::GCReason reason) {
+ if (heap.bytes() >= threshold.startBytes()) {
+ gc::MaybeMallocTriggerZoneGC(runtimeFromAnyThread(), this, heap,
+ threshold, reason);
+ }
+ }
+
+ void updateCollectionRate(mozilla::TimeDuration mainThreadGCTime,
+ size_t initialBytesForAllZones);
+
+ void updateAllocationRate(mozilla::TimeDuration mutatorTime);
+
+ public:
+ // The size of allocated GC arenas in this zone.
+ gc::PerZoneGCHeapSize gcHeapSize;
+
+ // Threshold used to trigger GC based on GC heap size.
+ gc::GCHeapThreshold gcHeapThreshold;
+
+ // Amount of malloc data owned by tenured GC things in this zone, including
+ // external allocations supplied by JS::AddAssociatedMemory.
+ gc::HeapSize mallocHeapSize;
+
+ // Threshold used to trigger GC based on malloc allocations.
+ gc::MallocHeapThreshold mallocHeapThreshold;
+
+ // Amount of exectuable JIT code owned by GC things in this zone.
+ gc::HeapSize jitHeapSize;
+
+ // Threshold used to trigger GC based on JIT allocations.
+ gc::JitHeapThreshold jitHeapThreshold;
+
+ // Use counts for memory that can be referenced by more than one GC thing.
+ // Memory recorded here is also recorded in mallocHeapSize. This structure
+ // is used to avoid over-counting in mallocHeapSize.
+ gc::SharedMemoryMap sharedMemoryUseCounts;
+
+ // Collection rate estimate for this zone in MB/s, and state used to calculate
+ // it. Updated every time this zone is collected.
+ MainThreadData<mozilla::Maybe<double>> smoothedCollectionRate;
+ MainThreadOrGCTaskData<mozilla::TimeDuration> perZoneGCTime;
+
+ // Allocation rate estimate in MB/s of mutator time, and state used to
+ // calculate it.
+ MainThreadData<mozilla::Maybe<double>> smoothedAllocationRate;
+ MainThreadData<size_t> prevGCHeapSize;
+
+ private:
+#ifdef DEBUG
+ // In debug builds, malloc allocations can be tracked to make debugging easier
+ // (possible?) if allocation and free sizes don't balance.
+ gc::MemoryTracker mallocTracker;
+#endif
+
+ friend class gc::GCRuntime;
+};
+
+// Whether memory is associated with a single cell or whether it is associated
+// with the zone as a whole (for memory used by the system).
+enum class TrackingKind { Cell, Zone };
+
+/*
+ * Allocation policy that performs memory tracking for malloced memory. This
+ * should be used for all containers associated with a GC thing or a zone.
+ *
+ * Since it doesn't hold a JSContext (those may not live long enough), it can't
+ * report out-of-memory conditions itself; the caller must check for OOM and
+ * take the appropriate action.
+ *
+ * FIXME bug 647103 - replace these *AllocPolicy names.
+ */
+template <TrackingKind kind>
+class TrackedAllocPolicy : public MallocProvider<TrackedAllocPolicy<kind>> {
+ ZoneAllocator* zone_;
+
+#ifdef DEBUG
+ friend class js::gc::MemoryTracker; // Can clear |zone_| on merge.
+#endif
+
+ public:
+ MOZ_IMPLICIT TrackedAllocPolicy(ZoneAllocator* z) : zone_(z) {
+ zone()->registerNonGCMemory(this, MemoryUse::TrackedAllocPolicy);
+ }
+ MOZ_IMPLICIT TrackedAllocPolicy(JS::Zone* z)
+ : TrackedAllocPolicy(ZoneAllocator::from(z)) {}
+ TrackedAllocPolicy(TrackedAllocPolicy& other)
+ : TrackedAllocPolicy(other.zone_) {}
+ TrackedAllocPolicy(TrackedAllocPolicy&& other) : zone_(other.zone_) {
+ zone()->moveOtherMemory(this, &other, MemoryUse::TrackedAllocPolicy);
+ other.zone_ = nullptr;
+ }
+ ~TrackedAllocPolicy() {
+ if (zone_) {
+ zone_->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy);
+ }
+ }
+
+ TrackedAllocPolicy& operator=(const TrackedAllocPolicy& other) {
+ zone()->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy);
+ zone_ = other.zone();
+ zone()->registerNonGCMemory(this, MemoryUse::TrackedAllocPolicy);
+ return *this;
+ }
+ TrackedAllocPolicy& operator=(TrackedAllocPolicy&& other) {
+ MOZ_ASSERT(this != &other);
+ zone()->unregisterNonGCMemory(this, MemoryUse::TrackedAllocPolicy);
+ zone_ = other.zone();
+ zone()->moveOtherMemory(this, &other, MemoryUse::TrackedAllocPolicy);
+ other.zone_ = nullptr;
+ return *this;
+ }
+
+ // Public methods required to fulfill the AllocPolicy interface.
+
+ template <typename T>
+ void free_(T* p, size_t numElems) {
+ if (p) {
+ decMemory(numElems * sizeof(T));
+ js_free(p);
+ }
+ }
+
+ [[nodiscard]] bool checkSimulatedOOM() const {
+ return !js::oom::ShouldFailWithOOM();
+ }
+
+ void reportAllocOverflow() const { reportAllocationOverflow(); }
+
+ // Internal methods called by the MallocProvider implementation.
+
+ [[nodiscard]] void* onOutOfMemory(js::AllocFunction allocFunc,
+ arena_id_t arena, size_t nbytes,
+ void* reallocPtr = nullptr) {
+ return zone()->onOutOfMemory(allocFunc, arena, nbytes, reallocPtr);
+ }
+ void reportAllocationOverflow() const { zone()->reportAllocationOverflow(); }
+ void updateMallocCounter(size_t nbytes) {
+ zone()->incNonGCMemory(this, nbytes, MemoryUse::TrackedAllocPolicy);
+ }
+
+ private:
+ ZoneAllocator* zone() const {
+ MOZ_ASSERT(zone_);
+ return zone_;
+ }
+ void decMemory(size_t nbytes);
+};
+
+using ZoneAllocPolicy = TrackedAllocPolicy<TrackingKind::Zone>;
+using CellAllocPolicy = TrackedAllocPolicy<TrackingKind::Cell>;
+
+// Functions for memory accounting on the zone.
+
+// Associate malloc memory with a GC thing. This call should be matched by a
+// following call to RemoveCellMemory with the same size and use. The total
+// amount of malloc memory associated with a zone is used to trigger GC.
+//
+// You should use InitReservedSlot / InitObjectPrivate in preference to this
+// where possible.
+
+inline void AddCellMemory(gc::TenuredCell* cell, size_t nbytes, MemoryUse use) {
+ if (nbytes) {
+ ZoneAllocator::from(cell->zone())->addCellMemory(cell, nbytes, use);
+ }
+}
+inline void AddCellMemory(gc::Cell* cell, size_t nbytes, MemoryUse use) {
+ if (cell->isTenured()) {
+ AddCellMemory(&cell->asTenured(), nbytes, use);
+ }
+}
+
+// Remove association between malloc memory and a GC thing. This call should
+// follow a call to AddCellMemory with the same size and use.
+
+inline void RemoveCellMemory(gc::TenuredCell* cell, size_t nbytes,
+ MemoryUse use) {
+ MOZ_ASSERT(!CurrentThreadIsGCFinalizing(),
+ "Use GCContext methods to remove associated memory in finalizers");
+
+ if (nbytes) {
+ auto zoneBase = ZoneAllocator::from(cell->zoneFromAnyThread());
+ zoneBase->removeCellMemory(cell, nbytes, use, false);
+ }
+}
+inline void RemoveCellMemory(gc::Cell* cell, size_t nbytes, MemoryUse use) {
+ if (cell->isTenured()) {
+ RemoveCellMemory(&cell->asTenured(), nbytes, use);
+ }
+}
+
+} // namespace js
+
+#endif // gc_ZoneAllocator_h
diff --git a/js/src/gc/moz.build b/js/src/gc/moz.build
new file mode 100644
index 0000000000..489b5a96ae
--- /dev/null
+++ b/js/src/gc/moz.build
@@ -0,0 +1,61 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_LIBRARY = "js"
+
+# Includes should be relative to parent path
+LOCAL_INCLUDES += ["!..", ".."]
+
+include("../js-config.mozbuild")
+include("../js-cxxflags.mozbuild")
+
+# Generate GC statistics phase data.
+GeneratedFile(
+ "StatsPhasesGenerated.h",
+ script="GenerateStatsPhases.py",
+ entry_point="generateHeader",
+)
+GeneratedFile(
+ "StatsPhasesGenerated.inc",
+ script="GenerateStatsPhases.py",
+ entry_point="generateCpp",
+)
+
+UNIFIED_SOURCES += [
+ "Allocator.cpp",
+ "AtomMarking.cpp",
+ "Barrier.cpp",
+ "Compacting.cpp",
+ "FinalizationObservers.cpp",
+ "GC.cpp",
+ "GCAPI.cpp",
+ "GCParallelTask.cpp",
+ "Heap.cpp",
+ "MallocedBlockCache.cpp",
+ "Marking.cpp",
+ "Memory.cpp",
+ "Nursery.cpp",
+ "ParallelMarking.cpp",
+ "Pretenuring.cpp",
+ "PublicIterators.cpp",
+ "RootMarking.cpp",
+ "Scheduling.cpp",
+ "Statistics.cpp",
+ "Sweeping.cpp",
+ "Tenuring.cpp",
+ "Tracer.cpp",
+ "Verifier.cpp",
+ "WeakMap.cpp",
+ "WeakMapPtr.cpp",
+ "Zone.cpp",
+]
+
+# StoreBuffer.cpp cannot be built in unified mode because its template
+# instantiations may or may not be needed depending on what it gets bundled
+# with.
+SOURCES += [
+ "StoreBuffer.cpp",
+]