summaryrefslogtreecommitdiffstats
path: root/js/src/gc/Scheduling.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/gc/Scheduling.cpp')
-rw-r--r--js/src/gc/Scheduling.cpp890
1 files changed, 890 insertions, 0 deletions
diff --git a/js/src/gc/Scheduling.cpp b/js/src/gc/Scheduling.cpp
new file mode 100644
index 0000000000..2870e0a97b
--- /dev/null
+++ b/js/src/gc/Scheduling.cpp
@@ -0,0 +1,890 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gc/Scheduling.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/TimeStamp.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "gc/Memory.h"
+#include "gc/Nursery.h"
+#include "gc/RelocationOverlay.h"
+#include "gc/ZoneAllocator.h"
+#include "util/DifferentialTesting.h"
+#include "vm/MutexIDs.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::CheckedInt;
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+using mozilla::TimeDuration;
+
+/*
+ * We may start to collect a zone before its trigger threshold is reached if
+ * GCRuntime::maybeGC() is called for that zone or we start collecting other
+ * zones. These eager threshold factors are not configurable.
+ */
+static constexpr double HighFrequencyEagerAllocTriggerFactor = 0.85;
+static constexpr double LowFrequencyEagerAllocTriggerFactor = 0.9;
+
+/*
+ * Don't allow heap growth factors to be set so low that eager collections could
+ * reduce the trigger threshold.
+ */
+static constexpr double MinHeapGrowthFactor =
+ 1.0f / std::min(HighFrequencyEagerAllocTriggerFactor,
+ LowFrequencyEagerAllocTriggerFactor);
+
+// Limit various parameters to reasonable levels to catch errors.
+static constexpr double MaxHeapGrowthFactor = 100;
+static constexpr size_t MaxNurseryBytesParam = 128 * 1024 * 1024;
+
+namespace {
+
+// Helper classes to marshal GC parameter values to/from uint32_t.
+
+template <typename T>
+struct ConvertGeneric {
+ static uint32_t toUint32(T value) {
+ static_assert(std::is_arithmetic_v<T>);
+ if constexpr (std::is_signed_v<T>) {
+ MOZ_ASSERT(value >= 0);
+ }
+ if constexpr (!std::is_same_v<T, bool> &&
+ std::numeric_limits<T>::max() >
+ std::numeric_limits<uint32_t>::max()) {
+ MOZ_ASSERT(value <= UINT32_MAX);
+ }
+ return uint32_t(value);
+ }
+ static Maybe<T> fromUint32(uint32_t param) {
+ // Currently we use explicit conversion and don't range check.
+ return Some(T(param));
+ }
+};
+
+using ConvertBool = ConvertGeneric<bool>;
+using ConvertSize = ConvertGeneric<size_t>;
+using ConvertDouble = ConvertGeneric<double>;
+
+struct ConvertTimes100 {
+ static uint32_t toUint32(double value) { return uint32_t(value * 100.0); }
+ static Maybe<double> fromUint32(uint32_t param) {
+ return Some(double(param) / 100.0);
+ }
+};
+
+struct ConvertNurseryBytes : ConvertSize {
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ return Some(Nursery::roundSize(param));
+ }
+};
+
+struct ConvertKB {
+ static uint32_t toUint32(size_t value) { return value / 1024; }
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ // Parameters which represent heap sizes in bytes are restricted to values
+ // which can be represented on 32 bit platforms.
+ CheckedInt<uint32_t> size = CheckedInt<uint32_t>(param) * 1024;
+ return size.isValid() ? Some(size_t(size.value())) : Nothing();
+ }
+};
+
+struct ConvertMB {
+ static uint32_t toUint32(size_t value) { return value / (1024 * 1024); }
+ static Maybe<size_t> fromUint32(uint32_t param) {
+ // Parameters which represent heap sizes in bytes are restricted to values
+ // which can be represented on 32 bit platforms.
+ CheckedInt<uint32_t> size = CheckedInt<uint32_t>(param) * 1024 * 1024;
+ return size.isValid() ? Some(size_t(size.value())) : Nothing();
+ }
+};
+
+struct ConvertMillis {
+ static uint32_t toUint32(TimeDuration value) {
+ return uint32_t(value.ToMilliseconds());
+ }
+ static Maybe<TimeDuration> fromUint32(uint32_t param) {
+ return Some(TimeDuration::FromMilliseconds(param));
+ }
+};
+
+struct ConvertSeconds {
+ static uint32_t toUint32(TimeDuration value) {
+ return uint32_t(value.ToSeconds());
+ }
+ static Maybe<TimeDuration> fromUint32(uint32_t param) {
+ return Some(TimeDuration::FromSeconds(param));
+ }
+};
+
+} // anonymous namespace
+
+// Helper functions to check GC parameter values
+
+template <typename T>
+static bool NoCheck(T value) {
+ return true;
+}
+
+template <typename T>
+static bool CheckNonZero(T value) {
+ return value != 0;
+}
+
+static bool CheckNurserySize(size_t bytes) {
+ return bytes >= SystemPageSize() && bytes <= MaxNurseryBytesParam;
+}
+
+static bool CheckHeapGrowth(double growth) {
+ return growth >= MinHeapGrowthFactor && growth <= MaxHeapGrowthFactor;
+}
+
+static bool CheckIncrementalLimit(double factor) {
+ return factor >= 1.0 && factor <= MaxHeapGrowthFactor;
+}
+
+static bool CheckNonZeroUnitRange(double value) {
+ return value > 0.0 && value <= 100.0;
+}
+
+GCSchedulingTunables::GCSchedulingTunables() {
+#define INIT_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ name##_ = default; \
+ MOZ_ASSERT(check(name##_));
+ FOR_EACH_GC_TUNABLE(INIT_TUNABLE_FIELD)
+#undef INIT_TUNABLE_FIELD
+
+ checkInvariants();
+}
+
+uint32_t GCSchedulingTunables::getParameter(JSGCParamKey key) {
+ switch (key) {
+#define GET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: \
+ return convert::toUint32(name##_);
+ FOR_EACH_GC_TUNABLE(GET_TUNABLE_FIELD)
+#undef GET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown parameter key");
+ }
+}
+
+bool GCSchedulingTunables::setParameter(JSGCParamKey key, uint32_t value) {
+ auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); });
+
+ switch (key) {
+#define SET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: { \
+ Maybe<type> converted = convert::fromUint32(value); \
+ if (!converted || !check(converted.value())) { \
+ return false; \
+ } \
+ name##_ = converted.value(); \
+ break; \
+ }
+ FOR_EACH_GC_TUNABLE(SET_TUNABLE_FIELD)
+#undef SET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown GC parameter.");
+ }
+
+ maintainInvariantsAfterUpdate(key);
+ return true;
+}
+
+void GCSchedulingTunables::resetParameter(JSGCParamKey key) {
+ auto guard = mozilla::MakeScopeExit([this] { checkInvariants(); });
+
+ switch (key) {
+#define RESET_TUNABLE_FIELD(key, type, name, convert, check, default) \
+ case key: \
+ name##_ = default; \
+ MOZ_ASSERT(check(name##_)); \
+ break;
+ FOR_EACH_GC_TUNABLE(RESET_TUNABLE_FIELD)
+#undef RESET_TUNABLE_FIELD
+
+ default:
+ MOZ_CRASH("Unknown GC parameter.");
+ }
+
+ maintainInvariantsAfterUpdate(key);
+}
+
+void GCSchedulingTunables::maintainInvariantsAfterUpdate(JSGCParamKey updated) {
+ // Check whether a change to parameter |updated| has broken an invariant in
+ // relation to another parameter. If it has, adjust that other parameter to
+ // restore the invariant.
+ switch (updated) {
+ case JSGC_MIN_NURSERY_BYTES:
+ if (gcMaxNurseryBytes_ < gcMinNurseryBytes_) {
+ gcMaxNurseryBytes_ = gcMinNurseryBytes_;
+ }
+ break;
+ case JSGC_MAX_NURSERY_BYTES:
+ if (gcMinNurseryBytes_ > gcMaxNurseryBytes_) {
+ gcMinNurseryBytes_ = gcMaxNurseryBytes_;
+ }
+ break;
+ case JSGC_SMALL_HEAP_SIZE_MAX:
+ if (smallHeapSizeMaxBytes_ >= largeHeapSizeMinBytes_) {
+ largeHeapSizeMinBytes_ = smallHeapSizeMaxBytes_ + 1;
+ }
+ break;
+ case JSGC_LARGE_HEAP_SIZE_MIN:
+ if (largeHeapSizeMinBytes_ <= smallHeapSizeMaxBytes_) {
+ smallHeapSizeMaxBytes_ = largeHeapSizeMinBytes_ - 1;
+ }
+ break;
+ case JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH:
+ if (highFrequencySmallHeapGrowth_ < highFrequencyLargeHeapGrowth_) {
+ highFrequencyLargeHeapGrowth_ = highFrequencySmallHeapGrowth_;
+ }
+ break;
+ case JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH:
+ if (highFrequencyLargeHeapGrowth_ > highFrequencySmallHeapGrowth_) {
+ highFrequencySmallHeapGrowth_ = highFrequencyLargeHeapGrowth_;
+ }
+ break;
+ case JSGC_SMALL_HEAP_INCREMENTAL_LIMIT:
+ if (smallHeapIncrementalLimit_ < largeHeapIncrementalLimit_) {
+ largeHeapIncrementalLimit_ = smallHeapIncrementalLimit_;
+ }
+ break;
+ case JSGC_LARGE_HEAP_INCREMENTAL_LIMIT:
+ if (largeHeapIncrementalLimit_ > smallHeapIncrementalLimit_) {
+ smallHeapIncrementalLimit_ = largeHeapIncrementalLimit_;
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+void GCSchedulingTunables::checkInvariants() {
+ MOZ_ASSERT(gcMinNurseryBytes_ == Nursery::roundSize(gcMinNurseryBytes_));
+ MOZ_ASSERT(gcMaxNurseryBytes_ == Nursery::roundSize(gcMaxNurseryBytes_));
+ MOZ_ASSERT(gcMinNurseryBytes_ <= gcMaxNurseryBytes_);
+ MOZ_ASSERT(gcMinNurseryBytes_ >= SystemPageSize());
+ MOZ_ASSERT(gcMaxNurseryBytes_ <= MaxNurseryBytesParam);
+
+ MOZ_ASSERT(largeHeapSizeMinBytes_ > smallHeapSizeMaxBytes_);
+
+ MOZ_ASSERT(lowFrequencyHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(lowFrequencyHeapGrowth_ <= MaxHeapGrowthFactor);
+
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencyLargeHeapGrowth_ <= highFrequencySmallHeapGrowth_);
+ MOZ_ASSERT(highFrequencyLargeHeapGrowth_ >= MinHeapGrowthFactor);
+ MOZ_ASSERT(highFrequencySmallHeapGrowth_ <= MaxHeapGrowthFactor);
+
+ MOZ_ASSERT(smallHeapIncrementalLimit_ >= largeHeapIncrementalLimit_);
+}
+
+void GCSchedulingState::updateHighFrequencyMode(
+ const mozilla::TimeStamp& lastGCTime, const mozilla::TimeStamp& currentTime,
+ const GCSchedulingTunables& tunables) {
+ if (js::SupportDifferentialTesting()) {
+ return;
+ }
+
+ inHighFrequencyGCMode_ =
+ !lastGCTime.IsNull() &&
+ lastGCTime + tunables.highFrequencyThreshold() > currentTime;
+}
+
+void GCSchedulingState::updateHighFrequencyModeForReason(JS::GCReason reason) {
+ // These reason indicate that the embedding isn't triggering GC slices often
+ // enough and allocation rate is high.
+ if (reason == JS::GCReason::ALLOC_TRIGGER ||
+ reason == JS::GCReason::TOO_MUCH_MALLOC) {
+ inHighFrequencyGCMode_ = true;
+ }
+}
+
+static constexpr size_t BytesPerMB = 1024 * 1024;
+static constexpr double CollectionRateSmoothingFactor = 0.5;
+static constexpr double AllocationRateSmoothingFactor = 0.5;
+
+static double ExponentialMovingAverage(double prevAverage, double newData,
+ double smoothingFactor) {
+ MOZ_ASSERT(smoothingFactor > 0.0 && smoothingFactor <= 1.0);
+ return smoothingFactor * newData + (1.0 - smoothingFactor) * prevAverage;
+}
+
+void js::ZoneAllocator::updateCollectionRate(
+ mozilla::TimeDuration mainThreadGCTime, size_t initialBytesForAllZones) {
+ MOZ_ASSERT(initialBytesForAllZones != 0);
+ MOZ_ASSERT(gcHeapSize.initialBytes() <= initialBytesForAllZones);
+
+ double zoneFraction =
+ double(gcHeapSize.initialBytes()) / double(initialBytesForAllZones);
+ double zoneDuration = mainThreadGCTime.ToSeconds() * zoneFraction +
+ perZoneGCTime.ref().ToSeconds();
+ double collectionRate =
+ double(gcHeapSize.initialBytes()) / (zoneDuration * BytesPerMB);
+
+ if (!smoothedCollectionRate.ref()) {
+ smoothedCollectionRate = Some(collectionRate);
+ } else {
+ double prevRate = smoothedCollectionRate.ref().value();
+ smoothedCollectionRate = Some(ExponentialMovingAverage(
+ prevRate, collectionRate, CollectionRateSmoothingFactor));
+ }
+}
+
+void js::ZoneAllocator::updateAllocationRate(TimeDuration mutatorTime) {
+ // To get the total size allocated since the last collection we have to
+ // take account of how much memory got freed in the meantime.
+ size_t freedBytes = gcHeapSize.freedBytes();
+
+ size_t sizeIncludingFreedBytes = gcHeapSize.bytes() + freedBytes;
+
+ MOZ_ASSERT(prevGCHeapSize <= sizeIncludingFreedBytes);
+ size_t allocatedBytes = sizeIncludingFreedBytes - prevGCHeapSize;
+
+ double allocationRate =
+ double(allocatedBytes) / (mutatorTime.ToSeconds() * BytesPerMB);
+
+ if (!smoothedAllocationRate.ref()) {
+ smoothedAllocationRate = Some(allocationRate);
+ } else {
+ double prevRate = smoothedAllocationRate.ref().value();
+ smoothedAllocationRate = Some(ExponentialMovingAverage(
+ prevRate, allocationRate, AllocationRateSmoothingFactor));
+ }
+
+ gcHeapSize.clearFreedBytes();
+ prevGCHeapSize = gcHeapSize.bytes();
+}
+
+// GC thresholds may exceed the range of size_t on 32-bit platforms, so these
+// are calculated using 64-bit integers and clamped.
+static inline size_t ToClampedSize(uint64_t bytes) {
+ return std::min(bytes, uint64_t(SIZE_MAX));
+}
+
+void HeapThreshold::setIncrementalLimitFromStartBytes(
+ size_t retainedBytes, const GCSchedulingTunables& tunables) {
+ // Calculate the incremental limit for a heap based on its size and start
+ // threshold.
+ //
+ // This effectively classifies the heap size into small, medium or large, and
+ // uses the small heap incremental limit paramer, the large heap incremental
+ // limit parameter or an interpolation between them.
+ //
+ // The incremental limit is always set greater than the start threshold by at
+ // least the maximum nursery size to reduce the chance that tenuring a full
+ // nursery will send us straight into non-incremental collection.
+
+ MOZ_ASSERT(tunables.smallHeapIncrementalLimit() >=
+ tunables.largeHeapIncrementalLimit());
+
+ double factor = LinearInterpolate(double(retainedBytes),
+ double(tunables.smallHeapSizeMaxBytes()),
+ tunables.smallHeapIncrementalLimit(),
+ double(tunables.largeHeapSizeMinBytes()),
+ tunables.largeHeapIncrementalLimit());
+
+ uint64_t bytes =
+ std::max(uint64_t(double(startBytes_) * factor),
+ uint64_t(startBytes_) + tunables.gcMaxNurseryBytes());
+ incrementalLimitBytes_ = ToClampedSize(bytes);
+ MOZ_ASSERT(incrementalLimitBytes_ >= startBytes_);
+
+ // Maintain the invariant that the slice threshold is always less than the
+ // incremental limit when adjusting GC parameters.
+ if (hasSliceThreshold() && sliceBytes() > incrementalLimitBytes()) {
+ sliceBytes_ = incrementalLimitBytes();
+ }
+}
+
+size_t HeapThreshold::eagerAllocTrigger(bool highFrequencyGC) const {
+ double eagerTriggerFactor = highFrequencyGC
+ ? HighFrequencyEagerAllocTriggerFactor
+ : LowFrequencyEagerAllocTriggerFactor;
+ return size_t(eagerTriggerFactor * double(startBytes()));
+}
+
+void HeapThreshold::setSliceThreshold(ZoneAllocator* zone,
+ const HeapSize& heapSize,
+ const GCSchedulingTunables& tunables,
+ bool waitingOnBGTask) {
+ // Set the allocation threshold at which to trigger the a GC slice in an
+ // ongoing incremental collection. This is used to ensure progress in
+ // allocation heavy code that may not return to the main event loop.
+ //
+ // The threshold is based on the JSGC_ZONE_ALLOC_DELAY_KB parameter, but this
+ // is reduced to increase the slice frequency as we approach the incremental
+ // limit, in the hope that we never reach it. If collector is waiting for a
+ // background task to complete, don't trigger any slices until we reach the
+ // urgent threshold.
+
+ size_t bytesRemaining = incrementalBytesRemaining(heapSize);
+ bool isUrgent = bytesRemaining < tunables.urgentThresholdBytes();
+
+ size_t delayBeforeNextSlice = tunables.zoneAllocDelayBytes();
+ if (isUrgent) {
+ double fractionRemaining =
+ double(bytesRemaining) / double(tunables.urgentThresholdBytes());
+ delayBeforeNextSlice =
+ size_t(double(delayBeforeNextSlice) * fractionRemaining);
+ MOZ_ASSERT(delayBeforeNextSlice <= tunables.zoneAllocDelayBytes());
+ } else if (waitingOnBGTask) {
+ delayBeforeNextSlice = bytesRemaining - tunables.urgentThresholdBytes();
+ }
+
+ sliceBytes_ = ToClampedSize(
+ std::min(uint64_t(heapSize.bytes()) + uint64_t(delayBeforeNextSlice),
+ uint64_t(incrementalLimitBytes_)));
+}
+
+size_t HeapThreshold::incrementalBytesRemaining(
+ const HeapSize& heapSize) const {
+ if (heapSize.bytes() >= incrementalLimitBytes_) {
+ return 0;
+ }
+
+ return incrementalLimitBytes_ - heapSize.bytes();
+}
+
+/* static */
+double HeapThreshold::computeZoneHeapGrowthFactorForHeapSize(
+ size_t lastBytes, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state) {
+ // For small zones, our collection heuristics do not matter much: favor
+ // something simple in this case.
+ if (lastBytes < 1 * 1024 * 1024) {
+ return tunables.lowFrequencyHeapGrowth();
+ }
+
+ // The heap growth factor depends on the heap size after a GC and the GC
+ // frequency. If GC's are not triggering in rapid succession, use a lower
+ // threshold so that we will collect garbage sooner.
+ if (!state.inHighFrequencyGCMode()) {
+ return tunables.lowFrequencyHeapGrowth();
+ }
+
+ // For high frequency GCs we let the heap grow depending on whether we
+ // classify the heap as small, medium or large. There are parameters for small
+ // and large heap sizes and linear interpolation is used between them for
+ // medium sized heaps.
+
+ MOZ_ASSERT(tunables.smallHeapSizeMaxBytes() <=
+ tunables.largeHeapSizeMinBytes());
+ MOZ_ASSERT(tunables.highFrequencyLargeHeapGrowth() <=
+ tunables.highFrequencySmallHeapGrowth());
+
+ return LinearInterpolate(double(lastBytes),
+ double(tunables.smallHeapSizeMaxBytes()),
+ tunables.highFrequencySmallHeapGrowth(),
+ double(tunables.largeHeapSizeMinBytes()),
+ tunables.highFrequencyLargeHeapGrowth());
+}
+
+/* static */
+size_t GCHeapThreshold::computeZoneTriggerBytes(
+ double growthFactor, size_t lastBytes,
+ const GCSchedulingTunables& tunables) {
+ size_t base = std::max(lastBytes, tunables.gcZoneAllocThresholdBase());
+ double trigger = double(base) * growthFactor;
+ double triggerMax =
+ double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit();
+ return ToClampedSize(uint64_t(std::min(triggerMax, trigger)));
+}
+
+// Parameters for balanced heap limits computation.
+
+// The W0 parameter. How much memory can be traversed in the minimum collection
+// time.
+static constexpr double BalancedHeapBaseMB = 5.0;
+
+// The minimum heap limit. Do not constrain the heap to any less than this size.
+static constexpr double MinBalancedHeapLimitMB = 10.0;
+
+// The minimum amount of additional space to allow beyond the retained size.
+static constexpr double MinBalancedHeadroomMB = 3.0;
+
+// The maximum factor by which to expand the heap beyond the retained size.
+static constexpr double MaxHeapGrowth = 3.0;
+
+// The default allocation rate in MB/s allocated by the mutator to use before we
+// have an estimate. Used to set the heap limit for zones that have not yet been
+// collected.
+static constexpr double DefaultAllocationRate = 0.0;
+
+// The s0 parameter. The default collection rate in MB/s to use before we have
+// an estimate. Used to set the heap limit for zones that have not yet been
+// collected.
+static constexpr double DefaultCollectionRate = 200.0;
+
+double GCHeapThreshold::computeBalancedHeapLimit(
+ size_t lastBytes, double allocationRate, double collectionRate,
+ const GCSchedulingTunables& tunables) {
+ MOZ_ASSERT(tunables.balancedHeapLimitsEnabled());
+
+ // Optimal heap limits as described in https://arxiv.org/abs/2204.10455
+
+ double W = double(lastBytes) / BytesPerMB; // Retained size / MB.
+ double W0 = BalancedHeapBaseMB;
+ double d = tunables.heapGrowthFactor(); // Rearranged constant 'c'.
+ double g = allocationRate;
+ double s = collectionRate;
+ double f = d * sqrt((W + W0) * (g / s));
+ double M = W + std::min(f, MaxHeapGrowth * W);
+ M = std::max({MinBalancedHeapLimitMB, W + MinBalancedHeadroomMB, M});
+
+ return M * double(BytesPerMB);
+}
+
+void GCHeapThreshold::updateStartThreshold(
+ size_t lastBytes, mozilla::Maybe<double> allocationRate,
+ mozilla::Maybe<double> collectionRate, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state, bool isAtomsZone) {
+ if (!tunables.balancedHeapLimitsEnabled()) {
+ double growthFactor =
+ computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state);
+
+ startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes, tunables);
+ } else {
+ double threshold = computeBalancedHeapLimit(
+ lastBytes, allocationRate.valueOr(DefaultAllocationRate),
+ collectionRate.valueOr(DefaultCollectionRate), tunables);
+
+ double triggerMax =
+ double(tunables.gcMaxBytes()) / tunables.largeHeapIncrementalLimit();
+
+ startBytes_ = ToClampedSize(uint64_t(std::min(triggerMax, threshold)));
+ }
+
+ setIncrementalLimitFromStartBytes(lastBytes, tunables);
+}
+
+/* static */
+size_t MallocHeapThreshold::computeZoneTriggerBytes(double growthFactor,
+ size_t lastBytes,
+ size_t baseBytes) {
+ return ToClampedSize(
+ uint64_t(double(std::max(lastBytes, baseBytes)) * growthFactor));
+}
+
+void MallocHeapThreshold::updateStartThreshold(
+ size_t lastBytes, const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state) {
+ double growthFactor =
+ computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state);
+
+ startBytes_ = computeZoneTriggerBytes(growthFactor, lastBytes,
+ tunables.mallocThresholdBase());
+
+ setIncrementalLimitFromStartBytes(lastBytes, tunables);
+}
+
+#ifdef DEBUG
+
+static const char* MemoryUseName(MemoryUse use) {
+ switch (use) {
+# define DEFINE_CASE(Name) \
+ case MemoryUse::Name: \
+ return #Name;
+ JS_FOR_EACH_MEMORY_USE(DEFINE_CASE)
+# undef DEFINE_CASE
+ }
+
+ MOZ_CRASH("Unknown memory use");
+}
+
+MemoryTracker::MemoryTracker() : mutex(mutexid::MemoryTracker) {}
+
+void MemoryTracker::checkEmptyOnDestroy() {
+ bool ok = true;
+
+ if (!gcMap.empty()) {
+ ok = false;
+ fprintf(stderr, "Missing calls to JS::RemoveAssociatedMemory:\n");
+ for (auto r = gcMap.all(); !r.empty(); r.popFront()) {
+ fprintf(stderr, " %p 0x%zx %s\n", r.front().key().ptr(),
+ r.front().value(), MemoryUseName(r.front().key().use()));
+ }
+ }
+
+ if (!nonGCMap.empty()) {
+ ok = false;
+ fprintf(stderr, "Missing calls to Zone::decNonGCMemory:\n");
+ for (auto r = nonGCMap.all(); !r.empty(); r.popFront()) {
+ fprintf(stderr, " %p 0x%zx\n", r.front().key().ptr(), r.front().value());
+ }
+ }
+
+ MOZ_ASSERT(ok);
+}
+
+/* static */
+inline bool MemoryTracker::isGCMemoryUse(MemoryUse use) {
+ // Most memory uses are for memory associated with GC things but some are for
+ // memory associated with non-GC thing pointers.
+ return !isNonGCMemoryUse(use);
+}
+
+/* static */
+inline bool MemoryTracker::isNonGCMemoryUse(MemoryUse use) {
+ return use == MemoryUse::TrackedAllocPolicy;
+}
+
+/* static */
+inline bool MemoryTracker::allowMultipleAssociations(MemoryUse use) {
+ // For most uses only one association is possible for each GC thing. Allow a
+ // one-to-many relationship only where necessary.
+ return isNonGCMemoryUse(use) || use == MemoryUse::RegExpSharedBytecode ||
+ use == MemoryUse::BreakpointSite || use == MemoryUse::Breakpoint ||
+ use == MemoryUse::ForOfPICStub || use == MemoryUse::ICUObject;
+}
+
+void MemoryTracker::trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(cell->isTenured());
+ MOZ_ASSERT(isGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<Cell> key{cell, use};
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ auto ptr = gcMap.lookupForAdd(key);
+ if (ptr) {
+ if (!allowMultipleAssociations(use)) {
+ MOZ_CRASH_UNSAFE_PRINTF("Association already present: %p 0x%zx %s", cell,
+ nbytes, MemoryUseName(use));
+ }
+ ptr->value() += nbytes;
+ return;
+ }
+
+ if (!gcMap.add(ptr, key, nbytes)) {
+ oomUnsafe.crash("MemoryTracker::trackGCMemory");
+ }
+}
+
+void MemoryTracker::untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(cell->isTenured());
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<Cell> key{cell, use};
+ auto ptr = gcMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("Association not found: %p 0x%zx %s", cell, nbytes,
+ MemoryUseName(use));
+ }
+
+ if (!allowMultipleAssociations(use) && ptr->value() != nbytes) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "Association for %p %s has different size: "
+ "expected 0x%zx but got 0x%zx",
+ cell, MemoryUseName(use), ptr->value(), nbytes);
+ }
+
+ if (nbytes > ptr->value()) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "Association for %p %s size is too large: "
+ "expected at most 0x%zx but got 0x%zx",
+ cell, MemoryUseName(use), ptr->value(), nbytes);
+ }
+
+ ptr->value() -= nbytes;
+
+ if (ptr->value() == 0) {
+ gcMap.remove(ptr);
+ }
+}
+
+void MemoryTracker::swapGCMemory(Cell* a, Cell* b, MemoryUse use) {
+ Key<Cell> ka{a, use};
+ Key<Cell> kb{b, use};
+
+ LockGuard<Mutex> lock(mutex);
+
+ size_t sa = getAndRemoveEntry(ka, lock);
+ size_t sb = getAndRemoveEntry(kb, lock);
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ if ((sa && b->isTenured() && !gcMap.put(kb, sa)) ||
+ (sb && a->isTenured() && !gcMap.put(ka, sb))) {
+ oomUnsafe.crash("MemoryTracker::swapGCMemory");
+ }
+}
+
+size_t MemoryTracker::getAndRemoveEntry(const Key<Cell>& key,
+ LockGuard<Mutex>& lock) {
+ auto ptr = gcMap.lookup(key);
+ if (!ptr) {
+ return 0;
+ }
+
+ size_t size = ptr->value();
+ gcMap.remove(ptr);
+ return size;
+}
+
+void MemoryTracker::registerNonGCMemory(void* mem, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookupForAdd(key);
+ if (ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s assocaition %p already registered",
+ MemoryUseName(use), mem);
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!nonGCMap.add(ptr, key, 0)) {
+ oomUnsafe.crash("MemoryTracker::registerNonGCMemory");
+ }
+}
+
+void MemoryTracker::unregisterNonGCMemory(void* mem, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ if (ptr->value() != 0) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "%s association %p still has 0x%zx bytes associated",
+ MemoryUseName(use), mem, ptr->value());
+ }
+
+ nonGCMap.remove(ptr);
+}
+
+void MemoryTracker::moveNonGCMemory(void* dst, void* src, MemoryUse use) {
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> srcKey{src, use};
+ auto srcPtr = nonGCMap.lookup(srcKey);
+ if (!srcPtr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s association %p not found", MemoryUseName(use),
+ src);
+ }
+
+ size_t nbytes = srcPtr->value();
+ nonGCMap.remove(srcPtr);
+
+ Key<void> dstKey{dst, use};
+ auto dstPtr = nonGCMap.lookupForAdd(dstKey);
+ if (dstPtr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s %p already registered", MemoryUseName(use),
+ dst);
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!nonGCMap.add(dstPtr, dstKey, nbytes)) {
+ oomUnsafe.crash("MemoryTracker::moveNonGCMemory");
+ }
+}
+
+void MemoryTracker::incNonGCMemory(void* mem, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(isNonGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ ptr->value() += nbytes;
+}
+
+void MemoryTracker::decNonGCMemory(void* mem, size_t nbytes, MemoryUse use) {
+ MOZ_ASSERT(isNonGCMemoryUse(use));
+
+ LockGuard<Mutex> lock(mutex);
+
+ Key<void> key{mem, use};
+ auto ptr = nonGCMap.lookup(key);
+ if (!ptr) {
+ MOZ_CRASH_UNSAFE_PRINTF("%s allocation %p not found", MemoryUseName(use),
+ mem);
+ }
+
+ size_t& value = ptr->value();
+ if (nbytes > value) {
+ MOZ_CRASH_UNSAFE_PRINTF(
+ "%s allocation %p is too large: "
+ "expected at most 0x%zx but got 0x%zx bytes",
+ MemoryUseName(use), mem, value, nbytes);
+ }
+
+ value -= nbytes;
+}
+
+void MemoryTracker::fixupAfterMovingGC() {
+ // Update the table after we move GC things. We don't use StableCellHasher
+ // because that would create a difference between debug and release builds.
+ for (GCMap::Enum e(gcMap); !e.empty(); e.popFront()) {
+ const auto& key = e.front().key();
+ Cell* cell = key.ptr();
+ if (cell->isForwarded()) {
+ cell = gc::RelocationOverlay::fromCell(cell)->forwardingAddress();
+ e.rekeyFront(Key<Cell>{cell, key.use()});
+ }
+ }
+}
+
+template <typename Ptr>
+inline MemoryTracker::Key<Ptr>::Key(Ptr* ptr, MemoryUse use)
+ : ptr_(uint64_t(ptr)), use_(uint64_t(use)) {
+# ifdef JS_64BIT
+ static_assert(sizeof(Key) == 8,
+ "MemoryTracker::Key should be packed into 8 bytes");
+# endif
+ MOZ_ASSERT(this->ptr() == ptr);
+ MOZ_ASSERT(this->use() == use);
+}
+
+template <typename Ptr>
+inline Ptr* MemoryTracker::Key<Ptr>::ptr() const {
+ return reinterpret_cast<Ptr*>(ptr_);
+}
+template <typename Ptr>
+inline MemoryUse MemoryTracker::Key<Ptr>::use() const {
+ return static_cast<MemoryUse>(use_);
+}
+
+template <typename Ptr>
+inline HashNumber MemoryTracker::Hasher<Ptr>::hash(const Lookup& l) {
+ return mozilla::HashGeneric(DefaultHasher<Ptr*>::hash(l.ptr()),
+ DefaultHasher<unsigned>::hash(unsigned(l.use())));
+}
+
+template <typename Ptr>
+inline bool MemoryTracker::Hasher<Ptr>::match(const KeyT& k, const Lookup& l) {
+ return k.ptr() == l.ptr() && k.use() == l.use();
+}
+
+template <typename Ptr>
+inline void MemoryTracker::Hasher<Ptr>::rekey(KeyT& k, const KeyT& newKey) {
+ k = newKey;
+}
+
+#endif // DEBUG