1 files changed, 85 insertions, 53 deletions

diff --git a/js/src/gc/GC.cpp b/js/src/gc/GC.cpp
index c01dfe3660..68dd66898c 100644
--- a/js/src/gc/GC.cpp
+++ b/js/src/gc/GC.cpp
@@ -444,7 +444,9 @@ GCRuntime::GCRuntime(JSRuntime* rt)
 #endif
       requestSliceAfterBackgroundTask(false),
       lifoBlocksToFree((size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
-      lifoBlocksToFreeAfterMinorGC(
+      lifoBlocksToFreeAfterFullMinorGC(
+          (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+      lifoBlocksToFreeAfterNextMinorGC(
           (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
       sweepGroupIndex(0),
       sweepGroups(nullptr),
@@ -658,7 +660,7 @@ void GCRuntime::setZeal(uint8_t zeal, uint32_t frequency) {
 
   if (zeal == 0) {
     if (hasZealMode(ZealMode::GenerationalGC)) {
-      evictNursery(JS::GCReason::DEBUG_GC);
+      evictNursery();
       nursery().leaveZealMode();
     }
 
@@ -669,7 +671,7 @@ void GCRuntime::setZeal(uint8_t zeal, uint32_t frequency) {
 
   ZealMode zealMode = ZealMode(zeal);
   if (zealMode == ZealMode::GenerationalGC) {
-    evictNursery(JS::GCReason::DEBUG_GC);
+    evictNursery(JS::GCReason::EVICT_NURSERY);
     nursery().enterZealMode();
   }
 
@@ -704,7 +706,7 @@ void GCRuntime::unsetZeal(uint8_t zeal) {
   }
 
   if (zealMode == ZealMode::GenerationalGC) {
-    evictNursery(JS::GCReason::DEBUG_GC);
+    evictNursery();
     nursery().leaveZealMode();
   }
 
@@ -1073,6 +1075,11 @@ bool GCRuntime::setParameter(JSGCParamKey key, uint32_t value,
         marker->incrementalWeakMapMarkingEnabled = value != 0;
       }
       break;
+    case JSGC_SEMISPACE_NURSERY_ENABLED: {
+      AutoUnlockGC unlock(lock);
+      nursery().setSemispaceEnabled(value);
+      break;
+    }
     case JSGC_MIN_EMPTY_CHUNK_COUNT:
       setMinEmptyChunkCount(value, lock);
       break;
@@ -1160,6 +1167,11 @@ void GCRuntime::resetParameter(JSGCParamKey key, AutoLockGC& lock) {
             TuningDefaults::IncrementalWeakMapMarkingEnabled;
       }
       break;
+    case JSGC_SEMISPACE_NURSERY_ENABLED: {
+      AutoUnlockGC unlock(lock);
+      nursery().setSemispaceEnabled(TuningDefaults::SemispaceNurseryEnabled);
+      break;
+    }
     case JSGC_MIN_EMPTY_CHUNK_COUNT:
       setMinEmptyChunkCount(TuningDefaults::MinEmptyChunkCount, lock);
       break;
@@ -1241,6 +1253,8 @@ uint32_t GCRuntime::getParameter(JSGCParamKey key, const AutoLockGC& lock) {
       return parallelMarkingEnabled;
     case JSGC_INCREMENTAL_WEAKMAP_ENABLED:
       return marker().incrementalWeakMapMarkingEnabled;
+    case JSGC_SEMISPACE_NURSERY_ENABLED:
+      return nursery().semispaceEnabled();
     case JSGC_CHUNK_BYTES:
       return ChunkSize;
     case JSGC_HELPER_THREAD_RATIO:
@@ -2135,7 +2149,7 @@ void GCRuntime::queueUnusedLifoBlocksForFree(LifoAlloc* lifo) {
 }
 
 void GCRuntime::queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo) {
-  lifoBlocksToFreeAfterMinorGC.ref().transferFrom(lifo);
+  lifoBlocksToFreeAfterFullMinorGC.ref().transferFrom(lifo);
 }
 
 void GCRuntime::queueBuffersForFreeAfterMinorGC(Nursery::BufferSet& buffers) {
@@ -2741,24 +2755,7 @@ void GCRuntime::endPreparePhase(JS::GCReason reason) {
   MOZ_ASSERT(unmarkTask.isIdle());
 
   for (GCZonesIter zone(this); !zone.done(); zone.next()) {
-    /*
-     * In an incremental GC, clear the area free lists to ensure that subsequent
-     * allocations refill them and end up marking new cells back. See
-     * arenaAllocatedDuringGC().
-     */
-    zone->arenas.clearFreeLists();
-
     zone->setPreservingCode(false);
-
-#ifdef JS_GC_ZEAL
-    if (hasZealMode(ZealMode::YieldBeforeRootMarking)) {
-      for (auto kind : AllAllocKinds()) {
-        for (ArenaIterInGC arena(zone, kind); !arena.done(); arena.next()) {
-          arena->checkNoMarkedCells();
-        }
-      }
-    }
-#endif
   }
 
   // Discard JIT code more aggressively if the process is approaching its
@@ -2800,7 +2797,7 @@ void GCRuntime::endPreparePhase(JS::GCReason reason) {
    */
 
   {
-    gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::PREPARE);
+    gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PREPARE);
 
     AutoLockHelperThreadState helperLock;
 
@@ -2817,20 +2814,6 @@ void GCRuntime::endPreparePhase(JS::GCReason reason) {
     haveDiscardedJITCodeThisSlice = true;
 
     /*
-     * Relazify functions after discarding JIT code (we can't relazify
-     * functions with JIT code) and before the actual mark phase, so that
-     * the current GC can collect the JSScripts we're unlinking here.  We do
-     * this only when we're performing a shrinking GC, as too much
-     * relazification can cause performance issues when we have to reparse
-     * the same functions over and over.
-     */
-    if (isShrinkingGC()) {
-      relazifyFunctionsForShrinkingGC();
-      purgePropMapTablesForShrinkingGC();
-      purgeSourceURLsForShrinkingGC();
-    }
-
-    /*
      * We must purge the runtime at the beginning of an incremental GC. The
      * danger if we purge later is that the snapshot invariant of
      * incremental GC will be broken, as follows. If some object is
@@ -2840,8 +2823,25 @@ void GCRuntime::endPreparePhase(JS::GCReason reason) {
      * it. This object might never be marked, so a GC hazard would exist.
      */
     purgeRuntime();
+  }
+
+  // This will start background free for lifo blocks queued by purgeRuntime,
+  // even if there's nothing in the nursery.
+  collectNurseryFromMajorGC(reason);
 
-    startBackgroundFreeAfterMinorGC();
+  {
+    gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::PREPARE);
+    // Relazify functions after discarding JIT code (we can't relazify functions
+    // with JIT code) and before the actual mark phase, so that the current GC
+    // can collect the JSScripts we're unlinking here.  We do this only when
+    // we're performing a shrinking GC, as too much relazification can cause
+    // performance issues when we have to reparse the same functions over and
+    // over.
+    if (isShrinkingGC()) {
+      relazifyFunctionsForShrinkingGC();
+      purgePropMapTablesForShrinkingGC();
+      purgeSourceURLsForShrinkingGC();
+    }
 
     if (isShutdownGC()) {
       /* Clear any engine roots that may hold external data live. */
@@ -2903,6 +2903,21 @@ void GCRuntime::beginMarkPhase(AutoGCSession& session) {
 #endif
 
   for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+    // In an incremental GC, clear the arena free lists to ensure that
+    // subsequent allocations refill them and end up marking new cells black.
+    // See arenaAllocatedDuringGC().
+    zone->arenas.clearFreeLists();
+
+#ifdef JS_GC_ZEAL
+    if (hasZealMode(ZealMode::YieldBeforeRootMarking)) {
+      for (auto kind : AllAllocKinds()) {
+        for (ArenaIter arena(zone, kind); !arena.done(); arena.next()) {
+          arena->checkNoMarkedCells();
+        }
+      }
+    }
+#endif
+
     // Incremental marking barriers are enabled at this point.
     zone->changeGCState(Zone::Prepare, zone->initialMarkingState());
 
@@ -3715,11 +3730,8 @@ void GCRuntime::incrementalSlice(SliceBudget& budget, JS::GCReason reason,
       [[fallthrough]];
 
     case State::MarkRoots:
-      if (NeedToCollectNursery(this)) {
-        collectNurseryFromMajorGC(reason);
-      }
-
       endPreparePhase(reason);
+
       beginMarkPhase(session);
       incrementalState = State::Mark;
 
@@ -3878,8 +3890,11 @@ void GCRuntime::incrementalSlice(SliceBudget& budget, JS::GCReason reason,
 }
 
 void GCRuntime::collectNurseryFromMajorGC(JS::GCReason reason) {
-  collectNursery(gcOptions(), reason,
+  collectNursery(gcOptions(), JS::GCReason::EVICT_NURSERY,
                  gcstats::PhaseKind::EVICT_NURSERY_FOR_MAJOR_GC);
+
+  MOZ_ASSERT(nursery().isEmpty());
+  MOZ_ASSERT(storeBuffer().isEmpty());
 }
 
 bool GCRuntime::hasForegroundWork() const {
@@ -4733,26 +4748,43 @@ void GCRuntime::collectNursery(JS::GCOptions options, JS::GCReason reason,
   gcstats::AutoPhase ap(stats(), phase);
 
   nursery().collect(options, reason);
-  MOZ_ASSERT(nursery().isEmpty());
 
   startBackgroundFreeAfterMinorGC();
+
+  // We ignore gcMaxBytes when allocating for minor collection. However, if we
+  // overflowed, we disable the nursery. The next time we allocate, we'll fail
+  // because bytes >= gcMaxBytes.
+  if (heapSize.bytes() >= tunables.gcMaxBytes()) {
+    if (!nursery().isEmpty()) {
+      nursery().collect(options, JS::GCReason::DISABLE_GENERATIONAL_GC);
+      MOZ_ASSERT(nursery().isEmpty());
+      startBackgroundFreeAfterMinorGC();
+    }
+    nursery().disable();
+  }
 }
 
 void GCRuntime::startBackgroundFreeAfterMinorGC() {
-  MOZ_ASSERT(nursery().isEmpty());
+  // Called after nursery collection. Free whatever blocks are safe to free now.
 
-  {
-    AutoLockHelperThreadState lock;
+  AutoLockHelperThreadState lock;
 
-    lifoBlocksToFree.ref().transferFrom(&lifoBlocksToFreeAfterMinorGC.ref());
+  lifoBlocksToFree.ref().transferFrom(&lifoBlocksToFreeAfterNextMinorGC.ref());
 
-    if (lifoBlocksToFree.ref().isEmpty() &&
-        buffersToFreeAfterMinorGC.ref().empty()) {
-      return;
-    }
+  if (nursery().tenuredEverything) {
+    lifoBlocksToFree.ref().transferFrom(
+        &lifoBlocksToFreeAfterFullMinorGC.ref());
+  } else {
+    lifoBlocksToFreeAfterNextMinorGC.ref().transferFrom(
+        &lifoBlocksToFreeAfterFullMinorGC.ref());
+  }
+
+  if (lifoBlocksToFree.ref().isEmpty() &&
+      buffersToFreeAfterMinorGC.ref().empty()) {
+    return;
   }
 
-  startBackgroundFree();
+  freeTask.startOrRunIfIdle(lock);
 }
 
 bool GCRuntime::gcIfRequestedImpl(bool eagerOk) {