Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 961 insertions, 0 deletions

diff --git a/js/src/gc/Compacting.cpp b/js/src/gc/Compacting.cpp
new file mode 100644
index 0000000000..ad3651b348
--- /dev/null
+++ b/js/src/gc/Compacting.cpp
@@ -0,0 +1,961 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Implementation of compacting GC.
+ */
+
+#include "mozilla/Maybe.h"
+
+#include "debugger/DebugAPI.h"
+#include "gc/ArenaList.h"
+#include "gc/GCInternals.h"
+#include "gc/GCLock.h"
+#include "gc/ParallelWork.h"
+#include "gc/Zone.h"
+#include "jit/JitCode.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "js/GCAPI.h"
+#include "vm/HelperThreads.h"
+#include "vm/Realm.h"
+#include "wasm/WasmGcObject.h"
+
+#include "gc/Heap-inl.h"
+#include "gc/Marking-inl.h"
+#include "gc/PrivateIterators-inl.h"
+#include "gc/TraceMethods-inl.h"
+#include "gc/Zone-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::Maybe;
+
+bool GCRuntime::canRelocateZone(Zone* zone) const {
+  return !zone->isAtomsZone();
+}
+
+void GCRuntime::beginCompactPhase() {
+  MOZ_ASSERT(!isBackgroundSweeping());
+  assertBackgroundSweepingFinished();
+
+  gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT);
+
+  MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty());
+  for (GCZonesIter zone(this); !zone.done(); zone.next()) {
+    if (canRelocateZone(zone)) {
+      zonesToMaybeCompact.ref().append(zone);
+    }
+  }
+
+  startedCompacting = true;
+  zonesCompacted = 0;
+
+#ifdef DEBUG
+  AutoLockGC lock(this);
+  MOZ_ASSERT(!relocatedArenasToRelease);
+#endif
+}
+
+IncrementalProgress GCRuntime::compactPhase(JS::GCReason reason,
+                                            SliceBudget& sliceBudget,
+                                            AutoGCSession& session) {
+  assertBackgroundSweepingFinished();
+  MOZ_ASSERT(startedCompacting);
+
+  AutoMajorGCProfilerEntry s(this);
+  gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT);
+
+  // TODO: JSScripts can move. If the sampler interrupts the GC in the
+  // middle of relocating an arena, invalid JSScript pointers may be
+  // accessed. Suppress all sampling until a finer-grained solution can be
+  // found. See bug 1295775.
+  AutoSuppressProfilerSampling suppressSampling(rt->mainContextFromOwnThread());
+
+  ZoneList relocatedZones;
+  Arena* relocatedArenas = nullptr;
+  while (!zonesToMaybeCompact.ref().isEmpty()) {
+    Zone* zone = zonesToMaybeCompact.ref().front();
+    zonesToMaybeCompact.ref().removeFront();
+
+    MOZ_ASSERT(nursery().isEmpty());
+    zone->changeGCState(Zone::Finished, Zone::Compact);
+
+    if (relocateArenas(zone, reason, relocatedArenas, sliceBudget)) {
+      updateZonePointersToRelocatedCells(zone);
+      relocatedZones.append(zone);
+      zonesCompacted++;
+    } else {
+      zone->changeGCState(Zone::Compact, Zone::Finished);
+    }
+
+    if (sliceBudget.isOverBudget()) {
+      break;
+    }
+  }
+
+  if (!relocatedZones.isEmpty()) {
+    updateRuntimePointersToRelocatedCells(session);
+
+    do {
+      Zone* zone = relocatedZones.front();
+      relocatedZones.removeFront();
+      zone->changeGCState(Zone::Compact, Zone::Finished);
+    } while (!relocatedZones.isEmpty());
+  }
+
+  clearRelocatedArenas(relocatedArenas, reason);
+
+#ifdef DEBUG
+  protectOrReleaseRelocatedArenas(relocatedArenas, reason);
+#else
+  releaseRelocatedArenas(relocatedArenas);
+#endif
+
+  // Clear caches that can contain cell pointers.
+  rt->caches().purgeForCompaction();
+
+#ifdef DEBUG
+  checkHashTablesAfterMovingGC();
+#endif
+
+  return zonesToMaybeCompact.ref().isEmpty() ? Finished : NotFinished;
+}
+
+void GCRuntime::endCompactPhase() { startedCompacting = false; }
+
+static bool ShouldRelocateAllArenas(JS::GCReason reason) {
+  return reason == JS::GCReason::DEBUG_GC;
+}
+
+/*
+ * Choose which arenas to relocate all cells from. Return an arena cursor that
+ * can be passed to removeRemainingArenas().
+ */
+Arena** ArenaList::pickArenasToRelocate(size_t& arenaTotalOut,
+                                        size_t& relocTotalOut) {
+  // Relocate the greatest number of arenas such that the number of used cells
+  // in relocated arenas is less than or equal to the number of free cells in
+  // unrelocated arenas. In other words we only relocate cells we can move
+  // into existing arenas, and we choose the least full areans to relocate.
+  //
+  // This is made easier by the fact that the arena list has been sorted in
+  // descending order of number of used cells, so we will always relocate a
+  // tail of the arena list. All we need to do is find the point at which to
+  // start relocating.
+
+  check();
+
+  if (isCursorAtEnd()) {
+    return nullptr;
+  }
+
+  Arena** arenap = cursorp_;      // Next arena to consider for relocation.
+  size_t previousFreeCells = 0;   // Count of free cells before arenap.
+  size_t followingUsedCells = 0;  // Count of used cells after arenap.
+  size_t fullArenaCount = 0;      // Number of full arenas (not relocated).
+  size_t nonFullArenaCount =
+      0;  // Number of non-full arenas (considered for relocation).
+  size_t arenaIndex = 0;  // Index of the next arena to consider.
+
+  for (Arena* arena = head_; arena != *cursorp_; arena = arena->next) {
+    fullArenaCount++;
+  }
+
+  for (Arena* arena = *cursorp_; arena; arena = arena->next) {
+    followingUsedCells += arena->countUsedCells();
+    nonFullArenaCount++;
+  }
+
+  mozilla::DebugOnly<size_t> lastFreeCells(0);
+  size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getAllocKind());
+
+  while (*arenap) {
+    Arena* arena = *arenap;
+    if (followingUsedCells <= previousFreeCells) {
+      break;
+    }
+
+    size_t freeCells = arena->countFreeCells();
+    size_t usedCells = cellsPerArena - freeCells;
+    followingUsedCells -= usedCells;
+#ifdef DEBUG
+    MOZ_ASSERT(freeCells >= lastFreeCells);
+    lastFreeCells = freeCells;
+#endif
+    previousFreeCells += freeCells;
+    arenap = &arena->next;
+    arenaIndex++;
+  }
+
+  size_t relocCount = nonFullArenaCount - arenaIndex;
+  MOZ_ASSERT(relocCount < nonFullArenaCount);
+  MOZ_ASSERT((relocCount == 0) == (!*arenap));
+  arenaTotalOut += fullArenaCount + nonFullArenaCount;
+  relocTotalOut += relocCount;
+
+  return arenap;
+}
+
+#ifdef DEBUG
+inline bool PtrIsInRange(const void* ptr, const void* start, size_t length) {
+  return uintptr_t(ptr) - uintptr_t(start) < length;
+}
+#endif
+
+static void RelocateCell(Zone* zone, TenuredCell* src, AllocKind thingKind,
+                         size_t thingSize) {
+  JS::AutoSuppressGCAnalysis nogc;
+
+  // Allocate a new cell.
+  MOZ_ASSERT(zone == src->zone());
+  TenuredCell* dst = AllocateCellInGC(zone, thingKind);
+
+  // Copy source cell contents to destination.
+  memcpy(dst, src, thingSize);
+
+  // Move any uid attached to the object.
+  src->zone()->transferUniqueId(dst, src);
+
+  if (IsObjectAllocKind(thingKind)) {
+    auto* srcObj = static_cast<JSObject*>(static_cast<Cell*>(src));
+    auto* dstObj = static_cast<JSObject*>(static_cast<Cell*>(dst));
+
+    if (srcObj->is<NativeObject>()) {
+      NativeObject* srcNative = &srcObj->as<NativeObject>();
+      NativeObject* dstNative = &dstObj->as<NativeObject>();
+
+      // Fixup the pointer to inline object elements if necessary.
+      if (srcNative->hasFixedElements()) {
+        uint32_t numShifted =
+            srcNative->getElementsHeader()->numShiftedElements();
+        dstNative->setFixedElements(numShifted);
+      }
+    } else if (srcObj->is<ProxyObject>()) {
+      if (srcObj->as<ProxyObject>().usingInlineValueArray()) {
+        dstObj->as<ProxyObject>().setInlineValueArray();
+      }
+    }
+
+    // Call object moved hook if present.
+    if (JSObjectMovedOp op = srcObj->getClass()->extObjectMovedOp()) {
+      op(dstObj, srcObj);
+    }
+
+    MOZ_ASSERT_IF(
+        dstObj->is<NativeObject>(),
+        !PtrIsInRange(
+            (const Value*)dstObj->as<NativeObject>().getDenseElements(), src,
+            thingSize));
+  }
+
+  // Copy the mark bits.
+  dst->copyMarkBitsFrom(src);
+
+  // Poison the source cell contents except for the forwarding flag and pointer
+  // which will be stored in the first word. We can't do this for native object
+  // with fixed elements because this would overwrite the element flags and
+  // these are needed when updating COW elements referred to by other objects.
+#ifdef DEBUG
+  JSObject* srcObj = IsObjectAllocKind(thingKind)
+                         ? static_cast<JSObject*>(static_cast<Cell*>(src))
+                         : nullptr;
+  if (!srcObj || !srcObj->is<NativeObject>() ||
+      !srcObj->as<NativeObject>().hasFixedElements()) {
+    AlwaysPoison(reinterpret_cast<uint8_t*>(src) + sizeof(uintptr_t),
+                 JS_MOVED_TENURED_PATTERN, thingSize - sizeof(uintptr_t),
+                 MemCheckKind::MakeNoAccess);
+  }
+#endif
+
+  // Mark source cell as forwarded and leave a pointer to the destination.
+  RelocationOverlay::forwardCell(src, dst);
+}
+
+static void RelocateArena(Arena* arena, SliceBudget& sliceBudget) {
+  MOZ_ASSERT(arena->allocated());
+  MOZ_ASSERT(!arena->onDelayedMarkingList());
+  MOZ_ASSERT(arena->bufferedCells()->isEmpty());
+
+  Zone* zone = arena->zone;
+
+  AllocKind thingKind = arena->getAllocKind();
+  size_t thingSize = arena->getThingSize();
+
+  for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+    RelocateCell(zone, cell, thingKind, thingSize);
+    sliceBudget.step();
+  }
+
+#ifdef DEBUG
+  for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+    TenuredCell* src = cell;
+    MOZ_ASSERT(src->isForwarded());
+    TenuredCell* dest = Forwarded(src);
+    MOZ_ASSERT(src->isMarkedBlack() == dest->isMarkedBlack());
+    MOZ_ASSERT(src->isMarkedGray() == dest->isMarkedGray());
+  }
+#endif
+}
+
+/*
+ * Relocate all arenas identified by pickArenasToRelocate: for each arena,
+ * relocate each cell within it, then add it to a list of relocated arenas.
+ */
+Arena* ArenaList::relocateArenas(Arena* toRelocate, Arena* relocated,
+                                 SliceBudget& sliceBudget,
+                                 gcstats::Statistics& stats) {
+  check();
+
+  while (Arena* arena = toRelocate) {
+    toRelocate = arena->next;
+    RelocateArena(arena, sliceBudget);
+    // Prepend to list of relocated arenas
+    arena->next = relocated;
+    relocated = arena;
+    stats.count(gcstats::COUNT_ARENA_RELOCATED);
+  }
+
+  check();
+
+  return relocated;
+}
+
+// Skip compacting zones unless we can free a certain proportion of their GC
+// heap memory.
+static const float MIN_ZONE_RECLAIM_PERCENT = 2.0;
+
+static bool ShouldRelocateZone(size_t arenaCount, size_t relocCount,
+                               JS::GCReason reason) {
+  if (relocCount == 0) {
+    return false;
+  }
+
+  if (IsOOMReason(reason)) {
+    return true;
+  }
+
+  return (relocCount * 100.0f) / arenaCount >= MIN_ZONE_RECLAIM_PERCENT;
+}
+
+static AllocKinds CompactingAllocKinds() {
+  AllocKinds result;
+  for (AllocKind kind : AllAllocKinds()) {
+    if (IsCompactingKind(kind)) {
+      result += kind;
+    }
+  }
+  return result;
+}
+
+bool ArenaLists::relocateArenas(Arena*& relocatedListOut, JS::GCReason reason,
+                                SliceBudget& sliceBudget,
+                                gcstats::Statistics& stats) {
+  // This is only called from the main thread while we are doing a GC, so
+  // there is no need to lock.
+  MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+  MOZ_ASSERT(runtime()->gc.isHeapCompacting());
+  MOZ_ASSERT(!runtime()->gc.isBackgroundSweeping());
+
+  // Relocate all compatible kinds
+  AllocKinds allocKindsToRelocate = CompactingAllocKinds();
+
+  // Clear all the free lists.
+  clearFreeLists();
+
+  if (ShouldRelocateAllArenas(reason)) {
+    zone_->prepareForCompacting();
+    for (auto kind : allocKindsToRelocate) {
+      ArenaList& al = arenaList(kind);
+      Arena* allArenas = al.head();
+      al.clear();
+      relocatedListOut =
+          al.relocateArenas(allArenas, relocatedListOut, sliceBudget, stats);
+    }
+  } else {
+    size_t arenaCount = 0;
+    size_t relocCount = 0;
+    AllAllocKindArray<Arena**> toRelocate;
+
+    for (auto kind : allocKindsToRelocate) {
+      toRelocate[kind] =
+          arenaList(kind).pickArenasToRelocate(arenaCount, relocCount);
+    }
+
+    if (!ShouldRelocateZone(arenaCount, relocCount, reason)) {
+      return false;
+    }
+
+    zone_->prepareForCompacting();
+    for (auto kind : allocKindsToRelocate) {
+      if (toRelocate[kind]) {
+        ArenaList& al = arenaList(kind);
+        Arena* arenas = al.removeRemainingArenas(toRelocate[kind]);
+        relocatedListOut =
+            al.relocateArenas(arenas, relocatedListOut, sliceBudget, stats);
+      }
+    }
+  }
+
+  return true;
+}
+
+bool GCRuntime::relocateArenas(Zone* zone, JS::GCReason reason,
+                               Arena*& relocatedListOut,
+                               SliceBudget& sliceBudget) {
+  gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_MOVE);
+
+  MOZ_ASSERT(!zone->isPreservingCode());
+  MOZ_ASSERT(canRelocateZone(zone));
+
+  js::CancelOffThreadIonCompile(rt, JS::Zone::Compact);
+
+  if (!zone->arenas.relocateArenas(relocatedListOut, reason, sliceBudget,
+                                   stats())) {
+    return false;
+  }
+
+#ifdef DEBUG
+  // Check that we did as much compaction as we should have. There
+  // should always be less than one arena's worth of free cells.
+  for (auto kind : CompactingAllocKinds()) {
+    ArenaList& al = zone->arenas.arenaList(kind);
+    size_t freeCells = 0;
+    for (Arena* arena = al.arenaAfterCursor(); arena; arena = arena->next) {
+      freeCells += arena->countFreeCells();
+    }
+    MOZ_ASSERT(freeCells < Arena::thingsPerArena(kind));
+  }
+#endif
+
+  return true;
+}
+
+MovingTracer::MovingTracer(JSRuntime* rt)
+    : GenericTracerImpl(rt, JS::TracerKind::Moving,
+                        JS::WeakMapTraceAction::TraceKeysAndValues) {}
+
+template <typename T>
+inline void MovingTracer::onEdge(T** thingp, const char* name) {
+  T* thing = *thingp;
+  if (thing->runtimeFromAnyThread() == runtime() && IsForwarded(thing)) {
+    *thingp = Forwarded(thing);
+  }
+}
+
+void Zone::prepareForCompacting() {
+  JS::GCContext* gcx = runtimeFromMainThread()->gcContext();
+  discardJitCode(gcx);
+}
+
+void GCRuntime::sweepZoneAfterCompacting(MovingTracer* trc, Zone* zone) {
+  MOZ_ASSERT(zone->isCollecting());
+  traceWeakFinalizationObserverEdges(trc, zone);
+
+  zone->traceWeakMaps(trc);
+
+  for (auto* cache : zone->weakCaches()) {
+    cache->traceWeak(trc, nullptr);
+  }
+
+  if (jit::JitZone* jitZone = zone->jitZone()) {
+    jitZone->traceWeak(trc);
+  }
+
+  for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
+    c->traceWeakNativeIterators(trc);
+
+    for (RealmsInCompartmentIter r(c); !r.done(); r.next()) {
+      r->traceWeakRegExps(trc);
+      r->traceWeakSavedStacks(trc);
+      r->traceWeakGlobalEdge(trc);
+      r->traceWeakDebugEnvironmentEdges(trc);
+      r->traceWeakEdgesInJitRealm(trc);
+    }
+  }
+}
+
+template <typename T>
+static inline void UpdateCellPointers(MovingTracer* trc, T* cell) {
+  // We only update unmoved GC things or the new copy of moved GC things, never
+  // the old copy. If this happened it could clear the forwarded flag which
+  // could lead to pointers to the old copy not being updated.
+  MOZ_ASSERT(!cell->isForwarded());
+
+  cell->fixupAfterMovingGC();
+  cell->traceChildren(trc);
+}
+
+template <typename T>
+static void UpdateArenaPointersTyped(MovingTracer* trc, Arena* arena) {
+  for (ArenaCellIterUnderGC cell(arena); !cell.done(); cell.next()) {
+    UpdateCellPointers(trc, cell.as<T>());
+  }
+}
+
+static bool CanUpdateKindInBackground(AllocKind kind) {
+  // We try to update as many GC things in parallel as we can, but there are
+  // kinds for which this might not be safe:
+  //  - we assume JSObjects that are foreground finalized are not safe to
+  //    update in parallel
+  //  - updating a SharedPropMap touches child maps in
+  //    SharedPropMap::fixupAfterMovingGC
+  return js::gc::IsBackgroundFinalized(kind) && !IsShapeAllocKind(kind) &&
+         kind != AllocKind::BASE_SHAPE;
+}
+
+/*
+ * Update the internal pointers for all cells in an arena.
+ */
+static void UpdateArenaPointers(MovingTracer* trc, Arena* arena) {
+  AllocKind kind = arena->getAllocKind();
+
+  MOZ_ASSERT_IF(!CanUpdateKindInBackground(kind),
+                CurrentThreadCanAccessRuntime(trc->runtime()));
+
+  switch (kind) {
+#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery, \
+                    compact)                                                 \
+  case AllocKind::allocKind:                                                 \
+    UpdateArenaPointersTyped<type>(trc, arena);                              \
+    return;
+    FOR_EACH_ALLOCKIND(EXPAND_CASE)
+#undef EXPAND_CASE
+
+    default:
+      MOZ_CRASH("Invalid alloc kind for UpdateArenaPointers");
+  }
+}
+
+struct ArenaListSegment {
+  Arena* begin;
+  Arena* end;
+};
+
+/*
+ * Update the internal pointers for all arenas in a segment of an arena list.
+ *
+ * Returns the number of steps to count against the slice budget.
+ */
+static size_t UpdateArenaListSegmentPointers(GCRuntime* gc,
+                                             const ArenaListSegment& arenas) {
+  MOZ_ASSERT(arenas.begin);
+  MovingTracer trc(gc->rt);
+  size_t count = 0;
+  for (Arena* arena = arenas.begin; arena != arenas.end; arena = arena->next) {
+    UpdateArenaPointers(&trc, arena);
+    count++;
+  }
+  return count * 256;
+}
+
+class ArenasToUpdate {
+  // Maximum number of arenas to update in one block.
+#ifdef DEBUG
+  static const unsigned MaxArenasToProcess = 16;
+#else
+  static const unsigned MaxArenasToProcess = 256;
+#endif
+
+ public:
+  explicit ArenasToUpdate(Zone* zone);
+  ArenasToUpdate(Zone* zone, const AllocKinds& kinds);
+
+  bool done() const { return !segmentBegin; }
+
+  ArenaListSegment get() const {
+    MOZ_ASSERT(!done());
+    return {segmentBegin, segmentEnd};
+  }
+
+  void next();
+
+ private:
+  Maybe<AllocKinds> kinds;            // Selects which thing kinds to update.
+  Zone* zone;                         // Zone to process.
+  AllocKind kind = AllocKind::FIRST;  // Current alloc kind to process.
+  Arena* segmentBegin = nullptr;
+  Arena* segmentEnd = nullptr;
+
+  static AllocKind nextAllocKind(AllocKind i) {
+    return AllocKind(uint8_t(i) + 1);
+  }
+
+  void settle();
+  void findSegmentEnd();
+};
+
+ArenasToUpdate::ArenasToUpdate(Zone* zone) : zone(zone) { settle(); }
+
+ArenasToUpdate::ArenasToUpdate(Zone* zone, const AllocKinds& kinds)
+    : kinds(Some(kinds)), zone(zone) {
+  settle();
+}
+
+void ArenasToUpdate::settle() {
+  // Called when we have set |kind| to a new kind. Sets |arena| to the next
+  // arena or null if there are no more arenas to update.
+
+  MOZ_ASSERT(!segmentBegin);
+
+  for (; kind < AllocKind::LIMIT; kind = nextAllocKind(kind)) {
+    if (kinds && !kinds.ref().contains(kind)) {
+      continue;
+    }
+
+    Arena* arena = zone->arenas.getFirstArena(kind);
+    if (arena) {
+      segmentBegin = arena;
+      findSegmentEnd();
+      break;
+    }
+  }
+}
+
+void ArenasToUpdate::findSegmentEnd() {
+  // Take up to MaxArenasToProcess arenas from the list starting at
+  // |segmentBegin| and set |segmentEnd|.
+  Arena* arena = segmentBegin;
+  for (size_t i = 0; arena && i < MaxArenasToProcess; i++) {
+    arena = arena->next;
+  }
+  segmentEnd = arena;
+}
+
+void ArenasToUpdate::next() {
+  MOZ_ASSERT(!done());
+
+  segmentBegin = segmentEnd;
+  if (segmentBegin) {
+    findSegmentEnd();
+    return;
+  }
+
+  kind = nextAllocKind(kind);
+  settle();
+}
+
+static AllocKinds ForegroundUpdateKinds(AllocKinds kinds) {
+  AllocKinds result;
+  for (AllocKind kind : kinds) {
+    if (!CanUpdateKindInBackground(kind)) {
+      result += kind;
+    }
+  }
+  return result;
+}
+
+void GCRuntime::updateCellPointers(Zone* zone, AllocKinds kinds) {
+  AllocKinds fgKinds = ForegroundUpdateKinds(kinds);
+  AllocKinds bgKinds = kinds - fgKinds;
+
+  ArenasToUpdate fgArenas(zone, fgKinds);
+  ArenasToUpdate bgArenas(zone, bgKinds);
+
+  AutoLockHelperThreadState lock;
+
+  AutoRunParallelWork bgTasks(this, UpdateArenaListSegmentPointers,
+                              gcstats::PhaseKind::COMPACT_UPDATE_CELLS,
+                              GCUse::Unspecified, bgArenas,
+                              SliceBudget::unlimited(), lock);
+
+  AutoUnlockHelperThreadState unlock(lock);
+
+  for (; !fgArenas.done(); fgArenas.next()) {
+    UpdateArenaListSegmentPointers(this, fgArenas.get());
+  }
+}
+
+// After cells have been relocated any pointers to a cell's old locations must
+// be updated to point to the new location.  This happens by iterating through
+// all cells in heap and tracing their children (non-recursively) to update
+// them.
+//
+// This is complicated by the fact that updating a GC thing sometimes depends on
+// making use of other GC things.  After a moving GC these things may not be in
+// a valid state since they may contain pointers which have not been updated
+// yet.
+//
+// The main dependencies are:
+//
+//   - Updating a JSObject makes use of its shape
+//   - Updating a typed object makes use of its type descriptor object
+//
+// This means we require at least three phases for update:
+//
+//  1) shapes
+//  2) typed object type descriptor objects
+//  3) all other objects
+//
+// Also, there can be data races calling IsForwarded() on the new location of a
+// cell whose first word is being updated in parallel on another thread. This
+// easiest way to avoid this is to not store a GC pointer in the first word of a
+// cell. Otherwise this can be avoided by updating different kinds of cell in
+// different phases.
+//
+// Since we want to minimize the number of phases, arrange kinds into three
+// arbitrary phases.
+
+static constexpr AllocKinds UpdatePhaseOne{AllocKind::SCRIPT,
+                                           AllocKind::BASE_SHAPE,
+                                           AllocKind::SHAPE,
+                                           AllocKind::STRING,
+                                           AllocKind::JITCODE,
+                                           AllocKind::REGEXP_SHARED,
+                                           AllocKind::SCOPE,
+                                           AllocKind::GETTER_SETTER,
+                                           AllocKind::COMPACT_PROP_MAP,
+                                           AllocKind::NORMAL_PROP_MAP,
+                                           AllocKind::DICT_PROP_MAP};
+
+// UpdatePhaseTwo is typed object descriptor objects.
+
+static constexpr AllocKinds UpdatePhaseThree{AllocKind::FUNCTION,
+                                             AllocKind::FUNCTION_EXTENDED,
+                                             AllocKind::OBJECT0,
+                                             AllocKind::OBJECT0_BACKGROUND,
+                                             AllocKind::OBJECT2,
+                                             AllocKind::OBJECT2_BACKGROUND,
+                                             AllocKind::ARRAYBUFFER4,
+                                             AllocKind::OBJECT4,
+                                             AllocKind::OBJECT4_BACKGROUND,
+                                             AllocKind::ARRAYBUFFER8,
+                                             AllocKind::OBJECT8,
+                                             AllocKind::OBJECT8_BACKGROUND,
+                                             AllocKind::ARRAYBUFFER12,
+                                             AllocKind::OBJECT12,
+                                             AllocKind::OBJECT12_BACKGROUND,
+                                             AllocKind::ARRAYBUFFER16,
+                                             AllocKind::OBJECT16,
+                                             AllocKind::OBJECT16_BACKGROUND};
+
+void GCRuntime::updateAllCellPointers(MovingTracer* trc, Zone* zone) {
+  updateCellPointers(zone, UpdatePhaseOne);
+
+  updateCellPointers(zone, UpdatePhaseThree);
+}
+
+/*
+ * Update pointers to relocated cells in a single zone by doing a traversal of
+ * that zone's arenas and calling per-zone sweep hooks.
+ *
+ * The latter is necessary to update weak references which are not marked as
+ * part of the traversal.
+ */
+void GCRuntime::updateZonePointersToRelocatedCells(Zone* zone) {
+  MOZ_ASSERT(!rt->isBeingDestroyed());
+  MOZ_ASSERT(zone->isGCCompacting());
+
+  AutoTouchingGrayThings tgt;
+
+  gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT_UPDATE);
+  MovingTracer trc(rt);
+
+  zone->fixupAfterMovingGC();
+  zone->fixupScriptMapsAfterMovingGC(&trc);
+
+  // Fixup compartment global pointers as these get accessed during marking.
+  for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+    comp->fixupAfterMovingGC(&trc);
+  }
+
+  zone->externalStringCache().purge();
+  zone->functionToStringCache().purge();
+  zone->shapeZone().purgeShapeCaches(rt->gcContext());
+  rt->caches().stringToAtomCache.purge();
+
+  // Iterate through all cells that can contain relocatable pointers to update
+  // them. Since updating each cell is independent we try to parallelize this
+  // as much as possible.
+  updateAllCellPointers(&trc, zone);
+
+  // Sweep everything to fix up weak pointers.
+  sweepZoneAfterCompacting(&trc, zone);
+
+  // Call callbacks to get the rest of the system to fixup other untraced
+  // pointers.
+  for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+    callWeakPointerCompartmentCallbacks(&trc, comp);
+  }
+}
+
+/*
+ * Update runtime-wide pointers to relocated cells.
+ */
+void GCRuntime::updateRuntimePointersToRelocatedCells(AutoGCSession& session) {
+  MOZ_ASSERT(!rt->isBeingDestroyed());
+
+  gcstats::AutoPhase ap1(stats(), gcstats::PhaseKind::COMPACT_UPDATE);
+  MovingTracer trc(rt);
+
+  Zone::fixupAllCrossCompartmentWrappersAfterMovingGC(&trc);
+
+  rt->geckoProfiler().fixupStringsMapAfterMovingGC();
+
+  // Mark roots to update them.
+
+  traceRuntimeForMajorGC(&trc, session);
+
+  {
+    gcstats::AutoPhase ap2(stats(), gcstats::PhaseKind::MARK_ROOTS);
+    DebugAPI::traceAllForMovingGC(&trc);
+    DebugAPI::traceCrossCompartmentEdges(&trc);
+
+    // Mark all gray roots.
+    traceEmbeddingGrayRoots(&trc);
+    Compartment::traceIncomingCrossCompartmentEdgesForZoneGC(
+        &trc, Compartment::GrayEdges);
+  }
+
+  // Sweep everything to fix up weak pointers.
+  jit::JitRuntime::TraceWeakJitcodeGlobalTable(rt, &trc);
+  for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) {
+    cache->traceWeak(&trc, nullptr);
+  }
+
+  // Type inference may put more blocks here to free.
+  {
+    AutoLockHelperThreadState lock;
+    lifoBlocksToFree.ref().freeAll();
+  }
+
+  // Call callbacks to get the rest of the system to fixup other untraced
+  // pointers.
+  callWeakPointerZonesCallbacks(&trc);
+}
+
+void GCRuntime::clearRelocatedArenas(Arena* arenaList, JS::GCReason reason) {
+  AutoLockGC lock(this);
+  clearRelocatedArenasWithoutUnlocking(arenaList, reason, lock);
+}
+
+void GCRuntime::clearRelocatedArenasWithoutUnlocking(Arena* arenaList,
+                                                     JS::GCReason reason,
+                                                     const AutoLockGC& lock) {
+  // Clear the relocated arenas, now containing only forwarding pointers
+  while (arenaList) {
+    Arena* arena = arenaList;
+    arenaList = arenaList->next;
+
+    // Clear the mark bits
+    arena->unmarkAll();
+
+    // Mark arena as empty
+    arena->setAsFullyUnused();
+
+#ifdef DEBUG
+    // The cell contents have been partially marked no access in RelocateCell,
+    // so we need to mark the region as undefined again so we can poison it.
+    SetMemCheckKind(reinterpret_cast<void*>(arena->thingsStart()),
+                    arena->getThingsSpan(), MemCheckKind::MakeUndefined);
+#endif
+
+    AlwaysPoison(reinterpret_cast<void*>(arena->thingsStart()),
+                 JS_MOVED_TENURED_PATTERN, arena->getThingsSpan(),
+                 MemCheckKind::MakeNoAccess);
+
+    // Don't count emptied arenas as being freed by the current GC:
+    //  - if we purposely moved everything to new arenas, as that will already
+    //    have allocated a similar number of arenas. (This only happens for
+    //    collections triggered by GC zeal.)
+    //  - if they were allocated since the start of the GC.
+    bool allArenasRelocated = ShouldRelocateAllArenas(reason);
+    bool updateRetainedSize = !allArenasRelocated && !arena->isNewlyCreated();
+    arena->zone->gcHeapSize.removeBytes(ArenaSize, updateRetainedSize,
+                                        heapSize);
+
+    // Release the arena but don't return it to the chunk yet.
+    arena->release(lock);
+  }
+}
+
+#ifdef DEBUG
+
+// In debug mode we don't always release relocated arenas straight away.
+// Sometimes protect them instead and hold onto them until the next GC sweep
+// phase to catch any pointers to them that didn't get forwarded.
+
+static inline bool CanProtectArenas() {
+  // On some systems the page size is larger than the size of an arena so we
+  // can't change the mapping permissions per arena.
+  return SystemPageSize() <= ArenaSize;
+}
+
+static inline bool ShouldProtectRelocatedArenas(JS::GCReason reason) {
+  // For zeal mode collections we don't release the relocated arenas
+  // immediately. Instead we protect them and keep them around until the next
+  // collection so we can catch any stray accesses to them.
+  return reason == JS::GCReason::DEBUG_GC && CanProtectArenas();
+}
+
+void GCRuntime::protectOrReleaseRelocatedArenas(Arena* arenaList,
+                                                JS::GCReason reason) {
+  if (ShouldProtectRelocatedArenas(reason)) {
+    protectAndHoldArenas(arenaList);
+    return;
+  }
+
+  releaseRelocatedArenas(arenaList);
+}
+
+void GCRuntime::protectAndHoldArenas(Arena* arenaList) {
+  for (Arena* arena = arenaList; arena;) {
+    MOZ_ASSERT(!arena->allocated());
+    Arena* next = arena->next;
+    if (!next) {
+      // Prepend to hold list before we protect the memory.
+      AutoLockGC lock(this);
+      arena->next = relocatedArenasToRelease;
+      relocatedArenasToRelease = arenaList;
+    }
+    ProtectPages(arena, ArenaSize);
+    arena = next;
+  }
+}
+
+void GCRuntime::unprotectHeldRelocatedArenas(const AutoLockGC& lock) {
+  for (Arena* arena = relocatedArenasToRelease; arena; arena = arena->next) {
+    UnprotectPages(arena, ArenaSize);
+    MOZ_ASSERT(!arena->allocated());
+  }
+}
+
+void GCRuntime::releaseHeldRelocatedArenas() {
+  AutoLockGC lock(this);
+  unprotectHeldRelocatedArenas(lock);
+  Arena* arenas = relocatedArenasToRelease;
+  relocatedArenasToRelease = nullptr;
+  releaseRelocatedArenasWithoutUnlocking(arenas, lock);
+}
+
+void GCRuntime::releaseHeldRelocatedArenasWithoutUnlocking(
+    const AutoLockGC& lock) {
+  unprotectHeldRelocatedArenas(lock);
+  releaseRelocatedArenasWithoutUnlocking(relocatedArenasToRelease, lock);
+  relocatedArenasToRelease = nullptr;
+}
+
+#endif
+
+void GCRuntime::releaseRelocatedArenas(Arena* arenaList) {
+  AutoLockGC lock(this);
+  releaseRelocatedArenasWithoutUnlocking(arenaList, lock);
+}
+
+void GCRuntime::releaseRelocatedArenasWithoutUnlocking(Arena* arenaList,
+                                                       const AutoLockGC& lock) {
+  // Release relocated arenas previously cleared with clearRelocatedArenas().
+  while (arenaList) {
+    Arena* arena = arenaList;
+    arenaList = arenaList->next;
+
+    // We already updated the memory accounting so just call
+    // Chunk::releaseArena.
+    arena->chunk()->releaseArena(this, arena, lock);
+  }
+}