Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1735 insertions, 0 deletions

diff --git a/js/src/vm/HelperThreads.cpp b/js/src/vm/HelperThreads.cpp
new file mode 100644
index 0000000000..36792b1d09
--- /dev/null
+++ b/js/src/vm/HelperThreads.cpp
@@ -0,0 +1,1735 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "vm/HelperThreads.h"
+
+#include "mozilla/ReverseIterator.h"  // mozilla::Reversed(...)
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Span.h"  // mozilla::Span<TaggedScriptThingIndex>
+#include "mozilla/Utf8.h"  // mozilla::Utf8Unit
+
+#include <algorithm>
+
+#include "frontend/CompilationStencil.h"  // frontend::CompilationStencil
+#include "gc/GC.h"
+#include "jit/Ion.h"
+#include "jit/IonCompileTask.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitScript.h"
+#include "js/CompileOptions.h"  // JS::PrefableCompileOptions, JS::ReadOnlyCompileOptions
+#include "js/experimental/CompileScript.h"  // JS::ThreadStackQuotaForSize
+#include "js/friend/StackLimits.h"          // js::ReportOverRecursed
+#include "js/HelperThreadAPI.h"
+#include "js/Stack.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "threading/CpuCount.h"
+#include "vm/ErrorReporting.h"
+#include "vm/HelperThreadState.h"
+#include "vm/InternalThreadPool.h"
+#include "vm/MutexIDs.h"
+#include "wasm/WasmGenerator.h"
+
+using namespace js;
+
+using mozilla::TimeDuration;
+using mozilla::TimeStamp;
+using mozilla::Utf8Unit;
+
+using JS::DispatchReason;
+
+namespace js {
+
+Mutex gHelperThreadLock(mutexid::GlobalHelperThreadState);
+GlobalHelperThreadState* gHelperThreadState = nullptr;
+
+}  // namespace js
+
+bool js::CreateHelperThreadsState() {
+  MOZ_ASSERT(!gHelperThreadState);
+  gHelperThreadState = js_new<GlobalHelperThreadState>();
+  return gHelperThreadState;
+}
+
+void js::DestroyHelperThreadsState() {
+  AutoLockHelperThreadState lock;
+
+  if (!gHelperThreadState) {
+    return;
+  }
+
+  gHelperThreadState->finish(lock);
+  js_delete(gHelperThreadState);
+  gHelperThreadState = nullptr;
+}
+
+bool js::EnsureHelperThreadsInitialized() {
+  MOZ_ASSERT(gHelperThreadState);
+  return gHelperThreadState->ensureInitialized();
+}
+
+static size_t ClampDefaultCPUCount(size_t cpuCount) {
+  // It's extremely rare for SpiderMonkey to have more than a few cores worth
+  // of work. At higher core counts, performance can even decrease due to NUMA
+  // (and SpiderMonkey's lack of NUMA-awareness), contention, and general lack
+  // of optimization for high core counts. So to avoid wasting thread stack
+  // resources (and cluttering gdb and core dumps), clamp to 8 cores for now.
+  return std::min<size_t>(cpuCount, 8);
+}
+
+static size_t ThreadCountForCPUCount(size_t cpuCount) {
+  // We need at least two threads for tier-2 wasm compilations, because
+  // there's a master task that holds a thread while other threads do the
+  // compilation.
+  return std::max<size_t>(cpuCount, 2);
+}
+
+bool js::SetFakeCPUCount(size_t count) {
+  HelperThreadState().setCpuCount(count);
+  return true;
+}
+
+void GlobalHelperThreadState::setCpuCount(size_t count) {
+  // This must be called before any threads have been initialized.
+  AutoLockHelperThreadState lock;
+  MOZ_ASSERT(!isInitialized(lock));
+
+  // We can't do this if an external thread pool is in use.
+  MOZ_ASSERT(!dispatchTaskCallback);
+
+  cpuCount = count;
+  threadCount = ThreadCountForCPUCount(count);
+}
+
+size_t js::GetHelperThreadCount() { return HelperThreadState().threadCount; }
+
+size_t js::GetHelperThreadCPUCount() { return HelperThreadState().cpuCount; }
+
+size_t js::GetMaxWasmCompilationThreads() {
+  return HelperThreadState().maxWasmCompilationThreads();
+}
+
+void JS::SetProfilingThreadCallbacks(
+    JS::RegisterThreadCallback registerThread,
+    JS::UnregisterThreadCallback unregisterThread) {
+  HelperThreadState().registerThread = registerThread;
+  HelperThreadState().unregisterThread = unregisterThread;
+}
+
+// Bug 1630189: Without MOZ_NEVER_INLINE, Windows PGO builds have a linking
+// error for HelperThreadTaskCallback.
+JS_PUBLIC_API MOZ_NEVER_INLINE void JS::SetHelperThreadTaskCallback(
+    HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize) {
+  AutoLockHelperThreadState lock;
+  HelperThreadState().setDispatchTaskCallback(callback, threadCount, stackSize,
+                                              lock);
+}
+
+void GlobalHelperThreadState::setDispatchTaskCallback(
+    JS::HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize,
+    const AutoLockHelperThreadState& lock) {
+  MOZ_ASSERT(!isInitialized(lock));
+  MOZ_ASSERT(!dispatchTaskCallback);
+  MOZ_ASSERT(threadCount != 0);
+  MOZ_ASSERT(stackSize >= 16 * 1024);
+
+  dispatchTaskCallback = callback;
+  this->threadCount = threadCount;
+  this->stackQuota = JS::ThreadStackQuotaForSize(stackSize);
+}
+
+bool js::StartOffThreadWasmCompile(wasm::CompileTask* task,
+                                   wasm::CompileMode mode) {
+  return HelperThreadState().submitTask(task, mode);
+}
+
+bool GlobalHelperThreadState::submitTask(wasm::CompileTask* task,
+                                         wasm::CompileMode mode) {
+  AutoLockHelperThreadState lock;
+  if (!wasmWorklist(lock, mode).pushBack(task)) {
+    return false;
+  }
+
+  dispatch(DispatchReason::NewTask, lock);
+  return true;
+}
+
+size_t js::RemovePendingWasmCompileTasks(
+    const wasm::CompileTaskState& taskState, wasm::CompileMode mode,
+    const AutoLockHelperThreadState& lock) {
+  wasm::CompileTaskPtrFifo& worklist =
+      HelperThreadState().wasmWorklist(lock, mode);
+  return worklist.eraseIf([&taskState](wasm::CompileTask* task) {
+    return &task->state == &taskState;
+  });
+}
+
+void js::StartOffThreadWasmTier2Generator(wasm::UniqueTier2GeneratorTask task) {
+  (void)HelperThreadState().submitTask(std::move(task));
+}
+
+bool GlobalHelperThreadState::submitTask(wasm::UniqueTier2GeneratorTask task) {
+  AutoLockHelperThreadState lock;
+
+  MOZ_ASSERT(isInitialized(lock));
+
+  if (!wasmTier2GeneratorWorklist(lock).append(task.get())) {
+    return false;
+  }
+  (void)task.release();
+
+  dispatch(DispatchReason::NewTask, lock);
+  return true;
+}
+
+static void CancelOffThreadWasmTier2GeneratorLocked(
+    AutoLockHelperThreadState& lock) {
+  if (!HelperThreadState().isInitialized(lock)) {
+    return;
+  }
+
+  // Remove pending tasks from the tier2 generator worklist and cancel and
+  // delete them.
+  {
+    wasm::Tier2GeneratorTaskPtrVector& worklist =
+        HelperThreadState().wasmTier2GeneratorWorklist(lock);
+    for (size_t i = 0; i < worklist.length(); i++) {
+      wasm::Tier2GeneratorTask* task = worklist[i];
+      HelperThreadState().remove(worklist, &i);
+      js_delete(task);
+    }
+  }
+
+  // There is at most one running Tier2Generator task and we assume that
+  // below.
+  static_assert(GlobalHelperThreadState::MaxTier2GeneratorTasks == 1,
+                "code must be generalized");
+
+  // If there is a running Tier2 generator task, shut it down in a predictable
+  // way.  The task will be deleted by the normal deletion logic.
+  for (auto* helper : HelperThreadState().helperTasks(lock)) {
+    if (helper->is<wasm::Tier2GeneratorTask>()) {
+      // Set a flag that causes compilation to shortcut itself.
+      helper->as<wasm::Tier2GeneratorTask>()->cancel();
+
+      // Wait for the generator task to finish.  This avoids a shutdown race
+      // where the shutdown code is trying to shut down helper threads and the
+      // ongoing tier2 compilation is trying to finish, which requires it to
+      // have access to helper threads.
+      uint32_t oldFinishedCount =
+          HelperThreadState().wasmTier2GeneratorsFinished(lock);
+      while (HelperThreadState().wasmTier2GeneratorsFinished(lock) ==
+             oldFinishedCount) {
+        HelperThreadState().wait(lock);
+      }
+
+      // At most one of these tasks.
+      break;
+    }
+  }
+}
+
+void js::CancelOffThreadWasmTier2Generator() {
+  AutoLockHelperThreadState lock;
+  CancelOffThreadWasmTier2GeneratorLocked(lock);
+}
+
+bool js::StartOffThreadIonCompile(jit::IonCompileTask* task,
+                                  const AutoLockHelperThreadState& lock) {
+  return HelperThreadState().submitTask(task, lock);
+}
+
+bool GlobalHelperThreadState::submitTask(
+    jit::IonCompileTask* task, const AutoLockHelperThreadState& locked) {
+  MOZ_ASSERT(isInitialized(locked));
+
+  if (!ionWorklist(locked).append(task)) {
+    return false;
+  }
+
+  // The build is moving off-thread. Freeze the LifoAlloc to prevent any
+  // unwanted mutations.
+  task->alloc().lifoAlloc()->setReadOnly();
+
+  dispatch(DispatchReason::NewTask, locked);
+  return true;
+}
+
+bool js::AutoStartIonFreeTask::addIonCompileToFreeTaskBatch(
+    jit::IonCompileTask* task) {
+  return jitRuntime_->addIonCompileToFreeTaskBatch(task);
+}
+
+js::AutoStartIonFreeTask::~AutoStartIonFreeTask() {
+  jitRuntime_->maybeStartIonFreeTask(force_);
+}
+
+void jit::JitRuntime::maybeStartIonFreeTask(bool force) {
+  IonFreeCompileTasks& tasks = ionFreeTaskBatch_.ref();
+  if (tasks.empty()) {
+    return;
+  }
+
+  // Start an IonFreeTask if we have at least eight tasks. If |force| is true we
+  // always start an IonFreeTask.
+  if (!force) {
+    constexpr size_t MinBatchSize = 8;
+    static_assert(IonFreeCompileTasks::InlineLength >= MinBatchSize,
+                  "Minimum batch size shouldn't require malloc");
+    if (tasks.length() < MinBatchSize) {
+      return;
+    }
+  }
+
+  auto freeTask = js::MakeUnique<jit::IonFreeTask>(std::move(tasks));
+  if (!freeTask) {
+    // Free compilation data on the main thread instead.
+    MOZ_ASSERT(!tasks.empty(), "shouldn't have moved tasks on OOM");
+    jit::FreeIonCompileTasks(tasks);
+    tasks.clearAndFree();
+    return;
+  }
+
+  AutoLockHelperThreadState lock;
+  if (!HelperThreadState().submitTask(std::move(freeTask), lock)) {
+    // If submitTask OOMs, then freeTask hasn't been moved so we can still use
+    // its task list.
+    jit::FreeIonCompileTasks(freeTask->compileTasks());
+  }
+
+  tasks.clearAndFree();
+}
+
+bool GlobalHelperThreadState::submitTask(
+    UniquePtr<jit::IonFreeTask>&& task,
+    const AutoLockHelperThreadState& locked) {
+  MOZ_ASSERT(isInitialized(locked));
+
+  if (!ionFreeList(locked).append(std::move(task))) {
+    return false;
+  }
+
+  dispatch(DispatchReason::NewTask, locked);
+  return true;
+}
+
+/*
+ * Move an IonCompilationTask for which compilation has either finished, failed,
+ * or been cancelled into the global finished compilation list. All off thread
+ * compilations which are started must eventually be finished.
+ */
+void js::FinishOffThreadIonCompile(jit::IonCompileTask* task,
+                                   const AutoLockHelperThreadState& lock) {
+  AutoEnterOOMUnsafeRegion oomUnsafe;
+  if (!HelperThreadState().ionFinishedList(lock).append(task)) {
+    oomUnsafe.crash("FinishOffThreadIonCompile");
+  }
+  task->script()
+      ->runtimeFromAnyThread()
+      ->jitRuntime()
+      ->numFinishedOffThreadTasksRef(lock)++;
+}
+
+static JSRuntime* GetSelectorRuntime(const CompilationSelector& selector) {
+  struct Matcher {
+    JSRuntime* operator()(JSScript* script) {
+      return script->runtimeFromMainThread();
+    }
+    JSRuntime* operator()(Zone* zone) { return zone->runtimeFromMainThread(); }
+    JSRuntime* operator()(ZonesInState zbs) { return zbs.runtime; }
+    JSRuntime* operator()(JSRuntime* runtime) { return runtime; }
+  };
+
+  return selector.match(Matcher());
+}
+
+static bool JitDataStructuresExist(const CompilationSelector& selector) {
+  struct Matcher {
+    bool operator()(JSScript* script) { return !!script->zone()->jitZone(); }
+    bool operator()(Zone* zone) { return !!zone->jitZone(); }
+    bool operator()(ZonesInState zbs) { return zbs.runtime->hasJitRuntime(); }
+    bool operator()(JSRuntime* runtime) { return runtime->hasJitRuntime(); }
+  };
+
+  return selector.match(Matcher());
+}
+
+static bool IonCompileTaskMatches(const CompilationSelector& selector,
+                                  jit::IonCompileTask* task) {
+  struct TaskMatches {
+    jit::IonCompileTask* task_;
+
+    bool operator()(JSScript* script) { return script == task_->script(); }
+    bool operator()(Zone* zone) {
+      return zone == task_->script()->zoneFromAnyThread();
+    }
+    bool operator()(JSRuntime* runtime) {
+      return runtime == task_->script()->runtimeFromAnyThread();
+    }
+    bool operator()(ZonesInState zbs) {
+      return zbs.runtime == task_->script()->runtimeFromAnyThread() &&
+             zbs.state == task_->script()->zoneFromAnyThread()->gcState();
+    }
+  };
+
+  return selector.match(TaskMatches{task});
+}
+
+// If we're canceling Ion compilations for a zone/runtime, force a new
+// IonFreeTask even if there are just a few tasks. This lets us free as much
+// memory as possible.
+static bool ShouldForceIonFreeTask(const CompilationSelector& selector) {
+  struct Matcher {
+    bool operator()(JSScript* script) { return false; }
+    bool operator()(Zone* zone) { return true; }
+    bool operator()(ZonesInState zbs) { return true; }
+    bool operator()(JSRuntime* runtime) { return true; }
+  };
+
+  return selector.match(Matcher());
+}
+
+void js::CancelOffThreadIonCompile(const CompilationSelector& selector) {
+  if (!JitDataStructuresExist(selector)) {
+    return;
+  }
+
+  if (jit::IsPortableBaselineInterpreterEnabled()) {
+    return;
+  }
+
+  jit::JitRuntime* jitRuntime = GetSelectorRuntime(selector)->jitRuntime();
+  MOZ_ASSERT(jitRuntime);
+
+  AutoStartIonFreeTask freeTask(jitRuntime, ShouldForceIonFreeTask(selector));
+
+  {
+    AutoLockHelperThreadState lock;
+    if (!HelperThreadState().isInitialized(lock)) {
+      return;
+    }
+
+    /* Cancel any pending entries for which processing hasn't started. */
+    GlobalHelperThreadState::IonCompileTaskVector& worklist =
+        HelperThreadState().ionWorklist(lock);
+    for (size_t i = 0; i < worklist.length(); i++) {
+      jit::IonCompileTask* task = worklist[i];
+      if (IonCompileTaskMatches(selector, task)) {
+        // Once finished, tasks are added to a Linked list which is
+        // allocated with the IonCompileTask class. The IonCompileTask is
+        // allocated in the LifoAlloc so we need the LifoAlloc to be mutable.
+        worklist[i]->alloc().lifoAlloc()->setReadWrite();
+
+        FinishOffThreadIonCompile(task, lock);
+        HelperThreadState().remove(worklist, &i);
+      }
+    }
+
+    /* Wait for in progress entries to finish up. */
+    bool cancelled;
+    do {
+      cancelled = false;
+      for (auto* helper : HelperThreadState().helperTasks(lock)) {
+        if (!helper->is<jit::IonCompileTask>()) {
+          continue;
+        }
+
+        jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
+        if (IonCompileTaskMatches(selector, ionCompileTask)) {
+          ionCompileTask->mirGen().cancel();
+          cancelled = true;
+        }
+      }
+      if (cancelled) {
+        HelperThreadState().wait(lock);
+      }
+    } while (cancelled);
+
+    /* Cancel code generation for any completed entries. */
+    GlobalHelperThreadState::IonCompileTaskVector& finished =
+        HelperThreadState().ionFinishedList(lock);
+    for (size_t i = 0; i < finished.length(); i++) {
+      jit::IonCompileTask* task = finished[i];
+      if (IonCompileTaskMatches(selector, task)) {
+        JSRuntime* rt = task->script()->runtimeFromAnyThread();
+        rt->jitRuntime()->numFinishedOffThreadTasksRef(lock)--;
+        jit::FinishOffThreadTask(rt, freeTask, task);
+        HelperThreadState().remove(finished, &i);
+      }
+    }
+  }
+
+  /* Cancel lazy linking for pending tasks (attached to the ionScript). */
+  JSRuntime* runtime = GetSelectorRuntime(selector);
+  jit::IonCompileTask* task =
+      runtime->jitRuntime()->ionLazyLinkList(runtime).getFirst();
+  while (task) {
+    jit::IonCompileTask* next = task->getNext();
+    if (IonCompileTaskMatches(selector, task)) {
+      jit::FinishOffThreadTask(runtime, freeTask, task);
+    }
+    task = next;
+  }
+}
+
+#ifdef DEBUG
+bool js::HasOffThreadIonCompile(Zone* zone) {
+  if (jit::IsPortableBaselineInterpreterEnabled()) {
+    return false;
+  }
+
+  AutoLockHelperThreadState lock;
+
+  if (!HelperThreadState().isInitialized(lock)) {
+    return false;
+  }
+
+  GlobalHelperThreadState::IonCompileTaskVector& worklist =
+      HelperThreadState().ionWorklist(lock);
+  for (size_t i = 0; i < worklist.length(); i++) {
+    jit::IonCompileTask* task = worklist[i];
+    if (task->script()->zoneFromAnyThread() == zone) {
+      return true;
+    }
+  }
+
+  for (auto* helper : HelperThreadState().helperTasks(lock)) {
+    if (!helper->is<jit::IonCompileTask>()) {
+      continue;
+    }
+    JSScript* script = helper->as<jit::IonCompileTask>()->script();
+    if (script->zoneFromAnyThread() == zone) {
+      return true;
+    }
+  }
+
+  GlobalHelperThreadState::IonCompileTaskVector& finished =
+      HelperThreadState().ionFinishedList(lock);
+  for (size_t i = 0; i < finished.length(); i++) {
+    jit::IonCompileTask* task = finished[i];
+    if (task->script()->zoneFromAnyThread() == zone) {
+      return true;
+    }
+  }
+
+  JSRuntime* rt = zone->runtimeFromMainThread();
+  if (rt->hasJitRuntime()) {
+    jit::IonCompileTask* task =
+        rt->jitRuntime()->ionLazyLinkList(rt).getFirst();
+    while (task) {
+      if (task->script()->zone() == zone) {
+        return true;
+      }
+      task = task->getNext();
+    }
+  }
+
+  return false;
+}
+#endif
+
+void js::StartOffThreadDelazification(
+    JSContext* maybeCx, const JS::ReadOnlyCompileOptions& options,
+    const frontend::CompilationStencil& stencil) {
+  // Skip delazify tasks if we parse everything on-demand or ahead.
+  auto strategy = options.eagerDelazificationStrategy();
+  if (strategy == JS::DelazificationOption::OnDemandOnly ||
+      strategy == JS::DelazificationOption::ParseEverythingEagerly) {
+    return;
+  }
+
+  // Skip delazify task if code coverage is enabled.
+  if (maybeCx && maybeCx->realm()->collectCoverageForDebug()) {
+    return;
+  }
+
+  if (!CanUseExtraThreads()) {
+    return;
+  }
+
+  JSRuntime* maybeRuntime = maybeCx ? maybeCx->runtime() : nullptr;
+  UniquePtr<DelazifyTask> task;
+  task = DelazifyTask::Create(maybeRuntime, options, stencil);
+  if (!task) {
+    return;
+  }
+
+  // Schedule delazification task if there is any function to delazify.
+  if (!task->done()) {
+    AutoLockHelperThreadState lock;
+    HelperThreadState().submitTask(task.release(), lock);
+  }
+}
+
+UniquePtr<DelazifyTask> DelazifyTask::Create(
+    JSRuntime* maybeRuntime, const JS::ReadOnlyCompileOptions& options,
+    const frontend::CompilationStencil& stencil) {
+  UniquePtr<DelazifyTask> task;
+  task.reset(js_new<DelazifyTask>(maybeRuntime, options.prefableOptions()));
+  if (!task) {
+    return nullptr;
+  }
+
+  if (!task->init(options, stencil)) {
+    // In case of errors, skip this and delazify on-demand.
+    return nullptr;
+  }
+
+  return task;
+}
+
+DelazifyTask::DelazifyTask(
+    JSRuntime* maybeRuntime,
+    const JS::PrefableCompileOptions& initialPrefableOptions)
+    : maybeRuntime(maybeRuntime),
+      delazificationCx(initialPrefableOptions, HelperThreadState().stackQuota) {
+}
+
+DelazifyTask::~DelazifyTask() {
+  // The LinkedListElement destructor will remove us from any list we are part
+  // of without synchronization, so ensure that doesn't happen.
+  MOZ_DIAGNOSTIC_ASSERT(!isInList());
+}
+
+bool DelazifyTask::init(const JS::ReadOnlyCompileOptions& options,
+                        const frontend::CompilationStencil& stencil) {
+  return delazificationCx.init(options, stencil);
+}
+
+size_t DelazifyTask::sizeOfExcludingThis(
+    mozilla::MallocSizeOf mallocSizeOf) const {
+  return delazificationCx.sizeOfExcludingThis(mallocSizeOf);
+}
+
+void DelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
+  {
+    AutoUnlockHelperThreadState unlock(lock);
+    // NOTE: We do not report errors beyond this scope, as there is no where
+    // to report these errors to. In the mean time, prevent the eager
+    // delazification from running after any kind of errors.
+    (void)runTask();
+  }
+
+  // If we should continue to delazify even more functions, then re-add this
+  // task to the vector of delazification tasks. This might happen when the
+  // DelazifyTask is interrupted by a higher priority task. (see
+  // mozilla::TaskController & mozilla::Task)
+  if (!delazificationCx.done()) {
+    HelperThreadState().submitTask(this, lock);
+  } else {
+    UniquePtr<FreeDelazifyTask> freeTask(js_new<FreeDelazifyTask>(this));
+    if (freeTask) {
+      HelperThreadState().submitTask(std::move(freeTask), lock);
+    }
+  }
+}
+
+bool DelazifyTask::runTask() { return delazificationCx.delazify(); }
+
+bool DelazifyTask::done() const { return delazificationCx.done(); }
+
+void FreeDelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
+  {
+    AutoUnlockHelperThreadState unlock(locked);
+    js_delete(task);
+    task = nullptr;
+  }
+
+  js_delete(this);
+}
+
+static void CancelPendingDelazifyTask(JSRuntime* rt,
+                                      AutoLockHelperThreadState& lock) {
+  auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
+
+  auto end = delazifyList.end();
+  for (auto iter = delazifyList.begin(); iter != end;) {
+    DelazifyTask* task = *iter;
+    ++iter;
+    if (task->runtimeMatchesOrNoRuntime(rt)) {
+      task->removeFrom(delazifyList);
+      js_delete(task);
+    }
+  }
+}
+
+static void WaitUntilCancelledDelazifyTasks(JSRuntime* rt,
+                                            AutoLockHelperThreadState& lock) {
+  if (!HelperThreadState().isInitialized(lock)) {
+    return;
+  }
+
+  while (true) {
+    CancelPendingDelazifyTask(rt, lock);
+
+    // If running tasks are delazifying any functions, then we have to wait
+    // until they complete to remove them from the pending list. DelazifyTask
+    // are inserting themself back to be processed once more after delazifying a
+    // function.
+    bool inProgress = false;
+    for (auto* helper : HelperThreadState().helperTasks(lock)) {
+      if (helper->is<DelazifyTask>() &&
+          helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
+        inProgress = true;
+        break;
+      }
+    }
+    if (!inProgress) {
+      break;
+    }
+
+    HelperThreadState().wait(lock);
+  }
+
+#ifdef DEBUG
+  for (DelazifyTask* task : HelperThreadState().delazifyWorklist(lock)) {
+    MOZ_ASSERT(!task->runtimeMatchesOrNoRuntime(rt));
+  }
+  for (auto* helper : HelperThreadState().helperTasks(lock)) {
+    MOZ_ASSERT_IF(helper->is<DelazifyTask>(),
+                  !helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt));
+  }
+#endif
+}
+
+static void WaitUntilEmptyFreeDelazifyTaskVector(
+    AutoLockHelperThreadState& lock) {
+  if (!HelperThreadState().isInitialized(lock)) {
+    return;
+  }
+
+  while (true) {
+    bool inProgress = false;
+    auto& freeList = HelperThreadState().freeDelazifyTaskVector(lock);
+    if (!freeList.empty()) {
+      inProgress = true;
+    }
+
+    // If running tasks are delazifying any functions, then we have to wait
+    // until they complete to remove them from the pending list. DelazifyTask
+    // are inserting themself back to be processed once more after delazifying a
+    // function.
+    for (auto* helper : HelperThreadState().helperTasks(lock)) {
+      if (helper->is<FreeDelazifyTask>()) {
+        inProgress = true;
+        break;
+      }
+    }
+    if (!inProgress) {
+      break;
+    }
+
+    HelperThreadState().wait(lock);
+  }
+}
+
+void js::CancelOffThreadDelazify(JSRuntime* runtime) {
+  AutoLockHelperThreadState lock;
+
+  // Cancel all Delazify tasks from the given runtime, and wait if tasks are
+  // from the given runtime are being executed.
+  WaitUntilCancelledDelazifyTasks(runtime, lock);
+
+  // Empty the free list of delazify task, in case one of the delazify task
+  // ended and therefore did not returned to the pending list of delazify tasks.
+  WaitUntilEmptyFreeDelazifyTaskVector(lock);
+}
+
+static bool HasAnyDelazifyTask(JSRuntime* rt, AutoLockHelperThreadState& lock) {
+  auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
+  for (auto task : delazifyList) {
+    if (task->runtimeMatchesOrNoRuntime(rt)) {
+      return true;
+    }
+  }
+
+  for (auto* helper : HelperThreadState().helperTasks(lock)) {
+    if (helper->is<DelazifyTask>() &&
+        helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void js::WaitForAllDelazifyTasks(JSRuntime* rt) {
+  AutoLockHelperThreadState lock;
+  if (!HelperThreadState().isInitialized(lock)) {
+    return;
+  }
+
+  while (true) {
+    if (!HasAnyDelazifyTask(rt, lock)) {
+      break;
+    }
+
+    HelperThreadState().wait(lock);
+  }
+}
+
+void GlobalHelperThreadState::submitTask(
+    DelazifyTask* task, const AutoLockHelperThreadState& locked) {
+  delazifyWorklist(locked).insertBack(task);
+  dispatch(DispatchReason::NewTask, locked);
+}
+
+bool GlobalHelperThreadState::submitTask(
+    UniquePtr<FreeDelazifyTask> task, const AutoLockHelperThreadState& locked) {
+  if (!freeDelazifyTaskVector(locked).append(std::move(task))) {
+    return false;
+  }
+  dispatch(DispatchReason::NewTask, locked);
+  return true;
+}
+
+bool GlobalHelperThreadState::ensureInitialized() {
+  MOZ_ASSERT(CanUseExtraThreads());
+  MOZ_ASSERT(this == &HelperThreadState());
+
+  AutoLockHelperThreadState lock;
+
+  if (isInitialized(lock)) {
+    return true;
+  }
+
+  for (size_t& i : runningTaskCount) {
+    i = 0;
+  }
+
+  useInternalThreadPool_ = !dispatchTaskCallback;
+  if (useInternalThreadPool(lock)) {
+    if (!InternalThreadPool::Initialize(threadCount, lock)) {
+      return false;
+    }
+  }
+
+  MOZ_ASSERT(dispatchTaskCallback);
+
+  if (!ensureThreadCount(threadCount, lock)) {
+    finishThreads(lock);
+    return false;
+  }
+
+  MOZ_ASSERT(threadCount != 0);
+  isInitialized_ = true;
+  return true;
+}
+
+bool GlobalHelperThreadState::ensureThreadCount(
+    size_t count, AutoLockHelperThreadState& lock) {
+  if (!helperTasks_.reserve(count)) {
+    return false;
+  }
+
+  if (useInternalThreadPool(lock)) {
+    InternalThreadPool& pool = InternalThreadPool::Get();
+    if (pool.threadCount(lock) < count) {
+      if (!pool.ensureThreadCount(count, lock)) {
+        return false;
+      }
+
+      threadCount = pool.threadCount(lock);
+    }
+  }
+
+  return true;
+}
+
+GlobalHelperThreadState::GlobalHelperThreadState()
+    : cpuCount(0),
+      threadCount(0),
+      totalCountRunningTasks(0),
+      registerThread(nullptr),
+      unregisterThread(nullptr),
+      wasmTier2GeneratorsFinished_(0) {
+  MOZ_ASSERT(!gHelperThreadState);
+
+  cpuCount = ClampDefaultCPUCount(GetCPUCount());
+  threadCount = ThreadCountForCPUCount(cpuCount);
+
+  MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
+}
+
+void GlobalHelperThreadState::finish(AutoLockHelperThreadState& lock) {
+  if (!isInitialized(lock)) {
+    return;
+  }
+
+  finishThreads(lock);
+
+  // Make sure there are no Ion free tasks left. We check this here because,
+  // unlike the other tasks, we don't explicitly block on this when
+  // destroying a runtime.
+  auto& freeList = ionFreeList(lock);
+  while (!freeList.empty()) {
+    UniquePtr<jit::IonFreeTask> task = std::move(freeList.back());
+    freeList.popBack();
+    jit::FreeIonCompileTasks(task->compileTasks());
+  }
+}
+
+void GlobalHelperThreadState::finishThreads(AutoLockHelperThreadState& lock) {
+  waitForAllTasksLocked(lock);
+  terminating_ = true;
+
+  if (InternalThreadPool::IsInitialized()) {
+    InternalThreadPool::ShutDown(lock);
+  }
+}
+
+#ifdef DEBUG
+void GlobalHelperThreadState::assertIsLockedByCurrentThread() const {
+  gHelperThreadLock.assertOwnedByCurrentThread();
+}
+#endif  // DEBUG
+
+void GlobalHelperThreadState::dispatch(
+    DispatchReason reason, const AutoLockHelperThreadState& locked) {
+  if (canStartTasks(locked) && tasksPending_ < threadCount) {
+    // This doesn't guarantee that we don't dispatch more tasks to the external
+    // pool than necessary if tasks are taking a long time to start, but it does
+    // limit the number.
+    tasksPending_++;
+
+    // The hazard analysis can't tell that the callback doesn't GC.
+    JS::AutoSuppressGCAnalysis nogc;
+
+    dispatchTaskCallback(reason);
+  }
+}
+
+void GlobalHelperThreadState::wait(
+    AutoLockHelperThreadState& locked,
+    TimeDuration timeout /* = TimeDuration::Forever() */) {
+  consumerWakeup.wait_for(locked, timeout);
+}
+
+void GlobalHelperThreadState::notifyAll(const AutoLockHelperThreadState&) {
+  consumerWakeup.notify_all();
+}
+
+void GlobalHelperThreadState::notifyOne(const AutoLockHelperThreadState&) {
+  consumerWakeup.notify_one();
+}
+
+bool GlobalHelperThreadState::hasActiveThreads(
+    const AutoLockHelperThreadState& lock) {
+  return !helperTasks(lock).empty();
+}
+
+void js::WaitForAllHelperThreads() { HelperThreadState().waitForAllTasks(); }
+
+void js::WaitForAllHelperThreads(AutoLockHelperThreadState& lock) {
+  HelperThreadState().waitForAllTasksLocked(lock);
+}
+
+void GlobalHelperThreadState::waitForAllTasks() {
+  AutoLockHelperThreadState lock;
+  waitForAllTasksLocked(lock);
+}
+
+void GlobalHelperThreadState::waitForAllTasksLocked(
+    AutoLockHelperThreadState& lock) {
+  CancelOffThreadWasmTier2GeneratorLocked(lock);
+
+  while (canStartTasks(lock) || tasksPending_ || hasActiveThreads(lock)) {
+    wait(lock);
+  }
+
+  MOZ_ASSERT(gcParallelWorklist().isEmpty(lock));
+  MOZ_ASSERT(ionWorklist(lock).empty());
+  MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier1).empty());
+  MOZ_ASSERT(promiseHelperTasks(lock).empty());
+  MOZ_ASSERT(compressionWorklist(lock).empty());
+  MOZ_ASSERT(ionFreeList(lock).empty());
+  MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier2).empty());
+  MOZ_ASSERT(wasmTier2GeneratorWorklist(lock).empty());
+  MOZ_ASSERT(!tasksPending_);
+  MOZ_ASSERT(!hasActiveThreads(lock));
+}
+
+// A task can be a "master" task, ie, it will block waiting for other worker
+// threads that perform work on its behalf.  If so it must not take the last
+// available thread; there must always be at least one worker thread able to do
+// the actual work.  (Or the system may deadlock.)
+//
+// If a task is a master task it *must* pass isMaster=true here, or perform a
+// similar calculation to avoid deadlock from starvation.
+//
+// isMaster should only be true if the thread calling checkTaskThreadLimit() is
+// a helper thread.
+//
+// NOTE: Calling checkTaskThreadLimit() from a helper thread in the dynamic
+// region after currentTask.emplace() and before currentTask.reset() may cause
+// it to return a different result than if it is called outside that dynamic
+// region, as the predicate inspects the values of the threads' currentTask
+// members.
+
+bool GlobalHelperThreadState::checkTaskThreadLimit(
+    ThreadType threadType, size_t maxThreads, bool isMaster,
+    const AutoLockHelperThreadState& lock) const {
+  MOZ_ASSERT(maxThreads > 0);
+
+  if (!isMaster && maxThreads >= threadCount) {
+    return true;
+  }
+
+  size_t count = runningTaskCount[threadType];
+  if (count >= maxThreads) {
+    return false;
+  }
+
+  MOZ_ASSERT(threadCount >= totalCountRunningTasks);
+  size_t idle = threadCount - totalCountRunningTasks;
+
+  // It is possible for the number of idle threads to be zero here, because
+  // checkTaskThreadLimit() can be called from non-helper threads.  Notably,
+  // the compression task scheduler invokes it, and runs off a helper thread.
+  if (idle == 0) {
+    return false;
+  }
+
+  // A master thread that's the last available thread must not be allowed to
+  // run.
+  if (isMaster && idle == 1) {
+    return false;
+  }
+
+  return true;
+}
+
+static inline bool IsHelperThreadSimulatingOOM(js::ThreadType threadType) {
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+  return js::oom::simulator.targetThread() == threadType;
+#else
+  return false;
+#endif
+}
+
+void GlobalHelperThreadState::addSizeOfIncludingThis(
+    JS::GlobalStats* stats, const AutoLockHelperThreadState& lock) const {
+#ifdef DEBUG
+  assertIsLockedByCurrentThread();
+#endif
+
+  mozilla::MallocSizeOf mallocSizeOf = stats->mallocSizeOf_;
+  JS::HelperThreadStats& htStats = stats->helperThread;
+
+  htStats.stateData += mallocSizeOf(this);
+
+  if (InternalThreadPool::IsInitialized()) {
+    htStats.stateData +=
+        InternalThreadPool::Get().sizeOfIncludingThis(mallocSizeOf, lock);
+  }
+
+  // Report memory used by various containers
+  htStats.stateData +=
+      ionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
+      ionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
+      ionFreeList_.sizeOfExcludingThis(mallocSizeOf) +
+      wasmWorklist_tier1_.sizeOfExcludingThis(mallocSizeOf) +
+      wasmWorklist_tier2_.sizeOfExcludingThis(mallocSizeOf) +
+      wasmTier2GeneratorWorklist_.sizeOfExcludingThis(mallocSizeOf) +
+      promiseHelperTasks_.sizeOfExcludingThis(mallocSizeOf) +
+      compressionPendingList_.sizeOfExcludingThis(mallocSizeOf) +
+      compressionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
+      compressionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
+      gcParallelWorklist_.sizeOfExcludingThis(mallocSizeOf, lock) +
+      helperTasks_.sizeOfExcludingThis(mallocSizeOf);
+
+  // Report IonCompileTasks on wait lists
+  for (auto task : ionWorklist_) {
+    htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
+  }
+  for (auto task : ionFinishedList_) {
+    htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
+  }
+  for (const auto& task : ionFreeList_) {
+    for (auto* compileTask : task->compileTasks()) {
+      htStats.ionCompileTask += compileTask->sizeOfExcludingThis(mallocSizeOf);
+    }
+  }
+
+  // Report wasm::CompileTasks on wait lists
+  for (auto task : wasmWorklist_tier1_) {
+    htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
+  }
+  for (auto task : wasmWorklist_tier2_) {
+    htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
+  }
+
+  // Report number of helper threads.
+  MOZ_ASSERT(htStats.idleThreadCount == 0);
+  MOZ_ASSERT(threadCount >= totalCountRunningTasks);
+  htStats.activeThreadCount = totalCountRunningTasks;
+  htStats.idleThreadCount = threadCount - totalCountRunningTasks;
+}
+
+size_t GlobalHelperThreadState::maxIonCompilationThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_ION)) {
+    return 1;
+  }
+  return threadCount;
+}
+
+size_t GlobalHelperThreadState::maxIonFreeThreads() const {
+  // IonFree tasks are low priority. Limit to one thread to help avoid jemalloc
+  // lock contention.
+  return 1;
+}
+
+size_t GlobalHelperThreadState::maxWasmCompilationThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
+      IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
+    return 1;
+  }
+  return std::min(cpuCount, threadCount);
+}
+
+size_t GlobalHelperThreadState::maxWasmTier2GeneratorThreads() const {
+  return MaxTier2GeneratorTasks;
+}
+
+size_t GlobalHelperThreadState::maxPromiseHelperThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_PROMISE_TASK)) {
+    return 1;
+  }
+  return std::min(cpuCount, threadCount);
+}
+
+size_t GlobalHelperThreadState::maxDelazifyThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_DELAZIFY)) {
+    return 1;
+  }
+  return std::min(cpuCount, threadCount);
+}
+
+size_t GlobalHelperThreadState::maxCompressionThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_COMPRESS)) {
+    return 1;
+  }
+
+  // Compression is triggered on major GCs to compress ScriptSources. It is
+  // considered low priority work.
+  return 1;
+}
+
+size_t GlobalHelperThreadState::maxGCParallelThreads() const {
+  if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_GCPARALLEL)) {
+    return 1;
+  }
+  return threadCount;
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier1CompileTask(
+    const AutoLockHelperThreadState& lock) {
+  return maybeGetWasmCompile(lock, wasm::CompileMode::Tier1);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2CompileTask(
+    const AutoLockHelperThreadState& lock) {
+  return maybeGetWasmCompile(lock, wasm::CompileMode::Tier2);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetWasmCompile(
+    const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
+  if (!canStartWasmCompile(lock, mode)) {
+    return nullptr;
+  }
+
+  return wasmWorklist(lock, mode).popCopyFront();
+}
+
+bool GlobalHelperThreadState::canStartWasmTier1CompileTask(
+    const AutoLockHelperThreadState& lock) {
+  return canStartWasmCompile(lock, wasm::CompileMode::Tier1);
+}
+
+bool GlobalHelperThreadState::canStartWasmTier2CompileTask(
+    const AutoLockHelperThreadState& lock) {
+  return canStartWasmCompile(lock, wasm::CompileMode::Tier2);
+}
+
+bool GlobalHelperThreadState::canStartWasmCompile(
+    const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
+  if (wasmWorklist(lock, mode).empty()) {
+    return false;
+  }
+
+  // Parallel compilation and background compilation should be disabled on
+  // unicore systems.
+
+  MOZ_RELEASE_ASSERT(cpuCount > 1);
+
+  // If Tier2 is very backlogged we must give priority to it, since the Tier2
+  // queue holds onto Tier1 tasks.  Indeed if Tier2 is backlogged we will
+  // devote more resources to Tier2 and not start any Tier1 work at all.
+
+  bool tier2oversubscribed = wasmTier2GeneratorWorklist(lock).length() > 20;
+
+  // For Tier1 and Once compilation, honor the maximum allowed threads to
+  // compile wasm jobs at once, to avoid oversaturating the machine.
+  //
+  // For Tier2 compilation we need to allow other things to happen too, so we
+  // do not allow all logical cores to be used for background work; instead we
+  // wish to use a fraction of the physical cores.  We can't directly compute
+  // the physical cores from the logical cores, but 1/3 of the logical cores
+  // is a safe estimate for the number of physical cores available for
+  // background work.
+
+  size_t physCoresAvailable = size_t(ceil(cpuCount / 3.0));
+
+  size_t threads;
+  ThreadType threadType;
+  if (mode == wasm::CompileMode::Tier2) {
+    if (tier2oversubscribed) {
+      threads = maxWasmCompilationThreads();
+    } else {
+      threads = physCoresAvailable;
+    }
+    threadType = THREAD_TYPE_WASM_COMPILE_TIER2;
+  } else {
+    if (tier2oversubscribed) {
+      threads = 0;
+    } else {
+      threads = maxWasmCompilationThreads();
+    }
+    threadType = THREAD_TYPE_WASM_COMPILE_TIER1;
+  }
+
+  return threads != 0 && checkTaskThreadLimit(threadType, threads, lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartWasmTier2GeneratorTask(lock)) {
+    return nullptr;
+  }
+
+  return wasmTier2GeneratorWorklist(lock).popCopy();
+}
+
+bool GlobalHelperThreadState::canStartWasmTier2GeneratorTask(
+    const AutoLockHelperThreadState& lock) {
+  return !wasmTier2GeneratorWorklist(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_WASM_GENERATOR_TIER2,
+                              maxWasmTier2GeneratorThreads(),
+                              /*isMaster=*/true, lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetPromiseHelperTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartPromiseHelperTask(lock)) {
+    return nullptr;
+  }
+
+  return promiseHelperTasks(lock).popCopy();
+}
+
+bool GlobalHelperThreadState::canStartPromiseHelperTask(
+    const AutoLockHelperThreadState& lock) {
+  // PromiseHelperTasks can be wasm compilation tasks that in turn block on
+  // wasm compilation so set isMaster = true.
+  return !promiseHelperTasks(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_PROMISE_TASK,
+                              maxPromiseHelperThreads(),
+                              /*isMaster=*/true, lock);
+}
+
+static bool IonCompileTaskHasHigherPriority(jit::IonCompileTask* first,
+                                            jit::IonCompileTask* second) {
+  // Return true if priority(first) > priority(second).
+  //
+  // This method can return whatever it wants, though it really ought to be a
+  // total order. The ordering is allowed to race (change on the fly), however.
+
+  // A higher warm-up counter indicates a higher priority.
+  jit::JitScript* firstJitScript = first->script()->jitScript();
+  jit::JitScript* secondJitScript = second->script()->jitScript();
+  return firstJitScript->warmUpCount() / first->script()->length() >
+         secondJitScript->warmUpCount() / second->script()->length();
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetIonCompileTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartIonCompileTask(lock)) {
+    return nullptr;
+  }
+
+  return highestPriorityPendingIonCompile(lock,
+                                          /* checkExecutionStatus */ true);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetLowPrioIonCompileTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartIonCompileTask(lock)) {
+    return nullptr;
+  }
+
+  return highestPriorityPendingIonCompile(lock,
+                                          /* checkExecutionStatus */ false);
+}
+
+bool GlobalHelperThreadState::canStartIonCompileTask(
+    const AutoLockHelperThreadState& lock) {
+  return !ionWorklist(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_ION, maxIonCompilationThreads(),
+                              lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetIonFreeTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartIonFreeTask(lock)) {
+    return nullptr;
+  }
+
+  UniquePtr<jit::IonFreeTask> task = std::move(ionFreeList(lock).back());
+  ionFreeList(lock).popBack();
+  return task.release();
+}
+
+bool GlobalHelperThreadState::canStartIonFreeTask(
+    const AutoLockHelperThreadState& lock) {
+  return !ionFreeList(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_ION_FREE, maxIonFreeThreads(), lock);
+}
+
+jit::IonCompileTask* GlobalHelperThreadState::highestPriorityPendingIonCompile(
+    const AutoLockHelperThreadState& lock, bool checkExecutionStatus) {
+  auto& worklist = ionWorklist(lock);
+  MOZ_ASSERT(!worklist.empty());
+
+  // Get the highest priority IonCompileTask which has not started compilation
+  // yet.
+  size_t index = worklist.length();
+  for (size_t i = 0; i < worklist.length(); i++) {
+    if (checkExecutionStatus && !worklist[i]->isMainThreadRunningJS()) {
+      continue;
+    }
+    if (i < index ||
+        IonCompileTaskHasHigherPriority(worklist[i], worklist[index])) {
+      index = i;
+    }
+  }
+
+  if (index == worklist.length()) {
+    return nullptr;
+  }
+  jit::IonCompileTask* task = worklist[index];
+  worklist.erase(&worklist[index]);
+  return task;
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetFreeDelazifyTask(
+    const AutoLockHelperThreadState& lock) {
+  auto& freeList = freeDelazifyTaskVector(lock);
+  if (!freeList.empty()) {
+    UniquePtr<FreeDelazifyTask> task = std::move(freeList.back());
+    freeList.popBack();
+    return task.release();
+  }
+  return nullptr;
+}
+
+bool GlobalHelperThreadState::canStartFreeDelazifyTask(
+    const AutoLockHelperThreadState& lock) {
+  return !freeDelazifyTaskVector(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_DELAZIFY_FREE, maxDelazifyThreads(),
+                              /*isMaster=*/true, lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetDelazifyTask(
+    const AutoLockHelperThreadState& lock) {
+  // NOTE: We want to span all cores availables with delazification tasks, in
+  // order to parse a maximum number of functions ahead of their executions.
+  // Thus, as opposed to parse task which have a higher priority, we are not
+  // exclusively executing these task on parse threads.
+  auto& worklist = delazifyWorklist(lock);
+  if (worklist.isEmpty()) {
+    return nullptr;
+  }
+  return worklist.popFirst();
+}
+
+bool GlobalHelperThreadState::canStartDelazifyTask(
+    const AutoLockHelperThreadState& lock) {
+  return !delazifyWorklist(lock).isEmpty() &&
+         checkTaskThreadLimit(THREAD_TYPE_DELAZIFY, maxDelazifyThreads(),
+                              /*isMaster=*/true, lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetCompressionTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartCompressionTask(lock)) {
+    return nullptr;
+  }
+
+  auto& worklist = compressionWorklist(lock);
+  UniquePtr<SourceCompressionTask> task = std::move(worklist.back());
+  worklist.popBack();
+  return task.release();
+}
+
+bool GlobalHelperThreadState::canStartCompressionTask(
+    const AutoLockHelperThreadState& lock) {
+  return !compressionWorklist(lock).empty() &&
+         checkTaskThreadLimit(THREAD_TYPE_COMPRESS, maxCompressionThreads(),
+                              lock);
+}
+
+void GlobalHelperThreadState::startHandlingCompressionTasks(
+    ScheduleCompressionTask schedule, JSRuntime* maybeRuntime,
+    const AutoLockHelperThreadState& lock) {
+  MOZ_ASSERT((schedule == ScheduleCompressionTask::GC) ==
+             (maybeRuntime != nullptr));
+
+  auto& pending = compressionPendingList(lock);
+
+  for (size_t i = 0; i < pending.length(); i++) {
+    UniquePtr<SourceCompressionTask>& task = pending[i];
+    if (schedule == ScheduleCompressionTask::API ||
+        (task->runtimeMatches(maybeRuntime) && task->shouldStart())) {
+      // OOMing during appending results in the task not being scheduled
+      // and deleted.
+      (void)submitTask(std::move(task), lock);
+      remove(pending, &i);
+    }
+  }
+}
+
+bool GlobalHelperThreadState::submitTask(
+    UniquePtr<SourceCompressionTask> task,
+    const AutoLockHelperThreadState& locked) {
+  if (!compressionWorklist(locked).append(std::move(task))) {
+    return false;
+  }
+
+  dispatch(DispatchReason::NewTask, locked);
+  return true;
+}
+
+bool GlobalHelperThreadState::submitTask(
+    GCParallelTask* task, const AutoLockHelperThreadState& locked) {
+  gcParallelWorklist().insertBack(task, locked);
+  dispatch(DispatchReason::NewTask, locked);
+  return true;
+}
+
+HelperThreadTask* GlobalHelperThreadState::maybeGetGCParallelTask(
+    const AutoLockHelperThreadState& lock) {
+  if (!canStartGCParallelTask(lock)) {
+    return nullptr;
+  }
+
+  return gcParallelWorklist().popFirst(lock);
+}
+
+bool GlobalHelperThreadState::canStartGCParallelTask(
+    const AutoLockHelperThreadState& lock) {
+  return !gcParallelWorklist().isEmpty(lock) &&
+         checkTaskThreadLimit(THREAD_TYPE_GCPARALLEL, maxGCParallelThreads(),
+                              lock);
+}
+
+bool js::EnqueueOffThreadCompression(JSContext* cx,
+                                     UniquePtr<SourceCompressionTask> task) {
+  AutoLockHelperThreadState lock;
+
+  auto& pending = HelperThreadState().compressionPendingList(lock);
+  if (!pending.append(std::move(task))) {
+    ReportOutOfMemory(cx);
+    return false;
+  }
+
+  return true;
+}
+
+void js::StartHandlingCompressionsOnGC(JSRuntime* runtime) {
+  AutoLockHelperThreadState lock;
+  HelperThreadState().startHandlingCompressionTasks(
+      GlobalHelperThreadState::ScheduleCompressionTask::GC, runtime, lock);
+}
+
+template <typename T>
+static void ClearCompressionTaskList(T& list, JSRuntime* runtime) {
+  for (size_t i = 0; i < list.length(); i++) {
+    if (list[i]->runtimeMatches(runtime)) {
+      HelperThreadState().remove(list, &i);
+    }
+  }
+}
+
+void js::CancelOffThreadCompressions(JSRuntime* runtime) {
+  if (!CanUseExtraThreads()) {
+    return;
+  }
+
+  AutoLockHelperThreadState lock;
+
+  // Cancel all pending compression tasks.
+  ClearCompressionTaskList(HelperThreadState().compressionPendingList(lock),
+                           runtime);
+  ClearCompressionTaskList(HelperThreadState().compressionWorklist(lock),
+                           runtime);
+
+  // Cancel all in-process compression tasks and wait for them to join so we
+  // clean up the finished tasks.
+  while (true) {
+    bool inProgress = false;
+    for (auto* helper : HelperThreadState().helperTasks(lock)) {
+      if (!helper->is<SourceCompressionTask>()) {
+        continue;
+      }
+
+      if (helper->as<SourceCompressionTask>()->runtimeMatches(runtime)) {
+        inProgress = true;
+      }
+    }
+
+    if (!inProgress) {
+      break;
+    }
+
+    HelperThreadState().wait(lock);
+  }
+
+  // Clean up finished tasks.
+  ClearCompressionTaskList(HelperThreadState().compressionFinishedList(lock),
+                           runtime);
+}
+
+void js::AttachFinishedCompressions(JSRuntime* runtime,
+                                    AutoLockHelperThreadState& lock) {
+  auto& finished = HelperThreadState().compressionFinishedList(lock);
+  for (size_t i = 0; i < finished.length(); i++) {
+    if (finished[i]->runtimeMatches(runtime)) {
+      UniquePtr<SourceCompressionTask> compressionTask(std::move(finished[i]));
+      HelperThreadState().remove(finished, &i);
+      compressionTask->complete();
+    }
+  }
+}
+
+void js::SweepPendingCompressions(AutoLockHelperThreadState& lock) {
+  auto& pending = HelperThreadState().compressionPendingList(lock);
+  for (size_t i = 0; i < pending.length(); i++) {
+    if (pending[i]->shouldCancel()) {
+      HelperThreadState().remove(pending, &i);
+    }
+  }
+}
+
+void js::RunPendingSourceCompressions(JSRuntime* runtime) {
+  if (!CanUseExtraThreads()) {
+    return;
+  }
+
+  AutoLockHelperThreadState lock;
+
+  HelperThreadState().startHandlingCompressionTasks(
+      GlobalHelperThreadState::ScheduleCompressionTask::API, nullptr, lock);
+
+  // Wait until all tasks have started compression.
+  while (!HelperThreadState().compressionWorklist(lock).empty()) {
+    HelperThreadState().wait(lock);
+  }
+
+  // Wait for all in-process compression tasks to complete.
+  HelperThreadState().waitForAllTasksLocked(lock);
+
+  AttachFinishedCompressions(runtime, lock);
+}
+
+void PromiseHelperTask::executeAndResolveAndDestroy(JSContext* cx) {
+  execute();
+  run(cx, JS::Dispatchable::NotShuttingDown);
+}
+
+void PromiseHelperTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
+  {
+    AutoUnlockHelperThreadState unlock(lock);
+    execute();
+  }
+
+  // Don't release the lock between dispatching the resolve and destroy
+  // operation (which may start immediately on another thread) and returning
+  // from this method.
+
+  dispatchResolveAndDestroy(lock);
+}
+
+bool js::StartOffThreadPromiseHelperTask(JSContext* cx,
+                                         UniquePtr<PromiseHelperTask> task) {
+  // Execute synchronously if there are no helper threads.
+  if (!CanUseExtraThreads()) {
+    task.release()->executeAndResolveAndDestroy(cx);
+    return true;
+  }
+
+  if (!HelperThreadState().submitTask(task.get())) {
+    ReportOutOfMemory(cx);
+    return false;
+  }
+
+  (void)task.release();
+  return true;
+}
+
+bool js::StartOffThreadPromiseHelperTask(PromiseHelperTask* task) {
+  MOZ_ASSERT(CanUseExtraThreads());
+
+  return HelperThreadState().submitTask(task);
+}
+
+bool GlobalHelperThreadState::submitTask(PromiseHelperTask* task) {
+  AutoLockHelperThreadState lock;
+
+  if (!promiseHelperTasks(lock).append(task)) {
+    return false;
+  }
+
+  dispatch(DispatchReason::NewTask, lock);
+  return true;
+}
+
+void GlobalHelperThreadState::trace(JSTracer* trc) {
+  {
+    AutoLockHelperThreadState lock;
+
+#ifdef DEBUG
+    // Since we hold the helper thread lock here we must disable GCMarker's
+    // checking of the atom marking bitmap since that also relies on taking the
+    // lock.
+    GCMarker* marker = nullptr;
+    if (trc->isMarkingTracer()) {
+      marker = GCMarker::fromTracer(trc);
+      marker->setCheckAtomMarking(false);
+    }
+    auto reenableAtomMarkingCheck = mozilla::MakeScopeExit([marker] {
+      if (marker) {
+        marker->setCheckAtomMarking(true);
+      }
+    });
+#endif
+
+    for (auto task : ionWorklist(lock)) {
+      task->alloc().lifoAlloc()->setReadWrite();
+      task->trace(trc);
+      task->alloc().lifoAlloc()->setReadOnly();
+    }
+    for (auto task : ionFinishedList(lock)) {
+      task->trace(trc);
+    }
+
+    for (auto* helper : HelperThreadState().helperTasks(lock)) {
+      if (helper->is<jit::IonCompileTask>()) {
+        helper->as<jit::IonCompileTask>()->trace(trc);
+      }
+    }
+  }
+
+  // The lazy link list is only accessed on the main thread, so trace it after
+  // releasing the lock.
+  JSRuntime* rt = trc->runtime();
+  if (auto* jitRuntime = rt->jitRuntime()) {
+    jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(rt).getFirst();
+    while (task) {
+      task->trace(trc);
+      task = task->getNext();
+    }
+  }
+}
+
+// Definition of helper thread tasks.
+//
+// Priority is determined by the order they're listed here.
+const GlobalHelperThreadState::Selector GlobalHelperThreadState::selectors[] = {
+    &GlobalHelperThreadState::maybeGetGCParallelTask,
+    &GlobalHelperThreadState::maybeGetIonCompileTask,
+    &GlobalHelperThreadState::maybeGetWasmTier1CompileTask,
+    &GlobalHelperThreadState::maybeGetPromiseHelperTask,
+    &GlobalHelperThreadState::maybeGetFreeDelazifyTask,
+    &GlobalHelperThreadState::maybeGetDelazifyTask,
+    &GlobalHelperThreadState::maybeGetCompressionTask,
+    &GlobalHelperThreadState::maybeGetLowPrioIonCompileTask,
+    &GlobalHelperThreadState::maybeGetIonFreeTask,
+    &GlobalHelperThreadState::maybeGetWasmTier2CompileTask,
+    &GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask};
+
+bool GlobalHelperThreadState::canStartTasks(
+    const AutoLockHelperThreadState& lock) {
+  return canStartGCParallelTask(lock) || canStartIonCompileTask(lock) ||
+         canStartWasmTier1CompileTask(lock) ||
+         canStartPromiseHelperTask(lock) || canStartFreeDelazifyTask(lock) ||
+         canStartDelazifyTask(lock) || canStartCompressionTask(lock) ||
+         canStartIonFreeTask(lock) || canStartWasmTier2CompileTask(lock) ||
+         canStartWasmTier2GeneratorTask(lock);
+}
+
+void JS::RunHelperThreadTask() {
+  MOZ_ASSERT(CanUseExtraThreads());
+
+  AutoLockHelperThreadState lock;
+
+  if (!gHelperThreadState || HelperThreadState().isTerminating(lock)) {
+    return;
+  }
+
+  HelperThreadState().runOneTask(lock);
+}
+
+void GlobalHelperThreadState::runOneTask(AutoLockHelperThreadState& lock) {
+  MOZ_ASSERT(tasksPending_ > 0);
+  tasksPending_--;
+
+  // The selectors may depend on the HelperThreadState not changing between task
+  // selection and task execution, in particular, on new tasks not being added
+  // (because of the lifo structure of the work lists). Unlocking the
+  // HelperThreadState between task selection and execution is not well-defined.
+  HelperThreadTask* task = findHighestPriorityTask(lock);
+  if (task) {
+    runTaskLocked(task, lock);
+    dispatch(DispatchReason::FinishedTask, lock);
+  }
+
+  notifyAll(lock);
+}
+
+HelperThreadTask* GlobalHelperThreadState::findHighestPriorityTask(
+    const AutoLockHelperThreadState& locked) {
+  // Return the highest priority task that is ready to start, or nullptr.
+
+  for (const auto& selector : selectors) {
+    if (auto* task = (this->*(selector))(locked)) {
+      return task;
+    }
+  }
+
+  return nullptr;
+}
+
+void GlobalHelperThreadState::runTaskLocked(HelperThreadTask* task,
+                                            AutoLockHelperThreadState& locked) {
+  JS::AutoSuppressGCAnalysis nogc;
+
+  HelperThreadState().helperTasks(locked).infallibleEmplaceBack(task);
+
+  ThreadType threadType = task->threadType();
+  js::oom::SetThreadType(threadType);
+  runningTaskCount[threadType]++;
+  totalCountRunningTasks++;
+
+  task->runHelperThreadTask(locked);
+
+  // Delete task from helperTasks.
+  HelperThreadState().helperTasks(locked).eraseIfEqual(task);
+
+  totalCountRunningTasks--;
+  runningTaskCount[threadType]--;
+
+  js::oom::SetThreadType(js::THREAD_TYPE_NONE);
+}