/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * vim: set ts=8 sts=2 et sw=2 tw=80:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "jit/JitFrames-inl.h"

#include "mozilla/ScopeExit.h"

#include <algorithm>

#include "builtin/ModuleObject.h"
#include "gc/GC.h"
#include "jit/BaselineFrame.h"
#include "jit/BaselineIC.h"
#include "jit/BaselineJIT.h"
#include "jit/Ion.h"
#include "jit/IonScript.h"
#include "jit/JitRuntime.h"
#include "jit/JitSpewer.h"
#include "jit/LIR.h"
#include "jit/PcScriptCache.h"
#include "jit/Recover.h"
#include "jit/Safepoints.h"
#include "jit/ScriptFromCalleeToken.h"
#include "jit/Snapshots.h"
#include "jit/VMFunctions.h"
#include "js/Exception.h"
#include "js/friend/DumpFunctions.h"  // js::DumpObject, js::DumpValue
#include "vm/Interpreter.h"
#include "vm/JSContext.h"
#include "vm/JSFunction.h"
#include "vm/JSObject.h"
#include "vm/JSScript.h"
#include "wasm/WasmBuiltins.h"
#include "wasm/WasmInstance.h"

#include "debugger/DebugAPI-inl.h"
#include "jit/JSJitFrameIter-inl.h"
#include "vm/GeckoProfiler-inl.h"
#include "vm/JSScript-inl.h"
#include "vm/Probes-inl.h"

namespace js {
namespace jit {

// Given a slot index, returns the offset, in bytes, of that slot from an
// JitFrameLayout. Slot distances are uniform across architectures, however,
// the distance does depend on the size of the frame header.
static inline int32_t OffsetOfFrameSlot(int32_t slot) { return -slot; }

static inline uint8_t* AddressOfFrameSlot(JitFrameLayout* fp, int32_t slot) {
  return (uint8_t*)fp + OffsetOfFrameSlot(slot);
}

static inline uintptr_t ReadFrameSlot(JitFrameLayout* fp, int32_t slot) {
  return *(uintptr_t*)AddressOfFrameSlot(fp, slot);
}

static inline void WriteFrameSlot(JitFrameLayout* fp, int32_t slot,
                                  uintptr_t value) {
  *(uintptr_t*)AddressOfFrameSlot(fp, slot) = value;
}

static inline double ReadFrameDoubleSlot(JitFrameLayout* fp, int32_t slot) {
  return *(double*)AddressOfFrameSlot(fp, slot);
}

static inline float ReadFrameFloat32Slot(JitFrameLayout* fp, int32_t slot) {
  return *(float*)AddressOfFrameSlot(fp, slot);
}

static inline int32_t ReadFrameInt32Slot(JitFrameLayout* fp, int32_t slot) {
  return *(int32_t*)AddressOfFrameSlot(fp, slot);
}

static inline bool ReadFrameBooleanSlot(JitFrameLayout* fp, int32_t slot) {
  return *(bool*)AddressOfFrameSlot(fp, slot);
}

static uint32_t NumArgAndLocalSlots(const InlineFrameIterator& frame) {
  JSScript* script = frame.script();
  return CountArgSlots(script, frame.maybeCalleeTemplate()) + script->nfixed();
}

static void CloseLiveIteratorIon(JSContext* cx,
                                 const InlineFrameIterator& frame,
                                 const TryNote* tn) {
  MOZ_ASSERT(tn->kind() == TryNoteKind::ForIn ||
             tn->kind() == TryNoteKind::Destructuring);

  bool isDestructuring = tn->kind() == TryNoteKind::Destructuring;
  MOZ_ASSERT_IF(!isDestructuring, tn->stackDepth > 0);
  MOZ_ASSERT_IF(isDestructuring, tn->stackDepth > 1);

  // Save any pending exception, because some recover operations call into
  // AutoUnsafeCallWithABI functions, which don't allow pending exceptions.
  JS::AutoSaveExceptionState savedExc(cx);

  SnapshotIterator si = frame.snapshotIterator();

  // Skip stack slots until we reach the iterator object on the stack. For
  // the destructuring case, we also need to get the "done" value.
  uint32_t stackSlot = tn->stackDepth;
  uint32_t adjust = isDestructuring ? 2 : 1;
  uint32_t skipSlots = NumArgAndLocalSlots(frame) + stackSlot - adjust;

  for (unsigned i = 0; i < skipSlots; i++) {
    si.skip();
  }

  MaybeReadFallback recover(cx, cx->activation()->asJit(), &frame.frame(),
                            MaybeReadFallback::Fallback_DoNothing);
  Value v = si.maybeRead(recover);
  MOZ_RELEASE_ASSERT(v.isObject());
  RootedObject iterObject(cx, &v.toObject());

  if (isDestructuring) {
    RootedValue doneValue(cx, si.read());
    MOZ_RELEASE_ASSERT(!doneValue.isMagic());
    bool done = ToBoolean(doneValue);
    // Do not call IteratorClose if the destructuring iterator is already
    // done.
    if (done) {
      return;
    }
  }

  // Restore any pending exception before the closing the iterator.
  savedExc.restore();

  if (cx->isExceptionPending()) {
    if (tn->kind() == TryNoteKind::ForIn) {
      CloseIterator(iterObject);
    } else {
      IteratorCloseForException(cx, iterObject);
    }
  } else {
    UnwindIteratorForUncatchableException(iterObject);
  }
}

class IonTryNoteFilter {
  uint32_t depth_;

 public:
  explicit IonTryNoteFilter(const InlineFrameIterator& frame) {
    uint32_t base = NumArgAndLocalSlots(frame);
    SnapshotIterator si = frame.snapshotIterator();
    MOZ_ASSERT(si.numAllocations() >= base);
    depth_ = si.numAllocations() - base;
  }

  bool operator()(const TryNote* note) { return note->stackDepth <= depth_; }
};

class TryNoteIterIon : public TryNoteIter<IonTryNoteFilter> {
 public:
  TryNoteIterIon(JSContext* cx, const InlineFrameIterator& frame)
      : TryNoteIter(cx, frame.script(), frame.pc(), IonTryNoteFilter(frame)) {}
};

static bool ShouldBailoutForDebugger(JSContext* cx,
                                     const InlineFrameIterator& frame,
                                     bool hitBailoutException) {
  if (hitBailoutException) {
    MOZ_ASSERT(!cx->isPropagatingForcedReturn());
    return false;
  }

  // Bail out if we're propagating a forced return from an inlined frame,
  // even if the realm is no longer a debuggee.
  if (cx->isPropagatingForcedReturn() && frame.more()) {
    return true;
  }

  if (!cx->realm()->isDebuggee()) {
    return false;
  }

  // Bail out if there's a catchable exception and we are the debuggee of a
  // Debugger with a live onExceptionUnwind hook.
  if (cx->isExceptionPending() &&
      DebugAPI::hasExceptionUnwindHook(cx->global())) {
    return true;
  }

  // Bail out if a Debugger has observed this frame (e.g., for onPop).
  JitActivation* act = cx->activation()->asJit();
  RematerializedFrame* rematFrame =
      act->lookupRematerializedFrame(frame.frame().fp(), frame.frameNo());
  return rematFrame && rematFrame->isDebuggee();
}

static void OnLeaveIonFrame(JSContext* cx, const InlineFrameIterator& frame,
                            ResumeFromException* rfe) {
  bool returnFromThisFrame =
      cx->isPropagatingForcedReturn() || cx->isClosingGenerator();
  if (!returnFromThisFrame) {
    return;
  }

  JitActivation* act = cx->activation()->asJit();
  RematerializedFrame* rematFrame = nullptr;
  {
    JS::AutoSaveExceptionState savedExc(cx);

    // We can run recover instructions without invalidating because we're
    // already leaving the frame.
    MaybeReadFallback::FallbackConsequence consequence =
        MaybeReadFallback::Fallback_DoNothing;
    rematFrame = act->getRematerializedFrame(cx, frame.frame(), frame.frameNo(),
                                             consequence);
    if (!rematFrame) {
      return;
    }
  }

  MOZ_ASSERT(!frame.more());

  if (cx->isClosingGenerator()) {
    HandleClosingGeneratorReturn(cx, rematFrame, /*frameOk=*/true);
  } else {
    cx->clearPropagatingForcedReturn();
  }

  Value& rval = rematFrame->returnValue();
  MOZ_RELEASE_ASSERT(!rval.isMagic());

  // Set both framePointer and stackPointer to the address of the
  // JitFrameLayout.
  rfe->kind = ExceptionResumeKind::ForcedReturnIon;
  rfe->framePointer = frame.frame().fp();
  rfe->stackPointer = frame.frame().fp();
  rfe->exception = rval;

  act->removeIonFrameRecovery(frame.frame().jsFrame());
  act->removeRematerializedFrame(frame.frame().fp());
}

static void HandleExceptionIon(JSContext* cx, const InlineFrameIterator& frame,
                               ResumeFromException* rfe,
                               bool* hitBailoutException) {
  if (ShouldBailoutForDebugger(cx, frame, *hitBailoutException)) {
    // We do the following:
    //
    //   1. Bailout to baseline to reconstruct a baseline frame.
    //   2. Resume immediately into the exception tail afterwards, and
    //      handle the exception again with the top frame now a baseline
    //      frame.
    //
    // An empty exception info denotes that we're propagating an Ion
    // exception due to debug mode, which BailoutIonToBaseline needs to
    // know. This is because we might not be able to fully reconstruct up
    // to the stack depth at the snapshot, as we could've thrown in the
    // middle of a call.
    ExceptionBailoutInfo propagateInfo(cx);
    if (ExceptionHandlerBailout(cx, frame, rfe, propagateInfo)) {
      return;
    }
    *hitBailoutException = true;
  }

  RootedScript script(cx, frame.script());

  for (TryNoteIterIon tni(cx, frame); !tni.done(); ++tni) {
    const TryNote* tn = *tni;
    switch (tn->kind()) {
      case TryNoteKind::ForIn:
      case TryNoteKind::Destructuring:
        CloseLiveIteratorIon(cx, frame, tn);
        break;

      case TryNoteKind::Catch:
        // If we're closing a generator, we have to skip catch blocks.
        if (cx->isClosingGenerator()) {
          break;
        }

        if (cx->isExceptionPending()) {
          // Ion can compile try-catch, but bailing out to catch
          // exceptions is slow. Reset the warm-up counter so that if we
          // catch many exceptions we won't Ion-compile the script.
          script->resetWarmUpCounterToDelayIonCompilation();

          if (*hitBailoutException) {
            break;
          }

          // Bailout at the start of the catch block.
          jsbytecode* catchPC = script->offsetToPC(tn->start + tn->length);
          ExceptionBailoutInfo excInfo(cx, frame.frameNo(), catchPC,
                                       tn->stackDepth);
          if (ExceptionHandlerBailout(cx, frame, rfe, excInfo)) {
            // Record exception locations to allow scope unwinding in
            // |FinishBailoutToBaseline|
            MOZ_ASSERT(cx->isExceptionPending());
            rfe->bailoutInfo->tryPC =
                UnwindEnvironmentToTryPc(frame.script(), tn);
            rfe->bailoutInfo->faultPC = frame.pc();
            return;
          }

          *hitBailoutException = true;
          MOZ_ASSERT(cx->isExceptionPending());
        }
        break;

      case TryNoteKind::Finally: {
        if (!cx->isExceptionPending()) {
          // We don't catch uncatchable exceptions.
          break;
        }

        script->resetWarmUpCounterToDelayIonCompilation();

        if (*hitBailoutException) {
          break;
        }

        // Bailout at the start of the finally block.
        jsbytecode* finallyPC = script->offsetToPC(tn->start + tn->length);
        ExceptionBailoutInfo excInfo(cx, frame.frameNo(), finallyPC,
                                     tn->stackDepth);

        RootedValue exception(cx);
        if (!cx->getPendingException(&exception)) {
          exception = UndefinedValue();
        }
        excInfo.setFinallyException(exception.get());
        cx->clearPendingException();

        if (ExceptionHandlerBailout(cx, frame, rfe, excInfo)) {
          // Record exception locations to allow scope unwinding in
          // |FinishBailoutToBaseline|
          rfe->bailoutInfo->tryPC =
              UnwindEnvironmentToTryPc(frame.script(), tn);
          rfe->bailoutInfo->faultPC = frame.pc();
          return;
        }

        *hitBailoutException = true;
        MOZ_ASSERT(cx->isExceptionPending());
        break;
      }

      case TryNoteKind::ForOf:
      case TryNoteKind::Loop:
        break;

      // TryNoteKind::ForOfIterclose is handled internally by the try note
      // iterator.
      default:
        MOZ_CRASH("Unexpected try note");
    }
  }

  OnLeaveIonFrame(cx, frame, rfe);
}

static void OnLeaveBaselineFrame(JSContext* cx, const JSJitFrameIter& frame,
                                 jsbytecode* pc, ResumeFromException* rfe,
                                 bool frameOk) {
  BaselineFrame* baselineFrame = frame.baselineFrame();
  bool returnFromThisFrame = jit::DebugEpilogue(cx, baselineFrame, pc, frameOk);
  if (returnFromThisFrame) {
    rfe->kind = ExceptionResumeKind::ForcedReturnBaseline;
    rfe->framePointer = frame.fp();
    rfe->stackPointer = reinterpret_cast<uint8_t*>(baselineFrame);
  }
}

static inline void BaselineFrameAndStackPointersFromTryNote(
    const TryNote* tn, const JSJitFrameIter& frame, uint8_t** framePointer,
    uint8_t** stackPointer) {
  JSScript* script = frame.baselineFrame()->script();
  *framePointer = frame.fp();
  *stackPointer = *framePointer - BaselineFrame::Size() -
                  (script->nfixed() + tn->stackDepth) * sizeof(Value);
}

static void SettleOnTryNote(JSContext* cx, const TryNote* tn,
                            const JSJitFrameIter& frame, EnvironmentIter& ei,
                            ResumeFromException* rfe, jsbytecode** pc) {
  RootedScript script(cx, frame.baselineFrame()->script());

  // Unwind environment chain (pop block objects).
  if (cx->isExceptionPending()) {
    UnwindEnvironment(cx, ei, UnwindEnvironmentToTryPc(script, tn));
  }

  // Compute base pointer and stack pointer.
  BaselineFrameAndStackPointersFromTryNote(tn, frame, &rfe->framePointer,
                                           &rfe->stackPointer);

  // Compute the pc.
  *pc = script->offsetToPC(tn->start + tn->length);
}

class BaselineTryNoteFilter {
  const JSJitFrameIter& frame_;

 public:
  explicit BaselineTryNoteFilter(const JSJitFrameIter& frame) : frame_(frame) {}
  bool operator()(const TryNote* note) {
    BaselineFrame* frame = frame_.baselineFrame();

    uint32_t numValueSlots = frame_.baselineFrameNumValueSlots();
    MOZ_RELEASE_ASSERT(numValueSlots >= frame->script()->nfixed());

    uint32_t currDepth = numValueSlots - frame->script()->nfixed();
    return note->stackDepth <= currDepth;
  }
};

class TryNoteIterBaseline : public TryNoteIter<BaselineTryNoteFilter> {
 public:
  TryNoteIterBaseline(JSContext* cx, const JSJitFrameIter& frame,
                      jsbytecode* pc)
      : TryNoteIter(cx, frame.script(), pc, BaselineTryNoteFilter(frame)) {}
};

// Close all live iterators on a BaselineFrame due to exception unwinding. The
// pc parameter is updated to where the envs have been unwound to.
static void CloseLiveIteratorsBaselineForUncatchableException(
    JSContext* cx, const JSJitFrameIter& frame, jsbytecode* pc) {
  for (TryNoteIterBaseline tni(cx, frame, pc); !tni.done(); ++tni) {
    const TryNote* tn = *tni;
    switch (tn->kind()) {
      case TryNoteKind::ForIn: {
        uint8_t* framePointer;
        uint8_t* stackPointer;
        BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
                                                 &stackPointer);
        Value iterValue(*(Value*)stackPointer);
        RootedObject iterObject(cx, &iterValue.toObject());
        UnwindIteratorForUncatchableException(iterObject);
        break;
      }

      default:
        break;
    }
  }
}

static bool ProcessTryNotesBaseline(JSContext* cx, const JSJitFrameIter& frame,
                                    EnvironmentIter& ei,
                                    ResumeFromException* rfe, jsbytecode** pc) {
  MOZ_ASSERT(frame.baselineFrame()->runningInInterpreter(),
             "Caller must ensure frame is an interpreter frame");

  RootedScript script(cx, frame.baselineFrame()->script());

  for (TryNoteIterBaseline tni(cx, frame, *pc); !tni.done(); ++tni) {
    const TryNote* tn = *tni;

    MOZ_ASSERT(cx->isExceptionPending());
    switch (tn->kind()) {
      case TryNoteKind::Catch: {
        // If we're closing a generator, we have to skip catch blocks.
        if (cx->isClosingGenerator()) {
          break;
        }

        SettleOnTryNote(cx, tn, frame, ei, rfe, pc);

        // Ion can compile try-catch, but bailing out to catch
        // exceptions is slow. Reset the warm-up counter so that if we
        // catch many exceptions we won't Ion-compile the script.
        script->resetWarmUpCounterToDelayIonCompilation();

        // Resume at the start of the catch block.
        const BaselineInterpreter& interp =
            cx->runtime()->jitRuntime()->baselineInterpreter();
        frame.baselineFrame()->setInterpreterFields(*pc);
        rfe->kind = ExceptionResumeKind::Catch;
        rfe->target = interp.interpretOpAddr().value;
        return true;
      }

      case TryNoteKind::Finally: {
        SettleOnTryNote(cx, tn, frame, ei, rfe, pc);

        const BaselineInterpreter& interp =
            cx->runtime()->jitRuntime()->baselineInterpreter();
        frame.baselineFrame()->setInterpreterFields(*pc);
        rfe->kind = ExceptionResumeKind::Finally;
        rfe->target = interp.interpretOpAddr().value;

        // Drop the exception instead of leaking cross compartment data.
        if (!cx->getPendingException(
                MutableHandleValue::fromMarkedLocation(&rfe->exception))) {
          rfe->exception = UndefinedValue();
        }
        cx->clearPendingException();
        return true;
      }

      case TryNoteKind::ForIn: {
        uint8_t* framePointer;
        uint8_t* stackPointer;
        BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
                                                 &stackPointer);
        Value iterValue(*reinterpret_cast<Value*>(stackPointer));
        JSObject* iterObject = &iterValue.toObject();
        CloseIterator(iterObject);
        break;
      }

      case TryNoteKind::Destructuring: {
        uint8_t* framePointer;
        uint8_t* stackPointer;
        BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
                                                 &stackPointer);
        // Note: if this ever changes, also update the
        // TryNoteKind::Destructuring code in WarpBuilder.cpp!
        RootedValue doneValue(cx, *(reinterpret_cast<Value*>(stackPointer)));
        MOZ_RELEASE_ASSERT(!doneValue.isMagic());
        bool done = ToBoolean(doneValue);
        if (!done) {
          Value iterValue(*(reinterpret_cast<Value*>(stackPointer) + 1));
          RootedObject iterObject(cx, &iterValue.toObject());
          if (!IteratorCloseForException(cx, iterObject)) {
            SettleOnTryNote(cx, tn, frame, ei, rfe, pc);
            return false;
          }
        }
        break;
      }

      case TryNoteKind::ForOf:
      case TryNoteKind::Loop:
        break;

      // TryNoteKind::ForOfIterClose is handled internally by the try note
      // iterator.
      default:
        MOZ_CRASH("Invalid try note");
    }
  }
  return true;
}

static void HandleExceptionBaseline(JSContext* cx, JSJitFrameIter& frame,
                                    CommonFrameLayout* prevFrame,
                                    ResumeFromException* rfe) {
  MOZ_ASSERT(frame.isBaselineJS());
  MOZ_ASSERT(prevFrame);

  jsbytecode* pc;
  frame.baselineScriptAndPc(nullptr, &pc);

  // Ensure the BaselineFrame is an interpreter frame. This is easy to do and
  // simplifies the code below and interaction with DebugModeOSR.
  //
  // Note that we never return to this frame via the previous frame's return
  // address. We could set the return address to nullptr to ensure it's never
  // used, but the profiler expects a non-null return value for its JitCode map
  // lookup so we have to use an address in the interpreter code instead.
  if (!frame.baselineFrame()->runningInInterpreter()) {
    const BaselineInterpreter& interp =
        cx->runtime()->jitRuntime()->baselineInterpreter();
    uint8_t* retAddr = interp.codeRaw();
    BaselineFrame* baselineFrame = frame.baselineFrame();

    // Suppress profiler sampling while we fix up the frame to ensure the
    // sampler thread doesn't see an inconsistent state.
    AutoSuppressProfilerSampling suppressProfilerSampling(cx);
    baselineFrame->switchFromJitToInterpreterForExceptionHandler(cx, pc);
    prevFrame->setReturnAddress(retAddr);

    // Ensure the current iterator's resumePCInCurrentFrame_ isn't used
    // anywhere.
    frame.setResumePCInCurrentFrame(nullptr);
  }

  bool frameOk = false;
  RootedScript script(cx, frame.baselineFrame()->script());

  if (script->hasScriptCounts()) {
    PCCounts* counts = script->getThrowCounts(pc);
    // If we failed to allocate, then skip the increment and continue to
    // handle the exception.
    if (counts) {
      counts->numExec()++;
    }
  }

  bool hasTryNotes = !script->trynotes().empty();

again:
  if (cx->isExceptionPending()) {
    if (!cx->isClosingGenerator()) {
      if (!DebugAPI::onExceptionUnwind(cx, frame.baselineFrame())) {
        if (!cx->isExceptionPending()) {
          goto again;
        }
      }
      // Ensure that the debugger hasn't returned 'true' while clearing the
      // exception state.
      MOZ_ASSERT(cx->isExceptionPending());
    }

    if (hasTryNotes) {
      EnvironmentIter ei(cx, frame.baselineFrame(), pc);
      if (!ProcessTryNotesBaseline(cx, frame, ei, rfe, &pc)) {
        goto again;
      }
      if (rfe->kind != ExceptionResumeKind::EntryFrame) {
        // No need to increment the PCCounts number of execution here,
        // as the interpreter increments any PCCounts if present.
        MOZ_ASSERT_IF(script->hasScriptCounts(), script->maybeGetPCCounts(pc));
        return;
      }
    }

    frameOk = HandleClosingGeneratorReturn(cx, frame.baselineFrame(), frameOk);
  } else {
    if (hasTryNotes) {
      CloseLiveIteratorsBaselineForUncatchableException(cx, frame, pc);
    }

    // We may be propagating a forced return from a debugger hook function.
    if (MOZ_UNLIKELY(cx->isPropagatingForcedReturn())) {
      cx->clearPropagatingForcedReturn();
      frameOk = true;
    }
  }

  OnLeaveBaselineFrame(cx, frame, pc, rfe, frameOk);
}

static JitFrameLayout* GetLastProfilingFrame(ResumeFromException* rfe) {
  switch (rfe->kind) {
    case ExceptionResumeKind::EntryFrame:
    case ExceptionResumeKind::Wasm:
    case ExceptionResumeKind::WasmCatch:
      return nullptr;

    // The following all return into Baseline or Ion frames.
    case ExceptionResumeKind::Catch:
    case ExceptionResumeKind::Finally:
    case ExceptionResumeKind::ForcedReturnBaseline:
    case ExceptionResumeKind::ForcedReturnIon:
      return reinterpret_cast<JitFrameLayout*>(rfe->framePointer);

    // When resuming into a bailed-out ion frame, use the bailout info to
    // find the frame we are resuming into.
    case ExceptionResumeKind::Bailout:
      return reinterpret_cast<JitFrameLayout*>(rfe->bailoutInfo->incomingStack);
  }

  MOZ_CRASH("Invalid ResumeFromException type!");
  return nullptr;
}

void HandleExceptionWasm(JSContext* cx, wasm::WasmFrameIter* iter,
                         ResumeFromException* rfe) {
  MOZ_ASSERT(cx->activation()->asJit()->hasWasmExitFP());
  wasm::HandleThrow(cx, *iter, rfe);
  MOZ_ASSERT(iter->done());
}

void HandleException(ResumeFromException* rfe) {
  JSContext* cx = TlsContext.get();

#ifdef DEBUG
  cx->runtime()->jitRuntime()->clearDisallowArbitraryCode();

  // Reset the counter when we bailed after MDebugEnterGCUnsafeRegion, but
  // before the matching MDebugLeaveGCUnsafeRegion.
  //
  // NOTE: EnterJit ensures the counter is zero when we enter JIT code.
  cx->resetInUnsafeRegion();
#endif

  auto resetProfilerFrame = mozilla::MakeScopeExit([=] {
    if (!cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
            cx->runtime())) {
      return;
    }

    MOZ_ASSERT(cx->jitActivation == cx->profilingActivation());

    auto* lastProfilingFrame = GetLastProfilingFrame(rfe);
    cx->jitActivation->setLastProfilingFrame(lastProfilingFrame);
  });

  rfe->kind = ExceptionResumeKind::EntryFrame;

  JitSpew(JitSpew_IonInvalidate, "handling exception");

  JitActivation* activation = cx->activation()->asJit();

#ifdef CHECK_OSIPOINT_REGISTERS
  if (JitOptions.checkOsiPointRegisters) {
    activation->setCheckRegs(false);
  }
#endif

  JitFrameIter iter(cx->activation()->asJit(),
                    /* mustUnwindActivation = */ true);
  CommonFrameLayout* prevJitFrame = nullptr;
  while (!iter.done()) {
    if (iter.isWasm()) {
      prevJitFrame = nullptr;
      HandleExceptionWasm(cx, &iter.asWasm(), rfe);
      // If a wasm try-catch handler is found, we can immediately jump to it
      // and quit iterating through the stack.
      if (rfe->kind == ExceptionResumeKind::WasmCatch) {
        return;
      }
      if (!iter.done()) {
        ++iter;
      }
      continue;
    }

    JSJitFrameIter& frame = iter.asJSJit();

    // JIT code can enter same-compartment realms, so reset cx->realm to
    // this frame's realm.
    if (frame.isScripted()) {
      cx->setRealmForJitExceptionHandler(iter.realm());
    }

    if (frame.isIonJS()) {
      // Search each inlined frame for live iterator objects, and close
      // them.
      InlineFrameIterator frames(cx, &frame);

      // Invalidation state will be the same for all inlined scripts in the
      // frame.
      IonScript* ionScript = nullptr;
      bool invalidated = frame.checkInvalidation(&ionScript);

      // If we hit OOM or overrecursion while bailing out, we don't
      // attempt to bail out a second time for this Ion frame. Just unwind
      // and continue at the next frame.
      bool hitBailoutException = false;
      for (;;) {
        HandleExceptionIon(cx, frames, rfe, &hitBailoutException);

        if (rfe->kind == ExceptionResumeKind::Bailout ||
            rfe->kind == ExceptionResumeKind::ForcedReturnIon) {
          if (invalidated) {
            ionScript->decrementInvalidationCount(cx->gcContext());
          }
          return;
        }

        MOZ_ASSERT(rfe->kind == ExceptionResumeKind::EntryFrame);

        // When profiling, each frame popped needs a notification that
        // the function has exited, so invoke the probe that a function
        // is exiting.

        JSScript* script = frames.script();
        probes::ExitScript(cx, script, script->function(),
                           /* popProfilerFrame = */ false);
        if (!frames.more()) {
          break;
        }
        ++frames;
      }

      // Remove left-over state which might have been needed for bailout.
      activation->removeIonFrameRecovery(frame.jsFrame());
      activation->removeRematerializedFrame(frame.fp());

      // If invalidated, decrement the number of frames remaining on the
      // stack for the given IonScript.
      if (invalidated) {
        ionScript->decrementInvalidationCount(cx->gcContext());
      }

    } else if (frame.isBaselineJS()) {
      HandleExceptionBaseline(cx, frame, prevJitFrame, rfe);

      if (rfe->kind != ExceptionResumeKind::EntryFrame &&
          rfe->kind != ExceptionResumeKind::ForcedReturnBaseline) {
        return;
      }

      // Unwind profiler pseudo-stack
      JSScript* script = frame.script();
      probes::ExitScript(cx, script, script->function(),
                         /* popProfilerFrame = */ false);

      if (rfe->kind == ExceptionResumeKind::ForcedReturnBaseline) {
        return;
      }
    }

    prevJitFrame = frame.current();
    ++iter;
  }

  // Wasm sets its own value of SP in HandleExceptionWasm.
  if (iter.isJSJit()) {
    MOZ_ASSERT(rfe->kind == ExceptionResumeKind::EntryFrame);
    rfe->framePointer = iter.asJSJit().current()->callerFramePtr();
    rfe->stackPointer =
        iter.asJSJit().fp() + CommonFrameLayout::offsetOfReturnAddress();
  }
}

// Turns a JitFrameLayout into an UnwoundJit ExitFrameLayout.
void EnsureUnwoundJitExitFrame(JitActivation* act, JitFrameLayout* frame) {
  ExitFrameLayout* exitFrame = reinterpret_cast<ExitFrameLayout*>(frame);

  if (act->jsExitFP() == (uint8_t*)frame) {
    // If we already called this function for the current frame, do
    // nothing.
    MOZ_ASSERT(exitFrame->isUnwoundJitExit());
    return;
  }

#ifdef DEBUG
  JSJitFrameIter iter(act);
  while (!iter.isScripted()) {
    ++iter;
  }
  MOZ_ASSERT(iter.current() == frame, "|frame| must be the top JS frame");

  MOZ_ASSERT(!!act->jsExitFP());
  MOZ_ASSERT((uint8_t*)exitFrame->footer() >= act->jsExitFP(),
             "Must have space for ExitFooterFrame before jsExitFP");
#endif

  act->setJSExitFP((uint8_t*)frame);
  exitFrame->footer()->setUnwoundJitExitFrame();
  MOZ_ASSERT(exitFrame->isUnwoundJitExit());
}

JSScript* MaybeForwardedScriptFromCalleeToken(CalleeToken token) {
  switch (GetCalleeTokenTag(token)) {
    case CalleeToken_Script:
      return MaybeForwarded(CalleeTokenToScript(token));
    case CalleeToken_Function:
    case CalleeToken_FunctionConstructing: {
      JSFunction* fun = MaybeForwarded(CalleeTokenToFunction(token));
      return MaybeForwarded(fun)->nonLazyScript();
    }
  }
  MOZ_CRASH("invalid callee token tag");
}

CalleeToken TraceCalleeToken(JSTracer* trc, CalleeToken token) {
  switch (CalleeTokenTag tag = GetCalleeTokenTag(token)) {
    case CalleeToken_Function:
    case CalleeToken_FunctionConstructing: {
      JSFunction* fun = CalleeTokenToFunction(token);
      TraceRoot(trc, &fun, "jit-callee");
      return CalleeToToken(fun, tag == CalleeToken_FunctionConstructing);
    }
    case CalleeToken_Script: {
      JSScript* script = CalleeTokenToScript(token);
      TraceRoot(trc, &script, "jit-script");
      return CalleeToToken(script);
    }
    default:
      MOZ_CRASH("unknown callee token type");
  }
}

uintptr_t* JitFrameLayout::slotRef(SafepointSlotEntry where) {
  if (where.stack) {
    return (uintptr_t*)((uint8_t*)this - where.slot);
  }
  return (uintptr_t*)((uint8_t*)thisAndActualArgs() + where.slot);
}

#ifdef JS_NUNBOX32
static inline uintptr_t ReadAllocation(const JSJitFrameIter& frame,
                                       const LAllocation* a) {
  if (a->isGeneralReg()) {
    Register reg = a->toGeneralReg()->reg();
    return frame.machineState().read(reg);
  }
  return *frame.jsFrame()->slotRef(SafepointSlotEntry(a));
}
#endif

static void TraceThisAndArguments(JSTracer* trc, const JSJitFrameIter& frame,
                                  JitFrameLayout* layout) {
  // Trace |this| and any extra actual arguments for an Ion frame. Tracing
  // of formal arguments is taken care of by the frame's safepoint/snapshot,
  // except when the script might have lazy arguments or rest, in which case
  // we trace them as well. We also have to trace formals if we have a
  // LazyLink frame or an InterpreterStub frame or a special JSJit to wasm
  // frame (since wasm doesn't use snapshots).

  if (!CalleeTokenIsFunction(layout->calleeToken())) {
    return;
  }

  size_t nargs = layout->numActualArgs();
  size_t nformals = 0;

  JSFunction* fun = CalleeTokenToFunction(layout->calleeToken());
  if (frame.type() != FrameType::JSJitToWasm &&
      !frame.isExitFrameLayout<CalledFromJitExitFrameLayout>() &&
      !fun->nonLazyScript()->mayReadFrameArgsDirectly()) {
    nformals = fun->nargs();
  }

  size_t newTargetOffset = std::max(nargs, fun->nargs());

  Value* argv = layout->thisAndActualArgs();

  // Trace |this|.
  TraceRoot(trc, argv, "ion-thisv");

  // Trace actual arguments beyond the formals. Note + 1 for thisv.
  for (size_t i = nformals + 1; i < nargs + 1; i++) {
    TraceRoot(trc, &argv[i], "ion-argv");
  }

  // Always trace the new.target from the frame. It's not in the snapshots.
  // +1 to pass |this|
  if (CalleeTokenIsConstructing(layout->calleeToken())) {
    TraceRoot(trc, &argv[1 + newTargetOffset], "ion-newTarget");
  }
}

#ifdef JS_NUNBOX32
static inline void WriteAllocation(const JSJitFrameIter& frame,
                                   const LAllocation* a, uintptr_t value) {
  if (a->isGeneralReg()) {
    Register reg = a->toGeneralReg()->reg();
    frame.machineState().write(reg, value);
  } else {
    *frame.jsFrame()->slotRef(SafepointSlotEntry(a)) = value;
  }
}
#endif

static void TraceIonJSFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  JitFrameLayout* layout = (JitFrameLayout*)frame.fp();

  layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));

  IonScript* ionScript = nullptr;
  if (frame.checkInvalidation(&ionScript)) {
    // This frame has been invalidated, meaning that its IonScript is no
    // longer reachable through the callee token (JSFunction/JSScript->ion
    // is now nullptr or recompiled). Manually trace it here.
    ionScript->trace(trc);
  } else {
    ionScript = frame.ionScriptFromCalleeToken();
  }

  TraceThisAndArguments(trc, frame, frame.jsFrame());

  const SafepointIndex* si =
      ionScript->getSafepointIndex(frame.resumePCinCurrentFrame());

  SafepointReader safepoint(ionScript, si);

  // Scan through slots which contain pointers (or on punboxing systems,
  // actual values).
  SafepointSlotEntry entry;

  while (safepoint.getGcSlot(&entry)) {
    uintptr_t* ref = layout->slotRef(entry);
    TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(ref),
                            "ion-gc-slot");
  }

  uintptr_t* spill = frame.spillBase();
  LiveGeneralRegisterSet gcRegs = safepoint.gcSpills();
  LiveGeneralRegisterSet valueRegs = safepoint.valueSpills();
  for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills());
       iter.more(); ++iter) {
    --spill;
    if (gcRegs.has(*iter)) {
      TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(spill),
                              "ion-gc-spill");
    } else if (valueRegs.has(*iter)) {
      TraceRoot(trc, reinterpret_cast<Value*>(spill), "ion-value-spill");
    }
  }

#ifdef JS_PUNBOX64
  while (safepoint.getValueSlot(&entry)) {
    Value* v = (Value*)layout->slotRef(entry);
    TraceRoot(trc, v, "ion-gc-slot");
  }
#else
  LAllocation type, payload;
  while (safepoint.getNunboxSlot(&type, &payload)) {
    JSValueTag tag = JSValueTag(ReadAllocation(frame, &type));
    uintptr_t rawPayload = ReadAllocation(frame, &payload);

    Value v = Value::fromTagAndPayload(tag, rawPayload);
    TraceRoot(trc, &v, "ion-torn-value");

    if (v != Value::fromTagAndPayload(tag, rawPayload)) {
      // GC moved the value, replace the stored payload.
      rawPayload = v.toNunboxPayload();
      WriteAllocation(frame, &payload, rawPayload);
    }
  }
#endif
}

static void TraceBailoutFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  JitFrameLayout* layout = (JitFrameLayout*)frame.fp();

  layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));

  // We have to trace the list of actual arguments, as only formal arguments
  // are represented in the Snapshot.
  TraceThisAndArguments(trc, frame, frame.jsFrame());

  // Under a bailout, do not have a Safepoint to only iterate over GC-things.
  // Thus we use a SnapshotIterator to trace all the locations which would be
  // used to reconstruct the Baseline frame.
  //
  // Note that at the time where this function is called, we have not yet
  // started to reconstruct baseline frames.

  // The vector of recover instructions is already traced as part of the
  // JitActivation.
  SnapshotIterator snapIter(frame,
                            frame.activation()->bailoutData()->machineState());

  // For each instruction, we read the allocations without evaluating the
  // recover instruction, nor reconstructing the frame. We are only looking at
  // tracing readable allocations.
  while (true) {
    while (snapIter.moreAllocations()) {
      snapIter.traceAllocation(trc);
    }

    if (!snapIter.moreInstructions()) {
      break;
    }
    snapIter.nextInstruction();
  }
}

static void UpdateIonJSFrameForMinorGC(JSRuntime* rt,
                                       const JSJitFrameIter& frame) {
  // Minor GCs may move slots/elements allocated in the nursery. Update
  // any slots/elements pointers stored in this frame.

  JitFrameLayout* layout = (JitFrameLayout*)frame.fp();

  IonScript* ionScript = nullptr;
  if (frame.checkInvalidation(&ionScript)) {
    // This frame has been invalidated, meaning that its IonScript is no
    // longer reachable through the callee token (JSFunction/JSScript->ion
    // is now nullptr or recompiled).
  } else {
    ionScript = frame.ionScriptFromCalleeToken();
  }

  Nursery& nursery = rt->gc.nursery();

  const SafepointIndex* si =
      ionScript->getSafepointIndex(frame.resumePCinCurrentFrame());
  SafepointReader safepoint(ionScript, si);

  LiveGeneralRegisterSet slotsRegs = safepoint.slotsOrElementsSpills();
  uintptr_t* spill = frame.spillBase();
  for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills());
       iter.more(); ++iter) {
    --spill;
    if (slotsRegs.has(*iter)) {
      nursery.forwardBufferPointer(spill);
    }
  }

  // Skip to the right place in the safepoint
  SafepointSlotEntry entry;
  while (safepoint.getGcSlot(&entry)) {
  }

#ifdef JS_PUNBOX64
  while (safepoint.getValueSlot(&entry)) {
  }
#else
  LAllocation type, payload;
  while (safepoint.getNunboxSlot(&type, &payload)) {
  }
#endif

  while (safepoint.getSlotsOrElementsSlot(&entry)) {
    nursery.forwardBufferPointer(layout->slotRef(entry));
  }
}

static void TraceBaselineStubFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  // Trace the ICStub pointer stored in the stub frame. This is necessary
  // so that we don't destroy the stub code after unlinking the stub.

  MOZ_ASSERT(frame.type() == FrameType::BaselineStub);
  BaselineStubFrameLayout* layout = (BaselineStubFrameLayout*)frame.fp();

  if (ICStub* stub = layout->maybeStubPtr()) {
    if (stub->isFallback()) {
      // Fallback stubs use runtime-wide trampoline code we don't need to trace.
      MOZ_ASSERT(stub->usesTrampolineCode());
    } else {
      MOZ_ASSERT(stub->toCacheIRStub()->makesGCCalls());
      stub->toCacheIRStub()->trace(trc);
    }
  }
}

static void TraceIonICCallFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  MOZ_ASSERT(frame.type() == FrameType::IonICCall);
  IonICCallFrameLayout* layout = (IonICCallFrameLayout*)frame.fp();
  TraceRoot(trc, layout->stubCode(), "ion-ic-call-code");
}

#if defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32)
uint8_t* alignDoubleSpill(uint8_t* pointer) {
  uintptr_t address = reinterpret_cast<uintptr_t>(pointer);
  address &= ~(uintptr_t(ABIStackAlignment) - 1);
  return reinterpret_cast<uint8_t*>(address);
}
#endif

#ifdef JS_CODEGEN_MIPS32
static void TraceJitExitFrameCopiedArguments(JSTracer* trc,
                                             const VMFunctionData* f,
                                             ExitFooterFrame* footer) {
  uint8_t* doubleArgs = footer->alignedForABI();
  if (f->outParam == Type_Handle) {
    doubleArgs -= sizeof(Value);
  }
  doubleArgs -= f->doubleByRefArgs() * sizeof(double);

  for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) {
    if (f->argProperties(explicitArg) == VMFunctionData::DoubleByRef) {
      // Arguments with double size can only have RootValue type.
      if (f->argRootType(explicitArg) == VMFunctionData::RootValue) {
        TraceRoot(trc, reinterpret_cast<Value*>(doubleArgs), "ion-vm-args");
      } else {
        MOZ_ASSERT(f->argRootType(explicitArg) == VMFunctionData::RootNone);
      }
      doubleArgs += sizeof(double);
    }
  }
}
#else
static void TraceJitExitFrameCopiedArguments(JSTracer* trc,
                                             const VMFunctionData* f,
                                             ExitFooterFrame* footer) {
  // This is NO-OP on other platforms.
}
#endif

static void TraceJitExitFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  ExitFooterFrame* footer = frame.exitFrame()->footer();

  // This corresponds to the case where we have build a fake exit frame which
  // handles the case of a native function call. We need to trace the argument
  // vector of the function call, and also new.target if it was a constructing
  // call.
  if (frame.isExitFrameLayout<NativeExitFrameLayout>()) {
    NativeExitFrameLayout* native =
        frame.exitFrame()->as<NativeExitFrameLayout>();
    size_t len = native->argc() + 2;
    Value* vp = native->vp();
    TraceRootRange(trc, len, vp, "ion-native-args");
    if (frame.isExitFrameLayout<ConstructNativeExitFrameLayout>()) {
      TraceRoot(trc, vp + len, "ion-native-new-target");
    }
    return;
  }

  if (frame.isExitFrameLayout<IonOOLNativeExitFrameLayout>()) {
    IonOOLNativeExitFrameLayout* oolnative =
        frame.exitFrame()->as<IonOOLNativeExitFrameLayout>();
    TraceRoot(trc, oolnative->stubCode(), "ion-ool-native-code");
    TraceRoot(trc, oolnative->vp(), "iol-ool-native-vp");
    size_t len = oolnative->argc() + 1;
    TraceRootRange(trc, len, oolnative->thisp(), "ion-ool-native-thisargs");
    return;
  }

  if (frame.isExitFrameLayout<IonOOLProxyExitFrameLayout>()) {
    IonOOLProxyExitFrameLayout* oolproxy =
        frame.exitFrame()->as<IonOOLProxyExitFrameLayout>();
    TraceRoot(trc, oolproxy->stubCode(), "ion-ool-proxy-code");
    TraceRoot(trc, oolproxy->vp(), "ion-ool-proxy-vp");
    TraceRoot(trc, oolproxy->id(), "ion-ool-proxy-id");
    TraceRoot(trc, oolproxy->proxy(), "ion-ool-proxy-proxy");
    return;
  }

  if (frame.isExitFrameLayout<IonDOMExitFrameLayout>()) {
    IonDOMExitFrameLayout* dom = frame.exitFrame()->as<IonDOMExitFrameLayout>();
    TraceRoot(trc, dom->thisObjAddress(), "ion-dom-args");
    if (dom->isMethodFrame()) {
      IonDOMMethodExitFrameLayout* method =
          reinterpret_cast<IonDOMMethodExitFrameLayout*>(dom);
      size_t len = method->argc() + 2;
      Value* vp = method->vp();
      TraceRootRange(trc, len, vp, "ion-dom-args");
    } else {
      TraceRoot(trc, dom->vp(), "ion-dom-args");
    }
    return;
  }

  if (frame.isExitFrameLayout<CalledFromJitExitFrameLayout>()) {
    auto* layout = frame.exitFrame()->as<CalledFromJitExitFrameLayout>();
    JitFrameLayout* jsLayout = layout->jsFrame();
    jsLayout->replaceCalleeToken(
        TraceCalleeToken(trc, jsLayout->calleeToken()));
    TraceThisAndArguments(trc, frame, jsLayout);
    return;
  }

  if (frame.isExitFrameLayout<DirectWasmJitCallFrameLayout>()) {
    // Nothing needs to be traced here at the moment -- the arguments to the
    // callee are traced by the callee, and the inlined caller does not push
    // anything else.
    return;
  }

  if (frame.isBareExit() || frame.isUnwoundJitExit()) {
    // Nothing to trace. Fake exit frame pushed for VM functions with
    // nothing to trace on the stack or unwound JitFrameLayout.
    return;
  }

  MOZ_ASSERT(frame.exitFrame()->isWrapperExit());

  const VMFunctionData* f = footer->function();
  MOZ_ASSERT(f);

  // Trace arguments of the VM wrapper.
  uint8_t* argBase = frame.exitFrame()->argBase();
  for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) {
    switch (f->argRootType(explicitArg)) {
      case VMFunctionData::RootNone:
        break;
      case VMFunctionData::RootObject: {
        // Sometimes we can bake in HandleObjects to nullptr.
        JSObject** pobj = reinterpret_cast<JSObject**>(argBase);
        if (*pobj) {
          TraceRoot(trc, pobj, "ion-vm-args");
        }
        break;
      }
      case VMFunctionData::RootString:
        TraceRoot(trc, reinterpret_cast<JSString**>(argBase), "ion-vm-args");
        break;
      case VMFunctionData::RootValue:
        TraceRoot(trc, reinterpret_cast<Value*>(argBase), "ion-vm-args");
        break;
      case VMFunctionData::RootId:
        TraceRoot(trc, reinterpret_cast<jsid*>(argBase), "ion-vm-args");
        break;
      case VMFunctionData::RootCell:
        TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(argBase),
                                "ion-vm-args");
        break;
      case VMFunctionData::RootBigInt:
        TraceRoot(trc, reinterpret_cast<JS::BigInt**>(argBase), "ion-vm-args");
        break;
    }

    switch (f->argProperties(explicitArg)) {
      case VMFunctionData::WordByValue:
      case VMFunctionData::WordByRef:
        argBase += sizeof(void*);
        break;
      case VMFunctionData::DoubleByValue:
      case VMFunctionData::DoubleByRef:
        argBase += 2 * sizeof(void*);
        break;
    }
  }

  if (f->outParam == Type_Handle) {
    switch (f->outParamRootType) {
      case VMFunctionData::RootNone:
        MOZ_CRASH("Handle outparam must have root type");
      case VMFunctionData::RootObject:
        TraceRoot(trc, footer->outParam<JSObject*>(), "ion-vm-out");
        break;
      case VMFunctionData::RootString:
        TraceRoot(trc, footer->outParam<JSString*>(), "ion-vm-out");
        break;
      case VMFunctionData::RootValue:
        TraceRoot(trc, footer->outParam<Value>(), "ion-vm-outvp");
        break;
      case VMFunctionData::RootId:
        TraceRoot(trc, footer->outParam<jsid>(), "ion-vm-outvp");
        break;
      case VMFunctionData::RootCell:
        TraceGenericPointerRoot(trc, footer->outParam<gc::Cell*>(),
                                "ion-vm-out");
        break;
      case VMFunctionData::RootBigInt:
        TraceRoot(trc, footer->outParam<JS::BigInt*>(), "ion-vm-out");
        break;
    }
  }

  TraceJitExitFrameCopiedArguments(trc, f, footer);
}

static void TraceBaselineInterpreterEntryFrame(JSTracer* trc,
                                               const JSJitFrameIter& frame) {
  // Baseline Interpreter entry code generated under --emit-interpreter-entry.
  BaselineInterpreterEntryFrameLayout* layout =
      (BaselineInterpreterEntryFrameLayout*)frame.fp();
  layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));
  TraceThisAndArguments(trc, frame, layout);
}

static void TraceRectifierFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  // Trace thisv.
  //
  // Baseline JIT code generated as part of the ICCall_Fallback stub may use
  // it if we're calling a constructor that returns a primitive value.
  RectifierFrameLayout* layout = (RectifierFrameLayout*)frame.fp();
  TraceRoot(trc, &layout->thisv(), "rectifier-thisv");
}

static void TraceJSJitToWasmFrame(JSTracer* trc, const JSJitFrameIter& frame) {
  // This is doing a subset of TraceIonJSFrame, since the callee doesn't
  // have a script.
  JitFrameLayout* layout = (JitFrameLayout*)frame.fp();
  layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));
  TraceThisAndArguments(trc, frame, layout);
}

static void TraceJitActivation(JSTracer* trc, JitActivation* activation) {
#ifdef CHECK_OSIPOINT_REGISTERS
  if (JitOptions.checkOsiPointRegisters) {
    // GC can modify spilled registers, breaking our register checks.
    // To handle this, we disable these checks for the current VM call
    // when a GC happens.
    activation->setCheckRegs(false);
  }
#endif

  activation->traceRematerializedFrames(trc);
  activation->traceIonRecovery(trc);

  // This is used for sanity checking continuity of the sequence of wasm stack
  // maps as we unwind.  It has no functional purpose.
  uintptr_t highestByteVisitedInPrevWasmFrame = 0;

  for (JitFrameIter frames(activation); !frames.done(); ++frames) {
    if (frames.isJSJit()) {
      const JSJitFrameIter& jitFrame = frames.asJSJit();
      switch (jitFrame.type()) {
        case FrameType::Exit:
          TraceJitExitFrame(trc, jitFrame);
          break;
        case FrameType::BaselineJS:
          jitFrame.baselineFrame()->trace(trc, jitFrame);
          break;
        case FrameType::IonJS:
          TraceIonJSFrame(trc, jitFrame);
          break;
        case FrameType::BaselineStub:
          TraceBaselineStubFrame(trc, jitFrame);
          break;
        case FrameType::Bailout:
          TraceBailoutFrame(trc, jitFrame);
          break;
        case FrameType::BaselineInterpreterEntry:
          TraceBaselineInterpreterEntryFrame(trc, jitFrame);
          break;
        case FrameType::Rectifier:
          TraceRectifierFrame(trc, jitFrame);
          break;
        case FrameType::IonICCall:
          TraceIonICCallFrame(trc, jitFrame);
          break;
        case FrameType::WasmToJSJit:
          // Ignore: this is a special marker used to let the
          // JitFrameIter know the frame above is a wasm frame, handled
          // in the next iteration.
          break;
        case FrameType::JSJitToWasm:
          TraceJSJitToWasmFrame(trc, jitFrame);
          break;
        default:
          MOZ_CRASH("unexpected frame type");
      }
      highestByteVisitedInPrevWasmFrame = 0; /* "unknown" */
    } else {
      MOZ_ASSERT(frames.isWasm());
      uint8_t* nextPC = frames.resumePCinCurrentFrame();
      MOZ_ASSERT(nextPC != 0);
      wasm::WasmFrameIter& wasmFrameIter = frames.asWasm();
      wasm::Instance* instance = wasmFrameIter.instance();
      wasm::TraceInstanceEdge(trc, instance, "WasmFrameIter instance");
      highestByteVisitedInPrevWasmFrame = instance->traceFrame(
          trc, wasmFrameIter, nextPC, highestByteVisitedInPrevWasmFrame);
    }
  }
}

void TraceJitActivations(JSContext* cx, JSTracer* trc) {
  for (JitActivationIterator activations(cx); !activations.done();
       ++activations) {
    TraceJitActivation(trc, activations->asJit());
  }
}

void UpdateJitActivationsForMinorGC(JSRuntime* rt) {
  MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
  JSContext* cx = rt->mainContextFromOwnThread();
  for (JitActivationIterator activations(cx); !activations.done();
       ++activations) {
    for (OnlyJSJitFrameIter iter(activations); !iter.done(); ++iter) {
      if (iter.frame().type() == FrameType::IonJS) {
        UpdateIonJSFrameForMinorGC(rt, iter.frame());
      }
    }
  }
}

JSScript* GetTopJitJSScript(JSContext* cx) {
  JSJitFrameIter frame(cx->activation()->asJit());
  MOZ_ASSERT(frame.type() == FrameType::Exit);
  ++frame;

  if (frame.isBaselineStub()) {
    ++frame;
    MOZ_ASSERT(frame.isBaselineJS());
  }

  MOZ_ASSERT(frame.isScripted());
  return frame.script();
}

void GetPcScript(JSContext* cx, JSScript** scriptRes, jsbytecode** pcRes) {
  JitSpew(JitSpew_IonSnapshots, "Recover PC & Script from the last frame.");

  // Recover the return address so that we can look it up in the
  // PcScriptCache, as script/pc computation is expensive.
  JitActivationIterator actIter(cx);
  OnlyJSJitFrameIter it(actIter);
  uint8_t* retAddr;
  if (it.frame().isExitFrame()) {
    ++it;

    // Skip baseline interpreter entry frames.
    // Can exist before rectifier frames.
    if (it.frame().isBaselineInterpreterEntry()) {
      ++it;
    }

    // Skip rectifier frames.
    if (it.frame().isRectifier()) {
      ++it;
      MOZ_ASSERT(it.frame().isBaselineStub() || it.frame().isBaselineJS() ||
                 it.frame().isIonJS());
    }

    // Skip Baseline/Ion stub and IC call frames.
    if (it.frame().isBaselineStub()) {
      ++it;
      MOZ_ASSERT(it.frame().isBaselineJS());
    } else if (it.frame().isIonICCall()) {
      ++it;
      MOZ_ASSERT(it.frame().isIonJS());
    }

    MOZ_ASSERT(it.frame().isBaselineJS() || it.frame().isIonJS());

    // Don't use the return address and the cache if the BaselineFrame is
    // running in the Baseline Interpreter. In this case the bytecode pc is
    // cheap to get, so we won't benefit from the cache, and the return address
    // does not map to a single bytecode pc.
    if (it.frame().isBaselineJS() &&
        it.frame().baselineFrame()->runningInInterpreter()) {
      it.frame().baselineScriptAndPc(scriptRes, pcRes);
      return;
    }

    retAddr = it.frame().resumePCinCurrentFrame();
  } else {
    MOZ_ASSERT(it.frame().isBailoutJS());
    retAddr = it.frame().returnAddress();
  }

  MOZ_ASSERT(retAddr);

  uint32_t hash = PcScriptCache::Hash(retAddr);

  // Lazily initialize the cache. The allocation may safely fail and will not
  // GC.
  if (MOZ_UNLIKELY(cx->ionPcScriptCache == nullptr)) {
    cx->ionPcScriptCache =
        MakeUnique<PcScriptCache>(cx->runtime()->gc.gcNumber());
  }

  if (cx->ionPcScriptCache.ref() &&
      cx->ionPcScriptCache->get(cx->runtime(), hash, retAddr, scriptRes,
                                pcRes)) {
    return;
  }

  // Lookup failed: undertake expensive process to determine script and pc.
  if (it.frame().isIonJS() || it.frame().isBailoutJS()) {
    InlineFrameIterator ifi(cx, &it.frame());
    *scriptRes = ifi.script();
    *pcRes = ifi.pc();
  } else {
    MOZ_ASSERT(it.frame().isBaselineJS());
    it.frame().baselineScriptAndPc(scriptRes, pcRes);
  }

  // Add entry to cache.
  if (cx->ionPcScriptCache.ref()) {
    cx->ionPcScriptCache->add(hash, retAddr, *pcRes, *scriptRes);
  }
}

RInstructionResults::RInstructionResults(JitFrameLayout* fp)
    : results_(nullptr), fp_(fp), initialized_(false) {}

RInstructionResults::RInstructionResults(RInstructionResults&& src)
    : results_(std::move(src.results_)),
      fp_(src.fp_),
      initialized_(src.initialized_) {
  src.initialized_ = false;
}

RInstructionResults& RInstructionResults::operator=(RInstructionResults&& rhs) {
  MOZ_ASSERT(&rhs != this, "self-moves are prohibited");
  this->~RInstructionResults();
  new (this) RInstructionResults(std::move(rhs));
  return *this;
}

RInstructionResults::~RInstructionResults() {
  // results_ is freed by the UniquePtr.
}

bool RInstructionResults::init(JSContext* cx, uint32_t numResults) {
  if (numResults) {
    results_ = cx->make_unique<Values>();
    if (!results_) {
      return false;
    }
    if (!results_->growBy(numResults)) {
      ReportOutOfMemory(cx);
      return false;
    }

    Value guard = MagicValue(JS_ION_BAILOUT);
    for (size_t i = 0; i < numResults; i++) {
      (*results_)[i].init(guard);
    }
  }

  initialized_ = true;
  return true;
}

bool RInstructionResults::isInitialized() const { return initialized_; }

size_t RInstructionResults::length() const { return results_->length(); }

JitFrameLayout* RInstructionResults::frame() const {
  MOZ_ASSERT(fp_);
  return fp_;
}

HeapPtr<Value>& RInstructionResults::operator[](size_t index) {
  return (*results_)[index];
}

void RInstructionResults::trace(JSTracer* trc) {
  // Note: The vector necessary exists, otherwise this object would not have
  // been stored on the activation from where the trace function is called.
  TraceRange(trc, results_->length(), results_->begin(), "ion-recover-results");
}

SnapshotIterator::SnapshotIterator(const JSJitFrameIter& iter,
                                   const MachineState* machineState)
    : snapshot_(iter.ionScript()->snapshots(), iter.snapshotOffset(),
                iter.ionScript()->snapshotsRVATableSize(),
                iter.ionScript()->snapshotsListSize()),
      recover_(snapshot_, iter.ionScript()->recovers(),
               iter.ionScript()->recoversSize()),
      fp_(iter.jsFrame()),
      machine_(machineState),
      ionScript_(iter.ionScript()),
      instructionResults_(nullptr) {}

SnapshotIterator::SnapshotIterator()
    : snapshot_(nullptr, 0, 0, 0),
      recover_(snapshot_, nullptr, 0),
      fp_(nullptr),
      machine_(nullptr),
      ionScript_(nullptr),
      instructionResults_(nullptr) {}

uintptr_t SnapshotIterator::fromStack(int32_t offset) const {
  return ReadFrameSlot(fp_, offset);
}

static Value FromObjectPayload(uintptr_t payload) {
  MOZ_ASSERT(payload != 0);
  return ObjectValue(*reinterpret_cast<JSObject*>(payload));
}

static Value FromStringPayload(uintptr_t payload) {
  return StringValue(reinterpret_cast<JSString*>(payload));
}

static Value FromSymbolPayload(uintptr_t payload) {
  return SymbolValue(reinterpret_cast<JS::Symbol*>(payload));
}

static Value FromBigIntPayload(uintptr_t payload) {
  return BigIntValue(reinterpret_cast<JS::BigInt*>(payload));
}

static Value FromTypedPayload(JSValueType type, uintptr_t payload) {
  switch (type) {
    case JSVAL_TYPE_INT32:
      return Int32Value(payload);
    case JSVAL_TYPE_BOOLEAN:
      return BooleanValue(!!payload);
    case JSVAL_TYPE_STRING:
      return FromStringPayload(payload);
    case JSVAL_TYPE_SYMBOL:
      return FromSymbolPayload(payload);
    case JSVAL_TYPE_BIGINT:
      return FromBigIntPayload(payload);
    case JSVAL_TYPE_OBJECT:
      return FromObjectPayload(payload);
    default:
      MOZ_CRASH("unexpected type - needs payload");
  }
}

bool SnapshotIterator::allocationReadable(const RValueAllocation& alloc,
                                          ReadMethod rm) {
  // If we have to recover stores, and if we are not interested in the
  // default value of the instruction, then we have to check if the recover
  // instruction results are available.
  if (alloc.needSideEffect() && rm != ReadMethod::AlwaysDefault) {
    if (!hasInstructionResults()) {
      return false;
    }
  }

  switch (alloc.mode()) {
    case RValueAllocation::DOUBLE_REG:
      return hasRegister(alloc.fpuReg());

    case RValueAllocation::TYPED_REG:
      return hasRegister(alloc.reg2());

#if defined(JS_NUNBOX32)
    case RValueAllocation::UNTYPED_REG_REG:
      return hasRegister(alloc.reg()) && hasRegister(alloc.reg2());
    case RValueAllocation::UNTYPED_REG_STACK:
      return hasRegister(alloc.reg()) && hasStack(alloc.stackOffset2());
    case RValueAllocation::UNTYPED_STACK_REG:
      return hasStack(alloc.stackOffset()) && hasRegister(alloc.reg2());
    case RValueAllocation::UNTYPED_STACK_STACK:
      return hasStack(alloc.stackOffset()) && hasStack(alloc.stackOffset2());
#elif defined(JS_PUNBOX64)
    case RValueAllocation::UNTYPED_REG:
      return hasRegister(alloc.reg());
    case RValueAllocation::UNTYPED_STACK:
      return hasStack(alloc.stackOffset());
#endif

    case RValueAllocation::RECOVER_INSTRUCTION:
      return hasInstructionResult(alloc.index());
    case RValueAllocation::RI_WITH_DEFAULT_CST:
      return rm == ReadMethod::AlwaysDefault ||
             hasInstructionResult(alloc.index());

    default:
      return true;
  }
}

Value SnapshotIterator::allocationValue(const RValueAllocation& alloc,
                                        ReadMethod rm) {
  switch (alloc.mode()) {
    case RValueAllocation::CONSTANT:
      return ionScript_->getConstant(alloc.index());

    case RValueAllocation::CST_UNDEFINED:
      return UndefinedValue();

    case RValueAllocation::CST_NULL:
      return NullValue();

    case RValueAllocation::DOUBLE_REG:
      return DoubleValue(fromRegister<double>(alloc.fpuReg()));

    case RValueAllocation::ANY_FLOAT_REG:
      return Float32Value(fromRegister<float>(alloc.fpuReg()));

    case RValueAllocation::ANY_FLOAT_STACK:
      return Float32Value(ReadFrameFloat32Slot(fp_, alloc.stackOffset()));

    case RValueAllocation::TYPED_REG:
      return FromTypedPayload(alloc.knownType(), fromRegister(alloc.reg2()));

    case RValueAllocation::TYPED_STACK: {
      switch (alloc.knownType()) {
        case JSVAL_TYPE_DOUBLE:
          return DoubleValue(ReadFrameDoubleSlot(fp_, alloc.stackOffset2()));
        case JSVAL_TYPE_INT32:
          return Int32Value(ReadFrameInt32Slot(fp_, alloc.stackOffset2()));
        case JSVAL_TYPE_BOOLEAN:
          return BooleanValue(ReadFrameBooleanSlot(fp_, alloc.stackOffset2()));
        case JSVAL_TYPE_STRING:
          return FromStringPayload(fromStack(alloc.stackOffset2()));
        case JSVAL_TYPE_SYMBOL:
          return FromSymbolPayload(fromStack(alloc.stackOffset2()));
        case JSVAL_TYPE_BIGINT:
          return FromBigIntPayload(fromStack(alloc.stackOffset2()));
        case JSVAL_TYPE_OBJECT:
          return FromObjectPayload(fromStack(alloc.stackOffset2()));
        default:
          MOZ_CRASH("Unexpected type");
      }
    }

#if defined(JS_NUNBOX32)
    case RValueAllocation::UNTYPED_REG_REG: {
      return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())),
                                      fromRegister(alloc.reg2()));
    }

    case RValueAllocation::UNTYPED_REG_STACK: {
      return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())),
                                      fromStack(alloc.stackOffset2()));
    }

    case RValueAllocation::UNTYPED_STACK_REG: {
      return Value::fromTagAndPayload(
          JSValueTag(fromStack(alloc.stackOffset())),
          fromRegister(alloc.reg2()));
    }

    case RValueAllocation::UNTYPED_STACK_STACK: {
      return Value::fromTagAndPayload(
          JSValueTag(fromStack(alloc.stackOffset())),
          fromStack(alloc.stackOffset2()));
    }
#elif defined(JS_PUNBOX64)
    case RValueAllocation::UNTYPED_REG: {
      return Value::fromRawBits(fromRegister(alloc.reg()));
    }

    case RValueAllocation::UNTYPED_STACK: {
      return Value::fromRawBits(fromStack(alloc.stackOffset()));
    }
#endif

    case RValueAllocation::RECOVER_INSTRUCTION:
      return fromInstructionResult(alloc.index());

    case RValueAllocation::RI_WITH_DEFAULT_CST:
      if (rm == ReadMethod::Normal && hasInstructionResult(alloc.index())) {
        return fromInstructionResult(alloc.index());
      }
      MOZ_ASSERT(rm == ReadMethod::AlwaysDefault);
      return ionScript_->getConstant(alloc.index2());

    default:
      MOZ_CRASH("huh?");
  }
}

Value SnapshotIterator::maybeRead(const RValueAllocation& a,
                                  MaybeReadFallback& fallback) {
  if (allocationReadable(a)) {
    return allocationValue(a);
  }

  if (fallback.canRecoverResults()) {
    // Code paths which are calling maybeRead are not always capable of
    // returning an error code, as these code paths used to be infallible.
    AutoEnterOOMUnsafeRegion oomUnsafe;
    if (!initInstructionResults(fallback)) {
      oomUnsafe.crash("js::jit::SnapshotIterator::maybeRead");
    }

    if (allocationReadable(a)) {
      return allocationValue(a);
    }

    MOZ_ASSERT_UNREACHABLE("All allocations should be readable.");
  }

  return UndefinedValue();
}

bool SnapshotIterator::tryRead(Value* result) {
  RValueAllocation a = readAllocation();
  if (allocationReadable(a)) {
    *result = allocationValue(a);
    return true;
  }
  return false;
}

void SnapshotIterator::writeAllocationValuePayload(
    const RValueAllocation& alloc, const Value& v) {
  MOZ_ASSERT(v.isGCThing());

  switch (alloc.mode()) {
    case RValueAllocation::CONSTANT:
      ionScript_->getConstant(alloc.index()) = v;
      break;

    case RValueAllocation::CST_UNDEFINED:
    case RValueAllocation::CST_NULL:
    case RValueAllocation::DOUBLE_REG:
    case RValueAllocation::ANY_FLOAT_REG:
    case RValueAllocation::ANY_FLOAT_STACK:
      MOZ_CRASH("Not a GC thing: Unexpected write");
      break;

    case RValueAllocation::TYPED_REG:
      machine_->write(alloc.reg2(), uintptr_t(v.toGCThing()));
      break;

    case RValueAllocation::TYPED_STACK:
      switch (alloc.knownType()) {
        default:
          MOZ_CRASH("Not a GC thing: Unexpected write");
          break;
        case JSVAL_TYPE_STRING:
        case JSVAL_TYPE_SYMBOL:
        case JSVAL_TYPE_BIGINT:
        case JSVAL_TYPE_OBJECT:
          WriteFrameSlot(fp_, alloc.stackOffset2(), uintptr_t(v.toGCThing()));
          break;
      }
      break;

#if defined(JS_NUNBOX32)
    case RValueAllocation::UNTYPED_REG_REG:
    case RValueAllocation::UNTYPED_STACK_REG:
      machine_->write(alloc.reg2(), uintptr_t(v.toGCThing()));
      break;

    case RValueAllocation::UNTYPED_REG_STACK:
    case RValueAllocation::UNTYPED_STACK_STACK:
      WriteFrameSlot(fp_, alloc.stackOffset2(), uintptr_t(v.toGCThing()));
      break;
#elif defined(JS_PUNBOX64)
    case RValueAllocation::UNTYPED_REG:
      machine_->write(alloc.reg(), v.asRawBits());
      break;

    case RValueAllocation::UNTYPED_STACK:
      WriteFrameSlot(fp_, alloc.stackOffset(), v.asRawBits());
      break;
#endif

    case RValueAllocation::RECOVER_INSTRUCTION:
      MOZ_CRASH("Recover instructions are handled by the JitActivation.");
      break;

    case RValueAllocation::RI_WITH_DEFAULT_CST:
      // Assume that we are always going to be writing on the default value
      // while tracing.
      ionScript_->getConstant(alloc.index2()) = v;
      break;

    default:
      MOZ_CRASH("huh?");
  }
}

void SnapshotIterator::traceAllocation(JSTracer* trc) {
  RValueAllocation alloc = readAllocation();
  if (!allocationReadable(alloc, ReadMethod::AlwaysDefault)) {
    return;
  }

  Value v = allocationValue(alloc, ReadMethod::AlwaysDefault);
  if (!v.isGCThing()) {
    return;
  }

  Value copy = v;
  TraceRoot(trc, &v, "ion-typed-reg");
  if (v != copy) {
    MOZ_ASSERT(SameType(v, copy));
    writeAllocationValuePayload(alloc, v);
  }
}

const RResumePoint* SnapshotIterator::resumePoint() const {
  return instruction()->toResumePoint();
}

uint32_t SnapshotIterator::numAllocations() const {
  return instruction()->numOperands();
}

uint32_t SnapshotIterator::pcOffset() const {
  return resumePoint()->pcOffset();
}

ResumeMode SnapshotIterator::resumeMode() const {
  return resumePoint()->mode();
}

void SnapshotIterator::skipInstruction() {
  MOZ_ASSERT(snapshot_.numAllocationsRead() == 0);
  size_t numOperands = instruction()->numOperands();
  for (size_t i = 0; i < numOperands; i++) {
    skip();
  }
  nextInstruction();
}

bool SnapshotIterator::initInstructionResults(MaybeReadFallback& fallback) {
  MOZ_ASSERT(fallback.canRecoverResults());
  JSContext* cx = fallback.maybeCx;

  // If there is only one resume point in the list of instructions, then there
  // is no instruction to recover, and thus no need to register any results.
  if (recover_.numInstructions() == 1) {
    return true;
  }

  JitFrameLayout* fp = fallback.frame->jsFrame();
  RInstructionResults* results = fallback.activation->maybeIonFrameRecovery(fp);
  if (!results) {
    AutoRealm ar(cx, fallback.frame->script());

    // We are going to run recover instructions. To avoid problems where recover
    // instructions are not idempotent (for example, if we allocate an object,
    // object identity may be observable), we should not execute code in the
    // Ion stack frame afterwards. To avoid doing so, we invalidate the script.
    // This is not necessary for bailouts or other cases where we are leaving
    // the frame anyway. We only need it for niche cases like debugger
    // introspection or Function.arguments.
    if (fallback.consequence == MaybeReadFallback::Fallback_Invalidate) {
      ionScript_->invalidate(cx, fallback.frame->script(),
                             /* resetUses = */ false,
                             "Observe recovered instruction.");
    }

    // Register the list of result on the activation.  We need to do that
    // before we initialize the list such as if any recover instruction
    // cause a GC, we can ensure that the results are properly traced by the
    // activation.
    RInstructionResults tmp(fallback.frame->jsFrame());
    if (!fallback.activation->registerIonFrameRecovery(std::move(tmp))) {
      return false;
    }

    results = fallback.activation->maybeIonFrameRecovery(fp);

    // Start a new snapshot at the beginning of the JSJitFrameIter.  This
    // SnapshotIterator is used for evaluating the content of all recover
    // instructions.  The result is then saved on the JitActivation.
    MachineState machine = fallback.frame->machineState();
    SnapshotIterator s(*fallback.frame, &machine);
    if (!s.computeInstructionResults(cx, results)) {
      // If the evaluation failed because of OOMs, then we discard the
      // current set of result that we collected so far.
      fallback.activation->removeIonFrameRecovery(fp);
      return false;
    }
  }

  MOZ_ASSERT(results->isInitialized());
  MOZ_RELEASE_ASSERT(results->length() == recover_.numInstructions() - 1);
  instructionResults_ = results;
  return true;
}

bool SnapshotIterator::computeInstructionResults(
    JSContext* cx, RInstructionResults* results) const {
  MOZ_ASSERT(!results->isInitialized());
  MOZ_ASSERT(recover_.numInstructionsRead() == 1);

  // The last instruction will always be a resume point.
  size_t numResults = recover_.numInstructions() - 1;
  if (!results->isInitialized()) {
    if (!results->init(cx, numResults)) {
      return false;
    }

    // No need to iterate over the only resume point.
    if (!numResults) {
      MOZ_ASSERT(results->isInitialized());
      return true;
    }

    // Avoid invoking the object metadata callback, which could try to walk the
    // stack while bailing out.
    gc::AutoSuppressGC suppressGC(cx);
    js::AutoSuppressAllocationMetadataBuilder suppressMetadata(cx);

    // Fill with the results of recover instructions.
    SnapshotIterator s(*this);
    s.instructionResults_ = results;
    while (s.moreInstructions()) {
      // Skip resume point and only interpret recover instructions.
      if (s.instruction()->isResumePoint()) {
        s.skipInstruction();
        continue;
      }

      if (!s.instruction()->recover(cx, s)) {
        return false;
      }
      s.nextInstruction();
    }
  }

  MOZ_ASSERT(results->isInitialized());
  return true;
}

void SnapshotIterator::storeInstructionResult(const Value& v) {
  uint32_t currIns = recover_.numInstructionsRead() - 1;
  MOZ_ASSERT((*instructionResults_)[currIns].isMagic(JS_ION_BAILOUT));
  (*instructionResults_)[currIns] = v;
}

Value SnapshotIterator::fromInstructionResult(uint32_t index) const {
  MOZ_ASSERT(!(*instructionResults_)[index].isMagic(JS_ION_BAILOUT));
  return (*instructionResults_)[index];
}

void SnapshotIterator::settleOnFrame() {
  // Check that the current instruction can still be use.
  MOZ_ASSERT(snapshot_.numAllocationsRead() == 0);
  while (!instruction()->isResumePoint()) {
    skipInstruction();
  }
}

void SnapshotIterator::nextFrame() {
  nextInstruction();
  settleOnFrame();
}

Value SnapshotIterator::maybeReadAllocByIndex(size_t index) {
  while (index--) {
    MOZ_ASSERT(moreAllocations());
    skip();
  }

  Value s;
  {
    // This MaybeReadFallback method cannot GC.
    JS::AutoSuppressGCAnalysis nogc;
    MaybeReadFallback fallback;
    s = maybeRead(fallback);
  }

  while (moreAllocations()) {
    skip();
  }

  return s;
}

InlineFrameIterator::InlineFrameIterator(JSContext* cx,
                                         const JSJitFrameIter* iter)
    : calleeTemplate_(cx),
      calleeRVA_(),
      script_(cx),
      pc_(nullptr),
      numActualArgs_(0) {
  resetOn(iter);
}

InlineFrameIterator::InlineFrameIterator(JSContext* cx,
                                         const InlineFrameIterator* iter)
    : frame_(iter ? iter->frame_ : nullptr),
      framesRead_(0),
      frameCount_(iter ? iter->frameCount_ : UINT32_MAX),
      calleeTemplate_(cx),
      calleeRVA_(),
      script_(cx),
      pc_(nullptr),
      numActualArgs_(0) {
  if (frame_) {
    machine_ = iter->machine_;
    start_ = SnapshotIterator(*frame_, &machine_);

    // findNextFrame will iterate to the next frame and init. everything.
    // Therefore to settle on the same frame, we report one frame less readed.
    framesRead_ = iter->framesRead_ - 1;
    findNextFrame();
  }
}

void InlineFrameIterator::resetOn(const JSJitFrameIter* iter) {
  frame_ = iter;
  framesRead_ = 0;
  frameCount_ = UINT32_MAX;

  if (iter) {
    machine_ = iter->machineState();
    start_ = SnapshotIterator(*iter, &machine_);
    findNextFrame();
  }
}

void InlineFrameIterator::findNextFrame() {
  MOZ_ASSERT(more());

  si_ = start_;

  // Read the initial frame out of the C stack.
  calleeTemplate_ = frame_->maybeCallee();
  calleeRVA_ = RValueAllocation();
  script_ = frame_->script();
  MOZ_ASSERT(script_->hasBaselineScript());

  // Settle on the outermost frame without evaluating any instructions before
  // looking for a pc.
  si_.settleOnFrame();

  pc_ = script_->offsetToPC(si_.pcOffset());
  numActualArgs_ = 0xbadbad;

  // This unfortunately is O(n*m), because we must skip over outer frames
  // before reading inner ones.

  // The first time (frameCount_ == UINT32_MAX) we do not know the number of
  // frames that we are going to inspect.  So we are iterating until there is
  // no more frames, to settle on the inner most frame and to count the number
  // of frames.
  size_t remaining = (frameCount_ != UINT32_MAX) ? frameNo() - 1 : SIZE_MAX;

  size_t i = 1;
  for (; i <= remaining && si_.moreFrames(); i++) {
    ResumeMode mode = si_.resumeMode();
    MOZ_ASSERT(IsIonInlinableOp(JSOp(*pc_)));

    // Recover the number of actual arguments from the script.
    if (IsInvokeOp(JSOp(*pc_))) {
      MOZ_ASSERT(mode == ResumeMode::InlinedStandardCall ||
                 mode == ResumeMode::InlinedFunCall);
      numActualArgs_ = GET_ARGC(pc_);
      if (mode == ResumeMode::InlinedFunCall && numActualArgs_ > 0) {
        numActualArgs_--;
      }
    } else if (IsGetPropPC(pc_) || IsGetElemPC(pc_)) {
      MOZ_ASSERT(mode == ResumeMode::InlinedAccessor);
      numActualArgs_ = 0;
    } else {
      MOZ_RELEASE_ASSERT(IsSetPropPC(pc_));
      MOZ_ASSERT(mode == ResumeMode::InlinedAccessor);
      numActualArgs_ = 1;
    }

    // Skip over non-argument slots, as well as |this|.
    bool skipNewTarget = IsConstructPC(pc_);
    unsigned skipCount =
        (si_.numAllocations() - 1) - numActualArgs_ - 1 - skipNewTarget;
    for (unsigned j = 0; j < skipCount; j++) {
      si_.skip();
    }

    // This value should correspond to the function which is being inlined.
    // The value must be readable to iterate over the inline frame. Most of
    // the time, these functions are stored as JSFunction constants,
    // register which are holding the JSFunction pointer, or recover
    // instruction with Default value.
    Value funval = si_.readWithDefault(&calleeRVA_);

    // Skip extra value allocations.
    while (si_.moreAllocations()) {
      si_.skip();
    }

    si_.nextFrame();

    calleeTemplate_ = &funval.toObject().as<JSFunction>();
    script_ = calleeTemplate_->nonLazyScript();
    MOZ_ASSERT(script_->hasBaselineScript());

    pc_ = script_->offsetToPC(si_.pcOffset());
  }

  // The first time we do not know the number of frames, we only settle on the
  // last frame, and update the number of frames based on the number of
  // iteration that we have done.
  if (frameCount_ == UINT32_MAX) {
    MOZ_ASSERT(!si_.moreFrames());
    frameCount_ = i;
  }

  framesRead_++;
}

JSFunction* InlineFrameIterator::callee(MaybeReadFallback& fallback) const {
  MOZ_ASSERT(isFunctionFrame());
  if (calleeRVA_.mode() == RValueAllocation::INVALID ||
      !fallback.canRecoverResults()) {
    return calleeTemplate_;
  }

  SnapshotIterator s(si_);
  // :TODO: Handle allocation failures from recover instruction.
  Value funval = s.maybeRead(calleeRVA_, fallback);
  return &funval.toObject().as<JSFunction>();
}

JSObject* InlineFrameIterator::computeEnvironmentChain(
    const Value& envChainValue, MaybeReadFallback& fallback,
    bool* hasInitialEnv) const {
  if (envChainValue.isObject()) {
    if (hasInitialEnv) {
      if (fallback.canRecoverResults()) {
        RootedObject obj(fallback.maybeCx, &envChainValue.toObject());
        *hasInitialEnv = isFunctionFrame() &&
                         callee(fallback)->needsFunctionEnvironmentObjects();
        return obj;
      }
      JS::AutoSuppressGCAnalysis
          nogc;  // If we cannot recover then we cannot GC.
      *hasInitialEnv = isFunctionFrame() &&
                       callee(fallback)->needsFunctionEnvironmentObjects();
    }

    return &envChainValue.toObject();
  }

  // Note we can hit this case even for functions with a CallObject, in case
  // we are walking the frame during the function prologue, before the env
  // chain has been initialized.
  if (isFunctionFrame()) {
    return callee(fallback)->environment();
  }

  if (isModuleFrame()) {
    return script()->module()->environment();
  }

  // Ion does not handle non-function scripts that have anything other than
  // the global on their env chain.
  MOZ_ASSERT(!script()->isForEval());
  MOZ_ASSERT(!script()->hasNonSyntacticScope());
  return &script()->global().lexicalEnvironment();
}

bool InlineFrameIterator::isFunctionFrame() const { return !!calleeTemplate_; }

bool InlineFrameIterator::isModuleFrame() const { return script()->isModule(); }

uintptr_t* MachineState::SafepointState::addressOfRegister(Register reg) const {
  size_t offset = regs.offsetOfPushedRegister(reg);

  MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0);
  uint32_t index = offset / sizeof(uintptr_t);

#ifdef DEBUG
  // Assert correctness with a slower algorithm in debug builds.
  uint32_t expectedIndex = 0;
  bool found = false;
  for (GeneralRegisterBackwardIterator iter(regs); iter.more(); ++iter) {
    expectedIndex++;
    if (*iter == reg) {
      found = true;
      break;
    }
  }
  MOZ_ASSERT(found);
  MOZ_ASSERT(expectedIndex == index);
#endif

  return spillBase - index;
}

char* MachineState::SafepointState::addressOfRegister(FloatRegister reg) const {
  // Note: this could be optimized similar to the GPR case above by implementing
  // offsetOfPushedRegister for FloatRegisterSet. Float register sets are
  // complicated though and this case is very uncommon: it's only reachable for
  // exception bailouts with live float registers.
  MOZ_ASSERT(!reg.isSimd128());
  char* ptr = floatSpillBase;
  for (FloatRegisterBackwardIterator iter(floatRegs); iter.more(); ++iter) {
    ptr -= (*iter).size();
    for (uint32_t a = 0; a < (*iter).numAlignedAliased(); a++) {
      // Only say that registers that actually start here start here.
      // e.g. d0 should not start at s1, only at s0.
      FloatRegister ftmp = (*iter).alignedAliased(a);
      if (ftmp == reg) {
        return ptr;
      }
    }
  }
  MOZ_CRASH("Invalid register");
}

uintptr_t MachineState::read(Register reg) const {
  if (state_.is<BailoutState>()) {
    return state_.as<BailoutState>().regs[reg.code()].r;
  }
  if (state_.is<SafepointState>()) {
    uintptr_t* addr = state_.as<SafepointState>().addressOfRegister(reg);
    return *addr;
  }
  MOZ_CRASH("Invalid state");
}

template <typename T>
T MachineState::read(FloatRegister reg) const {
#if !defined(JS_CODEGEN_RISCV64)
  MOZ_ASSERT(reg.size() == sizeof(T));
#else
  // RISCV64 always store FloatRegister as 64bit.
  MOZ_ASSERT(reg.size() == sizeof(double));
#endif

#if !defined(JS_CODEGEN_NONE) && !defined(JS_CODEGEN_WASM32)
  if (state_.is<BailoutState>()) {
    uint32_t offset = reg.getRegisterDumpOffsetInBytes();
    MOZ_ASSERT((offset % sizeof(T)) == 0);
    MOZ_ASSERT((offset + sizeof(T)) <= sizeof(RegisterDump::FPUArray));

    const BailoutState& state = state_.as<BailoutState>();
    char* addr = reinterpret_cast<char*>(state.floatRegs.begin()) + offset;
    return *reinterpret_cast<T*>(addr);
  }
  if (state_.is<SafepointState>()) {
    char* addr = state_.as<SafepointState>().addressOfRegister(reg);
    return *reinterpret_cast<T*>(addr);
  }
#endif
  MOZ_CRASH("Invalid state");
}

void MachineState::write(Register reg, uintptr_t value) const {
  if (state_.is<SafepointState>()) {
    uintptr_t* addr = state_.as<SafepointState>().addressOfRegister(reg);
    *addr = value;
    return;
  }
  MOZ_CRASH("Invalid state");
}

bool InlineFrameIterator::isConstructing() const {
  // Skip the current frame and look at the caller's.
  if (more()) {
    InlineFrameIterator parent(TlsContext.get(), this);
    ++parent;

    // In the case of a JS frame, look up the pc from the snapshot.
    JSOp parentOp = JSOp(*parent.pc());

    // Inlined Getters and Setters are never constructing.
    if (IsIonInlinableGetterOrSetterOp(parentOp)) {
      return false;
    }

    MOZ_ASSERT(IsInvokeOp(parentOp) && !IsSpreadOp(parentOp));

    return IsConstructOp(parentOp);
  }

  return frame_->isConstructing();
}

void SnapshotIterator::warnUnreadableAllocation() {
  fprintf(stderr,
          "Warning! Tried to access unreadable value allocation (possible "
          "f.arguments).\n");
}

struct DumpOverflownOp {
  const unsigned numFormals_;
  unsigned i_ = 0;

  explicit DumpOverflownOp(unsigned numFormals) : numFormals_(numFormals) {}

  void operator()(const Value& v) {
    if (i_ >= numFormals_) {
      fprintf(stderr, "  actual (arg %u): ", i_);
#if defined(DEBUG) || defined(JS_JITSPEW)
      DumpValue(v);
#else
      fprintf(stderr, "?\n");
#endif
    }
    i_++;
  }
};

void InlineFrameIterator::dump() const {
  MaybeReadFallback fallback;

  if (more()) {
    fprintf(stderr, " JS frame (inlined)\n");
  } else {
    fprintf(stderr, " JS frame\n");
  }

  bool isFunction = false;
  if (isFunctionFrame()) {
    isFunction = true;
    fprintf(stderr, "  callee fun: ");
#if defined(DEBUG) || defined(JS_JITSPEW)
    DumpObject(callee(fallback));
#else
    fprintf(stderr, "?\n");
#endif
  } else {
    fprintf(stderr, "  global frame, no callee\n");
  }

  fprintf(stderr, "  file %s line %u\n", script()->filename(),
          script()->lineno());

  fprintf(stderr, "  script = %p, pc = %p\n", (void*)script(), pc());
  fprintf(stderr, "  current op: %s\n", CodeName(JSOp(*pc())));

  if (!more()) {
    numActualArgs();
  }

  SnapshotIterator si = snapshotIterator();
  fprintf(stderr, "  slots: %u\n", si.numAllocations() - 1);
  for (unsigned i = 0; i < si.numAllocations() - 1; i++) {
    if (isFunction) {
      if (i == 0) {
        fprintf(stderr, "  env chain: ");
      } else if (i == 1) {
        fprintf(stderr, "  this: ");
      } else if (i - 2 < calleeTemplate()->nargs()) {
        fprintf(stderr, "  formal (arg %u): ", i - 2);
      } else {
        if (i - 2 == calleeTemplate()->nargs() &&
            numActualArgs() > calleeTemplate()->nargs()) {
          DumpOverflownOp d(calleeTemplate()->nargs());
          unaliasedForEachActual(TlsContext.get(), d, fallback);
        }

        fprintf(stderr, "  slot %d: ", int(i - 2 - calleeTemplate()->nargs()));
      }
    } else
      fprintf(stderr, "  slot %u: ", i);
#if defined(DEBUG) || defined(JS_JITSPEW)
    DumpValue(si.maybeRead(fallback));
#else
    fprintf(stderr, "?\n");
#endif
  }

  fputc('\n', stderr);
}

JitFrameLayout* InvalidationBailoutStack::fp() const {
  return (JitFrameLayout*)(sp() + ionScript_->frameSize());
}

void InvalidationBailoutStack::checkInvariants() const {
#ifdef DEBUG
  JitFrameLayout* frame = fp();
  CalleeToken token = frame->calleeToken();
  MOZ_ASSERT(token);

  uint8_t* rawBase = ionScript()->method()->raw();
  uint8_t* rawLimit = rawBase + ionScript()->method()->instructionsSize();
  uint8_t* osiPoint = osiPointReturnAddress();
  MOZ_ASSERT(rawBase <= osiPoint && osiPoint <= rawLimit);
#endif
}

void AssertJitStackInvariants(JSContext* cx) {
  for (JitActivationIterator activations(cx); !activations.done();
       ++activations) {
    JitFrameIter iter(activations->asJit());
    if (iter.isJSJit()) {
      JSJitFrameIter& frames = iter.asJSJit();
      size_t prevFrameSize = 0;
      size_t frameSize = 0;
      bool isScriptedCallee = false;
      for (; !frames.done(); ++frames) {
        size_t calleeFp = reinterpret_cast<size_t>(frames.fp());
        size_t callerFp = reinterpret_cast<size_t>(frames.prevFp());
        MOZ_ASSERT(callerFp >= calleeFp);
        prevFrameSize = frameSize;
        frameSize = callerFp - calleeFp;

        if (frames.isScripted() &&
            (frames.prevType() == FrameType::Rectifier ||
             frames.prevType() == FrameType::BaselineInterpreterEntry)) {
          MOZ_RELEASE_ASSERT(
              frameSize % JitStackAlignment == 0,
              "The rectifier and bli entry frame should keep the alignment");

          size_t expectedFrameSize =
              sizeof(Value) *
                  (frames.callee()->nargs() + 1 /* |this| argument */ +
                   frames.isConstructing() /* new.target */) +
              sizeof(JitFrameLayout);
          MOZ_RELEASE_ASSERT(frameSize >= expectedFrameSize,
                             "The frame is large enough to hold all arguments");
          MOZ_RELEASE_ASSERT(expectedFrameSize + JitStackAlignment > frameSize,
                             "The frame size is optimal");
        }

        if (frames.isExitFrame()) {
          // For the moment, we do not keep the JitStackAlignment
          // alignment for exit frames.
          frameSize -= ExitFrameLayout::Size();
        }

        if (frames.isIonJS()) {
          // Ideally, we should not have such requirement, but keep the
          // alignment-delta as part of the Safepoint such that we can pad
          // accordingly when making out-of-line calls.  In the mean time,
          // let us have check-points where we can garantee that
          // everything can properly be aligned before adding complexity.
          MOZ_RELEASE_ASSERT(
              frames.ionScript()->frameSize() % JitStackAlignment == 0,
              "Ensure that if the Ion frame is aligned, then the spill base is "
              "also aligned");

          if (isScriptedCallee) {
            MOZ_RELEASE_ASSERT(prevFrameSize % JitStackAlignment == 0,
                               "The ion frame should keep the alignment");
          }
        }

        // The stack is dynamically aligned by baseline stubs before calling
        // any jitted code.
        if (frames.prevType() == FrameType::BaselineStub && isScriptedCallee) {
          MOZ_RELEASE_ASSERT(calleeFp % JitStackAlignment == 0,
                             "The baseline stub restores the stack alignment");
        }

        isScriptedCallee =
            frames.isScripted() || frames.type() == FrameType::Rectifier;
      }

      MOZ_RELEASE_ASSERT(
          JSJitFrameIter::isEntry(frames.type()),
          "The first frame of a Jit activation should be an entry frame");
      MOZ_RELEASE_ASSERT(
          reinterpret_cast<size_t>(frames.fp()) % JitStackAlignment == 0,
          "The entry frame should be properly aligned");
    } else {
      MOZ_ASSERT(iter.isWasm());
      wasm::WasmFrameIter& frames = iter.asWasm();
      while (!frames.done()) {
        ++frames;
      }
    }
  }
}

}  // namespace jit
}  // namespace js