Adding upstream version 115.7.0esr.upstream/115.7.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 3117 insertions, 0 deletions

diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp
new file mode 100644
index 0000000000..7fc61381b9
--- /dev/null
+++ b/js/src/wasm/WasmStubs.cpp
@@ -0,0 +1,3117 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright 2015 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "wasm/WasmStubs.h"
+
+#include <algorithm>
+#include <iterator>
+#include <type_traits>
+
+#include "jit/ABIArgGenerator.h"
+#include "jit/JitFrames.h"
+#include "jit/RegisterAllocator.h"
+#include "js/Printf.h"
+#include "util/Memory.h"
+#include "wasm/WasmCode.h"
+#include "wasm/WasmGenerator.h"
+#include "wasm/WasmInstance.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "wasm/WasmInstance-inl.h"
+
+using namespace js;
+using namespace js::jit;
+using namespace js::wasm;
+
+using MIRTypeVector = Vector<jit::MIRType, 8, SystemAllocPolicy>;
+using ABIArgMIRTypeIter = jit::ABIArgIter<MIRTypeVector>;
+
+/*****************************************************************************/
+// ABIResultIter implementation
+
+static uint32_t ResultStackSize(ValType type) {
+  switch (type.kind()) {
+    case ValType::I32:
+      return ABIResult::StackSizeOfInt32;
+    case ValType::I64:
+      return ABIResult::StackSizeOfInt64;
+    case ValType::F32:
+      return ABIResult::StackSizeOfFloat;
+    case ValType::F64:
+      return ABIResult::StackSizeOfDouble;
+#ifdef ENABLE_WASM_SIMD
+    case ValType::V128:
+      return ABIResult::StackSizeOfV128;
+#endif
+    case ValType::Ref:
+      return ABIResult::StackSizeOfPtr;
+    default:
+      MOZ_CRASH("Unexpected result type");
+  }
+}
+
+// Compute the size of the stack slot that the wasm ABI requires be allocated
+// for a particular MIRType.  Note that this sometimes differs from the
+// MIRType's natural size.  See also ResultStackSize above and ABIResult::size()
+// and ABIResultIter below.
+
+uint32_t js::wasm::MIRTypeToABIResultSize(jit::MIRType type) {
+  switch (type) {
+    case MIRType::Int32:
+      return ABIResult::StackSizeOfInt32;
+    case MIRType::Int64:
+      return ABIResult::StackSizeOfInt64;
+    case MIRType::Float32:
+      return ABIResult::StackSizeOfFloat;
+    case MIRType::Double:
+      return ABIResult::StackSizeOfDouble;
+#ifdef ENABLE_WASM_SIMD
+    case MIRType::Simd128:
+      return ABIResult::StackSizeOfV128;
+#endif
+    case MIRType::Pointer:
+    case MIRType::RefOrNull:
+      return ABIResult::StackSizeOfPtr;
+    default:
+      MOZ_CRASH("MIRTypeToABIResultSize - unhandled case");
+  }
+}
+
+uint32_t ABIResult::size() const { return ResultStackSize(type()); }
+
+void ABIResultIter::settleRegister(ValType type) {
+  MOZ_ASSERT(!done());
+  MOZ_ASSERT_IF(direction_ == Next, index() < MaxRegisterResults);
+  MOZ_ASSERT_IF(direction_ == Prev, index() >= count_ - MaxRegisterResults);
+  static_assert(MaxRegisterResults == 1, "expected a single register result");
+
+  switch (type.kind()) {
+    case ValType::I32:
+      cur_ = ABIResult(type, ReturnReg);
+      break;
+    case ValType::I64:
+      cur_ = ABIResult(type, ReturnReg64);
+      break;
+    case ValType::F32:
+      cur_ = ABIResult(type, ReturnFloat32Reg);
+      break;
+    case ValType::F64:
+      cur_ = ABIResult(type, ReturnDoubleReg);
+      break;
+    case ValType::Ref:
+      cur_ = ABIResult(type, ReturnReg);
+      break;
+#ifdef ENABLE_WASM_SIMD
+    case ValType::V128:
+      cur_ = ABIResult(type, ReturnSimd128Reg);
+      break;
+#endif
+    default:
+      MOZ_CRASH("Unexpected result type");
+  }
+}
+
+void ABIResultIter::settleNext() {
+  MOZ_ASSERT(direction_ == Next);
+  MOZ_ASSERT(!done());
+
+  uint32_t typeIndex = count_ - index_ - 1;
+  ValType type = type_[typeIndex];
+
+  if (index_ < MaxRegisterResults) {
+    settleRegister(type);
+    return;
+  }
+
+  cur_ = ABIResult(type, nextStackOffset_);
+  nextStackOffset_ += ResultStackSize(type);
+}
+
+void ABIResultIter::settlePrev() {
+  MOZ_ASSERT(direction_ == Prev);
+  MOZ_ASSERT(!done());
+  uint32_t typeIndex = index_;
+  ValType type = type_[typeIndex];
+
+  if (count_ - index_ - 1 < MaxRegisterResults) {
+    settleRegister(type);
+    return;
+  }
+
+  uint32_t size = ResultStackSize(type);
+  MOZ_ASSERT(nextStackOffset_ >= size);
+  nextStackOffset_ -= size;
+  cur_ = ABIResult(type, nextStackOffset_);
+}
+
+#ifdef WASM_CODEGEN_DEBUG
+template <class Closure>
+static void GenPrint(DebugChannel channel, MacroAssembler& masm,
+                     const Maybe<Register>& taken, Closure passArgAndCall) {
+  if (!IsCodegenDebugEnabled(channel)) {
+    return;
+  }
+
+  AllocatableRegisterSet regs(RegisterSet::All());
+  LiveRegisterSet save(regs.asLiveSet());
+  masm.PushRegsInMask(save);
+
+  if (taken) {
+    regs.take(taken.value());
+  }
+  Register temp = regs.takeAnyGeneral();
+
+  {
+    MOZ_ASSERT(MaybeGetJitContext(),
+               "codegen debug checks require a jit context");
+    masm.setupUnalignedABICall(temp);
+    passArgAndCall(IsCompilingWasm(), temp);
+  }
+
+  masm.PopRegsInMask(save);
+}
+
+static void GenPrintf(DebugChannel channel, MacroAssembler& masm,
+                      const char* fmt, ...) {
+  va_list ap;
+  va_start(ap, fmt);
+  UniqueChars str = JS_vsmprintf(fmt, ap);
+  va_end(ap);
+
+  GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) {
+    // If we've gone this far, it means we're actually using the debugging
+    // strings. In this case, we leak them! This is only for debugging, and
+    // doing the right thing is cumbersome (in Ion, it'd mean add a vec of
+    // strings to the IonScript; in wasm, it'd mean add it to the current
+    // Module and serialize it properly).
+    const char* text = str.release();
+
+    masm.movePtr(ImmPtr((void*)text, ImmPtr::NoCheckToken()), temp);
+    masm.passABIArg(temp);
+    if (inWasm) {
+      masm.callDebugWithABI(SymbolicAddress::PrintText);
+    } else {
+      using Fn = void (*)(const char* output);
+      masm.callWithABI<Fn, PrintText>(MoveOp::GENERAL,
+                                      CheckUnsafeCallWithABI::DontCheckOther);
+    }
+  });
+}
+
+static void GenPrintIsize(DebugChannel channel, MacroAssembler& masm,
+                          const Register& src) {
+  GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) {
+    masm.passABIArg(src);
+    if (inWasm) {
+      masm.callDebugWithABI(SymbolicAddress::PrintI32);
+    } else {
+      using Fn = void (*)(int32_t val);
+      masm.callWithABI<Fn, PrintI32>(MoveOp::GENERAL,
+                                     CheckUnsafeCallWithABI::DontCheckOther);
+    }
+  });
+}
+
+static void GenPrintPtr(DebugChannel channel, MacroAssembler& masm,
+                        const Register& src) {
+  GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) {
+    masm.passABIArg(src);
+    if (inWasm) {
+      masm.callDebugWithABI(SymbolicAddress::PrintPtr);
+    } else {
+      using Fn = void (*)(uint8_t* val);
+      masm.callWithABI<Fn, PrintPtr>(MoveOp::GENERAL,
+                                     CheckUnsafeCallWithABI::DontCheckOther);
+    }
+  });
+}
+
+static void GenPrintI64(DebugChannel channel, MacroAssembler& masm,
+                        const Register64& src) {
+#  if JS_BITS_PER_WORD == 64
+  GenPrintf(channel, masm, "i64 ");
+  GenPrintIsize(channel, masm, src.reg);
+#  else
+  GenPrintf(channel, masm, "i64(");
+  GenPrintIsize(channel, masm, src.low);
+  GenPrintIsize(channel, masm, src.high);
+  GenPrintf(channel, masm, ") ");
+#  endif
+}
+
+static void GenPrintF32(DebugChannel channel, MacroAssembler& masm,
+                        const FloatRegister& src) {
+  GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) {
+    masm.passABIArg(src, MoveOp::FLOAT32);
+    if (inWasm) {
+      masm.callDebugWithABI(SymbolicAddress::PrintF32);
+    } else {
+      using Fn = void (*)(float val);
+      masm.callWithABI<Fn, PrintF32>(MoveOp::GENERAL,
+                                     CheckUnsafeCallWithABI::DontCheckOther);
+    }
+  });
+}
+
+static void GenPrintF64(DebugChannel channel, MacroAssembler& masm,
+                        const FloatRegister& src) {
+  GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) {
+    masm.passABIArg(src, MoveOp::DOUBLE);
+    if (inWasm) {
+      masm.callDebugWithABI(SymbolicAddress::PrintF64);
+    } else {
+      using Fn = void (*)(double val);
+      masm.callWithABI<Fn, PrintF64>(MoveOp::GENERAL,
+                                     CheckUnsafeCallWithABI::DontCheckOther);
+    }
+  });
+}
+
+#  ifdef ENABLE_WASM_SIMD
+static void GenPrintV128(DebugChannel channel, MacroAssembler& masm,
+                         const FloatRegister& src) {
+  // TODO: We might try to do something meaningful here once SIMD data are
+  // aligned and hence C++-ABI compliant.  For now, just make ourselves visible.
+  GenPrintf(channel, masm, "v128");
+}
+#  endif
+#else
+static void GenPrintf(DebugChannel channel, MacroAssembler& masm,
+                      const char* fmt, ...) {}
+static void GenPrintIsize(DebugChannel channel, MacroAssembler& masm,
+                          const Register& src) {}
+static void GenPrintPtr(DebugChannel channel, MacroAssembler& masm,
+                        const Register& src) {}
+static void GenPrintI64(DebugChannel channel, MacroAssembler& masm,
+                        const Register64& src) {}
+static void GenPrintF32(DebugChannel channel, MacroAssembler& masm,
+                        const FloatRegister& src) {}
+static void GenPrintF64(DebugChannel channel, MacroAssembler& masm,
+                        const FloatRegister& src) {}
+#  ifdef ENABLE_WASM_SIMD
+static void GenPrintV128(DebugChannel channel, MacroAssembler& masm,
+                         const FloatRegister& src) {}
+#  endif
+#endif
+
+static bool FinishOffsets(MacroAssembler& masm, Offsets* offsets) {
+  // On old ARM hardware, constant pools could be inserted and they need to
+  // be flushed before considering the size of the masm.
+  masm.flushBuffer();
+  offsets->end = masm.size();
+  return !masm.oom();
+}
+
+static void AssertStackAlignment(MacroAssembler& masm, uint32_t alignment,
+                                 uint32_t addBeforeAssert = 0) {
+  MOZ_ASSERT(
+      (sizeof(Frame) + masm.framePushed() + addBeforeAssert) % alignment == 0);
+  masm.assertStackAlignment(alignment, addBeforeAssert);
+}
+
+template <class VectorT, template <class VecT> class ABIArgIterT>
+static unsigned StackArgBytesHelper(const VectorT& args) {
+  ABIArgIterT<VectorT> iter(args);
+  while (!iter.done()) {
+    iter++;
+  }
+  return iter.stackBytesConsumedSoFar();
+}
+
+template <class VectorT>
+static unsigned StackArgBytesForNativeABI(const VectorT& args) {
+  return StackArgBytesHelper<VectorT, ABIArgIter>(args);
+}
+
+template <class VectorT>
+static unsigned StackArgBytesForWasmABI(const VectorT& args) {
+  return StackArgBytesHelper<VectorT, WasmABIArgIter>(args);
+}
+
+static unsigned StackArgBytesForWasmABI(const FuncType& funcType) {
+  ArgTypeVector args(funcType);
+  return StackArgBytesForWasmABI(args);
+}
+
+static void Move64(MacroAssembler& masm, const Address& src,
+                   const Address& dest, Register scratch) {
+#if JS_BITS_PER_WORD == 32
+  MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
+  masm.load32(LowWord(src), scratch);
+  masm.store32(scratch, LowWord(dest));
+  masm.load32(HighWord(src), scratch);
+  masm.store32(scratch, HighWord(dest));
+#else
+  Register64 scratch64(scratch);
+  masm.load64(src, scratch64);
+  masm.store64(scratch64, dest);
+#endif
+}
+
+static void SetupABIArguments(MacroAssembler& masm, const FuncExport& fe,
+                              const FuncType& funcType, Register argv,
+                              Register scratch) {
+  // Copy parameters out of argv and into the registers/stack-slots specified by
+  // the wasm ABI.
+  //
+  // SetupABIArguments are only used for C++ -> wasm calls through callExport(),
+  // and V128 and Ref types (other than externref) are not currently allowed.
+  ArgTypeVector args(funcType);
+  for (WasmABIArgIter iter(args); !iter.done(); iter++) {
+    unsigned argOffset = iter.index() * sizeof(ExportArg);
+    Address src(argv, argOffset);
+    MIRType type = iter.mirType();
+    switch (iter->kind()) {
+      case ABIArg::GPR:
+        if (type == MIRType::Int32) {
+          masm.load32(src, iter->gpr());
+        } else if (type == MIRType::Int64) {
+          masm.load64(src, iter->gpr64());
+        } else if (type == MIRType::RefOrNull) {
+          masm.loadPtr(src, iter->gpr());
+        } else if (type == MIRType::StackResults) {
+          MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index()));
+          masm.loadPtr(src, iter->gpr());
+        } else {
+          MOZ_CRASH("unknown GPR type");
+        }
+        break;
+#ifdef JS_CODEGEN_REGISTER_PAIR
+      case ABIArg::GPR_PAIR:
+        if (type == MIRType::Int64) {
+          masm.load64(src, iter->gpr64());
+        } else {
+          MOZ_CRASH("wasm uses hardfp for function calls.");
+        }
+        break;
+#endif
+      case ABIArg::FPU: {
+        static_assert(sizeof(ExportArg) >= jit::Simd128DataSize,
+                      "ExportArg must be big enough to store SIMD values");
+        switch (type) {
+          case MIRType::Double:
+            masm.loadDouble(src, iter->fpu());
+            break;
+          case MIRType::Float32:
+            masm.loadFloat32(src, iter->fpu());
+            break;
+          case MIRType::Simd128:
+#ifdef ENABLE_WASM_SIMD
+            // This is only used by the testing invoke path,
+            // wasmLosslessInvoke, and is guarded against in normal JS-API
+            // call paths.
+            masm.loadUnalignedSimd128(src, iter->fpu());
+            break;
+#else
+            MOZ_CRASH("V128 not supported in SetupABIArguments");
+#endif
+          default:
+            MOZ_CRASH("unexpected FPU type");
+            break;
+        }
+        break;
+      }
+      case ABIArg::Stack:
+        switch (type) {
+          case MIRType::Int32:
+            masm.load32(src, scratch);
+            masm.storePtr(scratch, Address(masm.getStackPointer(),
+                                           iter->offsetFromArgBase()));
+            break;
+          case MIRType::Int64: {
+            RegisterOrSP sp = masm.getStackPointer();
+            Move64(masm, src, Address(sp, iter->offsetFromArgBase()), scratch);
+            break;
+          }
+          case MIRType::RefOrNull:
+            masm.loadPtr(src, scratch);
+            masm.storePtr(scratch, Address(masm.getStackPointer(),
+                                           iter->offsetFromArgBase()));
+            break;
+          case MIRType::Double: {
+            ScratchDoubleScope fpscratch(masm);
+            masm.loadDouble(src, fpscratch);
+            masm.storeDouble(fpscratch, Address(masm.getStackPointer(),
+                                                iter->offsetFromArgBase()));
+            break;
+          }
+          case MIRType::Float32: {
+            ScratchFloat32Scope fpscratch(masm);
+            masm.loadFloat32(src, fpscratch);
+            masm.storeFloat32(fpscratch, Address(masm.getStackPointer(),
+                                                 iter->offsetFromArgBase()));
+            break;
+          }
+          case MIRType::Simd128: {
+#ifdef ENABLE_WASM_SIMD
+            // This is only used by the testing invoke path,
+            // wasmLosslessInvoke, and is guarded against in normal JS-API
+            // call paths.
+            ScratchSimd128Scope fpscratch(masm);
+            masm.loadUnalignedSimd128(src, fpscratch);
+            masm.storeUnalignedSimd128(
+                fpscratch,
+                Address(masm.getStackPointer(), iter->offsetFromArgBase()));
+            break;
+#else
+            MOZ_CRASH("V128 not supported in SetupABIArguments");
+#endif
+          }
+          case MIRType::StackResults: {
+            MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index()));
+            masm.loadPtr(src, scratch);
+            masm.storePtr(scratch, Address(masm.getStackPointer(),
+                                           iter->offsetFromArgBase()));
+            break;
+          }
+          default:
+            MOZ_CRASH("unexpected stack arg type");
+        }
+        break;
+      case ABIArg::Uninitialized:
+        MOZ_CRASH("Uninitialized ABIArg kind");
+    }
+  }
+}
+
+static void StoreRegisterResult(MacroAssembler& masm, const FuncExport& fe,
+                                const FuncType& funcType, Register loc) {
+  ResultType results = ResultType::Vector(funcType.results());
+  DebugOnly<bool> sawRegisterResult = false;
+  for (ABIResultIter iter(results); !iter.done(); iter.next()) {
+    const ABIResult& result = iter.cur();
+    if (result.inRegister()) {
+      MOZ_ASSERT(!sawRegisterResult);
+      sawRegisterResult = true;
+      switch (result.type().kind()) {
+        case ValType::I32:
+          masm.store32(result.gpr(), Address(loc, 0));
+          break;
+        case ValType::I64:
+          masm.store64(result.gpr64(), Address(loc, 0));
+          break;
+        case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+          masm.storeUnalignedSimd128(result.fpr(), Address(loc, 0));
+          break;
+#else
+          MOZ_CRASH("V128 not supported in StoreABIReturn");
+#endif
+        case ValType::F32:
+          masm.storeFloat32(result.fpr(), Address(loc, 0));
+          break;
+        case ValType::F64:
+          masm.storeDouble(result.fpr(), Address(loc, 0));
+          break;
+        case ValType::Ref:
+          masm.storePtr(result.gpr(), Address(loc, 0));
+          break;
+      }
+    }
+  }
+  MOZ_ASSERT(sawRegisterResult == (results.length() > 0));
+}
+
+#if defined(JS_CODEGEN_ARM)
+// The ARM system ABI also includes d15 & s31 in the non volatile float
+// registers. Also exclude lr (a.k.a. r14) as we preserve it manually.
+static const LiveRegisterSet NonVolatileRegs = LiveRegisterSet(
+    GeneralRegisterSet(Registers::NonVolatileMask &
+                       ~(Registers::SetType(1) << Registers::lr)),
+    FloatRegisterSet(FloatRegisters::NonVolatileMask |
+                     (FloatRegisters::SetType(1) << FloatRegisters::d15) |
+                     (FloatRegisters::SetType(1) << FloatRegisters::s31)));
+#elif defined(JS_CODEGEN_ARM64)
+// Exclude the Link Register (x30) because it is preserved manually.
+//
+// Include x16 (scratch) to make a 16-byte aligned amount of integer registers.
+// Include d31 (scratch) to make a 16-byte aligned amount of floating registers.
+static const LiveRegisterSet NonVolatileRegs = LiveRegisterSet(
+    GeneralRegisterSet((Registers::NonVolatileMask &
+                        ~(Registers::SetType(1) << Registers::lr)) |
+                       (Registers::SetType(1) << Registers::x16)),
+    FloatRegisterSet(FloatRegisters::NonVolatileMask |
+                     FloatRegisters::NonAllocatableMask));
+#else
+static const LiveRegisterSet NonVolatileRegs =
+    LiveRegisterSet(GeneralRegisterSet(Registers::NonVolatileMask),
+                    FloatRegisterSet(FloatRegisters::NonVolatileMask));
+#endif
+
+static const unsigned NumExtraPushed = 2;  // instance and argv
+
+#ifdef JS_CODEGEN_ARM64
+static const unsigned WasmPushSize = 16;
+#else
+static const unsigned WasmPushSize = sizeof(void*);
+#endif
+
+static void AssertExpectedSP(MacroAssembler& masm) {
+#ifdef JS_CODEGEN_ARM64
+  MOZ_ASSERT(sp.Is(masm.GetStackPointer64()));
+#  ifdef DEBUG
+  // Since we're asserting that SP is the currently active stack pointer,
+  // let's also in effect assert that PSP is dead -- by setting it to 1, so as
+  // to cause to cause any attempts to use it to segfault in an easily
+  // identifiable way.
+  masm.asVIXL().Mov(PseudoStackPointer64, 1);
+#  endif
+#endif
+}
+
+template <class Operand>
+static void WasmPush(MacroAssembler& masm, const Operand& op) {
+#ifdef JS_CODEGEN_ARM64
+  // Allocate a pad word so that SP can remain properly aligned.  |op| will be
+  // written at the lower-addressed of the two words pushed here.
+  masm.reserveStack(WasmPushSize);
+  masm.storePtr(op, Address(masm.getStackPointer(), 0));
+#else
+  masm.Push(op);
+#endif
+}
+
+static void WasmPop(MacroAssembler& masm, Register r) {
+#ifdef JS_CODEGEN_ARM64
+  // Also pop the pad word allocated by WasmPush.
+  masm.loadPtr(Address(masm.getStackPointer(), 0), r);
+  masm.freeStack(WasmPushSize);
+#else
+  masm.Pop(r);
+#endif
+}
+
+static void MoveSPForJitABI(MacroAssembler& masm) {
+#ifdef JS_CODEGEN_ARM64
+  masm.moveStackPtrTo(PseudoStackPointer);
+#endif
+}
+
+static void CallFuncExport(MacroAssembler& masm, const FuncExport& fe,
+                           const Maybe<ImmPtr>& funcPtr) {
+  MOZ_ASSERT(fe.hasEagerStubs() == !funcPtr);
+  MoveSPForJitABI(masm);
+  if (funcPtr) {
+    masm.call(*funcPtr);
+  } else {
+    masm.call(CallSiteDesc(CallSiteDesc::Func), fe.funcIndex());
+  }
+}
+
+STATIC_ASSERT_ANYREF_IS_JSOBJECT;  // Strings are currently boxed
+
+// Unboxing is branchy and contorted because of Spectre mitigations - we don't
+// have enough scratch registers.  Were it not for the spectre mitigations in
+// branchTestObjClass, the branch nest below would be restructured significantly
+// by inverting branches and using fewer registers.
+
+// Unbox an anyref in src (clobbering src in the process) and then re-box it as
+// a Value in *dst.  See the definition of AnyRef for a discussion of pointer
+// representation.
+static void UnboxAnyrefIntoValue(MacroAssembler& masm, Register instance,
+                                 Register src, const Address& dst,
+                                 Register scratch) {
+  MOZ_ASSERT(src != scratch);
+
+  // Not actually the value we're passing, but we've no way of
+  // decoding anything better.
+  GenPrintPtr(DebugChannel::Import, masm, src);
+
+  Label notNull, mustUnbox, done;
+  masm.branchTestPtr(Assembler::NonZero, src, src, &notNull);
+  masm.storeValue(NullValue(), dst);
+  masm.jump(&done);
+
+  masm.bind(&notNull);
+  // The type test will clear src if the test fails, so store early.
+  masm.storeValue(JSVAL_TYPE_OBJECT, src, dst);
+  // Spectre mitigations: see comment above about efficiency.
+  masm.branchTestObjClass(Assembler::Equal, src,
+                          Address(instance, Instance::offsetOfValueBoxClass()),
+                          scratch, src, &mustUnbox);
+  masm.jump(&done);
+
+  masm.bind(&mustUnbox);
+  Move64(masm, Address(src, WasmValueBox::offsetOfValue()), dst, scratch);
+
+  masm.bind(&done);
+}
+
+// Unbox an anyref in src and then re-box it as a Value in dst.
+// See the definition of AnyRef for a discussion of pointer representation.
+static void UnboxAnyrefIntoValueReg(MacroAssembler& masm, Register instance,
+                                    Register src, ValueOperand dst,
+                                    Register scratch) {
+  MOZ_ASSERT(src != scratch);
+#if JS_BITS_PER_WORD == 32
+  MOZ_ASSERT(dst.typeReg() != scratch);
+  MOZ_ASSERT(dst.payloadReg() != scratch);
+#else
+  MOZ_ASSERT(dst.valueReg() != scratch);
+#endif
+
+  // Not actually the value we're passing, but we've no way of
+  // decoding anything better.
+  GenPrintPtr(DebugChannel::Import, masm, src);
+
+  Label notNull, mustUnbox, done;
+  masm.branchTestPtr(Assembler::NonZero, src, src, &notNull);
+  masm.moveValue(NullValue(), dst);
+  masm.jump(&done);
+
+  masm.bind(&notNull);
+  // The type test will clear src if the test fails, so store early.
+  masm.moveValue(TypedOrValueRegister(MIRType::Object, AnyRegister(src)), dst);
+  // Spectre mitigations: see comment above about efficiency.
+  masm.branchTestObjClass(Assembler::Equal, src,
+                          Address(instance, Instance::offsetOfValueBoxClass()),
+                          scratch, src, &mustUnbox);
+  masm.jump(&done);
+
+  masm.bind(&mustUnbox);
+  masm.loadValue(Address(src, WasmValueBox::offsetOfValue()), dst);
+
+  masm.bind(&done);
+}
+
+// Box the Value in src as an anyref in dest.  src and dest must not overlap.
+// See the definition of AnyRef for a discussion of pointer representation.
+static void BoxValueIntoAnyref(MacroAssembler& masm, ValueOperand src,
+                               Register dest, Label* oolConvert) {
+  Label nullValue, objectValue, done;
+  {
+    ScratchTagScope tag(masm, src);
+    masm.splitTagForTest(src, tag);
+    masm.branchTestObject(Assembler::Equal, tag, &objectValue);
+    masm.branchTestNull(Assembler::Equal, tag, &nullValue);
+    masm.jump(oolConvert);
+  }
+
+  masm.bind(&nullValue);
+  masm.xorPtr(dest, dest);
+  masm.jump(&done);
+
+  masm.bind(&objectValue);
+  masm.unboxObject(src, dest);
+
+  masm.bind(&done);
+}
+
+// Generate a stub that enters wasm from a C++ caller via the native ABI. The
+// signature of the entry point is Module::ExportFuncPtr. The exported wasm
+// function has an ABI derived from its specific signature, so this function
+// must map from the ABI of ExportFuncPtr to the export's signature's ABI.
+static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe,
+                                const FuncType& funcType,
+                                const Maybe<ImmPtr>& funcPtr,
+                                Offsets* offsets) {
+  AutoCreatedBy acb(masm, "GenerateInterpEntry");
+
+  AssertExpectedSP(masm);
+  masm.haltingAlign(CodeAlignment);
+
+  offsets->begin = masm.currentOffset();
+
+  // Save the return address if it wasn't already saved by the call insn.
+#ifdef JS_USE_LINK_REGISTER
+#  if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS64) || \
+      defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64)
+  masm.pushReturnAddress();
+#  elif defined(JS_CODEGEN_ARM64)
+  // WasmPush updates framePushed() unlike pushReturnAddress(), but that's
+  // cancelled by the setFramePushed() below.
+  WasmPush(masm, lr);
+#  else
+  MOZ_CRASH("Implement this");
+#  endif
+#endif
+
+  // Save all caller non-volatile registers before we clobber them here and in
+  // the wasm callee (which does not preserve non-volatile registers).
+  masm.setFramePushed(0);
+  masm.PushRegsInMask(NonVolatileRegs);
+
+  const unsigned nonVolatileRegsPushSize =
+      masm.PushRegsInMaskSizeInBytes(NonVolatileRegs);
+
+  MOZ_ASSERT(masm.framePushed() == nonVolatileRegsPushSize);
+
+  // Put the 'argv' argument into a non-argument/return/instance register so
+  // that we can use 'argv' while we fill in the arguments for the wasm callee.
+  // Use a second non-argument/return register as temporary scratch.
+  Register argv = ABINonArgReturnReg0;
+  Register scratch = ABINonArgReturnReg1;
+
+  // scratch := SP
+  masm.moveStackPtrTo(scratch);
+
+  // Dynamically align the stack since ABIStackAlignment is not necessarily
+  // WasmStackAlignment. Preserve SP so it can be restored after the call.
+#ifdef JS_CODEGEN_ARM64
+  static_assert(WasmStackAlignment == 16, "ARM64 SP alignment");
+#else
+  masm.andToStackPtr(Imm32(~(WasmStackAlignment - 1)));
+#endif
+  masm.assertStackAlignment(WasmStackAlignment);
+
+  // Create a fake frame: just previous RA and an FP.
+  const size_t FakeFrameSize = 2 * sizeof(void*);
+#ifdef JS_CODEGEN_ARM64
+  masm.Ldr(ARMRegister(ABINonArgReturnReg0, 64),
+           MemOperand(ARMRegister(scratch, 64), nonVolatileRegsPushSize));
+#else
+  masm.Push(Address(scratch, nonVolatileRegsPushSize));
+#endif
+  // Store fake wasm register state. Ensure the frame pointer passed by the C++
+  // caller doesn't have the ExitFPTag bit set to not confuse frame iterators.
+  // This bit shouldn't be set if C++ code is using frame pointers, so this has
+  // no effect on native stack unwinders.
+  masm.andPtr(Imm32(int32_t(~ExitFPTag)), FramePointer);
+#ifdef JS_CODEGEN_ARM64
+  masm.asVIXL().Push(ARMRegister(ABINonArgReturnReg0, 64),
+                     ARMRegister(FramePointer, 64));
+  masm.moveStackPtrTo(FramePointer);
+#else
+  masm.Push(FramePointer);
+#endif
+
+  masm.moveStackPtrTo(FramePointer);
+  masm.setFramePushed(FakeFrameSize);
+#ifdef JS_CODEGEN_ARM64
+  const size_t FakeFramePushed = 0;
+#else
+  const size_t FakeFramePushed = sizeof(void*);
+  masm.Push(scratch);
+#endif
+
+  // Read the arguments of wasm::ExportFuncPtr according to the native ABI.
+  // The entry stub's frame is 1 word.
+  const unsigned argBase = sizeof(void*) + nonVolatileRegsPushSize;
+  ABIArgGenerator abi;
+  ABIArg arg;
+
+  // arg 1: ExportArg*
+  arg = abi.next(MIRType::Pointer);
+  if (arg.kind() == ABIArg::GPR) {
+    masm.movePtr(arg.gpr(), argv);
+  } else {
+    masm.loadPtr(Address(scratch, argBase + arg.offsetFromArgBase()), argv);
+  }
+
+  // Arg 2: Instance*
+  arg = abi.next(MIRType::Pointer);
+  if (arg.kind() == ABIArg::GPR) {
+    masm.movePtr(arg.gpr(), InstanceReg);
+  } else {
+    masm.loadPtr(Address(scratch, argBase + arg.offsetFromArgBase()),
+                 InstanceReg);
+  }
+
+  WasmPush(masm, InstanceReg);
+
+  // Save 'argv' on the stack so that we can recover it after the call.
+  WasmPush(masm, argv);
+
+  MOZ_ASSERT(masm.framePushed() ==
+             NumExtraPushed * WasmPushSize + FakeFrameSize + FakeFramePushed);
+
+  // Reserve stack space for the wasm call.
+  unsigned argDecrement =
+      StackDecrementForCall(WasmStackAlignment, masm.framePushed(),
+                            StackArgBytesForWasmABI(funcType));
+  masm.reserveStack(argDecrement);
+
+  // Copy parameters out of argv and into the wasm ABI registers/stack-slots.
+  SetupABIArguments(masm, fe, funcType, argv, scratch);
+
+  masm.loadWasmPinnedRegsFromInstance();
+
+  masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
+                                     WasmCalleeInstanceOffsetBeforeCall));
+
+  // Call into the real function. Note that, due to the throw stub, fp, instance
+  // and pinned registers may be clobbered.
+  masm.assertStackAlignment(WasmStackAlignment);
+  CallFuncExport(masm, fe, funcPtr);
+  masm.assertStackAlignment(WasmStackAlignment);
+
+  // Set the return value based on whether InstanceReg is the FailInstanceReg
+  // magic value (set by the throw stub).
+  Label success, join;
+  masm.branchPtr(Assembler::NotEqual, InstanceReg, Imm32(FailInstanceReg),
+                 &success);
+  masm.move32(Imm32(false), scratch);
+  masm.jump(&join);
+  masm.bind(&success);
+  masm.move32(Imm32(true), scratch);
+  masm.bind(&join);
+
+  // Pop the arguments pushed after the dynamic alignment.
+  masm.freeStack(argDecrement);
+
+  masm.setFramePushed(NumExtraPushed * WasmPushSize + FakeFrameSize +
+                      FakeFramePushed);
+
+  // Recover the 'argv' pointer which was saved before aligning the stack.
+  WasmPop(masm, argv);
+
+  WasmPop(masm, InstanceReg);
+
+  // Pop the stack pointer to its value right before dynamic alignment.
+#ifdef JS_CODEGEN_ARM64
+  static_assert(WasmStackAlignment == 16, "ARM64 SP alignment");
+  masm.freeStack(FakeFrameSize);
+#else
+  masm.PopStackPtr();
+#endif
+
+  // Store the register result, if any, in argv[0].
+  // No widening is required, as the value leaves ReturnReg.
+  StoreRegisterResult(masm, fe, funcType, argv);
+
+  masm.move32(scratch, ReturnReg);
+
+  // Restore clobbered non-volatile registers of the caller.
+  masm.setFramePushed(nonVolatileRegsPushSize);
+  masm.PopRegsInMask(NonVolatileRegs);
+  MOZ_ASSERT(masm.framePushed() == 0);
+
+#if defined(JS_CODEGEN_ARM64)
+  masm.setFramePushed(WasmPushSize);
+  WasmPop(masm, lr);
+  masm.abiret();
+#else
+  masm.ret();
+#endif
+
+  return FinishOffsets(masm, offsets);
+}
+
+#ifdef JS_PUNBOX64
+static const ValueOperand ScratchValIonEntry = ValueOperand(ABINonArgReg0);
+#else
+static const ValueOperand ScratchValIonEntry =
+    ValueOperand(ABINonArgReg0, ABINonArgReg1);
+#endif
+static const Register ScratchIonEntry = ABINonArgReg2;
+
+static void CallSymbolicAddress(MacroAssembler& masm, bool isAbsolute,
+                                SymbolicAddress sym) {
+  if (isAbsolute) {
+    masm.call(ImmPtr(SymbolicAddressTarget(sym), ImmPtr::NoCheckToken()));
+  } else {
+    masm.call(sym);
+  }
+}
+
+// Load instance's instance from the callee.
+static void GenerateJitEntryLoadInstance(MacroAssembler& masm) {
+  // ScratchIonEntry := callee => JSFunction*
+  unsigned offset = JitFrameLayout::offsetOfCalleeToken();
+  masm.loadFunctionFromCalleeToken(Address(FramePointer, offset),
+                                   ScratchIonEntry);
+
+  // ScratchIonEntry := callee->getExtendedSlot(WASM_INSTANCE_SLOT)->toPrivate()
+  //                 => Instance*
+  offset = FunctionExtended::offsetOfExtendedSlot(
+      FunctionExtended::WASM_INSTANCE_SLOT);
+  masm.loadPrivate(Address(ScratchIonEntry, offset), InstanceReg);
+}
+
+// Creates a JS fake exit frame for wasm, so the frame iterators just use
+// JSJit frame iteration.
+static void GenerateJitEntryThrow(MacroAssembler& masm, unsigned frameSize) {
+  AssertExpectedSP(masm);
+
+  MOZ_ASSERT(masm.framePushed() == frameSize);
+
+  masm.freeStack(frameSize);
+  MoveSPForJitABI(masm);
+
+  GenerateJitEntryLoadInstance(masm);
+
+  masm.loadPtr(Address(InstanceReg, Instance::offsetOfCx()), ScratchIonEntry);
+  masm.enterFakeExitFrameForWasm(ScratchIonEntry, ScratchIonEntry,
+                                 ExitFrameType::WasmGenericJitEntry);
+
+  masm.loadPtr(Address(InstanceReg, Instance::offsetOfJSJitExceptionHandler()),
+               ScratchIonEntry);
+  masm.jump(ScratchIonEntry);
+}
+
+// Helper function for allocating a BigInt and initializing it from an I64 in
+// GenerateJitEntry.  The return result is written to scratch.
+//
+// Note that this will create a new frame and must not - in its current form -
+// be called from a context where there is already another stub frame on the
+// stack, as that confuses unwinding during profiling.  This was a problem for
+// its use from GenerateImportJitExit, see bug 1754258.  Therefore,
+// FuncType::canHaveJitExit prevents the present function from being called for
+// exits.
+static void GenerateBigIntInitialization(MacroAssembler& masm,
+                                         unsigned bytesPushedByPrologue,
+                                         Register64 input, Register scratch,
+                                         const FuncExport& fe, Label* fail) {
+#if JS_BITS_PER_WORD == 32
+  MOZ_ASSERT(input.low != scratch);
+  MOZ_ASSERT(input.high != scratch);
+#else
+  MOZ_ASSERT(input.reg != scratch);
+#endif
+
+  // We need to avoid clobbering other argument registers and the input.
+  AllocatableRegisterSet regs(RegisterSet::Volatile());
+  LiveRegisterSet save(regs.asLiveSet());
+  masm.PushRegsInMask(save);
+
+  unsigned frameSize = StackDecrementForCall(
+      ABIStackAlignment, masm.framePushed() + bytesPushedByPrologue, 0);
+  masm.reserveStack(frameSize);
+  masm.assertStackAlignment(ABIStackAlignment);
+
+  CallSymbolicAddress(masm, !fe.hasEagerStubs(),
+                      SymbolicAddress::AllocateBigInt);
+  masm.storeCallPointerResult(scratch);
+
+  masm.assertStackAlignment(ABIStackAlignment);
+  masm.freeStack(frameSize);
+
+  LiveRegisterSet ignore;
+  ignore.add(scratch);
+  masm.PopRegsInMaskIgnore(save, ignore);
+
+  masm.branchTest32(Assembler::Zero, scratch, scratch, fail);
+  masm.initializeBigInt64(Scalar::BigInt64, scratch, input);
+}
+
+// Generate a stub that enters wasm from a jit code caller via the jit ABI.
+//
+// ARM64 note: This does not save the PseudoStackPointer so we must be sure to
+// recompute it on every return path, be it normal return or exception return.
+// The JIT code we return to assumes it is correct.
+
+static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex,
+                             const FuncExport& fe, const FuncType& funcType,
+                             const Maybe<ImmPtr>& funcPtr,
+                             CallableOffsets* offsets) {
+  AutoCreatedBy acb(masm, "GenerateJitEntry");
+
+  AssertExpectedSP(masm);
+
+  RegisterOrSP sp = masm.getStackPointer();
+
+  GenerateJitEntryPrologue(masm, offsets);
+
+  // The jit caller has set up the following stack layout (sp grows to the
+  // left):
+  // <-- retAddr | descriptor | callee | argc | this | arg1..N
+  //
+  // GenerateJitEntryPrologue has additionally pushed the caller's frame
+  // pointer. The stack pointer is now JitStackAlignment-aligned.
+
+  MOZ_ASSERT(masm.framePushed() == 0);
+
+  unsigned normalBytesNeeded = StackArgBytesForWasmABI(funcType);
+
+  MIRTypeVector coerceArgTypes;
+  MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Int32));
+  MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer));
+  MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer));
+  unsigned oolBytesNeeded = StackArgBytesForWasmABI(coerceArgTypes);
+
+  unsigned bytesNeeded = std::max(normalBytesNeeded, oolBytesNeeded);
+
+  // Note the jit caller ensures the stack is aligned *after* the call
+  // instruction.
+  unsigned frameSizeExclFP = StackDecrementForCall(
+      WasmStackAlignment, masm.framePushed(), bytesNeeded);
+
+  // Reserve stack space for wasm ABI arguments, set up like this:
+  // <-- ABI args | padding
+  masm.reserveStack(frameSizeExclFP);
+
+  uint32_t frameSize = masm.framePushed();
+
+  GenerateJitEntryLoadInstance(masm);
+
+  if (funcType.hasUnexposableArgOrRet()) {
+    CallSymbolicAddress(masm, !fe.hasEagerStubs(),
+                        SymbolicAddress::ReportV128JSCall);
+    GenerateJitEntryThrow(masm, frameSize);
+    return FinishOffsets(masm, offsets);
+  }
+
+  FloatRegister scratchF = ABINonArgDoubleReg;
+  Register scratchG = ScratchIonEntry;
+  ValueOperand scratchV = ScratchValIonEntry;
+
+  GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; arguments ",
+            fe.funcIndex());
+
+  // We do two loops:
+  // - one loop up-front will make sure that all the Value tags fit the
+  // expected signature argument types. If at least one inline conversion
+  // fails, we just jump to the OOL path which will call into C++. Inline
+  // conversions are ordered in the way we expect them to happen the most.
+  // - the second loop will unbox the arguments into the right registers.
+  Label oolCall;
+  for (size_t i = 0; i < funcType.args().length(); i++) {
+    Address jitArgAddr(FramePointer, JitFrameLayout::offsetOfActualArg(i));
+    masm.loadValue(jitArgAddr, scratchV);
+
+    Label next;
+    switch (funcType.args()[i].kind()) {
+      case ValType::I32: {
+        ScratchTagScope tag(masm, scratchV);
+        masm.splitTagForTest(scratchV, tag);
+
+        // For int32 inputs, just skip.
+        masm.branchTestInt32(Assembler::Equal, tag, &next);
+
+        // For double inputs, unbox, truncate and store back.
+        Label storeBack, notDouble;
+        masm.branchTestDouble(Assembler::NotEqual, tag, &notDouble);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.unboxDouble(scratchV, scratchF);
+          masm.branchTruncateDoubleMaybeModUint32(scratchF, scratchG, &oolCall);
+          masm.jump(&storeBack);
+        }
+        masm.bind(&notDouble);
+
+        // For null or undefined, store 0.
+        Label nullOrUndefined, notNullOrUndefined;
+        masm.branchTestUndefined(Assembler::Equal, tag, &nullOrUndefined);
+        masm.branchTestNull(Assembler::NotEqual, tag, &notNullOrUndefined);
+        masm.bind(&nullOrUndefined);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.storeValue(Int32Value(0), jitArgAddr);
+        }
+        masm.jump(&next);
+        masm.bind(&notNullOrUndefined);
+
+        // For booleans, store the number value back. Other types (symbol,
+        // object, strings) go to the C++ call.
+        masm.branchTestBoolean(Assembler::NotEqual, tag, &oolCall);
+        masm.unboxBoolean(scratchV, scratchG);
+        // fallthrough:
+
+        masm.bind(&storeBack);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.storeValue(JSVAL_TYPE_INT32, scratchG, jitArgAddr);
+        }
+        break;
+      }
+      case ValType::I64: {
+        ScratchTagScope tag(masm, scratchV);
+        masm.splitTagForTest(scratchV, tag);
+
+        // For BigInt inputs, just skip. Otherwise go to C++ for other
+        // types that require creating a new BigInt or erroring.
+        masm.branchTestBigInt(Assembler::NotEqual, tag, &oolCall);
+        masm.jump(&next);
+        break;
+      }
+      case ValType::F32:
+      case ValType::F64: {
+        // Note we can reuse the same code for f32/f64 here, since for the
+        // case of f32, the conversion of f64 to f32 will happen in the
+        // second loop.
+        ScratchTagScope tag(masm, scratchV);
+        masm.splitTagForTest(scratchV, tag);
+
+        // For double inputs, just skip.
+        masm.branchTestDouble(Assembler::Equal, tag, &next);
+
+        // For int32 inputs, convert and rebox.
+        Label storeBack, notInt32;
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.branchTestInt32(Assembler::NotEqual, scratchV, &notInt32);
+          masm.int32ValueToDouble(scratchV, scratchF);
+          masm.jump(&storeBack);
+        }
+        masm.bind(&notInt32);
+
+        // For undefined (missing argument), store NaN.
+        Label notUndefined;
+        masm.branchTestUndefined(Assembler::NotEqual, tag, &notUndefined);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.storeValue(DoubleValue(JS::GenericNaN()), jitArgAddr);
+          masm.jump(&next);
+        }
+        masm.bind(&notUndefined);
+
+        // +null is 0.
+        Label notNull;
+        masm.branchTestNull(Assembler::NotEqual, tag, &notNull);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.storeValue(DoubleValue(0.), jitArgAddr);
+        }
+        masm.jump(&next);
+        masm.bind(&notNull);
+
+        // For booleans, store the number value back. Other types (symbol,
+        // object, strings) go to the C++ call.
+        masm.branchTestBoolean(Assembler::NotEqual, tag, &oolCall);
+        masm.boolValueToDouble(scratchV, scratchF);
+        // fallthrough:
+
+        masm.bind(&storeBack);
+        {
+          ScratchTagScopeRelease _(&tag);
+          masm.boxDouble(scratchF, jitArgAddr);
+        }
+        break;
+      }
+      case ValType::Ref: {
+        // Guarded against by temporarilyUnsupportedReftypeForEntry()
+        MOZ_RELEASE_ASSERT(funcType.args()[i].refType().isExtern());
+        ScratchTagScope tag(masm, scratchV);
+        masm.splitTagForTest(scratchV, tag);
+
+        // For object inputs, we handle object and null inline, everything
+        // else requires an actual box and we go out of line to allocate
+        // that.
+        masm.branchTestObject(Assembler::Equal, tag, &next);
+        masm.branchTestNull(Assembler::Equal, tag, &next);
+        masm.jump(&oolCall);
+        break;
+      }
+      case ValType::V128: {
+        // Guarded against by hasUnexposableArgOrRet()
+        MOZ_CRASH("unexpected argument type when calling from the jit");
+      }
+      default: {
+        MOZ_CRASH("unexpected argument type when calling from the jit");
+      }
+    }
+    masm.nopAlign(CodeAlignment);
+    masm.bind(&next);
+  }
+
+  Label rejoinBeforeCall;
+  masm.bind(&rejoinBeforeCall);
+
+  // Convert all the expected values to unboxed values on the stack.
+  ArgTypeVector args(funcType);
+  for (WasmABIArgIter iter(args); !iter.done(); iter++) {
+    Address argv(FramePointer, JitFrameLayout::offsetOfActualArg(iter.index()));
+    bool isStackArg = iter->kind() == ABIArg::Stack;
+    switch (iter.mirType()) {
+      case MIRType::Int32: {
+        Register target = isStackArg ? ScratchIonEntry : iter->gpr();
+        masm.unboxInt32(argv, target);
+        GenPrintIsize(DebugChannel::Function, masm, target);
+        if (isStackArg) {
+          masm.storePtr(target, Address(sp, iter->offsetFromArgBase()));
+        }
+        break;
+      }
+      case MIRType::Int64: {
+        // The coercion has provided a BigInt value by this point, which
+        // we need to convert to an I64 here.
+        if (isStackArg) {
+          Address dst(sp, iter->offsetFromArgBase());
+          Register src = scratchV.payloadOrValueReg();
+#if JS_BITS_PER_WORD == 64
+          Register64 scratch64(scratchG);
+#else
+          Register64 scratch64(scratchG, ABINonArgReg3);
+#endif
+          masm.unboxBigInt(argv, src);
+          masm.loadBigInt64(src, scratch64);
+          GenPrintI64(DebugChannel::Function, masm, scratch64);
+          masm.store64(scratch64, dst);
+        } else {
+          Register src = scratchG;
+          Register64 target = iter->gpr64();
+          masm.unboxBigInt(argv, src);
+          masm.loadBigInt64(src, target);
+          GenPrintI64(DebugChannel::Function, masm, target);
+        }
+        break;
+      }
+      case MIRType::Float32: {
+        FloatRegister target = isStackArg ? ABINonArgDoubleReg : iter->fpu();
+        masm.unboxDouble(argv, ABINonArgDoubleReg);
+        masm.convertDoubleToFloat32(ABINonArgDoubleReg, target);
+        GenPrintF32(DebugChannel::Function, masm, target.asSingle());
+        if (isStackArg) {
+          masm.storeFloat32(target, Address(sp, iter->offsetFromArgBase()));
+        }
+        break;
+      }
+      case MIRType::Double: {
+        FloatRegister target = isStackArg ? ABINonArgDoubleReg : iter->fpu();
+        masm.unboxDouble(argv, target);
+        GenPrintF64(DebugChannel::Function, masm, target);
+        if (isStackArg) {
+          masm.storeDouble(target, Address(sp, iter->offsetFromArgBase()));
+        }
+        break;
+      }
+      case MIRType::RefOrNull: {
+        Register target = isStackArg ? ScratchIonEntry : iter->gpr();
+        masm.unboxObjectOrNull(argv, target);
+        GenPrintPtr(DebugChannel::Function, masm, target);
+        if (isStackArg) {
+          masm.storePtr(target, Address(sp, iter->offsetFromArgBase()));
+        }
+        break;
+      }
+      default: {
+        MOZ_CRASH("unexpected input argument when calling from jit");
+      }
+    }
+  }
+
+  GenPrintf(DebugChannel::Function, masm, "\n");
+
+  // Setup wasm register state.
+  masm.loadWasmPinnedRegsFromInstance();
+
+  masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
+                                     WasmCalleeInstanceOffsetBeforeCall));
+
+  // Call into the real function. Note that, due to the throw stub, instance
+  // and pinned registers may be clobbered.
+  masm.assertStackAlignment(WasmStackAlignment);
+  CallFuncExport(masm, fe, funcPtr);
+  masm.assertStackAlignment(WasmStackAlignment);
+
+  // If InstanceReg is equal to the FailInstanceReg magic value (set by the
+  // throw stub), then report the exception to the JIT caller by jumping into
+  // the exception stub.
+  Label exception;
+  masm.branchPtr(Assembler::Equal, InstanceReg, Imm32(FailInstanceReg),
+                 &exception);
+
+  // Pop arguments.
+  masm.freeStack(frameSizeExclFP);
+
+  GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; returns ",
+            fe.funcIndex());
+
+  // Store the return value in the JSReturnOperand.
+  const ValTypeVector& results = funcType.results();
+  if (results.length() == 0) {
+    GenPrintf(DebugChannel::Function, masm, "void");
+    masm.moveValue(UndefinedValue(), JSReturnOperand);
+  } else {
+    MOZ_ASSERT(results.length() == 1, "multi-value return to JS unimplemented");
+    switch (results[0].kind()) {
+      case ValType::I32:
+        GenPrintIsize(DebugChannel::Function, masm, ReturnReg);
+        // No widening is required, as the value is boxed.
+        masm.boxNonDouble(JSVAL_TYPE_INT32, ReturnReg, JSReturnOperand);
+        break;
+      case ValType::F32: {
+        masm.canonicalizeFloat(ReturnFloat32Reg);
+        masm.convertFloat32ToDouble(ReturnFloat32Reg, ReturnDoubleReg);
+        GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg);
+        ScratchDoubleScope fpscratch(masm);
+        masm.boxDouble(ReturnDoubleReg, JSReturnOperand, fpscratch);
+        break;
+      }
+      case ValType::F64: {
+        masm.canonicalizeDouble(ReturnDoubleReg);
+        GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg);
+        ScratchDoubleScope fpscratch(masm);
+        masm.boxDouble(ReturnDoubleReg, JSReturnOperand, fpscratch);
+        break;
+      }
+      case ValType::I64: {
+        Label fail, done;
+        GenPrintI64(DebugChannel::Function, masm, ReturnReg64);
+        GenerateBigIntInitialization(masm, 0, ReturnReg64, scratchG, fe, &fail);
+        masm.boxNonDouble(JSVAL_TYPE_BIGINT, scratchG, JSReturnOperand);
+        masm.jump(&done);
+        masm.bind(&fail);
+        // Fixup the stack for the exception tail so that we can share it.
+        masm.reserveStack(frameSizeExclFP);
+        masm.jump(&exception);
+        masm.bind(&done);
+        // Un-fixup the stack for the benefit of the assertion below.
+        masm.setFramePushed(0);
+        break;
+      }
+      case ValType::V128: {
+        MOZ_CRASH("unexpected return type when calling from ion to wasm");
+      }
+      case ValType::Ref: {
+        STATIC_ASSERT_ANYREF_IS_JSOBJECT;
+        // Per comment above, the call may have clobbered the instance
+        // register, so reload since unboxing will need it.
+        GenerateJitEntryLoadInstance(masm);
+        UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand,
+                                WasmJitEntryReturnScratch);
+        break;
+      }
+    }
+  }
+
+  GenPrintf(DebugChannel::Function, masm, "\n");
+
+  MOZ_ASSERT(masm.framePushed() == 0);
+
+  AssertExpectedSP(masm);
+  GenerateJitEntryEpilogue(masm, offsets);
+  MOZ_ASSERT(masm.framePushed() == 0);
+
+  // Generate an OOL call to the C++ conversion path.
+  if (funcType.args().length()) {
+    masm.bind(&oolCall);
+    masm.setFramePushed(frameSize);
+
+    // Baseline and Ion call C++ runtime via BuiltinThunk with wasm abi, so to
+    // unify the BuiltinThunk's interface we call it here with wasm abi.
+    jit::WasmABIArgIter<MIRTypeVector> argsIter(coerceArgTypes);
+
+    // argument 0: function export index.
+    if (argsIter->kind() == ABIArg::GPR) {
+      masm.movePtr(ImmWord(funcExportIndex), argsIter->gpr());
+    } else {
+      masm.storePtr(ImmWord(funcExportIndex),
+                    Address(sp, argsIter->offsetFromArgBase()));
+    }
+    argsIter++;
+
+    // argument 1: instance
+    if (argsIter->kind() == ABIArg::GPR) {
+      masm.movePtr(InstanceReg, argsIter->gpr());
+    } else {
+      masm.storePtr(InstanceReg, Address(sp, argsIter->offsetFromArgBase()));
+    }
+    argsIter++;
+
+    // argument 2: effective address of start of argv
+    Address argv(FramePointer, JitFrameLayout::offsetOfActualArgs());
+    if (argsIter->kind() == ABIArg::GPR) {
+      masm.computeEffectiveAddress(argv, argsIter->gpr());
+    } else {
+      masm.computeEffectiveAddress(argv, ScratchIonEntry);
+      masm.storePtr(ScratchIonEntry,
+                    Address(sp, argsIter->offsetFromArgBase()));
+    }
+    argsIter++;
+    MOZ_ASSERT(argsIter.done());
+
+    masm.assertStackAlignment(ABIStackAlignment);
+    CallSymbolicAddress(masm, !fe.hasEagerStubs(),
+                        SymbolicAddress::CoerceInPlace_JitEntry);
+    masm.assertStackAlignment(ABIStackAlignment);
+
+    // No widening is required, as the return value is used as a bool.
+    masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg,
+                      &rejoinBeforeCall);
+  }
+
+  // Prepare to throw: reload InstanceReg from the frame.
+  masm.bind(&exception);
+  masm.setFramePushed(frameSize);
+  GenerateJitEntryThrow(masm, frameSize);
+
+  return FinishOffsets(masm, offsets);
+}
+
+void wasm::GenerateDirectCallFromJit(MacroAssembler& masm, const FuncExport& fe,
+                                     const Instance& inst,
+                                     const JitCallStackArgVector& stackArgs,
+                                     Register scratch, uint32_t* callOffset) {
+  MOZ_ASSERT(!IsCompilingWasm());
+
+  const FuncType& funcType = inst.metadata().getFuncExportType(fe);
+
+  size_t framePushedAtStart = masm.framePushed();
+
+  // Note, if code here pushes a reference value into the frame for its own
+  // purposes (and not just as an argument to the callee) then the frame must be
+  // traced in TraceJitExitFrame, see the case there for DirectWasmJitCall.  The
+  // callee will trace values that are pushed as arguments, however.
+
+  // Push a special frame descriptor that indicates the frame size so we can
+  // directly iterate from the current JIT frame without an extra call.
+  // Note: buildFakeExitFrame pushes an ExitFrameLayout containing the current
+  // frame pointer. We also use this to restore the frame pointer after the
+  // call.
+  *callOffset = masm.buildFakeExitFrame(scratch);
+  // FP := ExitFrameLayout*
+  masm.moveStackPtrTo(FramePointer);
+  masm.loadJSContext(scratch);
+  masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::DirectWasmJitCall);
+
+  // Move stack arguments to their final locations.
+  unsigned bytesNeeded = StackArgBytesForWasmABI(funcType);
+  bytesNeeded = StackDecrementForCall(WasmStackAlignment, masm.framePushed(),
+                                      bytesNeeded);
+  if (bytesNeeded) {
+    masm.reserveStack(bytesNeeded);
+  }
+
+  GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; arguments ",
+            fe.funcIndex());
+
+  ArgTypeVector args(funcType);
+  for (WasmABIArgIter iter(args); !iter.done(); iter++) {
+    MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != scratch);
+    MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != FramePointer);
+    if (iter->kind() != ABIArg::Stack) {
+      switch (iter.mirType()) {
+        case MIRType::Int32:
+          GenPrintIsize(DebugChannel::Function, masm, iter->gpr());
+          break;
+        case MIRType::Int64:
+          GenPrintI64(DebugChannel::Function, masm, iter->gpr64());
+          break;
+        case MIRType::Float32:
+          GenPrintF32(DebugChannel::Function, masm, iter->fpu());
+          break;
+        case MIRType::Double:
+          GenPrintF64(DebugChannel::Function, masm, iter->fpu());
+          break;
+        case MIRType::RefOrNull:
+          GenPrintPtr(DebugChannel::Function, masm, iter->gpr());
+          break;
+        case MIRType::StackResults:
+          MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index()));
+          GenPrintPtr(DebugChannel::Function, masm, iter->gpr());
+          break;
+        default:
+          MOZ_CRASH("ion to wasm fast path can only handle i32/f32/f64");
+      }
+      continue;
+    }
+
+    Address dst(masm.getStackPointer(), iter->offsetFromArgBase());
+
+    const JitCallStackArg& stackArg = stackArgs[iter.index()];
+    switch (stackArg.tag()) {
+      case JitCallStackArg::Tag::Imm32:
+        GenPrintf(DebugChannel::Function, masm, "%d ", stackArg.imm32());
+        masm.storePtr(ImmWord(stackArg.imm32()), dst);
+        break;
+      case JitCallStackArg::Tag::GPR:
+        MOZ_ASSERT(stackArg.gpr() != scratch);
+        MOZ_ASSERT(stackArg.gpr() != FramePointer);
+        GenPrintIsize(DebugChannel::Function, masm, stackArg.gpr());
+        masm.storePtr(stackArg.gpr(), dst);
+        break;
+      case JitCallStackArg::Tag::FPU:
+        switch (iter.mirType()) {
+          case MIRType::Double:
+            GenPrintF64(DebugChannel::Function, masm, stackArg.fpu());
+            masm.storeDouble(stackArg.fpu(), dst);
+            break;
+          case MIRType::Float32:
+            GenPrintF32(DebugChannel::Function, masm, stackArg.fpu());
+            masm.storeFloat32(stackArg.fpu(), dst);
+            break;
+          default:
+            MOZ_CRASH(
+                "unexpected MIR type for a float register in wasm fast call");
+        }
+        break;
+      case JitCallStackArg::Tag::Address: {
+        // The address offsets were valid *before* we pushed our frame.
+        Address src = stackArg.addr();
+        MOZ_ASSERT(src.base == masm.getStackPointer());
+        src.offset += masm.framePushed() - framePushedAtStart;
+        switch (iter.mirType()) {
+          case MIRType::Double: {
+            ScratchDoubleScope fpscratch(masm);
+            GenPrintF64(DebugChannel::Function, masm, fpscratch);
+            masm.loadDouble(src, fpscratch);
+            masm.storeDouble(fpscratch, dst);
+            break;
+          }
+          case MIRType::Float32: {
+            ScratchFloat32Scope fpscratch(masm);
+            masm.loadFloat32(src, fpscratch);
+            GenPrintF32(DebugChannel::Function, masm, fpscratch);
+            masm.storeFloat32(fpscratch, dst);
+            break;
+          }
+          case MIRType::Int32: {
+            masm.loadPtr(src, scratch);
+            GenPrintIsize(DebugChannel::Function, masm, scratch);
+            masm.storePtr(scratch, dst);
+            break;
+          }
+          case MIRType::RefOrNull: {
+            masm.loadPtr(src, scratch);
+            GenPrintPtr(DebugChannel::Function, masm, scratch);
+            masm.storePtr(scratch, dst);
+            break;
+          }
+          case MIRType::StackResults: {
+            MOZ_CRASH("multi-value in ion to wasm fast path unimplemented");
+          }
+          default: {
+            MOZ_CRASH("unexpected MIR type for a stack slot in wasm fast call");
+          }
+        }
+        break;
+      }
+      case JitCallStackArg::Tag::Undefined: {
+        MOZ_CRASH("can't happen because of arg.kind() check");
+      }
+    }
+  }
+
+  GenPrintf(DebugChannel::Function, masm, "\n");
+
+  // Load instance; from now on, InstanceReg is live.
+  masm.movePtr(ImmPtr(&inst), InstanceReg);
+  masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
+                                     WasmCalleeInstanceOffsetBeforeCall));
+  masm.loadWasmPinnedRegsFromInstance();
+
+  // Actual call.
+  const CodeTier& codeTier = inst.code().codeTier(inst.code().bestTier());
+  const MetadataTier& metadata = codeTier.metadata();
+  const CodeRange& codeRange = metadata.codeRange(fe);
+  void* callee = codeTier.segment().base() + codeRange.funcUncheckedCallEntry();
+
+  masm.assertStackAlignment(WasmStackAlignment);
+  MoveSPForJitABI(masm);
+  masm.callJit(ImmPtr(callee));
+#ifdef JS_CODEGEN_ARM64
+  // WASM does not always keep PSP in sync with SP.  So reinitialize it as it
+  // might be clobbered either by WASM or by any C++ calls within.
+  masm.initPseudoStackPtr();
+#endif
+  masm.assertStackAlignment(WasmStackAlignment);
+
+  masm.branchPtr(Assembler::Equal, InstanceReg, Imm32(wasm::FailInstanceReg),
+                 masm.exceptionLabel());
+
+  // Store the return value in the appropriate place.
+  GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; returns ",
+            fe.funcIndex());
+  const ValTypeVector& results = funcType.results();
+  if (results.length() == 0) {
+    masm.moveValue(UndefinedValue(), JSReturnOperand);
+    GenPrintf(DebugChannel::Function, masm, "void");
+  } else {
+    MOZ_ASSERT(results.length() == 1, "multi-value return to JS unimplemented");
+    switch (results[0].kind()) {
+      case wasm::ValType::I32:
+        // The return value is in ReturnReg, which is what Ion expects.
+        GenPrintIsize(DebugChannel::Function, masm, ReturnReg);
+#ifdef JS_64BIT
+        masm.widenInt32(ReturnReg);
+#endif
+        break;
+      case wasm::ValType::I64:
+        // The return value is in ReturnReg64, which is what Ion expects.
+        GenPrintI64(DebugChannel::Function, masm, ReturnReg64);
+        break;
+      case wasm::ValType::F32:
+        masm.canonicalizeFloat(ReturnFloat32Reg);
+        GenPrintF32(DebugChannel::Function, masm, ReturnFloat32Reg);
+        break;
+      case wasm::ValType::F64:
+        masm.canonicalizeDouble(ReturnDoubleReg);
+        GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg);
+        break;
+      case wasm::ValType::Ref:
+        STATIC_ASSERT_ANYREF_IS_JSOBJECT;
+        // The call to wasm above preserves the InstanceReg, we don't
+        // need to reload it here.
+        UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand,
+                                WasmJitEntryReturnScratch);
+        break;
+      case wasm::ValType::V128:
+        MOZ_CRASH("unexpected return type when calling from ion to wasm");
+    }
+  }
+
+  GenPrintf(DebugChannel::Function, masm, "\n");
+
+  // Restore the frame pointer.
+  masm.loadPtr(Address(FramePointer, 0), FramePointer);
+
+  // Free args + frame descriptor.
+  masm.leaveExitFrame(bytesNeeded + ExitFrameLayout::Size());
+
+  MOZ_ASSERT(framePushedAtStart == masm.framePushed());
+}
+
+static void StackCopy(MacroAssembler& masm, MIRType type, Register scratch,
+                      Address src, Address dst) {
+  if (type == MIRType::Int32) {
+    masm.load32(src, scratch);
+    GenPrintIsize(DebugChannel::Import, masm, scratch);
+    masm.store32(scratch, dst);
+  } else if (type == MIRType::Int64) {
+#if JS_BITS_PER_WORD == 32
+    MOZ_RELEASE_ASSERT(src.base != scratch && dst.base != scratch);
+    GenPrintf(DebugChannel::Import, masm, "i64(");
+    masm.load32(LowWord(src), scratch);
+    GenPrintIsize(DebugChannel::Import, masm, scratch);
+    masm.store32(scratch, LowWord(dst));
+    masm.load32(HighWord(src), scratch);
+    GenPrintIsize(DebugChannel::Import, masm, scratch);
+    masm.store32(scratch, HighWord(dst));
+    GenPrintf(DebugChannel::Import, masm, ") ");
+#else
+    Register64 scratch64(scratch);
+    masm.load64(src, scratch64);
+    GenPrintIsize(DebugChannel::Import, masm, scratch);
+    masm.store64(scratch64, dst);
+#endif
+  } else if (type == MIRType::RefOrNull || type == MIRType::Pointer ||
+             type == MIRType::StackResults) {
+    masm.loadPtr(src, scratch);
+    GenPrintPtr(DebugChannel::Import, masm, scratch);
+    masm.storePtr(scratch, dst);
+  } else if (type == MIRType::Float32) {
+    ScratchFloat32Scope fpscratch(masm);
+    masm.loadFloat32(src, fpscratch);
+    GenPrintF32(DebugChannel::Import, masm, fpscratch);
+    masm.storeFloat32(fpscratch, dst);
+  } else if (type == MIRType::Double) {
+    ScratchDoubleScope fpscratch(masm);
+    masm.loadDouble(src, fpscratch);
+    GenPrintF64(DebugChannel::Import, masm, fpscratch);
+    masm.storeDouble(fpscratch, dst);
+#ifdef ENABLE_WASM_SIMD
+  } else if (type == MIRType::Simd128) {
+    ScratchSimd128Scope fpscratch(masm);
+    masm.loadUnalignedSimd128(src, fpscratch);
+    GenPrintV128(DebugChannel::Import, masm, fpscratch);
+    masm.storeUnalignedSimd128(fpscratch, dst);
+#endif
+  } else {
+    MOZ_CRASH("StackCopy: unexpected type");
+  }
+}
+
+static void FillArgumentArrayForInterpExit(MacroAssembler& masm,
+                                           unsigned funcImportIndex,
+                                           const FuncType& funcType,
+                                           unsigned argOffset,
+                                           Register scratch) {
+  // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter
+  // is accounted for by the ABIArgIter.
+  const unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances);
+
+  GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; arguments ",
+            funcImportIndex);
+
+  ArgTypeVector args(funcType);
+  for (ABIArgIter i(args); !i.done(); i++) {
+    Address dst(masm.getStackPointer(), argOffset + i.index() * sizeof(Value));
+
+    MIRType type = i.mirType();
+    MOZ_ASSERT(args.isSyntheticStackResultPointerArg(i.index()) ==
+               (type == MIRType::StackResults));
+    switch (i->kind()) {
+      case ABIArg::GPR:
+        if (type == MIRType::Int32) {
+          GenPrintIsize(DebugChannel::Import, masm, i->gpr());
+          masm.store32(i->gpr(), dst);
+        } else if (type == MIRType::Int64) {
+          GenPrintI64(DebugChannel::Import, masm, i->gpr64());
+          masm.store64(i->gpr64(), dst);
+        } else if (type == MIRType::RefOrNull) {
+          GenPrintPtr(DebugChannel::Import, masm, i->gpr());
+          masm.storePtr(i->gpr(), dst);
+        } else if (type == MIRType::StackResults) {
+          GenPrintPtr(DebugChannel::Import, masm, i->gpr());
+          masm.storePtr(i->gpr(), dst);
+        } else {
+          MOZ_CRASH(
+              "FillArgumentArrayForInterpExit, ABIArg::GPR: unexpected type");
+        }
+        break;
+#ifdef JS_CODEGEN_REGISTER_PAIR
+      case ABIArg::GPR_PAIR:
+        if (type == MIRType::Int64) {
+          GenPrintI64(DebugChannel::Import, masm, i->gpr64());
+          masm.store64(i->gpr64(), dst);
+        } else {
+          MOZ_CRASH("wasm uses hardfp for function calls.");
+        }
+        break;
+#endif
+      case ABIArg::FPU: {
+        FloatRegister srcReg = i->fpu();
+        if (type == MIRType::Double) {
+          GenPrintF64(DebugChannel::Import, masm, srcReg);
+          masm.storeDouble(srcReg, dst);
+        } else if (type == MIRType::Float32) {
+          // Preserve the NaN pattern in the input.
+          GenPrintF32(DebugChannel::Import, masm, srcReg);
+          masm.storeFloat32(srcReg, dst);
+        } else if (type == MIRType::Simd128) {
+          // The value should never escape; the call will be stopped later as
+          // the import is being called.  But we should generate something sane
+          // here for the boxed case since a debugger or the stack walker may
+          // observe something.
+          ScratchDoubleScope dscratch(masm);
+          masm.loadConstantDouble(0, dscratch);
+          GenPrintF64(DebugChannel::Import, masm, dscratch);
+          masm.storeDouble(dscratch, dst);
+        } else {
+          MOZ_CRASH("Unknown MIRType in wasm exit stub");
+        }
+        break;
+      }
+      case ABIArg::Stack: {
+        Address src(FramePointer,
+                    offsetFromFPToCallerStackArgs + i->offsetFromArgBase());
+        if (type == MIRType::Simd128) {
+          // As above.  StackCopy does not know this trick.
+          ScratchDoubleScope dscratch(masm);
+          masm.loadConstantDouble(0, dscratch);
+          GenPrintF64(DebugChannel::Import, masm, dscratch);
+          masm.storeDouble(dscratch, dst);
+        } else {
+          StackCopy(masm, type, scratch, src, dst);
+        }
+        break;
+      }
+      case ABIArg::Uninitialized:
+        MOZ_CRASH("Uninitialized ABIArg kind");
+    }
+  }
+  GenPrintf(DebugChannel::Import, masm, "\n");
+}
+
+// Note, this may destroy the values in incoming argument registers as a result
+// of Spectre mitigation.
+static void FillArgumentArrayForJitExit(MacroAssembler& masm, Register instance,
+                                        unsigned funcImportIndex,
+                                        const FuncType& funcType,
+                                        unsigned argOffset, Register scratch,
+                                        Register scratch2, Label* throwLabel) {
+  MOZ_ASSERT(scratch != scratch2);
+
+  // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter
+  // is accounted for by the ABIArgIter.
+  const unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances);
+
+  // This loop does not root the values that are being constructed in
+  // for the arguments. Allocations that are generated by code either
+  // in the loop or called from it should be NoGC allocations.
+  GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; arguments ",
+            funcImportIndex);
+
+  ArgTypeVector args(funcType);
+  for (ABIArgIter i(args); !i.done(); i++) {
+    Address dst(masm.getStackPointer(), argOffset + i.index() * sizeof(Value));
+
+    MIRType type = i.mirType();
+    MOZ_ASSERT(args.isSyntheticStackResultPointerArg(i.index()) ==
+               (type == MIRType::StackResults));
+    switch (i->kind()) {
+      case ABIArg::GPR:
+        if (type == MIRType::Int32) {
+          GenPrintIsize(DebugChannel::Import, masm, i->gpr());
+          masm.storeValue(JSVAL_TYPE_INT32, i->gpr(), dst);
+        } else if (type == MIRType::Int64) {
+          // FuncType::canHaveJitExit should prevent this.  Also see comments
+          // at GenerateBigIntInitialization.
+          MOZ_CRASH("Should not happen");
+        } else if (type == MIRType::RefOrNull) {
+          // This works also for FuncRef because it is distinguishable from
+          // a boxed AnyRef.
+          masm.movePtr(i->gpr(), scratch2);
+          UnboxAnyrefIntoValue(masm, instance, scratch2, dst, scratch);
+        } else if (type == MIRType::StackResults) {
+          MOZ_CRASH("Multi-result exit to JIT unimplemented");
+        } else {
+          MOZ_CRASH(
+              "FillArgumentArrayForJitExit, ABIArg::GPR: unexpected type");
+        }
+        break;
+#ifdef JS_CODEGEN_REGISTER_PAIR
+      case ABIArg::GPR_PAIR:
+        if (type == MIRType::Int64) {
+          // FuncType::canHaveJitExit should prevent this.  Also see comments
+          // at GenerateBigIntInitialization.
+          MOZ_CRASH("Should not happen");
+        } else {
+          MOZ_CRASH("wasm uses hardfp for function calls.");
+        }
+        break;
+#endif
+      case ABIArg::FPU: {
+        FloatRegister srcReg = i->fpu();
+        if (type == MIRType::Double) {
+          // Preserve the NaN pattern in the input.
+          ScratchDoubleScope fpscratch(masm);
+          masm.moveDouble(srcReg, fpscratch);
+          masm.canonicalizeDouble(fpscratch);
+          GenPrintF64(DebugChannel::Import, masm, fpscratch);
+          masm.boxDouble(fpscratch, dst);
+        } else if (type == MIRType::Float32) {
+          // JS::Values can't store Float32, so convert to a Double.
+          ScratchDoubleScope fpscratch(masm);
+          masm.convertFloat32ToDouble(srcReg, fpscratch);
+          masm.canonicalizeDouble(fpscratch);
+          GenPrintF64(DebugChannel::Import, masm, fpscratch);
+          masm.boxDouble(fpscratch, dst);
+        } else if (type == MIRType::Simd128) {
+          // The value should never escape; the call will be stopped later as
+          // the import is being called.  But we should generate something sane
+          // here for the boxed case since a debugger or the stack walker may
+          // observe something.
+          ScratchDoubleScope dscratch(masm);
+          masm.loadConstantDouble(0, dscratch);
+          GenPrintF64(DebugChannel::Import, masm, dscratch);
+          masm.boxDouble(dscratch, dst);
+        } else {
+          MOZ_CRASH("Unknown MIRType in wasm exit stub");
+        }
+        break;
+      }
+      case ABIArg::Stack: {
+        Address src(FramePointer,
+                    offsetFromFPToCallerStackArgs + i->offsetFromArgBase());
+        if (type == MIRType::Int32) {
+          masm.load32(src, scratch);
+          GenPrintIsize(DebugChannel::Import, masm, scratch);
+          masm.storeValue(JSVAL_TYPE_INT32, scratch, dst);
+        } else if (type == MIRType::Int64) {
+          // FuncType::canHaveJitExit should prevent this.  Also see comments
+          // at GenerateBigIntInitialization.
+          MOZ_CRASH("Should not happen");
+        } else if (type == MIRType::RefOrNull) {
+          // This works also for FuncRef because it is distinguishable from a
+          // boxed AnyRef.
+          masm.loadPtr(src, scratch);
+          UnboxAnyrefIntoValue(masm, instance, scratch, dst, scratch2);
+        } else if (IsFloatingPointType(type)) {
+          ScratchDoubleScope dscratch(masm);
+          FloatRegister fscratch = dscratch.asSingle();
+          if (type == MIRType::Float32) {
+            masm.loadFloat32(src, fscratch);
+            masm.convertFloat32ToDouble(fscratch, dscratch);
+          } else {
+            masm.loadDouble(src, dscratch);
+          }
+          masm.canonicalizeDouble(dscratch);
+          GenPrintF64(DebugChannel::Import, masm, dscratch);
+          masm.boxDouble(dscratch, dst);
+        } else if (type == MIRType::Simd128) {
+          // The value should never escape; the call will be stopped later as
+          // the import is being called.  But we should generate something
+          // sane here for the boxed case since a debugger or the stack walker
+          // may observe something.
+          ScratchDoubleScope dscratch(masm);
+          masm.loadConstantDouble(0, dscratch);
+          GenPrintF64(DebugChannel::Import, masm, dscratch);
+          masm.boxDouble(dscratch, dst);
+        } else {
+          MOZ_CRASH(
+              "FillArgumentArrayForJitExit, ABIArg::Stack: unexpected type");
+        }
+        break;
+      }
+      case ABIArg::Uninitialized:
+        MOZ_CRASH("Uninitialized ABIArg kind");
+    }
+  }
+  GenPrintf(DebugChannel::Import, masm, "\n");
+}
+
+// Generate a wrapper function with the standard intra-wasm call ABI which
+// simply calls an import. This wrapper function allows any import to be treated
+// like a normal wasm function for the purposes of exports and table calls. In
+// particular, the wrapper function provides:
+//  - a table entry, so JS imports can be put into tables
+//  - normal entries, so that, if the import is re-exported, an entry stub can
+//    be generated and called without any special cases
+static bool GenerateImportFunction(jit::MacroAssembler& masm,
+                                   const FuncImport& fi,
+                                   const FuncType& funcType,
+                                   CallIndirectId callIndirectId,
+                                   FuncOffsets* offsets) {
+  AutoCreatedBy acb(masm, "wasm::GenerateImportFunction");
+
+  AssertExpectedSP(masm);
+
+  GenerateFunctionPrologue(masm, callIndirectId, Nothing(), offsets);
+
+  MOZ_ASSERT(masm.framePushed() == 0);
+  const unsigned sizeOfInstanceSlot = sizeof(void*);
+  unsigned framePushed = StackDecrementForCall(
+      WasmStackAlignment,
+      sizeof(Frame),  // pushed by prologue
+      StackArgBytesForWasmABI(funcType) + sizeOfInstanceSlot);
+  masm.wasmReserveStackChecked(framePushed, BytecodeOffset(0));
+  MOZ_ASSERT(masm.framePushed() == framePushed);
+
+  masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
+                                     framePushed - sizeOfInstanceSlot));
+
+  // The argument register state is already setup by our caller. We just need
+  // to be sure not to clobber it before the call.
+  Register scratch = ABINonArgReg0;
+
+  // Copy our frame's stack arguments to the callee frame's stack argument.
+  //
+  // Note offsetFromFPToCallerStackArgs is sizeof(Frame) because the
+  // WasmABIArgIter accounts for both the ShadowStackSpace and the instance
+  // fields of FrameWithInstances.
+  unsigned offsetFromFPToCallerStackArgs = sizeof(Frame);
+  ArgTypeVector args(funcType);
+  for (WasmABIArgIter i(args); !i.done(); i++) {
+    if (i->kind() != ABIArg::Stack) {
+      continue;
+    }
+
+    Address src(FramePointer,
+                offsetFromFPToCallerStackArgs + i->offsetFromArgBase());
+    Address dst(masm.getStackPointer(), i->offsetFromArgBase());
+    GenPrintf(DebugChannel::Import, masm,
+              "calling exotic import function with arguments: ");
+    StackCopy(masm, i.mirType(), scratch, src, dst);
+    GenPrintf(DebugChannel::Import, masm, "\n");
+  }
+
+  // Call the import exit stub.
+  CallSiteDesc desc(CallSiteDesc::Import);
+  MoveSPForJitABI(masm);
+  masm.wasmCallImport(desc, CalleeDesc::import(fi.instanceOffset()));
+
+  // Restore the instance register and pinned regs, per wasm function ABI.
+  masm.loadPtr(
+      Address(masm.getStackPointer(), framePushed - sizeOfInstanceSlot),
+      InstanceReg);
+  masm.loadWasmPinnedRegsFromInstance();
+
+  // Restore cx->realm.
+  masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
+
+  GenerateFunctionEpilogue(masm, framePushed, offsets);
+  return FinishOffsets(masm, offsets);
+}
+
+static const unsigned STUBS_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024;
+
+bool wasm::GenerateImportFunctions(const ModuleEnvironment& env,
+                                   const FuncImportVector& imports,
+                                   CompiledCode* code) {
+  LifoAlloc lifo(STUBS_LIFO_DEFAULT_CHUNK_SIZE);
+  TempAllocator alloc(&lifo);
+  WasmMacroAssembler masm(alloc, env);
+
+  for (uint32_t funcIndex = 0; funcIndex < imports.length(); funcIndex++) {
+    const FuncImport& fi = imports[funcIndex];
+    const FuncType& funcType = *env.funcs[funcIndex].type;
+    CallIndirectId callIndirectId = CallIndirectId::forFunc(env, funcIndex);
+
+    FuncOffsets offsets;
+    if (!GenerateImportFunction(masm, fi, funcType, callIndirectId, &offsets)) {
+      return false;
+    }
+    if (!code->codeRanges.emplaceBack(funcIndex, /* bytecodeOffset = */ 0,
+                                      offsets)) {
+      return false;
+    }
+  }
+
+  masm.finish();
+  if (masm.oom()) {
+    return false;
+  }
+
+  return code->swap(masm);
+}
+
+// Generate a stub that is called via the internal ABI derived from the
+// signature of the import and calls into an appropriate callImport C++
+// function, having boxed all the ABI arguments into a homogeneous Value array.
+static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi,
+                                     const FuncType& funcType,
+                                     uint32_t funcImportIndex,
+                                     Label* throwLabel,
+                                     CallableOffsets* offsets) {
+  AutoCreatedBy acb(masm, "GenerateImportInterpExit");
+
+  AssertExpectedSP(masm);
+  masm.setFramePushed(0);
+
+  // Argument types for Instance::callImport_*:
+  static const MIRType typeArray[] = {MIRType::Pointer,   // Instance*
+                                      MIRType::Pointer,   // funcImportIndex
+                                      MIRType::Int32,     // argc
+                                      MIRType::Pointer};  // argv
+  MIRTypeVector invokeArgTypes;
+  MOZ_ALWAYS_TRUE(invokeArgTypes.append(typeArray, std::size(typeArray)));
+
+  // At the point of the call, the stack layout is:
+  //
+  //  | stack args | padding | argv[] | padding | retaddr | caller stack | ...
+  //  ^
+  //  +-- sp
+  //
+  // The padding between stack args and argv ensures that argv is aligned on a
+  // Value boundary. The padding between argv and retaddr ensures that sp is
+  // aligned.  The caller stack includes a ShadowStackArea and the instance
+  // fields before the args, see WasmFrame.h.
+  //
+  // The 'double' alignment is correct since the argv[] is a Value array.
+  unsigned argOffset =
+      AlignBytes(StackArgBytesForNativeABI(invokeArgTypes), sizeof(double));
+  // The abiArgCount includes a stack result pointer argument if needed.
+  unsigned abiArgCount = ArgTypeVector(funcType).lengthWithStackResults();
+  unsigned argBytes = std::max<size_t>(1, abiArgCount) * sizeof(Value);
+  unsigned framePushed =
+      StackDecrementForCall(ABIStackAlignment,
+                            sizeof(Frame),  // pushed by prologue
+                            argOffset + argBytes);
+
+  GenerateExitPrologue(masm, framePushed, ExitReason::Fixed::ImportInterp,
+                       offsets);
+
+  // Fill the argument array.
+  Register scratch = ABINonArgReturnReg0;
+  FillArgumentArrayForInterpExit(masm, funcImportIndex, funcType, argOffset,
+                                 scratch);
+
+  // Prepare the arguments for the call to Instance::callImport_*.
+  ABIArgMIRTypeIter i(invokeArgTypes);
+
+  // argument 0: Instance*
+  if (i->kind() == ABIArg::GPR) {
+    masm.movePtr(InstanceReg, i->gpr());
+  } else {
+    masm.storePtr(InstanceReg,
+                  Address(masm.getStackPointer(), i->offsetFromArgBase()));
+  }
+  i++;
+
+  // argument 1: funcImportIndex
+  if (i->kind() == ABIArg::GPR) {
+    masm.mov(ImmWord(funcImportIndex), i->gpr());
+  } else {
+    masm.store32(Imm32(funcImportIndex),
+                 Address(masm.getStackPointer(), i->offsetFromArgBase()));
+  }
+  i++;
+
+  // argument 2: argc
+  unsigned argc = abiArgCount;
+  if (i->kind() == ABIArg::GPR) {
+    masm.mov(ImmWord(argc), i->gpr());
+  } else {
+    masm.store32(Imm32(argc),
+                 Address(masm.getStackPointer(), i->offsetFromArgBase()));
+  }
+  i++;
+
+  // argument 3: argv
+  Address argv(masm.getStackPointer(), argOffset);
+  if (i->kind() == ABIArg::GPR) {
+    masm.computeEffectiveAddress(argv, i->gpr());
+  } else {
+    masm.computeEffectiveAddress(argv, scratch);
+    masm.storePtr(scratch,
+                  Address(masm.getStackPointer(), i->offsetFromArgBase()));
+  }
+  i++;
+  MOZ_ASSERT(i.done());
+
+  // Make the call, test whether it succeeded, and extract the return value.
+  AssertStackAlignment(masm, ABIStackAlignment);
+  masm.call(SymbolicAddress::CallImport_General);
+  masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+
+  ResultType resultType = ResultType::Vector(funcType.results());
+  ValType registerResultType;
+  for (ABIResultIter iter(resultType); !iter.done(); iter.next()) {
+    if (iter.cur().inRegister()) {
+      MOZ_ASSERT(!registerResultType.isValid());
+      registerResultType = iter.cur().type();
+    }
+  }
+  if (!registerResultType.isValid()) {
+    GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+              funcImportIndex);
+    GenPrintf(DebugChannel::Import, masm, "void");
+  } else {
+    switch (registerResultType.kind()) {
+      case ValType::I32:
+        masm.load32(argv, ReturnReg);
+        // No widening is required, as we know the value comes from an i32 load.
+        GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+                  funcImportIndex);
+        GenPrintIsize(DebugChannel::Import, masm, ReturnReg);
+        break;
+      case ValType::I64:
+        masm.load64(argv, ReturnReg64);
+        GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+                  funcImportIndex);
+        GenPrintI64(DebugChannel::Import, masm, ReturnReg64);
+        break;
+      case ValType::V128:
+        // Note, CallImport_Rtt/V128 currently always throws, so we should never
+        // reach this point.
+        masm.breakpoint();
+        break;
+      case ValType::F32:
+        masm.loadFloat32(argv, ReturnFloat32Reg);
+        GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+                  funcImportIndex);
+        GenPrintF32(DebugChannel::Import, masm, ReturnFloat32Reg);
+        break;
+      case ValType::F64:
+        masm.loadDouble(argv, ReturnDoubleReg);
+        GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+                  funcImportIndex);
+        GenPrintF64(DebugChannel::Import, masm, ReturnDoubleReg);
+        break;
+      case ValType::Ref:
+        STATIC_ASSERT_ANYREF_IS_JSOBJECT;
+        masm.loadPtr(argv, ReturnReg);
+        GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+                  funcImportIndex);
+        GenPrintPtr(DebugChannel::Import, masm, ReturnReg);
+        break;
+    }
+  }
+
+  GenPrintf(DebugChannel::Import, masm, "\n");
+
+  // The native ABI preserves the instance, heap and global registers since they
+  // are non-volatile.
+  MOZ_ASSERT(NonVolatileRegs.has(InstanceReg));
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) ||      \
+    defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \
+    defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64)
+  MOZ_ASSERT(NonVolatileRegs.has(HeapReg));
+#endif
+
+  GenerateExitEpilogue(masm, framePushed, ExitReason::Fixed::ImportInterp,
+                       offsets);
+
+  return FinishOffsets(masm, offsets);
+}
+
+// Generate a stub that is called via the internal ABI derived from the
+// signature of the import and calls into a compatible JIT function,
+// having boxed all the ABI arguments into the JIT stack frame layout.
+static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi,
+                                  const FuncType& funcType,
+                                  unsigned funcImportIndex, Label* throwLabel,
+                                  CallableOffsets* offsets) {
+  AutoCreatedBy acb(masm, "GenerateImportJitExit");
+
+  AssertExpectedSP(masm);
+  masm.setFramePushed(0);
+
+  // JIT calls use the following stack layout:
+  //
+  //   | WasmToJSJitFrameLayout | this | arg1..N | saved instance | ...
+  //   ^
+  //   +-- sp
+  //
+  // The JIT ABI requires that sp be JitStackAlignment-aligned after pushing
+  // the return address and frame pointer.
+  static_assert(WasmStackAlignment >= JitStackAlignment, "subsumes");
+  const unsigned sizeOfInstanceSlot = sizeof(void*);
+  const unsigned sizeOfRetAddrAndFP = 2 * sizeof(void*);
+  const unsigned sizeOfPreFrame =
+      WasmToJSJitFrameLayout::Size() - sizeOfRetAddrAndFP;
+  const unsigned sizeOfThisAndArgs =
+      (1 + funcType.args().length()) * sizeof(Value);
+  const unsigned totalJitFrameBytes = sizeOfRetAddrAndFP + sizeOfPreFrame +
+                                      sizeOfThisAndArgs + sizeOfInstanceSlot;
+  const unsigned jitFramePushed =
+      StackDecrementForCall(JitStackAlignment,
+                            sizeof(Frame),  // pushed by prologue
+                            totalJitFrameBytes) -
+      sizeOfRetAddrAndFP;
+
+  GenerateJitExitPrologue(masm, jitFramePushed, offsets);
+
+  // 1. Descriptor.
+  unsigned argc = funcType.args().length();
+  size_t argOffset = 0;
+  uint32_t descriptor =
+      MakeFrameDescriptorForJitCall(FrameType::WasmToJSJit, argc);
+  masm.storePtr(ImmWord(uintptr_t(descriptor)),
+                Address(masm.getStackPointer(), argOffset));
+  argOffset += sizeof(size_t);
+
+  // 2. Callee, part 1 -- need the callee register for argument filling, so
+  // record offset here and set up callee later.
+  size_t calleeArgOffset = argOffset;
+  argOffset += sizeof(size_t);
+  MOZ_ASSERT(argOffset == sizeOfPreFrame);
+
+  // 3. |this| value.
+  masm.storeValue(UndefinedValue(), Address(masm.getStackPointer(), argOffset));
+  argOffset += sizeof(Value);
+
+  // 4. Fill the arguments.
+  Register scratch = ABINonArgReturnReg1;   // Repeatedly clobbered
+  Register scratch2 = ABINonArgReturnReg0;  // Reused as callee below
+  FillArgumentArrayForJitExit(masm, InstanceReg, funcImportIndex, funcType,
+                              argOffset, scratch, scratch2, throwLabel);
+  argOffset += funcType.args().length() * sizeof(Value);
+  MOZ_ASSERT(argOffset == sizeOfThisAndArgs + sizeOfPreFrame);
+
+  // Preserve instance because the JIT callee clobbers it.
+  const size_t savedInstanceOffset = argOffset;
+  masm.storePtr(InstanceReg,
+                Address(masm.getStackPointer(), savedInstanceOffset));
+
+  // 2. Callee, part 2 -- now that the register is free, set up the callee.
+  Register callee = ABINonArgReturnReg0;  // Live until call
+
+  // 2.1. Get the callee. This must be a JSFunction if we're using this JIT
+  // exit.
+  masm.loadPtr(
+      Address(InstanceReg, Instance::offsetInData(
+                               fi.instanceOffset() +
+                               offsetof(FuncImportInstanceData, callable))),
+      callee);
+
+  // 2.2. Save callee.
+  masm.storePtr(callee, Address(masm.getStackPointer(), calleeArgOffset));
+
+  // 5. Check if we need to rectify arguments.
+  masm.loadFunctionArgCount(callee, scratch);
+
+  Label rectify;
+  masm.branch32(Assembler::Above, scratch, Imm32(funcType.args().length()),
+                &rectify);
+
+  // 6. If we haven't rectified arguments, load callee executable entry point.
+
+  masm.loadJitCodeRaw(callee, callee);
+
+  Label rejoinBeforeCall;
+  masm.bind(&rejoinBeforeCall);
+
+  AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddrAndFP);
+#ifdef JS_CODEGEN_ARM64
+  AssertExpectedSP(masm);
+  // Manually resync PSP.  Omitting this causes eg tests/wasm/import-export.js
+  // to segfault.
+  masm.moveStackPtrTo(PseudoStackPointer);
+#endif
+  masm.callJitNoProfiler(callee);
+
+  // Note that there might be a GC thing in the JSReturnOperand now.
+  // In all the code paths from here:
+  // - either the value is unboxed because it was a primitive and we don't
+  //   need to worry about rooting anymore.
+  // - or the value needs to be rooted, but nothing can cause a GC between
+  //   here and CoerceInPlace, which roots before coercing to a primitive.
+
+  // The JIT callee clobbers all registers other than the frame pointer, so
+  // restore InstanceReg here.
+  AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddrAndFP);
+  masm.loadPtr(Address(masm.getStackPointer(), savedInstanceOffset),
+               InstanceReg);
+
+  // The frame was aligned for the JIT ABI such that
+  //   (sp - 2 * sizeof(void*)) % JitStackAlignment == 0
+  // But now we possibly want to call one of several different C++ functions,
+  // so subtract 2 * sizeof(void*) so that sp is aligned for an ABI call.
+  static_assert(ABIStackAlignment <= JitStackAlignment, "subsumes");
+  masm.reserveStack(sizeOfRetAddrAndFP);
+  unsigned nativeFramePushed = masm.framePushed();
+  AssertStackAlignment(masm, ABIStackAlignment);
+
+#ifdef DEBUG
+  {
+    Label ok;
+    masm.branchTestMagic(Assembler::NotEqual, JSReturnOperand, &ok);
+    masm.breakpoint();
+    masm.bind(&ok);
+  }
+#endif
+
+  GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
+            funcImportIndex);
+
+  Label oolConvert;
+  const ValTypeVector& results = funcType.results();
+  if (results.length() == 0) {
+    GenPrintf(DebugChannel::Import, masm, "void");
+  } else {
+    MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented");
+    switch (results[0].kind()) {
+      case ValType::I32:
+        // No widening is required, as the return value does not come to us in
+        // ReturnReg.
+        masm.truncateValueToInt32(JSReturnOperand, ReturnDoubleReg, ReturnReg,
+                                  &oolConvert);
+        GenPrintIsize(DebugChannel::Import, masm, ReturnReg);
+        break;
+      case ValType::I64:
+        // No fastpath for now, go immediately to ool case
+        masm.jump(&oolConvert);
+        break;
+      case ValType::V128:
+        // Unreachable as callImport should not call the stub.
+        masm.breakpoint();
+        break;
+      case ValType::F32:
+        masm.convertValueToFloat(JSReturnOperand, ReturnFloat32Reg,
+                                 &oolConvert);
+        GenPrintF32(DebugChannel::Import, masm, ReturnFloat32Reg);
+        break;
+      case ValType::F64:
+        masm.convertValueToDouble(JSReturnOperand, ReturnDoubleReg,
+                                  &oolConvert);
+        GenPrintF64(DebugChannel::Import, masm, ReturnDoubleReg);
+        break;
+      case ValType::Ref:
+        // Guarded by temporarilyUnsupportedReftypeForExit()
+        MOZ_RELEASE_ASSERT(results[0].refType().isExtern());
+        BoxValueIntoAnyref(masm, JSReturnOperand, ReturnReg, &oolConvert);
+        GenPrintPtr(DebugChannel::Import, masm, ReturnReg);
+        break;
+    }
+  }
+
+  GenPrintf(DebugChannel::Import, masm, "\n");
+
+  Label done;
+  masm.bind(&done);
+
+  GenerateJitExitEpilogue(masm, masm.framePushed(), offsets);
+
+  {
+    // Call the arguments rectifier.
+    masm.bind(&rectify);
+    masm.loadPtr(Address(InstanceReg, Instance::offsetOfJSJitArgsRectifier()),
+                 callee);
+    masm.jump(&rejoinBeforeCall);
+  }
+
+  if (oolConvert.used()) {
+    masm.bind(&oolConvert);
+    masm.setFramePushed(nativeFramePushed);
+
+    // Coercion calls use the following stack layout (sp grows to the left):
+    //   | args | padding | Value argv[1] | padding | exit Frame |
+    MIRTypeVector coerceArgTypes;
+    MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer));
+    unsigned offsetToCoerceArgv =
+        AlignBytes(StackArgBytesForNativeABI(coerceArgTypes), sizeof(Value));
+    MOZ_ASSERT(nativeFramePushed >= offsetToCoerceArgv + sizeof(Value));
+    AssertStackAlignment(masm, ABIStackAlignment);
+
+    // Store return value into argv[0].
+    masm.storeValue(JSReturnOperand,
+                    Address(masm.getStackPointer(), offsetToCoerceArgv));
+
+    // From this point, it's safe to reuse the scratch register (which
+    // might be part of the JSReturnOperand).
+
+    // The JIT might have clobbered exitFP at this point. Since there's
+    // going to be a CoerceInPlace call, pretend we're still doing the JIT
+    // call by restoring our tagged exitFP.
+    SetExitFP(masm, ExitReason::Fixed::ImportJit, scratch);
+
+    // argument 0: argv
+    ABIArgMIRTypeIter i(coerceArgTypes);
+    Address argv(masm.getStackPointer(), offsetToCoerceArgv);
+    if (i->kind() == ABIArg::GPR) {
+      masm.computeEffectiveAddress(argv, i->gpr());
+    } else {
+      masm.computeEffectiveAddress(argv, scratch);
+      masm.storePtr(scratch,
+                    Address(masm.getStackPointer(), i->offsetFromArgBase()));
+    }
+    i++;
+    MOZ_ASSERT(i.done());
+
+    // Call coercion function. Note that right after the call, the value of
+    // FP is correct because FP is non-volatile in the native ABI.
+    AssertStackAlignment(masm, ABIStackAlignment);
+    const ValTypeVector& results = funcType.results();
+    if (results.length() > 0) {
+      // NOTE that once there can be more than one result and we can box some of
+      // the results (as we must for AnyRef), pointer and already-boxed results
+      // must be rooted while subsequent results are boxed.
+      MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented");
+      switch (results[0].kind()) {
+        case ValType::I32:
+          masm.call(SymbolicAddress::CoerceInPlace_ToInt32);
+          masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+          masm.unboxInt32(Address(masm.getStackPointer(), offsetToCoerceArgv),
+                          ReturnReg);
+          // No widening is required, as we generate a known-good value in a
+          // safe way here.
+          break;
+        case ValType::I64: {
+          masm.call(SymbolicAddress::CoerceInPlace_ToBigInt);
+          masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+          Address argv(masm.getStackPointer(), offsetToCoerceArgv);
+          masm.unboxBigInt(argv, scratch);
+          masm.loadBigInt64(scratch, ReturnReg64);
+          break;
+        }
+        case ValType::F64:
+        case ValType::F32:
+          masm.call(SymbolicAddress::CoerceInPlace_ToNumber);
+          masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+          masm.unboxDouble(Address(masm.getStackPointer(), offsetToCoerceArgv),
+                           ReturnDoubleReg);
+          if (results[0].kind() == ValType::F32) {
+            masm.convertDoubleToFloat32(ReturnDoubleReg, ReturnFloat32Reg);
+          }
+          break;
+        case ValType::Ref:
+          // Guarded by temporarilyUnsupportedReftypeForExit()
+          MOZ_RELEASE_ASSERT(results[0].refType().isExtern());
+          masm.call(SymbolicAddress::BoxValue_Anyref);
+          masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+          break;
+        default:
+          MOZ_CRASH("Unsupported convert type");
+      }
+    }
+
+    // Maintain the invariant that exitFP is either unset or not set to a
+    // wasm tagged exitFP, per the jit exit contract.
+    ClearExitFP(masm, scratch);
+
+    masm.jump(&done);
+    masm.setFramePushed(0);
+  }
+
+  MOZ_ASSERT(masm.framePushed() == 0);
+
+  return FinishOffsets(masm, offsets);
+}
+
+struct ABIFunctionArgs {
+  ABIFunctionType abiType;
+  size_t len;
+
+  explicit ABIFunctionArgs(ABIFunctionType sig)
+      : abiType(ABIFunctionType(sig >> ArgType_Shift)) {
+    len = 0;
+    uint64_t i = uint64_t(abiType);
+    while (i) {
+      i = i >> ArgType_Shift;
+      len++;
+    }
+  }
+
+  size_t length() const { return len; }
+
+  MIRType operator[](size_t i) const {
+    MOZ_ASSERT(i < len);
+    uint64_t abi = uint64_t(abiType);
+    while (i--) {
+      abi = abi >> ArgType_Shift;
+    }
+    return ToMIRType(ABIArgType(abi & ArgType_Mask));
+  }
+};
+
+bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIFunctionType abiType,
+                                ExitReason exitReason, void* funcPtr,
+                                CallableOffsets* offsets) {
+  AssertExpectedSP(masm);
+  masm.setFramePushed(0);
+
+  ABIFunctionArgs args(abiType);
+  uint32_t framePushed =
+      StackDecrementForCall(ABIStackAlignment,
+                            sizeof(Frame),  // pushed by prologue
+                            StackArgBytesForNativeABI(args));
+
+  GenerateExitPrologue(masm, framePushed, exitReason, offsets);
+
+  // Copy out and convert caller arguments, if needed.
+
+  // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter
+  // is accounted for by the ABIArgIter.
+  unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances);
+  Register scratch = ABINonArgReturnReg0;
+  for (ABIArgIter i(args); !i.done(); i++) {
+    if (i->argInRegister()) {
+#ifdef JS_CODEGEN_ARM
+      // Non hard-fp passes the args values in GPRs.
+      if (!UseHardFpABI() && IsFloatingPointType(i.mirType())) {
+        FloatRegister input = i->fpu();
+        if (i.mirType() == MIRType::Float32) {
+          masm.ma_vxfer(input, Register::FromCode(input.id()));
+        } else if (i.mirType() == MIRType::Double) {
+          uint32_t regId = input.singleOverlay().id();
+          masm.ma_vxfer(input, Register::FromCode(regId),
+                        Register::FromCode(regId + 1));
+        }
+      }
+#endif
+      continue;
+    }
+
+    Address src(FramePointer,
+                offsetFromFPToCallerStackArgs + i->offsetFromArgBase());
+    Address dst(masm.getStackPointer(), i->offsetFromArgBase());
+    StackCopy(masm, i.mirType(), scratch, src, dst);
+  }
+
+  AssertStackAlignment(masm, ABIStackAlignment);
+  MoveSPForJitABI(masm);
+  masm.call(ImmPtr(funcPtr, ImmPtr::NoCheckToken()));
+
+#if defined(JS_CODEGEN_X64)
+  // No widening is required, as the caller will widen.
+#elif defined(JS_CODEGEN_X86)
+  // x86 passes the return value on the x87 FP stack.
+  Operand op(esp, 0);
+  MIRType retType = ToMIRType(ABIArgType(
+      std::underlying_type_t<ABIFunctionType>(abiType) & ArgType_Mask));
+  if (retType == MIRType::Float32) {
+    masm.fstp32(op);
+    masm.loadFloat32(op, ReturnFloat32Reg);
+  } else if (retType == MIRType::Double) {
+    masm.fstp(op);
+    masm.loadDouble(op, ReturnDoubleReg);
+  }
+#elif defined(JS_CODEGEN_ARM)
+  // Non hard-fp passes the return values in GPRs.
+  MIRType retType = ToMIRType(ABIArgType(
+      std::underlying_type_t<ABIFunctionType>(abiType) & ArgType_Mask));
+  if (!UseHardFpABI() && IsFloatingPointType(retType)) {
+    masm.ma_vxfer(r0, r1, d0);
+  }
+#endif
+
+  GenerateExitEpilogue(masm, framePushed, exitReason, offsets);
+  return FinishOffsets(masm, offsets);
+}
+
+#if defined(JS_CODEGEN_ARM)
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~((Registers::SetType(1) << Registers::sp) |
+                         (Registers::SetType(1) << Registers::pc))),
+    FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  ifdef ENABLE_WASM_SIMD
+#    error "high lanes of SIMD registers need to be saved too."
+#  endif
+#elif defined(JS_CODEGEN_MIPS64)
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~((Registers::SetType(1) << Registers::k0) |
+                         (Registers::SetType(1) << Registers::k1) |
+                         (Registers::SetType(1) << Registers::sp) |
+                         (Registers::SetType(1) << Registers::zero))),
+    FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  ifdef ENABLE_WASM_SIMD
+#    error "high lanes of SIMD registers need to be saved too."
+#  endif
+#elif defined(JS_CODEGEN_LOONG64)
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~((uint32_t(1) << Registers::tp) |
+                         (uint32_t(1) << Registers::fp) |
+                         (uint32_t(1) << Registers::sp) |
+                         (uint32_t(1) << Registers::zero))),
+    FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  ifdef ENABLE_WASM_SIMD
+#    error "high lanes of SIMD registers need to be saved too."
+#  endif
+#elif defined(JS_CODEGEN_RISCV64)
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~((uint32_t(1) << Registers::tp) |
+                         (uint32_t(1) << Registers::fp) |
+                         (uint32_t(1) << Registers::sp) |
+                         (uint32_t(1) << Registers::zero))),
+    FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  ifdef ENABLE_WASM_SIMD
+#    error "high lanes of SIMD registers need to be saved too."
+#  endif
+#elif defined(JS_CODEGEN_ARM64)
+// We assume that traps do not happen while lr is live. This both ensures that
+// the size of RegsToPreserve is a multiple of 2 (preserving WasmStackAlignment)
+// and gives us a register to clobber in the return path.
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~((Registers::SetType(1) << RealStackPointer.code()) |
+                         (Registers::SetType(1) << Registers::lr))),
+#  ifdef ENABLE_WASM_SIMD
+    FloatRegisterSet(FloatRegisters::AllSimd128Mask));
+#  else
+    // If SIMD is not enabled, it's pointless to save/restore the upper 64
+    // bits of each vector register.
+    FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  endif
+#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+// It's correct to use FloatRegisters::AllMask even when SIMD is not enabled;
+// PushRegsInMask strips out the high lanes of the XMM registers in this case,
+// while the singles will be stripped as they are aliased by the larger doubles.
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(Registers::AllMask &
+                       ~(Registers::SetType(1) << Registers::StackPointer)),
+    FloatRegisterSet(FloatRegisters::AllMask));
+#else
+static const LiveRegisterSet RegsToPreserve(
+    GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllDoubleMask));
+#  ifdef ENABLE_WASM_SIMD
+#    error "no SIMD support"
+#  endif
+#endif
+
+// Generate a RegisterOffsets which describes the locations of the GPRs as saved
+// by GenerateTrapExit.  FP registers are ignored.  Note that the values
+// stored in the RegisterOffsets are offsets in words downwards from the top of
+// the save area.  That is, a higher value implies a lower address.
+void wasm::GenerateTrapExitRegisterOffsets(RegisterOffsets* offsets,
+                                           size_t* numWords) {
+  // This is the number of words pushed by the initial WasmPush().
+  *numWords = WasmPushSize / sizeof(void*);
+  MOZ_ASSERT(*numWords == TrapExitDummyValueOffsetFromTop + 1);
+
+  // And these correspond to the PushRegsInMask() that immediately follows.
+  for (GeneralRegisterBackwardIterator iter(RegsToPreserve.gprs()); iter.more();
+       ++iter) {
+    offsets->setOffset(*iter, *numWords);
+    (*numWords)++;
+  }
+}
+
+// Generate a stub which calls WasmReportTrap() and can be executed by having
+// the signal handler redirect PC from any trapping instruction.
+static bool GenerateTrapExit(MacroAssembler& masm, Label* throwLabel,
+                             Offsets* offsets) {
+  AssertExpectedSP(masm);
+  masm.haltingAlign(CodeAlignment);
+
+  masm.setFramePushed(0);
+
+  offsets->begin = masm.currentOffset();
+
+  // Traps can only happen at well-defined program points. However, since
+  // traps may resume and the optimal assumption for the surrounding code is
+  // that registers are not clobbered, we need to preserve all registers in
+  // the trap exit. One simplifying assumption is that flags may be clobbered.
+  // Push a dummy word to use as return address below.
+  WasmPush(masm, ImmWord(TrapExitDummyValue));
+  unsigned framePushedBeforePreserve = masm.framePushed();
+  masm.PushRegsInMask(RegsToPreserve);
+  unsigned offsetOfReturnWord = masm.framePushed() - framePushedBeforePreserve;
+
+  // We know that StackPointer is word-aligned, but not necessarily
+  // stack-aligned, so we need to align it dynamically.
+  Register preAlignStackPointer = ABINonVolatileReg;
+  masm.moveStackPtrTo(preAlignStackPointer);
+  masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1)));
+  if (ShadowStackSpace) {
+    masm.subFromStackPtr(Imm32(ShadowStackSpace));
+  }
+
+  masm.assertStackAlignment(ABIStackAlignment);
+  masm.call(SymbolicAddress::HandleTrap);
+
+  // WasmHandleTrap returns null if control should transfer to the throw stub.
+  masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+
+  // Otherwise, the return value is the TrapData::resumePC we must jump to.
+  // We must restore register state before jumping, which will clobber
+  // ReturnReg, so store ReturnReg in the above-reserved stack slot which we
+  // use to jump to via ret.
+  masm.moveToStackPtr(preAlignStackPointer);
+  masm.storePtr(ReturnReg, Address(masm.getStackPointer(), offsetOfReturnWord));
+  masm.PopRegsInMask(RegsToPreserve);
+#ifdef JS_CODEGEN_ARM64
+  WasmPop(masm, lr);
+  masm.abiret();
+#else
+  masm.ret();
+#endif
+
+  return FinishOffsets(masm, offsets);
+}
+
+static void ClobberWasmRegsForLongJmp(MacroAssembler& masm, Register jumpReg) {
+  // Get the set of all registers that are allocatable in wasm functions
+  AllocatableGeneralRegisterSet gprs(GeneralRegisterSet::All());
+  RegisterAllocator::takeWasmRegisters(gprs);
+  // Remove the instance register from this set as landing pads require it to be
+  // valid
+  gprs.take(InstanceReg);
+  // Remove a specified register that will be used for the longjmp
+  gprs.take(jumpReg);
+  // Set all of these registers to zero
+  for (GeneralRegisterIterator iter(gprs.asLiveSet()); iter.more(); ++iter) {
+    Register reg = *iter;
+    masm.xorPtr(reg, reg);
+  }
+
+  // Get the set of all floating point registers that are allocatable in wasm
+  // functions
+  AllocatableFloatRegisterSet fprs(FloatRegisterSet::All());
+  // Set all of these registers to NaN. We attempt for this to be a signalling
+  // NaN, but the bit format for signalling NaNs are implementation defined
+  // and so this is just best effort.
+  Maybe<FloatRegister> regNaN;
+  for (FloatRegisterIterator iter(fprs.asLiveSet()); iter.more(); ++iter) {
+    FloatRegister reg = *iter;
+    if (!reg.isDouble()) {
+      continue;
+    }
+    if (regNaN) {
+      masm.moveDouble(*regNaN, reg);
+      continue;
+    }
+    masm.loadConstantDouble(std::numeric_limits<double>::signaling_NaN(), reg);
+    regNaN = Some(reg);
+  }
+}
+
+// Generate a stub that restores the stack pointer to what it was on entry to
+// the wasm activation, sets the return register to 'false' and then executes a
+// return which will return from this wasm activation to the caller. This stub
+// should only be called after the caller has reported an error.
+static bool GenerateThrowStub(MacroAssembler& masm, Label* throwLabel,
+                              Offsets* offsets) {
+  Register scratch1 = ABINonArgReturnReg0;
+  Register scratch2 = ABINonArgReturnReg1;
+
+  AssertExpectedSP(masm);
+  masm.haltingAlign(CodeAlignment);
+  masm.setFramePushed(0);
+
+  masm.bind(throwLabel);
+
+  offsets->begin = masm.currentOffset();
+
+  // Conservatively, the stack pointer can be unaligned and we must align it
+  // dynamically.
+  masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1)));
+  if (ShadowStackSpace) {
+    masm.subFromStackPtr(Imm32(ShadowStackSpace));
+  }
+
+  // Allocate space for exception or regular resume information.
+  masm.reserveStack(sizeof(jit::ResumeFromException));
+  masm.moveStackPtrTo(scratch1);
+
+  MIRTypeVector handleThrowTypes;
+  MOZ_ALWAYS_TRUE(handleThrowTypes.append(MIRType::Pointer));
+
+  unsigned frameSize =
+      StackDecrementForCall(ABIStackAlignment, masm.framePushed(),
+                            StackArgBytesForNativeABI(handleThrowTypes));
+  masm.reserveStack(frameSize);
+  masm.assertStackAlignment(ABIStackAlignment);
+
+  ABIArgMIRTypeIter i(handleThrowTypes);
+  if (i->kind() == ABIArg::GPR) {
+    masm.movePtr(scratch1, i->gpr());
+  } else {
+    masm.storePtr(scratch1,
+                  Address(masm.getStackPointer(), i->offsetFromArgBase()));
+  }
+  i++;
+  MOZ_ASSERT(i.done());
+
+  // WasmHandleThrow unwinds JitActivation::wasmExitFP() and returns the
+  // address of the return address on the stack this stub should return to.
+  // Set the FramePointer to a magic value to indicate a return by throw.
+  //
+  // If there is a Wasm catch handler present, it will instead return the
+  // address of the handler to jump to and the FP/SP values to restore.
+  masm.call(SymbolicAddress::HandleThrow);
+
+  Label resumeCatch, leaveWasm;
+
+  masm.load32(Address(ReturnReg, offsetof(jit::ResumeFromException, kind)),
+              scratch1);
+
+  masm.branch32(Assembler::Equal, scratch1,
+                Imm32(jit::ExceptionResumeKind::WasmCatch), &resumeCatch);
+  masm.branch32(Assembler::Equal, scratch1,
+                Imm32(jit::ExceptionResumeKind::Wasm), &leaveWasm);
+
+  masm.breakpoint();
+
+  // The case where a Wasm catch handler was found while unwinding the stack.
+  masm.bind(&resumeCatch);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfInstance()),
+               InstanceReg);
+  masm.loadWasmPinnedRegsFromInstance();
+  masm.switchToWasmInstanceRealm(scratch1, scratch2);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfTarget()),
+               scratch1);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfFramePointer()),
+               FramePointer);
+  masm.loadStackPtr(
+      Address(ReturnReg, ResumeFromException::offsetOfStackPointer()));
+  MoveSPForJitABI(masm);
+  ClobberWasmRegsForLongJmp(masm, scratch1);
+  masm.jump(scratch1);
+
+  // No catch handler was found, so we will just return out.
+  masm.bind(&leaveWasm);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfFramePointer()),
+               FramePointer);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfInstance()),
+               InstanceReg);
+  masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfStackPointer()),
+               scratch1);
+  masm.moveToStackPtr(scratch1);
+#ifdef JS_CODEGEN_ARM64
+  masm.loadPtr(Address(scratch1, 0), lr);
+  masm.addToStackPtr(Imm32(8));
+  masm.abiret();
+#else
+  masm.ret();
+#endif
+
+  return FinishOffsets(masm, offsets);
+}
+
+static const LiveRegisterSet AllAllocatableRegs =
+    LiveRegisterSet(GeneralRegisterSet(Registers::AllocatableMask),
+                    FloatRegisterSet(FloatRegisters::AllMask));
+
+// Generate a stub that handle toggable enter/leave frame traps or breakpoints.
+// The trap records frame pointer (via GenerateExitPrologue) and saves most of
+// registers to not affect the code generated by WasmBaselineCompile.
+static bool GenerateDebugTrapStub(MacroAssembler& masm, Label* throwLabel,
+                                  CallableOffsets* offsets) {
+  AssertExpectedSP(masm);
+  masm.haltingAlign(CodeAlignment);
+  masm.setFramePushed(0);
+
+  GenerateExitPrologue(masm, 0, ExitReason::Fixed::DebugTrap, offsets);
+
+  // Save all registers used between baseline compiler operations.
+  masm.PushRegsInMask(AllAllocatableRegs);
+
+  uint32_t framePushed = masm.framePushed();
+
+  // This method might be called with unaligned stack -- aligning and
+  // saving old stack pointer at the top.
+#ifdef JS_CODEGEN_ARM64
+  // On ARM64 however the stack is always aligned.
+  static_assert(ABIStackAlignment == 16, "ARM64 SP alignment");
+#else
+  Register scratch = ABINonArgReturnReg0;
+  masm.moveStackPtrTo(scratch);
+  masm.subFromStackPtr(Imm32(sizeof(intptr_t)));
+  masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1)));
+  masm.storePtr(scratch, Address(masm.getStackPointer(), 0));
+#endif
+
+  if (ShadowStackSpace) {
+    masm.subFromStackPtr(Imm32(ShadowStackSpace));
+  }
+  masm.assertStackAlignment(ABIStackAlignment);
+  masm.call(SymbolicAddress::HandleDebugTrap);
+
+  masm.branchIfFalseBool(ReturnReg, throwLabel);
+
+  if (ShadowStackSpace) {
+    masm.addToStackPtr(Imm32(ShadowStackSpace));
+  }
+#ifndef JS_CODEGEN_ARM64
+  masm.Pop(scratch);
+  masm.moveToStackPtr(scratch);
+#endif
+
+  masm.setFramePushed(framePushed);
+  masm.PopRegsInMask(AllAllocatableRegs);
+
+  GenerateExitEpilogue(masm, 0, ExitReason::Fixed::DebugTrap, offsets);
+
+  return FinishOffsets(masm, offsets);
+}
+
+bool wasm::GenerateEntryStubs(MacroAssembler& masm, size_t funcExportIndex,
+                              const FuncExport& fe, const FuncType& funcType,
+                              const Maybe<ImmPtr>& callee, bool isAsmJS,
+                              CodeRangeVector* codeRanges) {
+  MOZ_ASSERT(!callee == fe.hasEagerStubs());
+  MOZ_ASSERT_IF(isAsmJS, fe.hasEagerStubs());
+
+  Offsets offsets;
+  if (!GenerateInterpEntry(masm, fe, funcType, callee, &offsets)) {
+    return false;
+  }
+  if (!codeRanges->emplaceBack(CodeRange::InterpEntry, fe.funcIndex(),
+                               offsets)) {
+    return false;
+  }
+
+  if (isAsmJS || !funcType.canHaveJitEntry()) {
+    return true;
+  }
+
+  CallableOffsets jitOffsets;
+  if (!GenerateJitEntry(masm, funcExportIndex, fe, funcType, callee,
+                        &jitOffsets)) {
+    return false;
+  }
+  if (!codeRanges->emplaceBack(CodeRange::JitEntry, fe.funcIndex(),
+                               jitOffsets)) {
+    return false;
+  }
+
+  return true;
+}
+
+bool wasm::GenerateProvisionalLazyJitEntryStub(MacroAssembler& masm,
+                                               Offsets* offsets) {
+  AssertExpectedSP(masm);
+  masm.setFramePushed(0);
+  offsets->begin = masm.currentOffset();
+
+#ifdef JS_CODEGEN_ARM64
+  // Unaligned ABI calls require SP+PSP, but our mode here is SP-only
+  masm.SetStackPointer64(PseudoStackPointer64);
+  masm.Mov(PseudoStackPointer64, sp);
+#endif
+
+#ifdef JS_USE_LINK_REGISTER
+  masm.pushReturnAddress();
+#endif
+
+  AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+  Register temp = regs.takeAny();
+
+  using Fn = void* (*)();
+  masm.setupUnalignedABICall(temp);
+  masm.callWithABI<Fn, GetContextSensitiveInterpreterStub>(
+      MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+#ifdef JS_USE_LINK_REGISTER
+  masm.popReturnAddress();
+#endif
+
+  masm.jump(ReturnReg);
+
+#ifdef JS_CODEGEN_ARM64
+  // Undo the SP+PSP mode
+  masm.SetStackPointer64(sp);
+#endif
+
+  return FinishOffsets(masm, offsets);
+}
+
+bool wasm::GenerateStubs(const ModuleEnvironment& env,
+                         const FuncImportVector& imports,
+                         const FuncExportVector& exports, CompiledCode* code) {
+  LifoAlloc lifo(STUBS_LIFO_DEFAULT_CHUNK_SIZE);
+  TempAllocator alloc(&lifo);
+  JitContext jcx;
+  WasmMacroAssembler masm(alloc, env);
+  AutoCreatedBy acb(masm, "wasm::GenerateStubs");
+
+  // Swap in already-allocated empty vectors to avoid malloc/free.
+  if (!code->swap(masm)) {
+    return false;
+  }
+
+  Label throwLabel;
+
+  JitSpew(JitSpew_Codegen, "# Emitting wasm import stubs");
+
+  for (uint32_t funcIndex = 0; funcIndex < imports.length(); funcIndex++) {
+    const FuncImport& fi = imports[funcIndex];
+    const FuncType& funcType = *env.funcs[funcIndex].type;
+
+    CallableOffsets interpOffsets;
+    if (!GenerateImportInterpExit(masm, fi, funcType, funcIndex, &throwLabel,
+                                  &interpOffsets)) {
+      return false;
+    }
+    if (!code->codeRanges.emplaceBack(CodeRange::ImportInterpExit, funcIndex,
+                                      interpOffsets)) {
+      return false;
+    }
+
+    // Skip if the function does not have a signature that allows for a JIT
+    // exit.
+    if (!funcType.canHaveJitExit()) {
+      continue;
+    }
+
+    CallableOffsets jitOffsets;
+    if (!GenerateImportJitExit(masm, fi, funcType, funcIndex, &throwLabel,
+                               &jitOffsets)) {
+      return false;
+    }
+    if (!code->codeRanges.emplaceBack(CodeRange::ImportJitExit, funcIndex,
+                                      jitOffsets)) {
+      return false;
+    }
+  }
+
+  JitSpew(JitSpew_Codegen, "# Emitting wasm export stubs");
+
+  Maybe<ImmPtr> noAbsolute;
+  for (size_t i = 0; i < exports.length(); i++) {
+    const FuncExport& fe = exports[i];
+    const FuncType& funcType = (*env.types)[fe.typeIndex()].funcType();
+    if (!fe.hasEagerStubs()) {
+      continue;
+    }
+    if (!GenerateEntryStubs(masm, i, fe, funcType, noAbsolute, env.isAsmJS(),
+                            &code->codeRanges)) {
+      return false;
+    }
+  }
+
+  JitSpew(JitSpew_Codegen, "# Emitting wasm exit stubs");
+
+  Offsets offsets;
+
+  if (!GenerateTrapExit(masm, &throwLabel, &offsets)) {
+    return false;
+  }
+  if (!code->codeRanges.emplaceBack(CodeRange::TrapExit, offsets)) {
+    return false;
+  }
+
+  CallableOffsets callableOffsets;
+  if (!GenerateDebugTrapStub(masm, &throwLabel, &callableOffsets)) {
+    return false;
+  }
+  if (!code->codeRanges.emplaceBack(CodeRange::DebugTrap, callableOffsets)) {
+    return false;
+  }
+
+  if (!GenerateThrowStub(masm, &throwLabel, &offsets)) {
+    return false;
+  }
+  if (!code->codeRanges.emplaceBack(CodeRange::Throw, offsets)) {
+    return false;
+  }
+
+  masm.finish();
+  if (masm.oom()) {
+    return false;
+  }
+
+  return code->swap(masm);
+}