Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 829 insertions, 0 deletions

diff --git a/js/src/jit/arm/Trampoline-arm.cpp b/js/src/jit/arm/Trampoline-arm.cpp
new file mode 100644
index 0000000000..c116a93c64
--- /dev/null
+++ b/js/src/jit/arm/Trampoline-arm.cpp
@@ -0,0 +1,829 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/SharedICHelpers-arm.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h"  // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+static const FloatRegisterSet NonVolatileFloatRegs = FloatRegisterSet(
+    (1ULL << FloatRegisters::d8) | (1ULL << FloatRegisters::d9) |
+    (1ULL << FloatRegisters::d10) | (1ULL << FloatRegisters::d11) |
+    (1ULL << FloatRegisters::d12) | (1ULL << FloatRegisters::d13) |
+    (1ULL << FloatRegisters::d14) | (1ULL << FloatRegisters::d15));
+
+static void GenerateReturn(MacroAssembler& masm, int returnCode) {
+  // Restore non-volatile floating point registers.
+  masm.transferMultipleByRuns(NonVolatileFloatRegs, IsLoad, StackPointer, IA);
+
+  // Get rid of padding word.
+  masm.addPtr(Imm32(sizeof(void*)), sp);
+
+  // Set up return value
+  masm.ma_mov(Imm32(returnCode), r0);
+
+  // Pop and return
+  masm.startDataTransferM(IsLoad, sp, IA, WriteBack);
+  masm.transferReg(r4);
+  masm.transferReg(r5);
+  masm.transferReg(r6);
+  masm.transferReg(r7);
+  masm.transferReg(r8);
+  masm.transferReg(r9);
+  masm.transferReg(r10);
+  masm.transferReg(r11);
+  // r12 isn't saved, so it shouldn't be restored.
+  masm.transferReg(pc);
+  masm.finishDataTransfer();
+  masm.flushBuffer();
+}
+
+struct EnterJITStack {
+  double d8;
+  double d9;
+  double d10;
+  double d11;
+  double d12;
+  double d13;
+  double d14;
+  double d15;
+
+  // Padding.
+  void* padding;
+
+  // Non-volatile registers.
+  void* r4;
+  void* r5;
+  void* r6;
+  void* r7;
+  void* r8;
+  void* r9;
+  void* r10;
+  void* r11;
+  // The abi does not expect r12 (ip) to be preserved
+  void* lr;
+
+  // Arguments.
+  // code == r0
+  // argc == r1
+  // argv == r2
+  // frame == r3
+  CalleeToken token;
+  JSObject* scopeChain;
+  size_t numStackValues;
+  Value* vp;
+};
+
+/*
+ * This method generates a trampoline for a c++ function with the following
+ * signature:
+ *   void enter(void* code, int argc, Value* argv, InterpreterFrame* fp,
+ *              CalleeToken calleeToken, JSObject* scopeChain, Value* vp)
+ *   ...using standard EABI calling convention
+ */
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+  enterJITOffset_ = startTrampolineCode(masm);
+
+  const Address slot_token(sp, offsetof(EnterJITStack, token));
+  const Address slot_vp(sp, offsetof(EnterJITStack, vp));
+
+  static_assert(OsrFrameReg == r3);
+
+  Assembler* aasm = &masm;
+
+  // Save non-volatile registers. These must be saved by the trampoline,
+  // rather than the JIT'd code, because they are scanned by the conservative
+  // scanner.
+  masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+  masm.transferReg(r4);   // [sp,0]
+  masm.transferReg(r5);   // [sp,4]
+  masm.transferReg(r6);   // [sp,8]
+  masm.transferReg(r7);   // [sp,12]
+  masm.transferReg(r8);   // [sp,16]
+  masm.transferReg(r9);   // [sp,20]
+  masm.transferReg(r10);  // [sp,24]
+  masm.transferReg(r11);  // [sp,28]
+  // The abi does not expect r12 (ip) to be preserved
+  masm.transferReg(lr);  // [sp,32]
+  // The 5th argument is located at [sp, 36]
+  masm.finishDataTransfer();
+
+  // Add padding word.
+  masm.subPtr(Imm32(sizeof(void*)), sp);
+
+  // Push the float registers.
+  masm.transferMultipleByRuns(NonVolatileFloatRegs, IsStore, sp, DB);
+
+  // Load calleeToken into r9.
+  masm.loadPtr(slot_token, r9);
+
+  // Save stack pointer.
+  masm.movePtr(sp, r11);
+
+  // Load the number of actual arguments into r10.
+  masm.loadPtr(slot_vp, r10);
+  masm.unboxInt32(Address(r10, 0), r10);
+
+  {
+    Label noNewTarget;
+    masm.branchTest32(Assembler::Zero, r9,
+                      Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+    masm.add32(Imm32(1), r1);
+
+    masm.bind(&noNewTarget);
+  }
+
+  // Guarantee stack alignment of Jit frames.
+  //
+  // This code moves the stack pointer to the location where it should be when
+  // we enter the Jit frame.  It moves the stack pointer such that we have
+  // enough space reserved for pushing the arguments, and the JitFrameLayout.
+  // The stack pointer is also aligned on the alignment expected by the Jit
+  // frames.
+  //
+  // At the end the register r4, is a pointer to the stack where the first
+  // argument is expected by the Jit frame.
+  //
+  aasm->as_sub(r4, sp, O2RegImmShift(r1, LSL, 3));  // r4 = sp - argc*8
+  aasm->as_bic(r4, r4, Imm8(JitStackAlignment - 1));
+  // r4 is now the aligned on the bottom of the list of arguments.
+  static_assert(
+      sizeof(JitFrameLayout) % JitStackAlignment == 0,
+      "No need to consider the JitFrameLayout for aligning the stack");
+  // sp' = ~(JitStackAlignment - 1) & (sp - argc * sizeof(Value))
+  masm.movePtr(r4, sp);
+
+  // Get a copy of the number of args to use as a decrement counter, also set
+  // the zero condition code.
+  aasm->as_mov(r5, O2Reg(r1), SetCC);
+
+  // Loop over arguments, copying them from an unknown buffer onto the Ion
+  // stack so they can be accessed from JIT'ed code.
+  {
+    Label header, footer;
+    // If there aren't any arguments, don't do anything.
+    aasm->as_b(&footer, Assembler::Zero);
+    // Get the top of the loop.
+    masm.bind(&header);
+    aasm->as_sub(r5, r5, Imm8(1), SetCC);
+    // We could be more awesome, and unroll this, using a loadm
+    // (particularly since the offset is effectively 0) but that seems more
+    // error prone, and complex.
+    // BIG FAT WARNING: this loads both r6 and r7.
+    aasm->as_extdtr(IsLoad, 64, true, PostIndex, r6,
+                    EDtrAddr(r2, EDtrOffImm(8)));
+    aasm->as_extdtr(IsStore, 64, true, PostIndex, r6,
+                    EDtrAddr(r4, EDtrOffImm(8)));
+    aasm->as_b(&header, Assembler::NonZero);
+    masm.bind(&footer);
+  }
+
+  // Push the callee token.
+  masm.push(r9);
+
+  // Push the frame descriptor.
+  masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, r10, r10);
+
+  Label returnLabel;
+  {
+    // Handle Interpreter -> Baseline OSR.
+    AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+    MOZ_ASSERT(!regs.has(r11));
+    regs.take(OsrFrameReg);
+    regs.take(r0);  // jitcode
+    MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches r0");
+
+    const Address slot_numStackValues(r11,
+                                      offsetof(EnterJITStack, numStackValues));
+
+    Label notOsr;
+    masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+    Register scratch = regs.takeAny();
+
+    Register numStackValues = regs.takeAny();
+    masm.load32(slot_numStackValues, numStackValues);
+
+    // Write return address. On ARM, CodeLabel is only used for tableswitch,
+    // so we can't use it here to get the return address. Instead, we use pc
+    // + a fixed offset to a jump to returnLabel. The pc register holds pc +
+    // 8, so we add the size of 2 instructions to skip the instructions
+    // emitted by push and jump(&skipJump).
+    {
+      AutoForbidPoolsAndNops afp(&masm, 5);
+      Label skipJump;
+      masm.mov(pc, scratch);
+      masm.addPtr(Imm32(2 * sizeof(uint32_t)), scratch);
+      masm.push(scratch);
+      masm.jump(&skipJump);
+      masm.jump(&returnLabel);
+      masm.bind(&skipJump);
+    }
+
+    // Frame prologue.
+    masm.push(FramePointer);
+    masm.mov(sp, FramePointer);
+
+    // Reserve frame.
+    masm.subPtr(Imm32(BaselineFrame::Size()), sp);
+
+    Register framePtrScratch = regs.takeAny();
+    masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+    masm.mov(sp, framePtrScratch);
+
+    // Reserve space for locals and stack values.
+    masm.ma_lsl(Imm32(3), numStackValues, scratch);
+    masm.ma_sub(sp, scratch, sp);
+
+    // Enter exit frame.
+    masm.pushFrameDescriptor(FrameType::BaselineJS);
+    masm.push(Imm32(0));  // Fake return address.
+    masm.push(FramePointer);
+    // No GC things to mark on the stack, push a bare token.
+    masm.loadJSContext(scratch);
+    masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+    masm.push(r0);  // jitcode
+
+    using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+                        uint32_t numStackValues);
+    masm.setupUnalignedABICall(scratch);
+    masm.passABIArg(framePtrScratch);  // BaselineFrame
+    masm.passABIArg(OsrFrameReg);      // InterpreterFrame
+    masm.passABIArg(numStackValues);
+    masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+        MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+    Register jitcode = regs.takeAny();
+    masm.pop(jitcode);
+
+    MOZ_ASSERT(jitcode != ReturnReg);
+
+    Label error;
+    masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), sp);
+    masm.branchIfFalseBool(ReturnReg, &error);
+
+    // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+    // if profiler instrumentation is enabled.
+    {
+      Label skipProfilingInstrumentation;
+      AbsoluteAddress addressOfEnabled(
+          cx->runtime()->geckoProfiler().addressOfEnabled());
+      masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+                    &skipProfilingInstrumentation);
+      masm.profilerEnterFrame(FramePointer, scratch);
+      masm.bind(&skipProfilingInstrumentation);
+    }
+
+    masm.jump(jitcode);
+
+    // OOM: frame epilogue, load error value, discard return address and return.
+    masm.bind(&error);
+    masm.mov(FramePointer, sp);
+    masm.pop(FramePointer);
+    masm.addPtr(Imm32(sizeof(uintptr_t)), sp);  // Return address.
+    masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+    masm.jump(&returnLabel);
+
+    masm.bind(&notOsr);
+    // Load the scope chain in R1.
+    MOZ_ASSERT(R1.scratchReg() != r0);
+    masm.loadPtr(Address(r11, offsetof(EnterJITStack, scopeChain)),
+                 R1.scratchReg());
+  }
+
+  // The callee will push the return address and frame pointer on the stack,
+  // thus we check that the stack would be aligned once the call is complete.
+  masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+  // Call the function.
+  masm.callJitNoProfiler(r0);
+
+  // Interpreter -> Baseline OSR will return here.
+  masm.bind(&returnLabel);
+
+  // Discard arguments and padding. Set sp to the address of the EnterJITStack
+  // on the stack.
+  masm.mov(r11, sp);
+
+  // Store the returned value into the slot_vp
+  masm.loadPtr(slot_vp, r5);
+  masm.storeValue(JSReturnOperand, Address(r5, 0));
+
+  // Restore non-volatile registers and return.
+  GenerateReturn(masm, true);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+  // Not supported, or not implemented yet.
+  // TODO: Implement along with the corresponding stack-walker changes, in
+  // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+  return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+  // See large comment in x86's JitRuntime::generateInvalidator.
+
+  AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+  invalidatorOffset_ = startTrampolineCode(masm);
+
+  // At this point, one of two things has happened:
+  // 1) Execution has just returned from C code, which left the stack aligned
+  // 2) Execution has just returned from Ion code, which left the stack
+  // unaligned. The old return address should not matter, but we still want the
+  // stack to be aligned, and there is no good reason to automatically align it
+  // with a call to setupUnalignedABICall.
+  masm.as_bic(sp, sp, Imm8(7));
+  masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+  // We don't have to push everything, but this is likely easier.
+  // Setting regs_.
+  for (uint32_t i = 0; i < Registers::Total; i++) {
+    masm.transferReg(Register::FromCode(i));
+  }
+  masm.finishDataTransfer();
+
+  // Since our datastructures for stack inspection are compile-time fixed,
+  // if there are only 16 double registers, then we need to reserve
+  // space on the stack for the missing 16.
+  if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+    ScratchRegisterScope scratch(masm);
+    int missingRegs =
+        FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+    masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch);
+  }
+
+  masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
+  for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) {
+    masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
+  }
+  masm.finishFloatTransfer();
+
+  masm.ma_mov(sp, r0);
+  // Reserve 8 bytes for the outparam to ensure alignment for
+  // setupAlignedABICall.
+  masm.reserveStack(sizeof(void*) * 2);
+  masm.mov(sp, r1);
+  using Fn =
+      bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+  masm.setupAlignedABICall();
+  masm.passABIArg(r0);
+  masm.passABIArg(r1);
+  masm.callWithABI<Fn, InvalidationBailout>(
+      MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+  masm.pop(r2);  // Get bailoutInfo outparam.
+
+  // Pop the machine state and the dead frame.
+  masm.moveToStackPtr(FramePointer);
+
+  // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+  masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+                                            ArgumentsRectifierKind kind) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+  switch (kind) {
+    case ArgumentsRectifierKind::Normal:
+      argumentsRectifierOffset_ = startTrampolineCode(masm);
+      break;
+    case ArgumentsRectifierKind::TrialInlining:
+      trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+      break;
+  }
+  masm.pushReturnAddress();
+
+  // Frame prologue.
+  //
+  // NOTE: if this changes, fix the Baseline bailout code too!
+  // See BaselineStackBuilder::calculatePrevFramePtr and
+  // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+  masm.push(FramePointer);
+  masm.mov(StackPointer, FramePointer);
+
+  static_assert(JitStackAlignment == sizeof(Value));
+
+  // Copy number of actual arguments into r0 and r8.
+  masm.loadNumActualArgs(FramePointer, r0);
+  masm.mov(r0, r8);
+
+  // Load the number of |undefined|s to push into r6.
+  masm.loadPtr(
+      Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()), r1);
+  {
+    ScratchRegisterScope scratch(masm);
+    masm.ma_and(Imm32(CalleeTokenMask), r1, r6, scratch);
+  }
+  masm.loadFunctionArgCount(r6, r6);
+
+  masm.ma_sub(r6, r8, r2);
+
+  // Get the topmost argument.
+  {
+    ScratchRegisterScope scratch(masm);
+    masm.ma_alu(sp, lsl(r8, 3), r3, OpAdd);  // r3 <- sp + nargs * 8
+    masm.ma_add(r3, Imm32(sizeof(RectifierFrameLayout)), r3, scratch);
+  }
+
+  {
+    Label notConstructing;
+
+    masm.branchTest32(Assembler::Zero, r1,
+                      Imm32(CalleeToken_FunctionConstructing),
+                      &notConstructing);
+
+    // Add sizeof(Value) to overcome |this|
+    masm.as_extdtr(IsLoad, 64, true, Offset, r4, EDtrAddr(r3, EDtrOffImm(8)));
+    masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+                   EDtrAddr(sp, EDtrOffImm(-8)));
+
+    masm.bind(&notConstructing);
+  }
+
+  // Push undefined.
+  masm.moveValue(UndefinedValue(), ValueOperand(r5, r4));
+  {
+    Label undefLoopTop;
+    masm.bind(&undefLoopTop);
+    masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+                   EDtrAddr(sp, EDtrOffImm(-8)));
+    masm.as_sub(r2, r2, Imm8(1), SetCC);
+
+    masm.ma_b(&undefLoopTop, Assembler::NonZero);
+  }
+
+  // Push arguments, |nargs| + 1 times (to include |this|).
+  {
+    Label copyLoopTop;
+    masm.bind(&copyLoopTop);
+    masm.as_extdtr(IsLoad, 64, true, PostIndex, r4,
+                   EDtrAddr(r3, EDtrOffImm(-8)));
+    masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+                   EDtrAddr(sp, EDtrOffImm(-8)));
+
+    masm.as_sub(r8, r8, Imm8(1), SetCC);
+    masm.ma_b(&copyLoopTop, Assembler::NotSigned);
+  }
+
+  // Construct JitFrameLayout.
+  masm.ma_push(r1);  // callee token
+  masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, r0, r0);
+
+  // Call the target function.
+  masm.andPtr(Imm32(CalleeTokenMask), r1);
+  switch (kind) {
+    case ArgumentsRectifierKind::Normal:
+      masm.loadJitCodeRaw(r1, r3);
+      argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r3);
+      break;
+    case ArgumentsRectifierKind::TrialInlining:
+      Label noBaselineScript, done;
+      masm.loadBaselineJitCodeRaw(r1, r3, &noBaselineScript);
+      masm.callJitNoProfiler(r3);
+      masm.jump(&done);
+
+      // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+      masm.bind(&noBaselineScript);
+      masm.loadJitCodeRaw(r1, r3);
+      masm.callJitNoProfiler(r3);
+      masm.bind(&done);
+      break;
+  }
+
+  masm.mov(FramePointer, StackPointer);
+  masm.pop(FramePointer);
+  masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+#ifdef ENABLE_WASM_SIMD
+#  error "Needs more careful logic if SIMD is enabled"
+#endif
+
+  // STEP 1a: Save our register sets to the stack so Bailout() can read
+  // everything.
+  // sp % 8 == 0
+
+  masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+  // We don't have to push everything, but this is likely easier.
+  // Setting regs_.
+  for (uint32_t i = 0; i < Registers::Total; i++) {
+    masm.transferReg(Register::FromCode(i));
+  }
+  masm.finishDataTransfer();
+
+  ScratchRegisterScope scratch(masm);
+
+  // Since our datastructures for stack inspection are compile-time fixed,
+  // if there are only 16 double registers, then we need to reserve
+  // space on the stack for the missing 16.
+  if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+    int missingRegs =
+        FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+    masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch);
+  }
+  masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
+  for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) {
+    masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
+  }
+  masm.finishFloatTransfer();
+
+  // The current stack pointer is the first argument to jit::Bailout.
+  masm.ma_mov(sp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+  PushBailoutFrame(masm, r0);
+
+  // Make space for Bailout's bailoutInfo outparam.
+  masm.reserveStack(sizeof(void*));
+  masm.mov(sp, r1);
+  using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+  masm.setupAlignedABICall();
+
+  masm.passABIArg(r0);
+  masm.passABIArg(r1);
+
+  masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+                                CheckUnsafeCallWithABI::DontCheckOther);
+  masm.pop(r2);  // Get the bailoutInfo outparam.
+
+  // Remove both the bailout frame and the topmost Ion frame's stack.
+  masm.moveToStackPtr(FramePointer);
+
+  // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+  masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+                                        Label* bailoutTail) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+  bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+  GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+                                   const VMFunctionData& f, DynFn nativeFun,
+                                   uint32_t* wrapperOffset) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+  *wrapperOffset = startTrampolineCode(masm);
+
+  AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+  static_assert(
+      (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+      "Wrapper register set must be a superset of Volatile register set.");
+
+  // The context is the first argument; r0 is the first argument register.
+  Register cxreg = r0;
+  regs.take(cxreg);
+
+  // Stack is:
+  //    ... frame ...
+  //  +8  [args] + argPadding
+  //  +0  ExitFrame
+  //
+  // If it isn't a tail call, then the return address needs to be saved.
+  // Push the frame pointer to finish the exit frame, then link it up.
+  if (f.expectTailCall == NonTailCall) {
+    masm.pushReturnAddress();
+  }
+  masm.Push(FramePointer);
+  masm.loadJSContext(cxreg);
+  masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+  // Save the base of the argument set stored on the stack.
+  Register argsBase = InvalidReg;
+  if (f.explicitArgs) {
+    argsBase = r5;
+    regs.take(argsBase);
+    ScratchRegisterScope scratch(masm);
+    masm.ma_add(sp, Imm32(ExitFrameLayout::SizeWithFooter()), argsBase,
+                scratch);
+  }
+
+  // Reserve space for the outparameter.
+  Register outReg = InvalidReg;
+  switch (f.outParam) {
+    case Type_Value:
+      outReg = r4;
+      regs.take(outReg);
+      masm.reserveStack(sizeof(Value));
+      masm.ma_mov(sp, outReg);
+      break;
+
+    case Type_Handle:
+      outReg = r4;
+      regs.take(outReg);
+      masm.PushEmptyRooted(f.outParamRootType);
+      masm.ma_mov(sp, outReg);
+      break;
+
+    case Type_Int32:
+    case Type_Pointer:
+    case Type_Bool:
+      outReg = r4;
+      regs.take(outReg);
+      masm.reserveStack(sizeof(int32_t));
+      masm.ma_mov(sp, outReg);
+      break;
+
+    case Type_Double:
+      outReg = r4;
+      regs.take(outReg);
+      masm.reserveStack(sizeof(double));
+      masm.ma_mov(sp, outReg);
+      break;
+
+    default:
+      MOZ_ASSERT(f.outParam == Type_Void);
+      break;
+  }
+
+  masm.setupUnalignedABICall(regs.getAny());
+  masm.passABIArg(cxreg);
+
+  size_t argDisp = 0;
+
+  // Copy any arguments.
+  for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+    switch (f.argProperties(explicitArg)) {
+      case VMFunctionData::WordByValue:
+        masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+        argDisp += sizeof(void*);
+        break;
+      case VMFunctionData::DoubleByValue:
+        // Values should be passed by reference, not by value, so we assert
+        // that the argument is a double-precision float.
+        MOZ_ASSERT(f.argPassedInFloatReg(explicitArg));
+        masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+        argDisp += sizeof(double);
+        break;
+      case VMFunctionData::WordByRef:
+        masm.passABIArg(
+            MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+            MoveOp::GENERAL);
+        argDisp += sizeof(void*);
+        break;
+      case VMFunctionData::DoubleByRef:
+        masm.passABIArg(
+            MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+            MoveOp::GENERAL);
+        argDisp += 2 * sizeof(void*);
+        break;
+    }
+  }
+
+  // Copy the implicit outparam, if any.
+  if (outReg != InvalidReg) {
+    masm.passABIArg(outReg);
+  }
+
+  masm.callWithABI(nativeFun, MoveOp::GENERAL,
+                   CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+  // Test for failure.
+  switch (f.failType()) {
+    case Type_Cell:
+      masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel());
+      break;
+    case Type_Bool:
+      masm.branchIfFalseBool(r0, masm.failureLabel());
+      break;
+    case Type_Void:
+      break;
+    default:
+      MOZ_CRASH("unknown failure kind");
+  }
+
+  // Load the outparam and free any allocated stack.
+  switch (f.outParam) {
+    case Type_Handle:
+      masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+      break;
+
+    case Type_Value:
+      masm.loadValue(Address(sp, 0), JSReturnOperand);
+      masm.freeStack(sizeof(Value));
+      break;
+
+    case Type_Int32:
+    case Type_Pointer:
+      masm.load32(Address(sp, 0), ReturnReg);
+      masm.freeStack(sizeof(int32_t));
+      break;
+
+    case Type_Bool:
+      masm.load8ZeroExtend(Address(sp, 0), ReturnReg);
+      masm.freeStack(sizeof(int32_t));
+      break;
+
+    case Type_Double:
+      masm.loadDouble(Address(sp, 0), ReturnDoubleReg);
+      masm.freeStack(sizeof(double));
+      break;
+
+    default:
+      MOZ_ASSERT(f.outParam == Type_Void);
+      break;
+  }
+
+  // Until C++ code is instrumented against Spectre, prevent speculative
+  // execution from returning any private data.
+  if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+    masm.speculationBarrier();
+  }
+
+  // Pop ExitFooterFrame and the frame pointer.
+  masm.leaveExitFrame(sizeof(void*));
+
+  // Return. Subtract sizeof(void*) for the frame pointer.
+  masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+                  f.explicitStackSlots() * sizeof(void*) +
+                  f.extraValuesToPop * sizeof(Value)));
+
+  return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+                                        MIRType type) {
+  AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+  uint32_t offset = startTrampolineCode(masm);
+
+  masm.pushReturnAddress();
+
+  static_assert(PreBarrierReg == r1);
+  Register temp1 = r2;
+  Register temp2 = r3;
+  Register temp3 = r4;
+  masm.push(temp1);
+  masm.push(temp2);
+  masm.push(temp3);
+
+  Label noBarrier;
+  masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+                              &noBarrier);
+
+  // Call into C++ to mark this GC thing.
+  masm.pop(temp3);
+  masm.pop(temp2);
+  masm.pop(temp1);
+
+  LiveRegisterSet save;
+  save.set() =
+      RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+                  FloatRegisterSet(FloatRegisters::VolatileDoubleMask));
+  masm.PushRegsInMask(save);
+
+  masm.movePtr(ImmPtr(cx->runtime()), r0);
+
+  masm.setupUnalignedABICall(r2);
+  masm.passABIArg(r0);
+  masm.passABIArg(r1);
+  masm.callWithABI(JitPreWriteBarrier(type));
+  masm.PopRegsInMask(save);
+  masm.ret();
+
+  masm.bind(&noBarrier);
+  masm.pop(temp3);
+  masm.pop(temp2);
+  masm.pop(temp1);
+  masm.ret();
+
+  return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+                                         Label* bailoutTail) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+  masm.bind(bailoutTail);
+  masm.generateBailoutTail(r1, r2);
+}