diff options
Diffstat (limited to 'js/src/jit/x64/Trampoline-x64.cpp')
| -rw-r--r-- | js/src/jit/x64/Trampoline-x64.cpp | 888 |
1 files changed, 888 insertions, 0 deletions
diff --git a/js/src/jit/x64/Trampoline-x64.cpp b/js/src/jit/x64/Trampoline-x64.cpp new file mode 100644 index 0000000000..dcc9291299 --- /dev/null +++ b/js/src/jit/x64/Trampoline-x64.cpp @@ -0,0 +1,888 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/CalleeToken.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/PerfSpewer.h" +#include "jit/VMFunctions.h" +#include "jit/x64/SharedICRegisters-x64.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::IsPowerOfTwo; + +// This struct reflects the contents of the stack entry. +// Given a `CommonFrameLayout* frame`: +// - `frame->prevType()` should be `FrameType::CppToJSJit`. +// - Then EnterJITStackEntry starts at: +// frame->callerFramePtr() + EnterJITStackEntry::offsetFromFP() +// (the offset is negative, so this subtracts from the frame pointer) +struct EnterJITStackEntry { + // Offset from frame pointer to EnterJITStackEntry*. + static constexpr int32_t offsetFromFP() { + return -int32_t(offsetof(EnterJITStackEntry, rbp)); + } + + void* result; + +#if defined(_WIN64) + struct XMM { + using XMM128 = char[16]; + XMM128 xmm6; + XMM128 xmm7; + XMM128 xmm8; + XMM128 xmm9; + XMM128 xmm10; + XMM128 xmm11; + XMM128 xmm12; + XMM128 xmm13; + XMM128 xmm14; + XMM128 xmm15; + } xmm; + + // 16-byte aligment for xmm registers above. + uint64_t xmmPadding; + + void* rsi; + void* rdi; +#endif + + void* r15; + void* r14; + void* r13; + void* r12; + void* rbx; + void* rbp; + + // Pushed by CALL. + void* rip; +}; + +// All registers to save and restore. This includes the stack pointer, since we +// use the ability to reference register values on the stack by index. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +// Generates a trampoline for calling Jit compiled code from a C++ function. +// The trampoline use the EnterJitCode signature, with the standard x64 fastcall +// calling convention. +void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) { + AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT"); + + enterJITOffset_ = startTrampolineCode(masm); + + masm.assertStackAlignment(ABIStackAlignment, + -int32_t(sizeof(uintptr_t)) /* return address */); + + const Register reg_code = IntArgReg0; + const Register reg_argc = IntArgReg1; + const Register reg_argv = IntArgReg2; + static_assert(OsrFrameReg == IntArgReg3); + +#if defined(_WIN64) + const Address token = Address(rbp, 16 + ShadowStackSpace); + const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace); + const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace); + const Operand result = Operand(rbp, 40 + ShadowStackSpace); +#else + const Register token = IntArgReg4; + const Register scopeChain = IntArgReg5; + const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace); + const Operand result = Operand(rbp, 24 + ShadowStackSpace); +#endif + + // Note: the stack pushes below must match the fields in EnterJITStackEntry. + + // Save old stack frame pointer, set new stack frame pointer. + masm.push(rbp); + masm.mov(rsp, rbp); + + // Save non-volatile registers. These must be saved by the trampoline, rather + // than by the JIT'd code, because they are scanned by the conservative + // scanner. + masm.push(rbx); + masm.push(r12); + masm.push(r13); + masm.push(r14); + masm.push(r15); +#if defined(_WIN64) + masm.push(rdi); + masm.push(rsi); + + // 16-byte aligment for vmovdqa + masm.subq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp); + + masm.vmovdqa(xmm6, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6))); + masm.vmovdqa(xmm7, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7))); + masm.vmovdqa(xmm8, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8))); + masm.vmovdqa(xmm9, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9))); + masm.vmovdqa(xmm10, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10))); + masm.vmovdqa(xmm11, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11))); + masm.vmovdqa(xmm12, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12))); + masm.vmovdqa(xmm13, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13))); + masm.vmovdqa(xmm14, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14))); + masm.vmovdqa(xmm15, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15))); +#endif + + // Save arguments passed in registers needed after function call. + masm.push(result); + + // End of pushes reflected in EnterJITStackEntry, i.e. EnterJITStackEntry + // starts at this rsp. + + // Remember number of bytes occupied by argument vector + masm.mov(reg_argc, r13); + + // if we are constructing, that also needs to include newTarget + { + Label noNewTarget; + masm.branchTest32(Assembler::Zero, token, + Imm32(CalleeToken_FunctionConstructing), &noNewTarget); + + masm.addq(Imm32(1), r13); + + masm.bind(&noNewTarget); + } + + masm.shll(Imm32(3), r13); // r13 = argc * sizeof(Value) + static_assert(sizeof(Value) == 1 << 3, "Constant is baked in assembly code"); + + // Guarantee stack alignment of Jit frames. + // + // This code compensates for the offset created by the copy of the vector of + // arguments, such that the jit frame will be aligned once the return + // address is pushed on the stack. + // + // In the computation of the offset, we omit the size of the JitFrameLayout + // which is pushed on the stack, as the JitFrameLayout size is a multiple of + // the JitStackAlignment. + masm.mov(rsp, r12); + masm.subq(r13, r12); + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + masm.andl(Imm32(JitStackAlignment - 1), r12); + masm.subq(r12, rsp); + + /*************************************************************** + Loop over argv vector, push arguments onto stack in reverse order + ***************************************************************/ + + // r13 still stores the number of bytes in the argument vector. + masm.addq(reg_argv, r13); // r13 points above last argument or newTarget + + // while r13 > rdx, push arguments. + { + Label header, footer; + masm.bind(&header); + + masm.cmpPtr(r13, reg_argv); + masm.j(AssemblerX86Shared::BelowOrEqual, &footer); + + masm.subq(Imm32(8), r13); + masm.push(Operand(r13, 0)); + masm.jmp(&header); + + masm.bind(&footer); + } + + // Load the number of actual arguments. |result| is used to store the + // actual number of arguments without adding an extra argument to the enter + // JIT. + masm.movq(result, reg_argc); + masm.unboxInt32(Operand(reg_argc, 0), reg_argc); + + // Push the callee token. + masm.push(token); + + // Push the descriptor. + masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, reg_argc, reg_argc); + + CodeLabel returnLabel; + Label oomReturnLabel; + { + // Handle Interpreter -> Baseline OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(rbp)); + regs.take(OsrFrameReg); + regs.take(reg_code); + + Register scratch = regs.takeAny(); + + Label notOsr; + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); + + Register numStackValues = regs.takeAny(); + masm.movq(numStackValuesAddr, numStackValues); + + // Push return address + masm.mov(&returnLabel, scratch); + masm.push(scratch); + + // Frame prologue. + masm.push(rbp); + masm.mov(rsp, rbp); + + // Reserve frame. + masm.subPtr(Imm32(BaselineFrame::Size()), rsp); + + Register framePtrScratch = regs.takeAny(); + masm.touchFrameValues(numStackValues, scratch, framePtrScratch); + masm.mov(rsp, framePtrScratch); + + // Reserve space for locals and stack values. + Register valuesSize = regs.takeAny(); + masm.mov(numStackValues, valuesSize); + masm.shll(Imm32(3), valuesSize); + masm.subPtr(valuesSize, rsp); + + // Enter exit frame. + masm.pushFrameDescriptor(FrameType::BaselineJS); + masm.push(Imm32(0)); // Fake return address. + masm.push(FramePointer); + // No GC things to mark, push a bare token. + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare); + + regs.add(valuesSize); + + masm.push(reg_code); + + using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame, + uint32_t numStackValues); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(framePtrScratch); // BaselineFrame + masm.passABIArg(OsrFrameReg); // InterpreterFrame + masm.passABIArg(numStackValues); + masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + masm.pop(reg_code); + + MOZ_ASSERT(reg_code != ReturnReg); + + Label error; + masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp); + masm.branchIfFalseBool(ReturnReg, &error); + + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.profilerEnterFrame(rbp, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(reg_code); + + // OOM: frame epilogue, load error value, discard return address and return. + masm.bind(&error); + masm.mov(rbp, rsp); + masm.pop(rbp); + masm.addPtr(Imm32(sizeof(uintptr_t)), rsp); // Return address. + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.jump(&oomReturnLabel); + + masm.bind(¬Osr); + masm.movq(scopeChain, R1.scratchReg()); + } + + // The call will push the return address and frame pointer on the stack, thus + // we check that the stack would be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t)); + + // Call function. + masm.callJitNoProfiler(reg_code); + + { + // Interpreter -> Baseline OSR will return here. + masm.bind(&returnLabel); + masm.addCodeLabel(returnLabel); + masm.bind(&oomReturnLabel); + } + + // Discard arguments and padding. Set rsp to the address of the + // EnterJITStackEntry on the stack. + masm.lea(Operand(rbp, EnterJITStackEntry::offsetFromFP()), rsp); + + /***************************************************************** + Place return value where it belongs, pop all saved registers + *****************************************************************/ + masm.pop(r12); // vp + masm.storeValue(JSReturnOperand, Operand(r12, 0)); + + // Restore non-volatile registers. +#if defined(_WIN64) + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)), xmm6); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)), xmm7); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)), xmm8); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)), xmm9); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)), xmm10); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)), xmm11); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)), xmm12); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)), xmm13); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)), xmm14); + masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)), xmm15); + + masm.addq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp); + + masm.pop(rsi); + masm.pop(rdi); +#endif + masm.pop(r15); + masm.pop(r14); + masm.pop(r13); + masm.pop(r12); + masm.pop(rbx); + + // Restore frame pointer and return. + masm.pop(rbp); + masm.ret(); +} + +// static +mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> +JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) { + if (frameStackAddress->prevType() != FrameType::CppToJSJit) { + // This is not a CppToJSJit frame, there are no C++ registers here. + return mozilla::Nothing{}; + } + + // Compute pointer to start of EnterJITStackEntry on the stack. + uint8_t* fp = frameStackAddress->callerFramePtr(); + auto* enterJITStackEntry = reinterpret_cast<EnterJITStackEntry*>( + fp + EnterJITStackEntry::offsetFromFP()); + + // Extract native function call registers. + ::JS::ProfilingFrameIterator::RegisterState registerState; + registerState.fp = enterJITStackEntry->rbp; + registerState.pc = enterJITStackEntry->rip; + // sp should be inside the caller's frame, so set sp to the value of the stack + // pointer before the call to the EnterJit trampoline. + registerState.sp = &enterJITStackEntry->rip + 1; + // No lr in this world. + registerState.lr = nullptr; + return mozilla::Some(registerState); +} + +// Push AllRegs in a way that is compatible with RegisterDump, regardless of +// what PushRegsInMask might do to reduce the set size. +static void DumpAllRegs(MacroAssembler& masm) { +#ifdef ENABLE_WASM_SIMD + masm.PushRegsInMask(AllRegs); +#else + // When SIMD isn't supported, PushRegsInMask reduces the set of float + // registers to be double-sized, while the RegisterDump expects each of + // the float registers to have the maximal possible size + // (Simd128DataSize). To work around this, we just spill the double + // registers by hand here, using the register dump offset directly. + for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more(); + ++iter) { + masm.Push(*iter); + } + + masm.reserveStack(sizeof(RegisterDump::FPUArray)); + for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more(); + ++iter) { + FloatRegister reg = *iter; + Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes()); + masm.storeDouble(reg, spillAddress); + } +#endif +} + +void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator"); + + // See explanatory comment in x86's JitRuntime::generateInvalidator. + + invalidatorOffset_ = startTrampolineCode(masm); + + // Push registers such that we can access them from [base + code]. + DumpAllRegs(masm); + + masm.movq(rsp, rax); // Argument to jit::InvalidationBailout. + + // Make space for InvalidationBailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movq(rsp, rbx); + + using Fn = + bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupUnalignedABICall(rdx); + masm.passABIArg(rax); + masm.passABIArg(rbx); + masm.callWithABI<Fn, InvalidationBailout>( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(r9); // Get the bailoutInfo outparam. + + // Pop the machine state and the dead frame. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9. + masm.jmp(bailoutTail); +} + +void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm, + ArgumentsRectifierKind kind) { + // Do not erase the frame pointer in this function. + + AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier"); + + switch (kind) { + case ArgumentsRectifierKind::Normal: + argumentsRectifierOffset_ = startTrampolineCode(masm); + break; + case ArgumentsRectifierKind::TrialInlining: + trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm); + break; + } + + // Caller: + // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- rsp + + // Frame prologue. + // + // NOTE: if this changes, fix the Baseline bailout code too! + // See BaselineStackBuilder::calculatePrevFramePtr and + // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp). + masm.push(FramePointer); + masm.movq(rsp, FramePointer); + + // Load argc. + masm.loadNumActualArgs(FramePointer, r8); + + // Load |nformals| into %rcx. + masm.loadPtr(Address(rbp, RectifierFrameLayout::offsetOfCalleeToken()), rax); + masm.mov(rax, rcx); + masm.andq(Imm32(uint32_t(CalleeTokenMask)), rcx); + masm.loadFunctionArgCount(rcx, rcx); + + // Stash another copy in r11, since we are going to do destructive operations + // on rcx + masm.mov(rcx, r11); + + static_assert( + CalleeToken_FunctionConstructing == 1, + "Ensure that we can use the constructing bit to count the value"); + masm.mov(rax, rdx); + masm.andq(Imm32(uint32_t(CalleeToken_FunctionConstructing)), rdx); + + // Including |this|, and |new.target|, there are (|nformals| + 1 + + // isConstructing) arguments to push to the stack. Then we push a + // JitFrameLayout. We compute the padding expressed in the number of extra + // |undefined| values to push on the stack. + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + static_assert( + JitStackAlignment % sizeof(Value) == 0, + "Ensure that we can pad the stack by pushing extra UndefinedValue"); + static_assert(IsPowerOfTwo(JitStackValueAlignment), + "must have power of two for masm.andl to do its job"); + + masm.addl( + Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */), + rcx); + masm.addl(rdx, rcx); + masm.andl(Imm32(~(JitStackValueAlignment - 1)), rcx); + + // Load the number of |undefined|s to push into %rcx. Subtract 1 for |this|. + masm.subl(r8, rcx); + masm.subl(Imm32(1), rcx); + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- rsp + // '--- #r8 ---' + // + // Rectifier frame: + // [rbp'] [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee] + // [descr] [raddr] ] + // '------- #rcx --------' '--- #r8 ---' + + // Copy the number of actual arguments into rdx. + masm.mov(r8, rdx); + + masm.moveValue(UndefinedValue(), ValueOperand(r10)); + + // Push undefined. (including the padding) + { + Label undefLoopTop; + masm.bind(&undefLoopTop); + + masm.push(r10); + masm.subl(Imm32(1), rcx); + masm.j(Assembler::NonZero, &undefLoopTop); + } + + // Get the topmost argument. + static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments"); + + // Get the topmost argument. + BaseIndex b(FramePointer, r8, TimesEight, sizeof(RectifierFrameLayout)); + masm.lea(Operand(b), rcx); + + // Push arguments, |nargs| + 1 times (to include |this|). + masm.addl(Imm32(1), r8); + { + Label copyLoopTop; + + masm.bind(©LoopTop); + masm.push(Operand(rcx, 0x0)); + masm.subq(Imm32(sizeof(Value)), rcx); + masm.subl(Imm32(1), r8); + masm.j(Assembler::NonZero, ©LoopTop); + } + + // if constructing, copy newTarget + { + Label notConstructing; + + masm.branchTest32(Assembler::Zero, rax, + Imm32(CalleeToken_FunctionConstructing), + ¬Constructing); + + // thisFrame[numFormals] = prevFrame[argc] + ValueOperand newTarget(r10); + + // Load vp[argc]. Add sizeof(Value) for |this|. + BaseIndex newTargetSrc(FramePointer, rdx, TimesEight, + sizeof(RectifierFrameLayout) + sizeof(Value)); + masm.loadValue(newTargetSrc, newTarget); + + // Again, 1 for |this| + BaseIndex newTargetDest(rsp, r11, TimesEight, sizeof(Value)); + masm.storeValue(newTarget, newTargetDest); + + masm.bind(¬Constructing); + } + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] + // + // + // Rectifier frame: + // [rbp'] <- rbp [undef] [undef] [undef] [arg2] [arg1] [this] <- rsp [ [argc] + // [callee] [descr] [raddr] ] + // + + // Construct JitFrameLayout. + masm.push(rax); // callee token + masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, rdx, rdx); + + // Call the target function. + masm.andq(Imm32(uint32_t(CalleeTokenMask)), rax); + switch (kind) { + case ArgumentsRectifierKind::Normal: + masm.loadJitCodeRaw(rax, rax); + argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(rax); + break; + case ArgumentsRectifierKind::TrialInlining: + Label noBaselineScript, done; + masm.loadBaselineJitCodeRaw(rax, rbx, &noBaselineScript); + masm.callJitNoProfiler(rbx); + masm.jump(&done); + + // See BaselineCacheIRCompiler::emitCallInlinedFunction. + masm.bind(&noBaselineScript); + masm.loadJitCodeRaw(rax, rax); + masm.callJitNoProfiler(rax); + masm.bind(&done); + break; + } + + masm.mov(FramePointer, StackPointer); + masm.pop(FramePointer); + masm.ret(); +} + +static void PushBailoutFrame(MacroAssembler& masm, Register spArg) { + // Push registers such that we can access them from [base + code]. + DumpAllRegs(masm); + + // Get the stack pointer into a register, pre-alignment. + masm.movq(rsp, spArg); +} + +static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) { + PushBailoutFrame(masm, r8); + + // Make space for Bailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movq(rsp, r9); + + // Call the bailout function. + using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupUnalignedABICall(rax); + masm.passABIArg(r8); + masm.passABIArg(r9); + masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(r9); // Get the bailoutInfo outparam. + + // Remove both the bailout frame and the topmost Ion frame's stack. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9. + masm.jmp(bailoutTail); +} + +void JitRuntime::generateBailoutHandler(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler"); + + bailoutHandlerOffset_ = startTrampolineCode(masm); + + GenerateBailoutThunk(masm, bailoutTail); +} + +bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm, + const VMFunctionData& f, DynFn nativeFun, + uint32_t* wrapperOffset) { + AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper"); + + *wrapperOffset = startTrampolineCode(masm); + + // Avoid conflicts with argument registers while discarding the result after + // the function call. + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + static_assert( + (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set must be a superset of Volatile register set"); + + // The context is the first argument. + Register cxreg = IntArgReg0; + regs.take(cxreg); + + // Stack is: + // ... frame ... + // +12 [args] + // +8 descriptor + // +0 returnAddress + // + // Push the frame pointer to finish the exit frame, then link it up. + masm.Push(FramePointer); + masm.moveStackPtrTo(FramePointer); + masm.loadJSContext(cxreg); + masm.enterExitFrame(cxreg, regs.getAny(), &f); + + // Save the current stack pointer as the base for copying arguments. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + argsBase = r10; + regs.take(argsBase); + masm.lea(Operand(rsp, ExitFrameLayout::SizeWithFooter()), argsBase); + } + + // Reserve space for the outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(Value)); + masm.movq(esp, outReg); + break; + + case Type_Handle: + outReg = regs.takeAny(); + masm.PushEmptyRooted(f.outParamRootType); + masm.movq(esp, outReg); + break; + + case Type_Int32: + case Type_Bool: + case Type_Pointer: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(uintptr_t)); + masm.movq(esp, outReg); + break; + + case Type_Double: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(double)); + masm.movq(esp, outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(cxreg); + + size_t argDisp = 0; + + // Copy arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + switch (f.argProperties(explicitArg)) { + case VMFunctionData::WordByValue: + if (f.argPassedInFloatReg(explicitArg)) { + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE); + } else { + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + } + argDisp += sizeof(void*); + break; + case VMFunctionData::WordByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByValue: + case VMFunctionData::DoubleByRef: + MOZ_CRASH("NYI: x64 callVM should not be used with 128bits values."); + } + } + + // Copy the implicit outparam, if any. + if (outReg != InvalidReg) { + masm.passABIArg(outReg); + } + + masm.callWithABI(nativeFun, MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + // Test for failure. + switch (f.failType()) { + case Type_Cell: + masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel()); + break; + case Type_Bool: + masm.testb(rax, rax); + masm.j(Assembler::Zero, masm.failureLabel()); + break; + case Type_Void: + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Value: + masm.loadValue(Address(esp, 0), JSReturnOperand); + masm.freeStack(sizeof(Value)); + break; + + case Type_Int32: + masm.load32(Address(esp, 0), ReturnReg); + masm.freeStack(sizeof(uintptr_t)); + break; + + case Type_Bool: + masm.load8ZeroExtend(Address(esp, 0), ReturnReg); + masm.freeStack(sizeof(uintptr_t)); + break; + + case Type_Double: + masm.loadDouble(Address(esp, 0), ReturnDoubleReg); + masm.freeStack(sizeof(double)); + break; + + case Type_Pointer: + masm.loadPtr(Address(esp, 0), ReturnReg); + masm.freeStack(sizeof(uintptr_t)); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + // Until C++ code is instrumented against Spectre, prevent speculative + // execution from returning any private data. + if (f.returnsData() && JitOptions.spectreJitToCxxCalls) { + masm.speculationBarrier(); + } + + // Pop ExitFooterFrame and the frame pointer. + masm.leaveExitFrame(0); + masm.pop(FramePointer); + + // Return. Subtract sizeof(void*) for the frame pointer. + masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + return true; +} + +uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type) { + AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier"); + + uint32_t offset = startTrampolineCode(masm); + + static_assert(PreBarrierReg == rdx); + Register temp1 = rax; + Register temp2 = rbx; + Register temp3 = rcx; + masm.push(temp1); + masm.push(temp2); + masm.push(temp3); + + Label noBarrier; + masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3, + &noBarrier); + + // Call into C++ to mark this GC thing. + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + + LiveRegisterSet regs = + LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileMask)); + masm.PushRegsInMask(regs); + + masm.mov(ImmPtr(cx->runtime()), rcx); + + masm.setupUnalignedABICall(rax); + masm.passABIArg(rcx); + masm.passABIArg(rdx); + masm.callWithABI(JitPreWriteBarrier(type)); + + masm.PopRegsInMask(regs); + masm.ret(); + + masm.bind(&noBarrier); + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + masm.ret(); + + return offset; +} + +void JitRuntime::generateBailoutTailStub(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub"); + + masm.bind(bailoutTail); + masm.generateBailoutTail(rdx, r9); +} |