diff options
Diffstat (limited to 'js/src/jit/arm/Trampoline-arm.cpp')
| -rw-r--r-- | js/src/jit/arm/Trampoline-arm.cpp | 831 |
1 files changed, 831 insertions, 0 deletions
diff --git a/js/src/jit/arm/Trampoline-arm.cpp b/js/src/jit/arm/Trampoline-arm.cpp new file mode 100644 index 0000000000..551f243bd3 --- /dev/null +++ b/js/src/jit/arm/Trampoline-arm.cpp @@ -0,0 +1,831 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/arm/SharedICHelpers-arm.h" +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/CalleeToken.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/PerfSpewer.h" +#include "jit/VMFunctions.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" +#include "vm/Realm.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +static const FloatRegisterSet NonVolatileFloatRegs = FloatRegisterSet( + (1ULL << FloatRegisters::d8) | (1ULL << FloatRegisters::d9) | + (1ULL << FloatRegisters::d10) | (1ULL << FloatRegisters::d11) | + (1ULL << FloatRegisters::d12) | (1ULL << FloatRegisters::d13) | + (1ULL << FloatRegisters::d14) | (1ULL << FloatRegisters::d15)); + +static void GenerateReturn(MacroAssembler& masm, int returnCode) { + // Restore non-volatile floating point registers. + masm.transferMultipleByRuns(NonVolatileFloatRegs, IsLoad, StackPointer, IA); + + // Get rid of padding word. + masm.addPtr(Imm32(sizeof(void*)), sp); + + // Set up return value + masm.ma_mov(Imm32(returnCode), r0); + + // Pop and return + masm.startDataTransferM(IsLoad, sp, IA, WriteBack); + masm.transferReg(r4); + masm.transferReg(r5); + masm.transferReg(r6); + masm.transferReg(r7); + masm.transferReg(r8); + masm.transferReg(r9); + masm.transferReg(r10); + masm.transferReg(r11); + // r12 isn't saved, so it shouldn't be restored. + masm.transferReg(pc); + masm.finishDataTransfer(); + masm.flushBuffer(); +} + +struct EnterJITStack { + double d8; + double d9; + double d10; + double d11; + double d12; + double d13; + double d14; + double d15; + + // Padding. + void* padding; + + // Non-volatile registers. + void* r4; + void* r5; + void* r6; + void* r7; + void* r8; + void* r9; + void* r10; + void* r11; + // The abi does not expect r12 (ip) to be preserved + void* lr; + + // Arguments. + // code == r0 + // argc == r1 + // argv == r2 + // frame == r3 + CalleeToken token; + JSObject* scopeChain; + size_t numStackValues; + Value* vp; +}; + +/* + * This method generates a trampoline for a c++ function with the following + * signature: + * void enter(void* code, int argc, Value* argv, InterpreterFrame* fp, + * CalleeToken calleeToken, JSObject* scopeChain, Value* vp) + * ...using standard EABI calling convention + */ +void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) { + AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT"); + + enterJITOffset_ = startTrampolineCode(masm); + + const Address slot_token(sp, offsetof(EnterJITStack, token)); + const Address slot_vp(sp, offsetof(EnterJITStack, vp)); + + static_assert(OsrFrameReg == r3); + + Assembler* aasm = &masm; + + // Save non-volatile registers. These must be saved by the trampoline, + // rather than the JIT'd code, because they are scanned by the conservative + // scanner. + masm.startDataTransferM(IsStore, sp, DB, WriteBack); + masm.transferReg(r4); // [sp,0] + masm.transferReg(r5); // [sp,4] + masm.transferReg(r6); // [sp,8] + masm.transferReg(r7); // [sp,12] + masm.transferReg(r8); // [sp,16] + masm.transferReg(r9); // [sp,20] + masm.transferReg(r10); // [sp,24] + masm.transferReg(r11); // [sp,28] + // The abi does not expect r12 (ip) to be preserved + masm.transferReg(lr); // [sp,32] + // The 5th argument is located at [sp, 36] + masm.finishDataTransfer(); + + // Add padding word. + masm.subPtr(Imm32(sizeof(void*)), sp); + + // Push the float registers. + masm.transferMultipleByRuns(NonVolatileFloatRegs, IsStore, sp, DB); + + // Load calleeToken into r9. + masm.loadPtr(slot_token, r9); + + // Save stack pointer. + masm.movePtr(sp, r11); + + // Load the number of actual arguments into r10. + masm.loadPtr(slot_vp, r10); + masm.unboxInt32(Address(r10, 0), r10); + + { + Label noNewTarget; + masm.branchTest32(Assembler::Zero, r9, + Imm32(CalleeToken_FunctionConstructing), &noNewTarget); + + masm.add32(Imm32(1), r1); + + masm.bind(&noNewTarget); + } + + // Guarantee stack alignment of Jit frames. + // + // This code moves the stack pointer to the location where it should be when + // we enter the Jit frame. It moves the stack pointer such that we have + // enough space reserved for pushing the arguments, and the JitFrameLayout. + // The stack pointer is also aligned on the alignment expected by the Jit + // frames. + // + // At the end the register r4, is a pointer to the stack where the first + // argument is expected by the Jit frame. + // + aasm->as_sub(r4, sp, O2RegImmShift(r1, LSL, 3)); // r4 = sp - argc*8 + aasm->as_bic(r4, r4, Imm8(JitStackAlignment - 1)); + // r4 is now the aligned on the bottom of the list of arguments. + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + // sp' = ~(JitStackAlignment - 1) & (sp - argc * sizeof(Value)) + masm.movePtr(r4, sp); + + // Get a copy of the number of args to use as a decrement counter, also set + // the zero condition code. + aasm->as_mov(r5, O2Reg(r1), SetCC); + + // Loop over arguments, copying them from an unknown buffer onto the Ion + // stack so they can be accessed from JIT'ed code. + { + Label header, footer; + // If there aren't any arguments, don't do anything. + aasm->as_b(&footer, Assembler::Zero); + // Get the top of the loop. + masm.bind(&header); + aasm->as_sub(r5, r5, Imm8(1), SetCC); + // We could be more awesome, and unroll this, using a loadm + // (particularly since the offset is effectively 0) but that seems more + // error prone, and complex. + // BIG FAT WARNING: this loads both r6 and r7. + aasm->as_extdtr(IsLoad, 64, true, PostIndex, r6, + EDtrAddr(r2, EDtrOffImm(8))); + aasm->as_extdtr(IsStore, 64, true, PostIndex, r6, + EDtrAddr(r4, EDtrOffImm(8))); + aasm->as_b(&header, Assembler::NonZero); + masm.bind(&footer); + } + + // Push the callee token. + masm.push(r9); + + // Push the frame descriptor. + masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, r10, r10); + + Label returnLabel; + { + // Handle Interpreter -> Baseline OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(r11)); + regs.take(OsrFrameReg); + regs.take(r0); // jitcode + MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches r0"); + + const Address slot_numStackValues(r11, + offsetof(EnterJITStack, numStackValues)); + + Label notOsr; + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); + + Register scratch = regs.takeAny(); + + Register numStackValues = regs.takeAny(); + masm.load32(slot_numStackValues, numStackValues); + + // Write return address. On ARM, CodeLabel is only used for tableswitch, + // so we can't use it here to get the return address. Instead, we use pc + // + a fixed offset to a jump to returnLabel. The pc register holds pc + + // 8, so we add the size of 2 instructions to skip the instructions + // emitted by push and jump(&skipJump). + { + AutoForbidPoolsAndNops afp(&masm, 5); + Label skipJump; + masm.mov(pc, scratch); + masm.addPtr(Imm32(2 * sizeof(uint32_t)), scratch); + masm.push(scratch); + masm.jump(&skipJump); + masm.jump(&returnLabel); + masm.bind(&skipJump); + } + + // Frame prologue. + masm.push(FramePointer); + masm.mov(sp, FramePointer); + + // Reserve frame. + masm.subPtr(Imm32(BaselineFrame::Size()), sp); + + Register framePtrScratch = regs.takeAny(); + masm.touchFrameValues(numStackValues, scratch, framePtrScratch); + masm.mov(sp, framePtrScratch); + + // Reserve space for locals and stack values. + masm.ma_lsl(Imm32(3), numStackValues, scratch); + masm.ma_sub(sp, scratch, sp); + + // Enter exit frame. + masm.pushFrameDescriptor(FrameType::BaselineJS); + masm.push(Imm32(0)); // Fake return address. + masm.push(FramePointer); + // No GC things to mark on the stack, push a bare token. + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare); + + masm.push(r0); // jitcode + + using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame, + uint32_t numStackValues); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(framePtrScratch); // BaselineFrame + masm.passABIArg(OsrFrameReg); // InterpreterFrame + masm.passABIArg(numStackValues); + masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + Register jitcode = regs.takeAny(); + masm.pop(jitcode); + + MOZ_ASSERT(jitcode != ReturnReg); + + Label error; + masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), sp); + masm.branchIfFalseBool(ReturnReg, &error); + + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.profilerEnterFrame(FramePointer, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(jitcode); + + // OOM: frame epilogue, load error value, discard return address and return. + masm.bind(&error); + masm.mov(FramePointer, sp); + masm.pop(FramePointer); + masm.addPtr(Imm32(sizeof(uintptr_t)), sp); // Return address. + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.jump(&returnLabel); + + masm.bind(¬Osr); + // Load the scope chain in R1. + MOZ_ASSERT(R1.scratchReg() != r0); + masm.loadPtr(Address(r11, offsetof(EnterJITStack, scopeChain)), + R1.scratchReg()); + } + + // The callee will push the return address and frame pointer on the stack, + // thus we check that the stack would be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t)); + + // Call the function. + masm.callJitNoProfiler(r0); + + // Interpreter -> Baseline OSR will return here. + masm.bind(&returnLabel); + + // Discard arguments and padding. Set sp to the address of the EnterJITStack + // on the stack. + masm.mov(r11, sp); + + // Store the returned value into the slot_vp + masm.loadPtr(slot_vp, r5); + masm.storeValue(JSReturnOperand, Address(r5, 0)); + + // Restore non-volatile registers and return. + GenerateReturn(masm, true); +} + +// static +mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> +JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) { + // Not supported, or not implemented yet. + // TODO: Implement along with the corresponding stack-walker changes, in + // coordination with the Gecko Profiler, see bug 1635987 and follow-ups. + return mozilla::Nothing{}; +} + +void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) { + // See large comment in x86's JitRuntime::generateInvalidator. + + AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator"); + + invalidatorOffset_ = startTrampolineCode(masm); + + // At this point, one of two things has happened: + // 1) Execution has just returned from C code, which left the stack aligned + // 2) Execution has just returned from Ion code, which left the stack + // unaligned. The old return address should not matter, but we still want the + // stack to be aligned, and there is no good reason to automatically align it + // with a call to setupUnalignedABICall. + masm.as_bic(sp, sp, Imm8(7)); + masm.startDataTransferM(IsStore, sp, DB, WriteBack); + // We don't have to push everything, but this is likely easier. + // Setting regs_. + for (uint32_t i = 0; i < Registers::Total; i++) { + masm.transferReg(Register::FromCode(i)); + } + masm.finishDataTransfer(); + + // Since our datastructures for stack inspection are compile-time fixed, + // if there are only 16 double registers, then we need to reserve + // space on the stack for the missing 16. + if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) { + ScratchRegisterScope scratch(masm); + int missingRegs = + FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys(); + masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch); + } + + masm.startFloatTransferM(IsStore, sp, DB, WriteBack); + for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) { + masm.transferFloatReg(FloatRegister(i, FloatRegister::Double)); + } + masm.finishFloatTransfer(); + + masm.ma_mov(sp, r0); + // Reserve 8 bytes for the outparam to ensure alignment for + // setupAlignedABICall. + masm.reserveStack(sizeof(void*) * 2); + masm.mov(sp, r1); + using Fn = + bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupAlignedABICall(); + masm.passABIArg(r0); + masm.passABIArg(r1); + masm.callWithABI<Fn, InvalidationBailout>( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(r2); // Get bailoutInfo outparam. + + // Pop the machine state and the dead frame. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2. + masm.jump(bailoutTail); +} + +void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm, + ArgumentsRectifierKind kind) { + AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier"); + + switch (kind) { + case ArgumentsRectifierKind::Normal: + argumentsRectifierOffset_ = startTrampolineCode(masm); + break; + case ArgumentsRectifierKind::TrialInlining: + trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm); + break; + } + masm.pushReturnAddress(); + + // Frame prologue. + // + // NOTE: if this changes, fix the Baseline bailout code too! + // See BaselineStackBuilder::calculatePrevFramePtr and + // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp). + masm.push(FramePointer); + masm.mov(StackPointer, FramePointer); + + static_assert(JitStackAlignment == sizeof(Value)); + + // Copy number of actual arguments into r0 and r8. + masm.loadNumActualArgs(FramePointer, r0); + masm.mov(r0, r8); + + // Load the number of |undefined|s to push into r6. + masm.loadPtr( + Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()), r1); + { + ScratchRegisterScope scratch(masm); + masm.ma_and(Imm32(CalleeTokenMask), r1, r6, scratch); + } + masm.loadFunctionArgCount(r6, r6); + + masm.ma_sub(r6, r8, r2); + + // Get the topmost argument. + { + ScratchRegisterScope scratch(masm); + masm.ma_alu(sp, lsl(r8, 3), r3, OpAdd); // r3 <- sp + nargs * 8 + masm.ma_add(r3, Imm32(sizeof(RectifierFrameLayout)), r3, scratch); + } + + { + Label notConstructing; + + masm.branchTest32(Assembler::Zero, r1, + Imm32(CalleeToken_FunctionConstructing), + ¬Constructing); + + // Add sizeof(Value) to overcome |this| + masm.as_extdtr(IsLoad, 64, true, Offset, r4, EDtrAddr(r3, EDtrOffImm(8))); + masm.as_extdtr(IsStore, 64, true, PreIndex, r4, + EDtrAddr(sp, EDtrOffImm(-8))); + + masm.bind(¬Constructing); + } + + // Push undefined. + masm.moveValue(UndefinedValue(), ValueOperand(r5, r4)); + { + Label undefLoopTop; + masm.bind(&undefLoopTop); + masm.as_extdtr(IsStore, 64, true, PreIndex, r4, + EDtrAddr(sp, EDtrOffImm(-8))); + masm.as_sub(r2, r2, Imm8(1), SetCC); + + masm.ma_b(&undefLoopTop, Assembler::NonZero); + } + + // Push arguments, |nargs| + 1 times (to include |this|). + { + Label copyLoopTop; + masm.bind(©LoopTop); + masm.as_extdtr(IsLoad, 64, true, PostIndex, r4, + EDtrAddr(r3, EDtrOffImm(-8))); + masm.as_extdtr(IsStore, 64, true, PreIndex, r4, + EDtrAddr(sp, EDtrOffImm(-8))); + + masm.as_sub(r8, r8, Imm8(1), SetCC); + masm.ma_b(©LoopTop, Assembler::NotSigned); + } + + // Construct JitFrameLayout. + masm.ma_push(r1); // callee token + masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, r0, r0); + + // Call the target function. + masm.andPtr(Imm32(CalleeTokenMask), r1); + switch (kind) { + case ArgumentsRectifierKind::Normal: + masm.loadJitCodeRaw(r1, r3); + argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r3); + break; + case ArgumentsRectifierKind::TrialInlining: + Label noBaselineScript, done; + masm.loadBaselineJitCodeRaw(r1, r3, &noBaselineScript); + masm.callJitNoProfiler(r3); + masm.jump(&done); + + // See BaselineCacheIRCompiler::emitCallInlinedFunction. + masm.bind(&noBaselineScript); + masm.loadJitCodeRaw(r1, r3); + masm.callJitNoProfiler(r3); + masm.bind(&done); + break; + } + + masm.mov(FramePointer, StackPointer); + masm.pop(FramePointer); + masm.ret(); +} + +static void PushBailoutFrame(MacroAssembler& masm, Register spArg) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + // STEP 1a: Save our register sets to the stack so Bailout() can read + // everything. + // sp % 8 == 0 + + masm.startDataTransferM(IsStore, sp, DB, WriteBack); + // We don't have to push everything, but this is likely easier. + // Setting regs_. + for (uint32_t i = 0; i < Registers::Total; i++) { + masm.transferReg(Register::FromCode(i)); + } + masm.finishDataTransfer(); + + ScratchRegisterScope scratch(masm); + + // Since our datastructures for stack inspection are compile-time fixed, + // if there are only 16 double registers, then we need to reserve + // space on the stack for the missing 16. + if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) { + int missingRegs = + FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys(); + masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch); + } + masm.startFloatTransferM(IsStore, sp, DB, WriteBack); + for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) { + masm.transferFloatReg(FloatRegister(i, FloatRegister::Double)); + } + masm.finishFloatTransfer(); + + // The current stack pointer is the first argument to jit::Bailout. + masm.ma_mov(sp, spArg); +} + +static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) { + PushBailoutFrame(masm, r0); + + // Make space for Bailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.mov(sp, r1); + using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupAlignedABICall(); + + masm.passABIArg(r0); + masm.passABIArg(r1); + + masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + masm.pop(r2); // Get the bailoutInfo outparam. + + // Remove both the bailout frame and the topmost Ion frame's stack. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2. + masm.jump(bailoutTail); +} + +void JitRuntime::generateBailoutHandler(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler"); + + bailoutHandlerOffset_ = startTrampolineCode(masm); + + GenerateBailoutThunk(masm, bailoutTail); +} + +bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm, + const VMFunctionData& f, DynFn nativeFun, + uint32_t* wrapperOffset) { + AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper"); + + *wrapperOffset = startTrampolineCode(masm); + + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + static_assert( + (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set must be a superset of Volatile register set."); + + // The context is the first argument; r0 is the first argument register. + Register cxreg = r0; + regs.take(cxreg); + + // Stack is: + // ... frame ... + // +8 [args] + argPadding + // +0 ExitFrame + // + // If it isn't a tail call, then the return address needs to be saved. + // Push the frame pointer to finish the exit frame, then link it up. + if (f.expectTailCall == NonTailCall) { + masm.pushReturnAddress(); + } + masm.Push(FramePointer); + masm.moveStackPtrTo(FramePointer); + masm.loadJSContext(cxreg); + masm.enterExitFrame(cxreg, regs.getAny(), &f); + + // Save the base of the argument set stored on the stack. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + argsBase = r5; + regs.take(argsBase); + ScratchRegisterScope scratch(masm); + masm.ma_add(sp, Imm32(ExitFrameLayout::SizeWithFooter()), argsBase, + scratch); + } + + // Reserve space for the outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = r4; + regs.take(outReg); + masm.reserveStack(sizeof(Value)); + masm.ma_mov(sp, outReg); + break; + + case Type_Handle: + outReg = r4; + regs.take(outReg); + masm.PushEmptyRooted(f.outParamRootType); + masm.ma_mov(sp, outReg); + break; + + case Type_Int32: + case Type_Pointer: + case Type_Bool: + outReg = r4; + regs.take(outReg); + masm.reserveStack(sizeof(int32_t)); + masm.ma_mov(sp, outReg); + break; + + case Type_Double: + outReg = r4; + regs.take(outReg); + masm.reserveStack(sizeof(double)); + masm.ma_mov(sp, outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(cxreg); + + size_t argDisp = 0; + + // Copy any arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + switch (f.argProperties(explicitArg)) { + case VMFunctionData::WordByValue: + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByValue: + // Values should be passed by reference, not by value, so we assert + // that the argument is a double-precision float. + MOZ_ASSERT(f.argPassedInFloatReg(explicitArg)); + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE); + argDisp += sizeof(double); + break; + case VMFunctionData::WordByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress), + MoveOp::GENERAL); + argDisp += 2 * sizeof(void*); + break; + } + } + + // Copy the implicit outparam, if any. + if (outReg != InvalidReg) { + masm.passABIArg(outReg); + } + + masm.callWithABI(nativeFun, MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + // Test for failure. + switch (f.failType()) { + case Type_Cell: + masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel()); + break; + case Type_Bool: + masm.branchIfFalseBool(r0, masm.failureLabel()); + break; + case Type_Void: + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Value: + masm.loadValue(Address(sp, 0), JSReturnOperand); + masm.freeStack(sizeof(Value)); + break; + + case Type_Int32: + case Type_Pointer: + masm.load32(Address(sp, 0), ReturnReg); + masm.freeStack(sizeof(int32_t)); + break; + + case Type_Bool: + masm.load8ZeroExtend(Address(sp, 0), ReturnReg); + masm.freeStack(sizeof(int32_t)); + break; + + case Type_Double: + masm.loadDouble(Address(sp, 0), ReturnDoubleReg); + masm.freeStack(sizeof(double)); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + // Until C++ code is instrumented against Spectre, prevent speculative + // execution from returning any private data. + if (f.returnsData() && JitOptions.spectreJitToCxxCalls) { + masm.speculationBarrier(); + } + + // Pop ExitFooterFrame and the frame pointer. + masm.leaveExitFrame(0); + masm.pop(FramePointer); + + // Return. Subtract sizeof(void*) for the frame pointer. + masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + return true; +} + +uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type) { + AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier"); + + uint32_t offset = startTrampolineCode(masm); + + masm.pushReturnAddress(); + + static_assert(PreBarrierReg == r1); + Register temp1 = r2; + Register temp2 = r3; + Register temp3 = r4; + masm.push(temp1); + masm.push(temp2); + masm.push(temp3); + + Label noBarrier; + masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3, + &noBarrier); + + // Call into C++ to mark this GC thing. + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + + LiveRegisterSet save; + save.set() = + RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileDoubleMask)); + masm.PushRegsInMask(save); + + masm.movePtr(ImmPtr(cx->runtime()), r0); + + masm.setupUnalignedABICall(r2); + masm.passABIArg(r0); + masm.passABIArg(r1); + masm.callWithABI(JitPreWriteBarrier(type)); + masm.PopRegsInMask(save); + masm.ret(); + + masm.bind(&noBarrier); + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + masm.ret(); + + return offset; +} + +void JitRuntime::generateBailoutTailStub(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub"); + + masm.bind(bailoutTail); + masm.generateBailoutTail(r1, r2); +} |