diff options
Diffstat (limited to 'js/src/jit/x86/Assembler-x86.h')
| -rw-r--r-- | js/src/jit/x86/Assembler-x86.h | 1079 |
1 files changed, 1079 insertions, 0 deletions
diff --git a/js/src/jit/x86/Assembler-x86.h b/js/src/jit/x86/Assembler-x86.h new file mode 100644 index 0000000000..12d790740e --- /dev/null +++ b/js/src/jit/x86/Assembler-x86.h @@ -0,0 +1,1079 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_Assembler_x86_h +#define jit_x86_Assembler_x86_h + +#include <iterator> + +#include "jit/CompactBuffer.h" +#include "jit/JitCode.h" +#include "jit/shared/Assembler-shared.h" +#include "jit/x86-shared/Constants-x86-shared.h" + +namespace js { +namespace jit { + +static constexpr Register eax{X86Encoding::rax}; +static constexpr Register ecx{X86Encoding::rcx}; +static constexpr Register edx{X86Encoding::rdx}; +static constexpr Register ebx{X86Encoding::rbx}; +static constexpr Register esp{X86Encoding::rsp}; +static constexpr Register ebp{X86Encoding::rbp}; +static constexpr Register esi{X86Encoding::rsi}; +static constexpr Register edi{X86Encoding::rdi}; + +static constexpr FloatRegister xmm0 = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); +static constexpr FloatRegister xmm1 = + FloatRegister(X86Encoding::xmm1, FloatRegisters::Double); +static constexpr FloatRegister xmm2 = + FloatRegister(X86Encoding::xmm2, FloatRegisters::Double); +static constexpr FloatRegister xmm3 = + FloatRegister(X86Encoding::xmm3, FloatRegisters::Double); +static constexpr FloatRegister xmm4 = + FloatRegister(X86Encoding::xmm4, FloatRegisters::Double); +static constexpr FloatRegister xmm5 = + FloatRegister(X86Encoding::xmm5, FloatRegisters::Double); +static constexpr FloatRegister xmm6 = + FloatRegister(X86Encoding::xmm6, FloatRegisters::Double); +static constexpr FloatRegister xmm7 = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Double); + +// Vector registers fixed for use with some instructions, e.g. PBLENDVB. +static constexpr FloatRegister vmm0 = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); + +static constexpr Register InvalidReg{X86Encoding::invalid_reg}; +static constexpr FloatRegister InvalidFloatReg = FloatRegister(); + +static constexpr Register JSReturnReg_Type = ecx; +static constexpr Register JSReturnReg_Data = edx; +static constexpr Register StackPointer = esp; +static constexpr Register FramePointer = ebp; +static constexpr Register ReturnReg = eax; +static constexpr FloatRegister ReturnFloat32Reg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Single); +static constexpr FloatRegister ReturnDoubleReg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); +static constexpr FloatRegister ReturnSimd128Reg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); +static constexpr FloatRegister ScratchFloat32Reg_ = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Single); +static constexpr FloatRegister ScratchDoubleReg_ = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Double); +static constexpr FloatRegister ScratchSimd128Reg = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Simd128); + +// Note, EDX:EAX is the system ABI 64-bit return register, and it is to our +// advantage to keep the SpiderMonkey ABI in sync with the system ABI. +// +// However, using EDX here means that we have to use a register that does not +// have a word or byte part (eg DX/DH/DL) in some other places; notably, +// ABINonArgReturnReg1 is EDI. If this becomes a problem and ReturnReg64 has to +// be something other than EDX:EAX, then jitted code that calls directly to C++ +// will need to shuffle the return value from EDX:EAX into ReturnReg64 directly +// after the call. See bug 1730161 for discussion and a patch that does that. +static constexpr Register64 ReturnReg64(edx, eax); + +// Avoid ebp, which is the FramePointer, which is unavailable in some modes. +static constexpr Register CallTempReg0 = edi; +static constexpr Register CallTempReg1 = eax; +static constexpr Register CallTempReg2 = ebx; +static constexpr Register CallTempReg3 = ecx; +static constexpr Register CallTempReg4 = esi; +static constexpr Register CallTempReg5 = edx; + +// We have no arg regs, so our NonArgRegs are just our CallTempReg* +static constexpr Register CallTempNonArgRegs[] = {edi, eax, ebx, ecx, esi, edx}; +static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); + +class ABIArgGenerator { + uint32_t stackOffset_; + ABIArg current_; + + public: + ABIArgGenerator(); + ABIArg next(MIRType argType); + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } +}; + +// These registers may be volatile or nonvolatile. +static constexpr Register ABINonArgReg0 = eax; +static constexpr Register ABINonArgReg1 = ebx; +static constexpr Register ABINonArgReg2 = ecx; +static constexpr Register ABINonArgReg3 = edx; + +// This register may be volatile or nonvolatile. Avoid xmm7 which is the +// ScratchDoubleReg_. +static constexpr FloatRegister ABINonArgDoubleReg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); + +// These registers may be volatile or nonvolatile. +// Note: these three registers are all guaranteed to be different +static constexpr Register ABINonArgReturnReg0 = ecx; +static constexpr Register ABINonArgReturnReg1 = edi; +static constexpr Register ABINonVolatileReg = ebx; + +// This register is guaranteed to be clobberable during the prologue and +// epilogue of an ABI call which must preserve both ABI argument, return +// and non-volatile registers. +static constexpr Register ABINonArgReturnVolatileReg = ecx; + +// Instance pointer argument register for WebAssembly functions. This must not +// alias any other register used for passing function arguments or return +// values. Preserved by WebAssembly functions. +static constexpr Register InstanceReg = esi; + +// Registers used for asm.js/wasm table calls. These registers must be disjoint +// from the ABI argument registers, InstanceReg and each other. +static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmTableCallSigReg = ABINonArgReg2; +static constexpr Register WasmTableCallIndexReg = ABINonArgReg3; + +// Registers used for ref calls. +static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmCallRefReg = ABINonArgReg3; + +// Register used as a scratch along the return path in the fast js -> wasm stub +// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg. +// It must be a volatile register. +static constexpr Register WasmJitEntryReturnScratch = ebx; + +static constexpr Register OsrFrameReg = edx; +static constexpr Register PreBarrierReg = edx; + +// Not enough registers for a PC register (R0-R2 use 2 registers each). +static constexpr Register InterpreterPCReg = InvalidReg; + +// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use +// JSReturnOperand). +static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; +static constexpr Register RegExpMatcherStringReg = CallTempReg1; +static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; + +// Registers used by RegExpExecTest stub (do not use ReturnReg). +static constexpr Register RegExpExecTestRegExpReg = CallTempReg0; +static constexpr Register RegExpExecTestStringReg = CallTempReg2; + +// Registers used by RegExpSearcher stub (do not use ReturnReg). +static constexpr Register RegExpSearcherRegExpReg = CallTempReg0; +static constexpr Register RegExpSearcherStringReg = CallTempReg2; +static constexpr Register RegExpSearcherLastIndexReg = CallTempReg3; + +// GCC stack is aligned on 16 bytes. Ion does not maintain this for internal +// calls. wasm code does. +#if defined(__GNUC__) && !defined(__MINGW32__) +static constexpr uint32_t ABIStackAlignment = 16; +#else +static constexpr uint32_t ABIStackAlignment = 4; +#endif +static constexpr uint32_t CodeAlignment = 16; +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = + JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && + JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +static constexpr uint32_t SimdMemoryAlignment = 16; + +static_assert(CodeAlignment % SimdMemoryAlignment == 0, + "Code alignment should be larger than any of the alignments " + "which are used for " + "the constant sections of the code buffer. Thus it should be " + "larger than the " + "alignment for SIMD constants."); + +static_assert(JitStackAlignment % SimdMemoryAlignment == 0, + "Stack alignment should be larger than any of the alignments " + "which are used for " + "spilled values. Thus it should be larger than the alignment " + "for SIMD accesses."); + +static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment; +static constexpr uint32_t WasmTrapInstructionLength = 2; + +// See comments in wasm::GenerateFunctionPrologue. The difference between these +// is the size of the largest callable prologue on the platform. (We could make +// the tail offset 3, but I have opted for 4 as that results in a better-aligned +// branch target.) +static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; + +struct ImmTag : public Imm32 { + explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {} +}; + +struct ImmType : public ImmTag { + explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {} +}; + +static constexpr Scale ScalePointer = TimesFour; + +} // namespace jit +} // namespace js + +#include "jit/x86-shared/Assembler-x86-shared.h" + +namespace js { +namespace jit { + +static inline Operand LowWord(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + return Operand(LowWord(op.toAddress())); + case Operand::MEM_SCALE: + return Operand(LowWord(op.toBaseIndex())); + default: + MOZ_CRASH("Invalid operand type"); + } +} + +static inline Operand HighWord(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + return Operand(HighWord(op.toAddress())); + case Operand::MEM_SCALE: + return Operand(HighWord(op.toBaseIndex())); + default: + MOZ_CRASH("Invalid operand type"); + } +} + +// Return operand from a JS -> JS call. +static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type, + JSReturnReg_Data}; + +class Assembler : public AssemblerX86Shared { + Vector<RelativePatch, 8, SystemAllocPolicy> jumps_; + + void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind kind) { + enoughMemory_ &= + jumps_.append(RelativePatch(src.offset(), target.value, kind)); + if (kind == RelocationKind::JITCODE) { + jumpRelocations_.writeUnsigned(src.offset()); + } + } + + public: + using AssemblerX86Shared::call; + using AssemblerX86Shared::cmpl; + using AssemblerX86Shared::j; + using AssemblerX86Shared::jmp; + using AssemblerX86Shared::movl; + using AssemblerX86Shared::pop; + using AssemblerX86Shared::push; + using AssemblerX86Shared::retarget; + using AssemblerX86Shared::vmovsd; + using AssemblerX86Shared::vmovss; + + static void TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + + // Copy the assembly code to the given buffer, and perform any pending + // relocations relying on the target address. + void executableCopy(uint8_t* buffer); + + void assertNoGCThings() const { +#ifdef DEBUG + MOZ_ASSERT(dataRelocations_.length() == 0); + for (auto& j : jumps_) { + MOZ_ASSERT(j.kind == RelocationKind::HARDCODED); + } +#endif + } + + // Actual assembly emitting functions. + + void push(ImmGCPtr ptr) { + masm.push_i32(int32_t(ptr.value)); + writeDataRelocation(ptr); + } + void push(const ImmWord imm) { push(Imm32(imm.value)); } + void push(const ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); } + void push(FloatRegister src) { + subl(Imm32(sizeof(double)), StackPointer); + vmovsd(src, Address(StackPointer, 0)); + } + + CodeOffset pushWithPatch(ImmWord word) { + masm.push_i32(int32_t(word.value)); + return CodeOffset(masm.currentOffset()); + } + + void pop(FloatRegister src) { + vmovsd(Address(StackPointer, 0), src); + addl(Imm32(sizeof(double)), StackPointer); + } + + CodeOffset movWithPatch(ImmWord word, Register dest) { + movl(Imm32(word.value), dest); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movWithPatch(ImmPtr imm, Register dest) { + return movWithPatch(ImmWord(uintptr_t(imm.value)), dest); + } + + void movl(ImmGCPtr ptr, Register dest) { + masm.movl_i32r(uintptr_t(ptr.value), dest.encoding()); + writeDataRelocation(ptr); + } + void movl(ImmGCPtr ptr, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.movl_i32r(uintptr_t(ptr.value), dest.reg()); + writeDataRelocation(ptr); + break; + case Operand::MEM_REG_DISP: + masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base()); + writeDataRelocation(ptr); + break; + case Operand::MEM_SCALE: + masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base(), + dest.index(), dest.scale()); + writeDataRelocation(ptr); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movl(ImmWord imm, Register dest) { + masm.movl_i32r(imm.value, dest.encoding()); + } + void movl(ImmPtr imm, Register dest) { + movl(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(ImmWord imm, Register dest) { + // Use xor for setting registers to zero, as it is specially optimized + // for this purpose on modern hardware. Note that it does clobber FLAGS + // though. + if (imm.value == 0) { + xorl(dest, dest); + } else { + movl(imm, dest); + } + } + void mov(ImmPtr imm, Register dest) { + mov(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(wasm::SymbolicAddress imm, Register dest) { + masm.movl_i32r(-1, dest.encoding()); + append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm)); + } + void mov(const Operand& src, Register dest) { movl(src, dest); } + void mov(Register src, const Operand& dest) { movl(src, dest); } + void mov(Imm32 imm, const Operand& dest) { movl(imm, dest); } + void mov(CodeLabel* label, Register dest) { + // Put a placeholder value in the instruction stream. + masm.movl_i32r(0, dest.encoding()); + label->patchAt()->bind(masm.size()); + } + void mov(Register src, Register dest) { movl(src, dest); } + void xchg(Register src, Register dest) { xchgl(src, dest); } + void lea(const Operand& src, Register dest) { return leal(src, dest); } + void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); } + void cmovzPtr(const Operand& src, Register dest) { return cmovzl(src, dest); } + + void fstp32(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fstp32_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void faddp() { masm.faddp(); } + + void cmpl(ImmWord rhs, Register lhs) { + masm.cmpl_ir(rhs.value, lhs.encoding()); + } + void cmpl(ImmPtr rhs, Register lhs) { + cmpl(ImmWord(uintptr_t(rhs.value)), lhs); + } + void cmpl(ImmGCPtr rhs, Register lhs) { + masm.cmpl_i32r(uintptr_t(rhs.value), lhs.encoding()); + writeDataRelocation(rhs); + } + void cmpl(Register rhs, Register lhs) { + masm.cmpl_rr(rhs.encoding(), lhs.encoding()); + } + void cmpl(ImmGCPtr rhs, const Operand& lhs) { + switch (lhs.kind()) { + case Operand::REG: + masm.cmpl_i32r(uintptr_t(rhs.value), lhs.reg()); + writeDataRelocation(rhs); + break; + case Operand::MEM_REG_DISP: + masm.cmpl_i32m(uintptr_t(rhs.value), lhs.disp(), lhs.base()); + writeDataRelocation(rhs); + break; + case Operand::MEM_ADDRESS32: + masm.cmpl_i32m(uintptr_t(rhs.value), lhs.address()); + writeDataRelocation(rhs); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void cmpl(Register rhs, wasm::SymbolicAddress lhs) { + masm.cmpl_rm_disp32(rhs.encoding(), (void*)-1); + append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), lhs)); + } + void cmpl(Imm32 rhs, wasm::SymbolicAddress lhs) { + JmpSrc src = masm.cmpl_im_disp32(rhs.value, (void*)-1); + append(wasm::SymbolicAccess(CodeOffset(src.offset()), lhs)); + } + + void adcl(Imm32 imm, Register dest) { + masm.adcl_ir(imm.value, dest.encoding()); + } + void adcl(Register src, Register dest) { + masm.adcl_rr(src.encoding(), dest.encoding()); + } + void adcl(Operand src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.adcl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.adcl_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void sbbl(Imm32 imm, Register dest) { + masm.sbbl_ir(imm.value, dest.encoding()); + } + void sbbl(Register src, Register dest) { + masm.sbbl_rr(src.encoding(), dest.encoding()); + } + void sbbl(Operand src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.sbbl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.sbbl_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void mull(Register multiplier) { masm.mull_r(multiplier.encoding()); } + + void shldl(const Imm32 imm, Register src, Register dest) { + masm.shldl_irr(imm.value, src.encoding(), dest.encoding()); + } + void shrdl(const Imm32 imm, Register src, Register dest) { + masm.shrdl_irr(imm.value, src.encoding(), dest.encoding()); + } + + void vhaddpd(FloatRegister rhs, FloatRegister lhsDest) { + MOZ_ASSERT(HasSSE3()); + MOZ_ASSERT(rhs.size() == 16); + MOZ_ASSERT(lhsDest.size() == 16); + masm.vhaddpd_rr(rhs.encoding(), lhsDest.encoding(), lhsDest.encoding()); + } + + void fild(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fild_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) { + JmpSrc src = masm.jmp(); + addPendingJump(src, target, reloc); + } + void j(Condition cond, ImmPtr target, + RelocationKind reloc = RelocationKind::HARDCODED) { + JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond)); + addPendingJump(src, target, reloc); + } + + void jmp(JitCode* target) { + jmp(ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void j(Condition cond, JitCode* target) { + j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void call(JitCode* target) { + JmpSrc src = masm.call(); + addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void call(ImmWord target) { call(ImmPtr((void*)target.value)); } + void call(ImmPtr target) { + JmpSrc src = masm.call(); + addPendingJump(src, target, RelocationKind::HARDCODED); + } + + // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch + // this instruction. + CodeOffset toggledCall(JitCode* target, bool enabled) { + CodeOffset offset(size()); + JmpSrc src = enabled ? masm.call() : masm.cmp_eax(); + addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); + MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr)); + return offset; + } + + static size_t ToggledCallSize(uint8_t* code) { + // Size of a call instruction. + return 5; + } + + // Re-routes pending jumps to an external target, flushing the label in the + // process. + void retarget(Label* label, ImmPtr target, RelocationKind reloc) { + if (label->used()) { + bool more; + X86Encoding::JmpSrc jmp(label->offset()); + do { + X86Encoding::JmpSrc next; + more = masm.nextJump(jmp, &next); + addPendingJump(jmp, target, reloc); + jmp = next; + } while (more); + } + label->reset(); + } + + // Move a 32-bit immediate into a register where the immediate can be + // patched. + CodeOffset movlWithPatch(Imm32 imm, Register dest) { + masm.movl_i32r(imm.value, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Load from *(base + disp32) where disp32 can be patched. + CodeOffset movsblWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movsbl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movsbl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzblWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movzbl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movzbl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movswlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movswl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movswl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzwlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movzwl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movzwl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovss(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_mr(src.address(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovss_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovdWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovd_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovsd(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_mr(src.address(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovsd_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovupsWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovups_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovdqu_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + + // Store to *(base + disp32) where disp32 can be patched. + CodeOffset movbWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movb_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movb_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movwWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movw_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movw_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movl_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatchLow(Register regLow, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: { + return movlWithPatch(regLow, LowWord(dest)); + } + case Operand::MEM_ADDRESS32: { + Operand low( + PatchedAbsoluteAddress(uint32_t(dest.address()) + INT64LOW_OFFSET)); + return movlWithPatch(regLow, low); + } + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset movlWithPatchHigh(Register regHigh, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: { + return movlWithPatch(regHigh, HighWord(dest)); + } + case Operand::MEM_ADDRESS32: { + Operand high(PatchedAbsoluteAddress(uint32_t(dest.address()) + + INT64HIGH_OFFSET)); + return movlWithPatch(regHigh, high); + } + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovdWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovd_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovss(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_rm(src.encoding(), dest.address()); + break; + case Operand::MEM_SCALE: + masm.vmovss_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), + dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovsdWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovsd(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_rm(src.encoding(), dest.address()); + break; + case Operand::MEM_SCALE: + masm.vmovsd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), + dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovupsWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovups_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovdqu_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + + // Load from *(addr + index*scale) where addr can be patched. + CodeOffset movlWithPatch(PatchedAbsoluteAddress addr, Register index, + Scale scale, Register dest) { + masm.movl_mr(addr.addr, index.encoding(), scale, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Load from *src where src can be patched. + CodeOffset movsblWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movsbl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzblWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movzbl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movswlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movswl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzwlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movzwl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovss_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovq_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovsd_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdqaWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqa_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqu_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovapsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovaps_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovupsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovups_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Store to *dest where dest can be patched. + CodeOffset movbWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movb_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movwWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movw_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movl_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovss_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovq_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovsd_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdqaWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqa_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovapsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovaps_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqu_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovupsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovups_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } +}; + +// Get a register in which we plan to put a quantity that will be used as an +// integer argument. This differs from GetIntArgReg in that if we have no more +// actual argument registers to use we will fall back on using whatever +// CallTempReg* don't overlap the argument registers, and only fail once those +// run out too. +static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, + uint32_t usedFloatArgs, Register* out) { + if (usedIntArgs >= NumCallTempNonArgRegs) { + return false; + } + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +} // namespace jit +} // namespace js + +#endif /* jit_x86_Assembler_x86_h */ |