From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- js/src/jit/x86/Assembler-x86.cpp | 85 ++ js/src/jit/x86/Assembler-x86.h | 1079 ++++++++++++++++++ js/src/jit/x86/BaseAssembler-x86.h | 190 ++++ js/src/jit/x86/CodeGenerator-x86.cpp | 1509 ++++++++++++++++++++++++ js/src/jit/x86/CodeGenerator-x86.h | 49 + js/src/jit/x86/LIR-x86.h | 308 +++++ js/src/jit/x86/Lowering-x86.cpp | 840 ++++++++++++++ js/src/jit/x86/Lowering-x86.h | 79 ++ js/src/jit/x86/MacroAssembler-x86-inl.h | 1386 ++++++++++++++++++++++ js/src/jit/x86/MacroAssembler-x86.cpp | 1829 ++++++++++++++++++++++++++++++ js/src/jit/x86/MacroAssembler-x86.h | 1149 +++++++++++++++++++ js/src/jit/x86/SharedICHelpers-x86-inl.h | 77 ++ js/src/jit/x86/SharedICHelpers-x86.h | 70 ++ js/src/jit/x86/SharedICRegisters-x86.h | 36 + js/src/jit/x86/Trampoline-x86.cpp | 796 +++++++++++++ 15 files changed, 9482 insertions(+) create mode 100644 js/src/jit/x86/Assembler-x86.cpp create mode 100644 js/src/jit/x86/Assembler-x86.h create mode 100644 js/src/jit/x86/BaseAssembler-x86.h create mode 100644 js/src/jit/x86/CodeGenerator-x86.cpp create mode 100644 js/src/jit/x86/CodeGenerator-x86.h create mode 100644 js/src/jit/x86/LIR-x86.h create mode 100644 js/src/jit/x86/Lowering-x86.cpp create mode 100644 js/src/jit/x86/Lowering-x86.h create mode 100644 js/src/jit/x86/MacroAssembler-x86-inl.h create mode 100644 js/src/jit/x86/MacroAssembler-x86.cpp create mode 100644 js/src/jit/x86/MacroAssembler-x86.h create mode 100644 js/src/jit/x86/SharedICHelpers-x86-inl.h create mode 100644 js/src/jit/x86/SharedICHelpers-x86.h create mode 100644 js/src/jit/x86/SharedICRegisters-x86.h create mode 100644 js/src/jit/x86/Trampoline-x86.cpp (limited to 'js/src/jit/x86') diff --git a/js/src/jit/x86/Assembler-x86.cpp b/js/src/jit/x86/Assembler-x86.cpp new file mode 100644 index 0000000000..7296f77291 --- /dev/null +++ b/js/src/jit/x86/Assembler-x86.cpp @@ -0,0 +1,85 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86/Assembler-x86.h" + +#include "gc/Marking.h" +#include "util/Memory.h" + +using namespace js; +using namespace js::jit; + +ABIArgGenerator::ABIArgGenerator() : stackOffset_(0), current_() {} + +ABIArg ABIArgGenerator::next(MIRType type) { + switch (type) { + case MIRType::Int32: + case MIRType::Float32: + case MIRType::Pointer: + case MIRType::RefOrNull: + case MIRType::StackResults: + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uint32_t); + break; + case MIRType::Double: + case MIRType::Int64: + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uint64_t); + break; + case MIRType::Simd128: + // On Win64, >64 bit args need to be passed by reference. However, wasm + // doesn't allow passing SIMD values to JS, so the only way to reach this + // is wasm to wasm calls. Ergo we can break the native ABI here and use + // the Wasm ABI instead. + stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment); + current_ = ABIArg(stackOffset_); + stackOffset_ += Simd128DataSize; + break; + default: + MOZ_CRASH("Unexpected argument type"); + } + return current_; +} + +void Assembler::executableCopy(uint8_t* buffer) { + AssemblerX86Shared::executableCopy(buffer); + for (RelativePatch& rp : jumps_) { + X86Encoding::SetRel32(buffer + rp.offset, rp.target); + } +} + +class RelocationIterator { + CompactBufferReader reader_; + uint32_t offset_; + + public: + explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {} + + bool read() { + if (!reader_.more()) { + return false; + } + offset_ = reader_.readUnsigned(); + return true; + } + + uint32_t offset() const { return offset_; } +}; + +static inline JitCode* CodeFromJump(uint8_t* jump) { + uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump); + return JitCode::FromExecutable(target); +} + +void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + RelocationIterator iter(reader); + while (iter.read()) { + JitCode* child = CodeFromJump(code->raw() + iter.offset()); + TraceManuallyBarrieredEdge(trc, &child, "rel32"); + MOZ_ASSERT(child == CodeFromJump(code->raw() + iter.offset())); + } +} diff --git a/js/src/jit/x86/Assembler-x86.h b/js/src/jit/x86/Assembler-x86.h new file mode 100644 index 0000000000..12d790740e --- /dev/null +++ b/js/src/jit/x86/Assembler-x86.h @@ -0,0 +1,1079 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_Assembler_x86_h +#define jit_x86_Assembler_x86_h + +#include + +#include "jit/CompactBuffer.h" +#include "jit/JitCode.h" +#include "jit/shared/Assembler-shared.h" +#include "jit/x86-shared/Constants-x86-shared.h" + +namespace js { +namespace jit { + +static constexpr Register eax{X86Encoding::rax}; +static constexpr Register ecx{X86Encoding::rcx}; +static constexpr Register edx{X86Encoding::rdx}; +static constexpr Register ebx{X86Encoding::rbx}; +static constexpr Register esp{X86Encoding::rsp}; +static constexpr Register ebp{X86Encoding::rbp}; +static constexpr Register esi{X86Encoding::rsi}; +static constexpr Register edi{X86Encoding::rdi}; + +static constexpr FloatRegister xmm0 = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); +static constexpr FloatRegister xmm1 = + FloatRegister(X86Encoding::xmm1, FloatRegisters::Double); +static constexpr FloatRegister xmm2 = + FloatRegister(X86Encoding::xmm2, FloatRegisters::Double); +static constexpr FloatRegister xmm3 = + FloatRegister(X86Encoding::xmm3, FloatRegisters::Double); +static constexpr FloatRegister xmm4 = + FloatRegister(X86Encoding::xmm4, FloatRegisters::Double); +static constexpr FloatRegister xmm5 = + FloatRegister(X86Encoding::xmm5, FloatRegisters::Double); +static constexpr FloatRegister xmm6 = + FloatRegister(X86Encoding::xmm6, FloatRegisters::Double); +static constexpr FloatRegister xmm7 = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Double); + +// Vector registers fixed for use with some instructions, e.g. PBLENDVB. +static constexpr FloatRegister vmm0 = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); + +static constexpr Register InvalidReg{X86Encoding::invalid_reg}; +static constexpr FloatRegister InvalidFloatReg = FloatRegister(); + +static constexpr Register JSReturnReg_Type = ecx; +static constexpr Register JSReturnReg_Data = edx; +static constexpr Register StackPointer = esp; +static constexpr Register FramePointer = ebp; +static constexpr Register ReturnReg = eax; +static constexpr FloatRegister ReturnFloat32Reg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Single); +static constexpr FloatRegister ReturnDoubleReg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); +static constexpr FloatRegister ReturnSimd128Reg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); +static constexpr FloatRegister ScratchFloat32Reg_ = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Single); +static constexpr FloatRegister ScratchDoubleReg_ = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Double); +static constexpr FloatRegister ScratchSimd128Reg = + FloatRegister(X86Encoding::xmm7, FloatRegisters::Simd128); + +// Note, EDX:EAX is the system ABI 64-bit return register, and it is to our +// advantage to keep the SpiderMonkey ABI in sync with the system ABI. +// +// However, using EDX here means that we have to use a register that does not +// have a word or byte part (eg DX/DH/DL) in some other places; notably, +// ABINonArgReturnReg1 is EDI. If this becomes a problem and ReturnReg64 has to +// be something other than EDX:EAX, then jitted code that calls directly to C++ +// will need to shuffle the return value from EDX:EAX into ReturnReg64 directly +// after the call. See bug 1730161 for discussion and a patch that does that. +static constexpr Register64 ReturnReg64(edx, eax); + +// Avoid ebp, which is the FramePointer, which is unavailable in some modes. +static constexpr Register CallTempReg0 = edi; +static constexpr Register CallTempReg1 = eax; +static constexpr Register CallTempReg2 = ebx; +static constexpr Register CallTempReg3 = ecx; +static constexpr Register CallTempReg4 = esi; +static constexpr Register CallTempReg5 = edx; + +// We have no arg regs, so our NonArgRegs are just our CallTempReg* +static constexpr Register CallTempNonArgRegs[] = {edi, eax, ebx, ecx, esi, edx}; +static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); + +class ABIArgGenerator { + uint32_t stackOffset_; + ABIArg current_; + + public: + ABIArgGenerator(); + ABIArg next(MIRType argType); + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } +}; + +// These registers may be volatile or nonvolatile. +static constexpr Register ABINonArgReg0 = eax; +static constexpr Register ABINonArgReg1 = ebx; +static constexpr Register ABINonArgReg2 = ecx; +static constexpr Register ABINonArgReg3 = edx; + +// This register may be volatile or nonvolatile. Avoid xmm7 which is the +// ScratchDoubleReg_. +static constexpr FloatRegister ABINonArgDoubleReg = + FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); + +// These registers may be volatile or nonvolatile. +// Note: these three registers are all guaranteed to be different +static constexpr Register ABINonArgReturnReg0 = ecx; +static constexpr Register ABINonArgReturnReg1 = edi; +static constexpr Register ABINonVolatileReg = ebx; + +// This register is guaranteed to be clobberable during the prologue and +// epilogue of an ABI call which must preserve both ABI argument, return +// and non-volatile registers. +static constexpr Register ABINonArgReturnVolatileReg = ecx; + +// Instance pointer argument register for WebAssembly functions. This must not +// alias any other register used for passing function arguments or return +// values. Preserved by WebAssembly functions. +static constexpr Register InstanceReg = esi; + +// Registers used for asm.js/wasm table calls. These registers must be disjoint +// from the ABI argument registers, InstanceReg and each other. +static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmTableCallSigReg = ABINonArgReg2; +static constexpr Register WasmTableCallIndexReg = ABINonArgReg3; + +// Registers used for ref calls. +static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmCallRefReg = ABINonArgReg3; + +// Register used as a scratch along the return path in the fast js -> wasm stub +// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg. +// It must be a volatile register. +static constexpr Register WasmJitEntryReturnScratch = ebx; + +static constexpr Register OsrFrameReg = edx; +static constexpr Register PreBarrierReg = edx; + +// Not enough registers for a PC register (R0-R2 use 2 registers each). +static constexpr Register InterpreterPCReg = InvalidReg; + +// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use +// JSReturnOperand). +static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; +static constexpr Register RegExpMatcherStringReg = CallTempReg1; +static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; + +// Registers used by RegExpExecTest stub (do not use ReturnReg). +static constexpr Register RegExpExecTestRegExpReg = CallTempReg0; +static constexpr Register RegExpExecTestStringReg = CallTempReg2; + +// Registers used by RegExpSearcher stub (do not use ReturnReg). +static constexpr Register RegExpSearcherRegExpReg = CallTempReg0; +static constexpr Register RegExpSearcherStringReg = CallTempReg2; +static constexpr Register RegExpSearcherLastIndexReg = CallTempReg3; + +// GCC stack is aligned on 16 bytes. Ion does not maintain this for internal +// calls. wasm code does. +#if defined(__GNUC__) && !defined(__MINGW32__) +static constexpr uint32_t ABIStackAlignment = 16; +#else +static constexpr uint32_t ABIStackAlignment = 4; +#endif +static constexpr uint32_t CodeAlignment = 16; +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = + JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && + JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +static constexpr uint32_t SimdMemoryAlignment = 16; + +static_assert(CodeAlignment % SimdMemoryAlignment == 0, + "Code alignment should be larger than any of the alignments " + "which are used for " + "the constant sections of the code buffer. Thus it should be " + "larger than the " + "alignment for SIMD constants."); + +static_assert(JitStackAlignment % SimdMemoryAlignment == 0, + "Stack alignment should be larger than any of the alignments " + "which are used for " + "spilled values. Thus it should be larger than the alignment " + "for SIMD accesses."); + +static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment; +static constexpr uint32_t WasmTrapInstructionLength = 2; + +// See comments in wasm::GenerateFunctionPrologue. The difference between these +// is the size of the largest callable prologue on the platform. (We could make +// the tail offset 3, but I have opted for 4 as that results in a better-aligned +// branch target.) +static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; + +struct ImmTag : public Imm32 { + explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {} +}; + +struct ImmType : public ImmTag { + explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {} +}; + +static constexpr Scale ScalePointer = TimesFour; + +} // namespace jit +} // namespace js + +#include "jit/x86-shared/Assembler-x86-shared.h" + +namespace js { +namespace jit { + +static inline Operand LowWord(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + return Operand(LowWord(op.toAddress())); + case Operand::MEM_SCALE: + return Operand(LowWord(op.toBaseIndex())); + default: + MOZ_CRASH("Invalid operand type"); + } +} + +static inline Operand HighWord(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + return Operand(HighWord(op.toAddress())); + case Operand::MEM_SCALE: + return Operand(HighWord(op.toBaseIndex())); + default: + MOZ_CRASH("Invalid operand type"); + } +} + +// Return operand from a JS -> JS call. +static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type, + JSReturnReg_Data}; + +class Assembler : public AssemblerX86Shared { + Vector jumps_; + + void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind kind) { + enoughMemory_ &= + jumps_.append(RelativePatch(src.offset(), target.value, kind)); + if (kind == RelocationKind::JITCODE) { + jumpRelocations_.writeUnsigned(src.offset()); + } + } + + public: + using AssemblerX86Shared::call; + using AssemblerX86Shared::cmpl; + using AssemblerX86Shared::j; + using AssemblerX86Shared::jmp; + using AssemblerX86Shared::movl; + using AssemblerX86Shared::pop; + using AssemblerX86Shared::push; + using AssemblerX86Shared::retarget; + using AssemblerX86Shared::vmovsd; + using AssemblerX86Shared::vmovss; + + static void TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + + // Copy the assembly code to the given buffer, and perform any pending + // relocations relying on the target address. + void executableCopy(uint8_t* buffer); + + void assertNoGCThings() const { +#ifdef DEBUG + MOZ_ASSERT(dataRelocations_.length() == 0); + for (auto& j : jumps_) { + MOZ_ASSERT(j.kind == RelocationKind::HARDCODED); + } +#endif + } + + // Actual assembly emitting functions. + + void push(ImmGCPtr ptr) { + masm.push_i32(int32_t(ptr.value)); + writeDataRelocation(ptr); + } + void push(const ImmWord imm) { push(Imm32(imm.value)); } + void push(const ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); } + void push(FloatRegister src) { + subl(Imm32(sizeof(double)), StackPointer); + vmovsd(src, Address(StackPointer, 0)); + } + + CodeOffset pushWithPatch(ImmWord word) { + masm.push_i32(int32_t(word.value)); + return CodeOffset(masm.currentOffset()); + } + + void pop(FloatRegister src) { + vmovsd(Address(StackPointer, 0), src); + addl(Imm32(sizeof(double)), StackPointer); + } + + CodeOffset movWithPatch(ImmWord word, Register dest) { + movl(Imm32(word.value), dest); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movWithPatch(ImmPtr imm, Register dest) { + return movWithPatch(ImmWord(uintptr_t(imm.value)), dest); + } + + void movl(ImmGCPtr ptr, Register dest) { + masm.movl_i32r(uintptr_t(ptr.value), dest.encoding()); + writeDataRelocation(ptr); + } + void movl(ImmGCPtr ptr, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.movl_i32r(uintptr_t(ptr.value), dest.reg()); + writeDataRelocation(ptr); + break; + case Operand::MEM_REG_DISP: + masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base()); + writeDataRelocation(ptr); + break; + case Operand::MEM_SCALE: + masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base(), + dest.index(), dest.scale()); + writeDataRelocation(ptr); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movl(ImmWord imm, Register dest) { + masm.movl_i32r(imm.value, dest.encoding()); + } + void movl(ImmPtr imm, Register dest) { + movl(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(ImmWord imm, Register dest) { + // Use xor for setting registers to zero, as it is specially optimized + // for this purpose on modern hardware. Note that it does clobber FLAGS + // though. + if (imm.value == 0) { + xorl(dest, dest); + } else { + movl(imm, dest); + } + } + void mov(ImmPtr imm, Register dest) { + mov(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(wasm::SymbolicAddress imm, Register dest) { + masm.movl_i32r(-1, dest.encoding()); + append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm)); + } + void mov(const Operand& src, Register dest) { movl(src, dest); } + void mov(Register src, const Operand& dest) { movl(src, dest); } + void mov(Imm32 imm, const Operand& dest) { movl(imm, dest); } + void mov(CodeLabel* label, Register dest) { + // Put a placeholder value in the instruction stream. + masm.movl_i32r(0, dest.encoding()); + label->patchAt()->bind(masm.size()); + } + void mov(Register src, Register dest) { movl(src, dest); } + void xchg(Register src, Register dest) { xchgl(src, dest); } + void lea(const Operand& src, Register dest) { return leal(src, dest); } + void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); } + void cmovzPtr(const Operand& src, Register dest) { return cmovzl(src, dest); } + + void fstp32(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fstp32_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void faddp() { masm.faddp(); } + + void cmpl(ImmWord rhs, Register lhs) { + masm.cmpl_ir(rhs.value, lhs.encoding()); + } + void cmpl(ImmPtr rhs, Register lhs) { + cmpl(ImmWord(uintptr_t(rhs.value)), lhs); + } + void cmpl(ImmGCPtr rhs, Register lhs) { + masm.cmpl_i32r(uintptr_t(rhs.value), lhs.encoding()); + writeDataRelocation(rhs); + } + void cmpl(Register rhs, Register lhs) { + masm.cmpl_rr(rhs.encoding(), lhs.encoding()); + } + void cmpl(ImmGCPtr rhs, const Operand& lhs) { + switch (lhs.kind()) { + case Operand::REG: + masm.cmpl_i32r(uintptr_t(rhs.value), lhs.reg()); + writeDataRelocation(rhs); + break; + case Operand::MEM_REG_DISP: + masm.cmpl_i32m(uintptr_t(rhs.value), lhs.disp(), lhs.base()); + writeDataRelocation(rhs); + break; + case Operand::MEM_ADDRESS32: + masm.cmpl_i32m(uintptr_t(rhs.value), lhs.address()); + writeDataRelocation(rhs); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void cmpl(Register rhs, wasm::SymbolicAddress lhs) { + masm.cmpl_rm_disp32(rhs.encoding(), (void*)-1); + append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), lhs)); + } + void cmpl(Imm32 rhs, wasm::SymbolicAddress lhs) { + JmpSrc src = masm.cmpl_im_disp32(rhs.value, (void*)-1); + append(wasm::SymbolicAccess(CodeOffset(src.offset()), lhs)); + } + + void adcl(Imm32 imm, Register dest) { + masm.adcl_ir(imm.value, dest.encoding()); + } + void adcl(Register src, Register dest) { + masm.adcl_rr(src.encoding(), dest.encoding()); + } + void adcl(Operand src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.adcl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.adcl_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void sbbl(Imm32 imm, Register dest) { + masm.sbbl_ir(imm.value, dest.encoding()); + } + void sbbl(Register src, Register dest) { + masm.sbbl_rr(src.encoding(), dest.encoding()); + } + void sbbl(Operand src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.sbbl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.sbbl_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void mull(Register multiplier) { masm.mull_r(multiplier.encoding()); } + + void shldl(const Imm32 imm, Register src, Register dest) { + masm.shldl_irr(imm.value, src.encoding(), dest.encoding()); + } + void shrdl(const Imm32 imm, Register src, Register dest) { + masm.shrdl_irr(imm.value, src.encoding(), dest.encoding()); + } + + void vhaddpd(FloatRegister rhs, FloatRegister lhsDest) { + MOZ_ASSERT(HasSSE3()); + MOZ_ASSERT(rhs.size() == 16); + MOZ_ASSERT(lhsDest.size() == 16); + masm.vhaddpd_rr(rhs.encoding(), lhsDest.encoding(), lhsDest.encoding()); + } + + void fild(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fild_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) { + JmpSrc src = masm.jmp(); + addPendingJump(src, target, reloc); + } + void j(Condition cond, ImmPtr target, + RelocationKind reloc = RelocationKind::HARDCODED) { + JmpSrc src = masm.jCC(static_cast(cond)); + addPendingJump(src, target, reloc); + } + + void jmp(JitCode* target) { + jmp(ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void j(Condition cond, JitCode* target) { + j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void call(JitCode* target) { + JmpSrc src = masm.call(); + addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); + } + void call(ImmWord target) { call(ImmPtr((void*)target.value)); } + void call(ImmPtr target) { + JmpSrc src = masm.call(); + addPendingJump(src, target, RelocationKind::HARDCODED); + } + + // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch + // this instruction. + CodeOffset toggledCall(JitCode* target, bool enabled) { + CodeOffset offset(size()); + JmpSrc src = enabled ? masm.call() : masm.cmp_eax(); + addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); + MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr)); + return offset; + } + + static size_t ToggledCallSize(uint8_t* code) { + // Size of a call instruction. + return 5; + } + + // Re-routes pending jumps to an external target, flushing the label in the + // process. + void retarget(Label* label, ImmPtr target, RelocationKind reloc) { + if (label->used()) { + bool more; + X86Encoding::JmpSrc jmp(label->offset()); + do { + X86Encoding::JmpSrc next; + more = masm.nextJump(jmp, &next); + addPendingJump(jmp, target, reloc); + jmp = next; + } while (more); + } + label->reset(); + } + + // Move a 32-bit immediate into a register where the immediate can be + // patched. + CodeOffset movlWithPatch(Imm32 imm, Register dest) { + masm.movl_i32r(imm.value, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Load from *(base + disp32) where disp32 can be patched. + CodeOffset movsblWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movsbl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movsbl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzblWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movzbl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movzbl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movswlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movswl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movswl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzwlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movzwl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movzwl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movl_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovss(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_mr(src.address(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovss_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovdWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovd_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovsd(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_mr(src.address(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovsd_mr(src.disp(), src.base(), src.index(), src.scale(), + dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovupsWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovups_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_mr_disp32(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovdqu_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + + // Store to *(base + disp32) where disp32 can be patched. + CodeOffset movbWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movb_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movb_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movwWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movw_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movw_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movl_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatchLow(Register regLow, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: { + return movlWithPatch(regLow, LowWord(dest)); + } + case Operand::MEM_ADDRESS32: { + Operand low( + PatchedAbsoluteAddress(uint32_t(dest.address()) + INT64LOW_OFFSET)); + return movlWithPatch(regLow, low); + } + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset movlWithPatchHigh(Register regHigh, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: { + return movlWithPatch(regHigh, HighWord(dest)); + } + case Operand::MEM_ADDRESS32: { + Operand high(PatchedAbsoluteAddress(uint32_t(dest.address()) + + INT64HIGH_OFFSET)); + return movlWithPatch(regHigh, high); + } + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovdWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovd_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovss(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovss_rm(src.encoding(), dest.address()); + break; + case Operand::MEM_SCALE: + masm.vmovss_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), + dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovsdWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + void vmovsd(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovsd_rm(src.encoding(), dest.address()); + break; + case Operand::MEM_SCALE: + masm.vmovsd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), + dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset vmovupsWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovups_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_rm_disp32(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovdqu_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + return CodeOffset(masm.currentOffset()); + } + + // Load from *(addr + index*scale) where addr can be patched. + CodeOffset movlWithPatch(PatchedAbsoluteAddress addr, Register index, + Scale scale, Register dest) { + masm.movl_mr(addr.addr, index.encoding(), scale, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Load from *src where src can be patched. + CodeOffset movsblWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movsbl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzblWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movzbl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movswlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movswl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movzwlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movzwl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(PatchedAbsoluteAddress src, Register dest) { + masm.movl_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovss_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovq_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovsd_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdqaWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqa_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqu_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovapsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovaps_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovupsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovups_mr(src.addr, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + + // Store to *dest where dest can be patched. + CodeOffset movbWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movb_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movwWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movw_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset movlWithPatch(Register src, PatchedAbsoluteAddress dest) { + masm.movl_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovssWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovss_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovqWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovq_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovsdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovsd_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdqaWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqa_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovapsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovaps_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovdquWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqu_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } + CodeOffset vmovupsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovups_rm(src.encoding(), dest.addr); + return CodeOffset(masm.currentOffset()); + } +}; + +// Get a register in which we plan to put a quantity that will be used as an +// integer argument. This differs from GetIntArgReg in that if we have no more +// actual argument registers to use we will fall back on using whatever +// CallTempReg* don't overlap the argument registers, and only fail once those +// run out too. +static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, + uint32_t usedFloatArgs, Register* out) { + if (usedIntArgs >= NumCallTempNonArgRegs) { + return false; + } + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +} // namespace jit +} // namespace js + +#endif /* jit_x86_Assembler_x86_h */ diff --git a/js/src/jit/x86/BaseAssembler-x86.h b/js/src/jit/x86/BaseAssembler-x86.h new file mode 100644 index 0000000000..a5a5f67bf2 --- /dev/null +++ b/js/src/jit/x86/BaseAssembler-x86.h @@ -0,0 +1,190 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_BaseAssembler_x86_h +#define jit_x86_BaseAssembler_x86_h + +#include "jit/x86-shared/BaseAssembler-x86-shared.h" + +namespace js { +namespace jit { + +namespace X86Encoding { + +class BaseAssemblerX86 : public BaseAssembler { + public: + // Arithmetic operations: + + void adcl_ir(int32_t imm, RegisterID dst) { + spew("adcl $%d, %s", imm, GPReg32Name(dst)); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_ADC); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_ADC); + m_formatter.immediate32(imm); + } + } + + void adcl_im(int32_t imm, const void* addr) { + spew("adcl %d, %p", imm, addr); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_ADC); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_ADC); + m_formatter.immediate32(imm); + } + } + + void adcl_rr(RegisterID src, RegisterID dst) { + spew("adcl %s, %s", GPReg32Name(src), GPReg32Name(dst)); + m_formatter.oneByteOp(OP_ADC_GvEv, src, dst); + } + + void adcl_mr(int32_t offset, RegisterID base, RegisterID dst) { + spew("adcl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst)); + m_formatter.oneByteOp(OP_ADC_GvEv, offset, base, dst); + } + + void adcl_mr(int32_t offset, RegisterID base, RegisterID index, int scale, + RegisterID dst) { + spew("adcl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale), + GPReg32Name(dst)); + m_formatter.oneByteOp(OP_ADC_GvEv, offset, base, index, scale, dst); + } + + void sbbl_ir(int32_t imm, RegisterID dst) { + spew("sbbl $%d, %s", imm, GPReg32Name(dst)); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_SBB); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_SBB); + m_formatter.immediate32(imm); + } + } + + void sbbl_rr(RegisterID src, RegisterID dst) { + spew("sbbl %s, %s", GPReg32Name(src), GPReg32Name(dst)); + m_formatter.oneByteOp(OP_SBB_GvEv, src, dst); + } + + void sbbl_mr(int32_t offset, RegisterID base, RegisterID dst) { + spew("sbbl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst)); + m_formatter.oneByteOp(OP_SBB_GvEv, offset, base, dst); + } + + void sbbl_mr(int32_t offset, RegisterID base, RegisterID index, int scale, + RegisterID dst) { + spew("sbbl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale), + GPReg32Name(dst)); + m_formatter.oneByteOp(OP_SBB_GvEv, offset, base, index, scale, dst); + } + + using BaseAssembler::andl_im; + void andl_im(int32_t imm, const void* addr) { + spew("andl $0x%x, %p", imm, addr); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_AND); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_AND); + m_formatter.immediate32(imm); + } + } + + using BaseAssembler::orl_im; + void orl_im(int32_t imm, const void* addr) { + spew("orl $0x%x, %p", imm, addr); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_OR); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_OR); + m_formatter.immediate32(imm); + } + } + + using BaseAssembler::subl_im; + void subl_im(int32_t imm, const void* addr) { + spew("subl $%d, %p", imm, addr); + if (CAN_SIGN_EXTEND_8_32(imm)) { + m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_SUB); + m_formatter.immediate8s(imm); + } else { + m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_SUB); + m_formatter.immediate32(imm); + } + } + + void shldl_irr(int32_t imm, RegisterID src, RegisterID dst) { + MOZ_ASSERT(imm < 32); + spew("shldl $%d, %s, %s", imm, GPReg32Name(src), GPReg32Name(dst)); + m_formatter.twoByteOp8(OP2_SHLD, dst, src); + m_formatter.immediate8u(imm); + } + + void shrdl_irr(int32_t imm, RegisterID src, RegisterID dst) { + MOZ_ASSERT(imm < 32); + spew("shrdl $%d, %s, %s", imm, GPReg32Name(src), GPReg32Name(dst)); + m_formatter.twoByteOp8(OP2_SHRD, dst, src); + m_formatter.immediate8u(imm); + } + + // SSE operations: + + using BaseAssembler::vcvtsi2sd_mr; + void vcvtsi2sd_mr(const void* address, XMMRegisterID src0, + XMMRegisterID dst) { + twoByteOpSimd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, address, src0, dst); + } + + using BaseAssembler::vmovaps_mr; + void vmovaps_mr(const void* address, XMMRegisterID dst) { + twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, address, invalid_xmm, + dst); + } + + using BaseAssembler::vmovdqa_mr; + void vmovdqa_mr(const void* address, XMMRegisterID dst) { + twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, address, invalid_xmm, + dst); + } + + void vhaddpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) { + twoByteOpSimd("vhaddpd", VEX_PD, OP2_HADDPD, src1, src0, dst); + } + + void vsubpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) { + twoByteOpSimd("vsubpd", VEX_PD, OP2_SUBPS_VpsWps, src1, src0, dst); + } + + void fild_m(int32_t offset, RegisterID base) { + m_formatter.oneByteOp(OP_FILD, offset, base, FILD_OP_64); + } + + // Misc instructions: + + void pusha() { + spew("pusha"); + m_formatter.oneByteOp(OP_PUSHA); + } + + void popa() { + spew("popa"); + m_formatter.oneByteOp(OP_POPA); + } +}; + +typedef BaseAssemblerX86 BaseAssemblerSpecific; + +} // namespace X86Encoding + +} // namespace jit +} // namespace js + +#endif /* jit_x86_BaseAssembler_x86_h */ diff --git a/js/src/jit/x86/CodeGenerator-x86.cpp b/js/src/jit/x86/CodeGenerator-x86.cpp new file mode 100644 index 0000000000..ce0fb01d9e --- /dev/null +++ b/js/src/jit/x86/CodeGenerator-x86.cpp @@ -0,0 +1,1509 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86/CodeGenerator-x86.h" + +#include "mozilla/Casting.h" +#include "mozilla/DebugOnly.h" + +#include + +#include "jsnum.h" + +#include "jit/CodeGenerator.h" +#include "jit/MIR.h" +#include "jit/MIRGraph.h" +#include "js/Conversions.h" +#include "vm/Shape.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmInstanceData.h" + +#include "jit/MacroAssembler-inl.h" +#include "jit/shared/CodeGenerator-shared-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using JS::GenericNaN; +using mozilla::BitwiseCast; +using mozilla::DebugOnly; +using mozilla::FloatingPoint; + +CodeGeneratorX86::CodeGeneratorX86(MIRGenerator* gen, LIRGraph* graph, + MacroAssembler* masm) + : CodeGeneratorX86Shared(gen, graph, masm) {} + +ValueOperand CodeGeneratorX86::ToValue(LInstruction* ins, size_t pos) { + Register typeReg = ToRegister(ins->getOperand(pos + TYPE_INDEX)); + Register payloadReg = ToRegister(ins->getOperand(pos + PAYLOAD_INDEX)); + return ValueOperand(typeReg, payloadReg); +} + +ValueOperand CodeGeneratorX86::ToTempValue(LInstruction* ins, size_t pos) { + Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX)); + Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX)); + return ValueOperand(typeReg, payloadReg); +} + +void CodeGenerator::visitValue(LValue* value) { + const ValueOperand out = ToOutValue(value); + masm.moveValue(value->value(), out); +} + +void CodeGenerator::visitBox(LBox* box) { + const LDefinition* type = box->getDef(TYPE_INDEX); + + DebugOnly a = box->getOperand(0); + MOZ_ASSERT(!a->isConstant()); + + // On x86, the input operand and the output payload have the same + // virtual register. All that needs to be written is the type tag for + // the type definition. + masm.mov(ImmWord(MIRTypeToTag(box->type())), ToRegister(type)); +} + +void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) { + const AnyRegister in = ToAnyRegister(box->getOperand(0)); + const ValueOperand out = ToOutValue(box); + + masm.moveValue(TypedOrValueRegister(box->type(), in), out); + + if (JitOptions.spectreValueMasking) { + Register scratch = ToRegister(box->spectreTemp()); + masm.move32(Imm32(JSVAL_TAG_CLEAR), scratch); + masm.cmp32Move32(Assembler::Below, scratch, out.typeReg(), scratch, + out.typeReg()); + } +} + +void CodeGenerator::visitUnbox(LUnbox* unbox) { + // Note that for unbox, the type and payload indexes are switched on the + // inputs. + Operand type = ToOperand(unbox->type()); + Operand payload = ToOperand(unbox->payload()); + Register output = ToRegister(unbox->output()); + MUnbox* mir = unbox->mir(); + + JSValueTag tag = MIRTypeToTag(mir->type()); + if (mir->fallible()) { + masm.cmp32(type, Imm32(tag)); + bailoutIf(Assembler::NotEqual, unbox->snapshot()); + } else { +#ifdef DEBUG + Label ok; + masm.branch32(Assembler::Equal, type, Imm32(tag), &ok); + masm.assumeUnreachable("Infallible unbox type mismatch"); + masm.bind(&ok); +#endif + } + + // Note: If spectreValueMasking is disabled, then this instruction will + // default to a no-op as long as the lowering allocate the same register for + // the output and the payload. + masm.unboxNonDouble(type, payload, output, ValueTypeFromMIRType(mir->type())); +} + +void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) { + Register elements = ToRegister(lir->elements()); + Register temp = ToRegister(lir->temp()); + Register64 temp64 = ToRegister64(lir->temp64()); + Register out = ToRegister(lir->output()); + + MOZ_ASSERT(out == ecx); + MOZ_ASSERT(temp == ebx); + MOZ_ASSERT(temp64 == Register64(edx, eax)); + + const MLoadUnboxedScalar* mir = lir->mir(); + + Scalar::Type storageType = mir->storageType(); + + if (lir->index()->isConstant()) { + Address source = + ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment()); + masm.atomicLoad64(Synchronization::Load(), source, Register64(ecx, ebx), + Register64(edx, eax)); + } else { + BaseIndex source(elements, ToRegister(lir->index()), + ScaleFromScalarType(storageType), mir->offsetAdjustment()); + masm.atomicLoad64(Synchronization::Load(), source, Register64(ecx, ebx), + Register64(edx, eax)); + } + + emitCreateBigInt(lir, storageType, temp64, out, temp); +} + +void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) { + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register64 temp2 = Register64(value, ToRegister(lir->tempLow())); + + MOZ_ASSERT(temp1 == Register64(ecx, ebx)); + MOZ_ASSERT(temp2 == Register64(edx, eax)); + + Scalar::Type writeType = lir->mir()->writeType(); + + masm.loadBigInt64(value, temp1); + + masm.push(value); + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), writeType); + masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(writeType)); + masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2); + } + masm.pop(value); +} + +void CodeGenerator::visitCompareExchangeTypedArrayElement64( + LCompareExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register oldval = ToRegister(lir->oldval()); + DebugOnly newval = ToRegister(lir->newval()); + DebugOnly temp = ToRegister(lir->tempLow()); + Register out = ToRegister(lir->output()); + + MOZ_ASSERT(elements == esi); + MOZ_ASSERT(oldval == eax); + MOZ_ASSERT(newval.inspect() == edx); + MOZ_ASSERT(temp.inspect() == ebx); + MOZ_ASSERT(out == ecx); + + Scalar::Type arrayType = lir->mir()->arrayType(); + + DebugOnly framePushed = masm.framePushed(); + + // Save eax and edx before they're clobbered below. + masm.push(eax); + masm.push(edx); + + auto restoreSavedRegisters = [&]() { + masm.pop(edx); + masm.pop(eax); + }; + + Register64 expected = Register64(edx, eax); + Register64 replacement = Register64(ecx, ebx); + + // Load |oldval| and |newval| into |expected| resp. |replacement|. + { + // Use `esi` as a temp register. + Register bigInt = esi; + masm.push(bigInt); + + masm.mov(oldval, bigInt); + masm.loadBigInt64(bigInt, expected); + + // |newval| is stored in `edx`, which is already pushed onto the stack. + masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), bigInt); + masm.loadBigInt64(bigInt, replacement); + + masm.pop(bigInt); + } + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.compareExchange64(Synchronization::Full(), dest, expected, replacement, + expected); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.compareExchange64(Synchronization::Full(), dest, expected, replacement, + expected); + } + + // Move the result from `edx:eax` to `ecx:ebx`. + masm.move64(expected, replacement); + + // OutOfLineCallVM tracks the currently pushed stack entries as reported by + // |masm.framePushed()|. We mustn't have any additional entries on the stack + // which weren't previously recorded by the safepoint, otherwise the GC + // complains when tracing the Ion frames, because the stack frames don't + // have their expected layout. + MOZ_ASSERT(framePushed == masm.framePushed()); + + OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, replacement, out); + + // Use `edx:eax`, which are both already on the stack, as temp registers. + Register bigInt = eax; + Register temp2 = edx; + + Label fail; + masm.newGCBigInt(bigInt, temp2, initialBigIntHeap(), &fail); + masm.initializeBigInt64(arrayType, bigInt, replacement); + masm.mov(bigInt, out); + restoreSavedRegisters(); + masm.jump(ool->rejoin()); + + // Couldn't create the BigInt. Restore `edx:eax` and call into the VM. + masm.bind(&fail); + restoreSavedRegisters(); + masm.jump(ool->entry()); + + // At this point `edx:eax` must have been restored to their original values. + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitAtomicExchangeTypedArrayElement64( + LAtomicExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register out = ToRegister(lir->output()); + Register64 temp2 = Register64(value, out); + + MOZ_ASSERT(value == edx); + MOZ_ASSERT(temp1 == Register64(ecx, ebx)); + MOZ_ASSERT(temp2 == Register64(edx, eax)); + MOZ_ASSERT(out == eax); + + Scalar::Type arrayType = lir->mir()->arrayType(); + + DebugOnly framePushed = masm.framePushed(); + + // Save edx before it's clobbered below. + masm.push(edx); + + auto restoreSavedRegisters = [&]() { masm.pop(edx); }; + + masm.loadBigInt64(value, temp1); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2); + } + + // Move the result from `edx:eax` to `ecx:ebx`. + masm.move64(temp2, temp1); + + // OutOfLineCallVM tracks the currently pushed stack entries as reported by + // |masm.framePushed()|. We mustn't have any additional entries on the stack + // which weren't previously recorded by the safepoint, otherwise the GC + // complains when tracing the Ion frames, because the stack frames don't + // have their expected layout. + MOZ_ASSERT(framePushed == masm.framePushed()); + + OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, temp1, out); + + // Use `edx`, which is already on the stack, as a temp register. + Register temp = edx; + + Label fail; + masm.newGCBigInt(out, temp, initialBigIntHeap(), &fail); + masm.initializeBigInt64(arrayType, out, temp1); + restoreSavedRegisters(); + masm.jump(ool->rejoin()); + + // Couldn't create the BigInt. Restore `edx` and call into the VM. + masm.bind(&fail); + restoreSavedRegisters(); + masm.jump(ool->entry()); + + // At this point `edx` must have been restored to its original value. + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinop64( + LAtomicTypedArrayElementBinop64* lir) { + MOZ_ASSERT(!lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register out = ToRegister(lir->output()); + Register64 temp2 = Register64(value, out); + + MOZ_ASSERT(value == edx); + MOZ_ASSERT(temp1 == Register64(ecx, ebx)); + MOZ_ASSERT(temp2 == Register64(edx, eax)); + MOZ_ASSERT(out == eax); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + DebugOnly framePushed = masm.framePushed(); + + // Save edx before it's clobbered below. + masm.push(edx); + + auto restoreSavedRegisters = [&]() { masm.pop(edx); }; + + masm.loadBigInt64(value, temp1); + + masm.Push(temp1); + + Address addr(masm.getStackPointer(), 0); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1, + temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1, + temp2); + } + + masm.freeStack(sizeof(uint64_t)); + + // Move the result from `edx:eax` to `ecx:ebx`. + masm.move64(temp2, temp1); + + // OutOfLineCallVM tracks the currently pushed stack entries as reported by + // |masm.framePushed()|. We mustn't have any additional entries on the stack + // which weren't previously recorded by the safepoint, otherwise the GC + // complains when tracing the Ion frames, because the stack frames don't + // have their expected layout. + MOZ_ASSERT(framePushed == masm.framePushed()); + + OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, temp1, out); + + // Use `edx`, which is already on the stack, as a temp register. + Register temp = edx; + + Label fail; + masm.newGCBigInt(out, temp, initialBigIntHeap(), &fail); + masm.initializeBigInt64(arrayType, out, temp1); + restoreSavedRegisters(); + masm.jump(ool->rejoin()); + + // Couldn't create the BigInt. Restore `edx` and call into the VM. + masm.bind(&fail); + restoreSavedRegisters(); + masm.jump(ool->entry()); + + // At this point `edx` must have been restored to its original value. + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64( + LAtomicTypedArrayElementBinopForEffect64* lir) { + MOZ_ASSERT(lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + Register64 temp1 = ToRegister64(lir->temp1()); + Register tempLow = ToRegister(lir->tempLow()); + Register64 temp2 = Register64(value, tempLow); + + MOZ_ASSERT(value == edx); + MOZ_ASSERT(temp1 == Register64(ecx, ebx)); + MOZ_ASSERT(temp2 == Register64(edx, eax)); + MOZ_ASSERT(tempLow == eax); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + // Save edx before it's clobbered below. + masm.push(edx); + + masm.loadBigInt64(value, temp1); + + masm.Push(temp1); + + Address addr(masm.getStackPointer(), 0); + + if (lir->index()->isConstant()) { + Address dest = ToAddress(elements, lir->index(), arrayType); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1, + temp2); + } else { + BaseIndex dest(elements, ToRegister(lir->index()), + ScaleFromScalarType(arrayType)); + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1, + temp2); + } + + masm.freeStack(sizeof(uint64_t)); + + masm.pop(edx); +} + +void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) { + Register input = ToRegister(lir->input()); + Register temp = ToRegister(lir->temp()); + + if (input != temp) { + masm.mov(input, temp); + } + + // Beware: convertUInt32ToDouble clobbers input. + masm.convertUInt32ToDouble(temp, ToFloatRegister(lir->output())); +} + +void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) { + Register input = ToRegister(lir->input()); + Register temp = ToRegister(lir->temp()); + FloatRegister output = ToFloatRegister(lir->output()); + + if (input != temp) { + masm.mov(input, temp); + } + + // Beware: convertUInt32ToFloat32 clobbers input. + masm.convertUInt32ToFloat32(temp, output); +} + +void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) { + masm.loadPtr(Address(ToRegister(ins->instance()), + wasm::Instance::offsetOfMemoryBase()), + ToRegister(ins->output())); +} + +template +void CodeGeneratorX86::emitWasmLoad(T* ins) { + const MWasmLoad* mir = ins->mir(); + + uint32_t offset = mir->access().offset(); + MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* ptr = ins->ptr(); + const LAllocation* memoryBase = ins->memoryBase(); + + // Lowering has set things up so that we can use a BaseIndex form if the + // pointer is constant and the offset is zero, or if the pointer is zero. + + Operand srcAddr = + ptr->isBogus() + ? Operand(ToRegister(memoryBase), + offset ? offset : mir->base()->toConstant()->toInt32()) + : Operand(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset); + + if (mir->type() == MIRType::Int64) { + MOZ_ASSERT_IF(mir->access().isAtomic(), + mir->access().type() != Scalar::Int64); + masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins)); + } else { + masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output())); + } +} + +void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); } + +void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); } + +template +void CodeGeneratorX86::emitWasmStore(T* ins) { + const MWasmStore* mir = ins->mir(); + + uint32_t offset = mir->access().offset(); + MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* ptr = ins->ptr(); + const LAllocation* memoryBase = ins->memoryBase(); + + // Lowering has set things up so that we can use a BaseIndex form if the + // pointer is constant and the offset is zero, or if the pointer is zero. + + Operand dstAddr = + ptr->isBogus() + ? Operand(ToRegister(memoryBase), + offset ? offset : mir->base()->toConstant()->toInt32()) + : Operand(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset); + + if (mir->access().type() == Scalar::Int64) { + Register64 value = + ToRegister64(ins->getInt64Operand(LWasmStoreI64::ValueIndex)); + masm.wasmStoreI64(mir->access(), value, dstAddr); + } else { + AnyRegister value = ToAnyRegister(ins->getOperand(LWasmStore::ValueIndex)); + masm.wasmStore(mir->access(), value, dstAddr); + } +} + +void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); } + +void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) { + emitWasmStore(ins); +} + +void CodeGenerator::visitWasmCompareExchangeHeap( + LWasmCompareExchangeHeap* ins) { + MWasmCompareExchangeHeap* mir = ins->mir(); + + Register ptrReg = ToRegister(ins->ptr()); + Register oldval = ToRegister(ins->oldValue()); + Register newval = ToRegister(ins->newValue()); + Register addrTemp = ToRegister(ins->addrTemp()); + Register memoryBase = ToRegister(ins->memoryBase()); + Register output = ToRegister(ins->output()); + + masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), + addrTemp); + + Address memAddr(addrTemp, 0); + masm.wasmCompareExchange(mir->access(), memAddr, oldval, newval, output); +} + +void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) { + MWasmAtomicExchangeHeap* mir = ins->mir(); + + Register ptrReg = ToRegister(ins->ptr()); + Register value = ToRegister(ins->value()); + Register addrTemp = ToRegister(ins->addrTemp()); + Register memoryBase = ToRegister(ins->memoryBase()); + Register output = ToRegister(ins->output()); + + masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), + addrTemp); + + Address memAddr(addrTemp, 0); + masm.wasmAtomicExchange(mir->access(), memAddr, value, output); +} + +void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) { + MWasmAtomicBinopHeap* mir = ins->mir(); + + Register ptrReg = ToRegister(ins->ptr()); + Register temp = + ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp()); + Register addrTemp = ToRegister(ins->addrTemp()); + Register out = ToRegister(ins->output()); + const LAllocation* value = ins->value(); + AtomicOp op = mir->operation(); + Register memoryBase = ToRegister(ins->memoryBase()); + + masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), + addrTemp); + + Address memAddr(addrTemp, 0); + if (value->isConstant()) { + masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), memAddr, + temp, out); + } else { + masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), memAddr, temp, + out); + } +} + +void CodeGenerator::visitWasmAtomicBinopHeapForEffect( + LWasmAtomicBinopHeapForEffect* ins) { + MWasmAtomicBinopHeap* mir = ins->mir(); + MOZ_ASSERT(!mir->hasUses()); + + Register ptrReg = ToRegister(ins->ptr()); + Register addrTemp = ToRegister(ins->addrTemp()); + const LAllocation* value = ins->value(); + AtomicOp op = mir->operation(); + Register memoryBase = ToRegister(ins->memoryBase()); + + masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), + addrTemp); + + Address memAddr(addrTemp, 0); + if (value->isConstant()) { + masm.wasmAtomicEffectOp(mir->access(), op, Imm32(ToInt32(value)), memAddr, + InvalidReg); + } else { + masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), memAddr, + InvalidReg); + } +} + +void CodeGenerator::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* ins) { + uint32_t offset = ins->mir()->access().offset(); + MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* memoryBase = ins->memoryBase(); + const LAllocation* ptr = ins->ptr(); + BaseIndex srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset); + + MOZ_ASSERT(ToRegister(ins->t1()) == ecx); + MOZ_ASSERT(ToRegister(ins->t2()) == ebx); + MOZ_ASSERT(ToOutRegister64(ins).high == edx); + MOZ_ASSERT(ToOutRegister64(ins).low == eax); + + masm.wasmAtomicLoad64(ins->mir()->access(), srcAddr, Register64(ecx, ebx), + Register64(edx, eax)); +} + +void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* ins) { + uint32_t offset = ins->mir()->access().offset(); + MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* memoryBase = ins->memoryBase(); + const LAllocation* ptr = ins->ptr(); + Operand srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset); + + MOZ_ASSERT(ToRegister64(ins->expected()).low == eax); + MOZ_ASSERT(ToRegister64(ins->expected()).high == edx); + MOZ_ASSERT(ToRegister64(ins->replacement()).low == ebx); + MOZ_ASSERT(ToRegister64(ins->replacement()).high == ecx); + MOZ_ASSERT(ToOutRegister64(ins).low == eax); + MOZ_ASSERT(ToOutRegister64(ins).high == edx); + + masm.append(ins->mir()->access(), masm.size()); + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, srcAddr); +} + +template +void CodeGeneratorX86::emitWasmStoreOrExchangeAtomicI64( + T* ins, const wasm::MemoryAccessDesc& access) { + MOZ_ASSERT(access.offset() < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* memoryBase = ins->memoryBase(); + const LAllocation* ptr = ins->ptr(); + Operand srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, + access.offset()); + + DebugOnly value = ins->value(); + MOZ_ASSERT(ToRegister64(value).low == ebx); + MOZ_ASSERT(ToRegister64(value).high == ecx); + + // eax and edx will be overwritten every time through the loop but + // memoryBase and ptr must remain live for a possible second iteration. + + MOZ_ASSERT(ToRegister(memoryBase) != edx && ToRegister(memoryBase) != eax); + MOZ_ASSERT(ToRegister(ptr) != edx && ToRegister(ptr) != eax); + + Label again; + masm.bind(&again); + masm.append(access, masm.size()); + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, srcAddr); + masm.j(Assembler::Condition::NonZero, &again); +} + +void CodeGenerator::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* ins) { + MOZ_ASSERT(ToRegister(ins->t1()) == edx); + MOZ_ASSERT(ToRegister(ins->t2()) == eax); + + emitWasmStoreOrExchangeAtomicI64(ins, ins->mir()->access()); +} + +void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* ins) { + MOZ_ASSERT(ToOutRegister64(ins).high == edx); + MOZ_ASSERT(ToOutRegister64(ins).low == eax); + + emitWasmStoreOrExchangeAtomicI64(ins, ins->access()); +} + +void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* ins) { + uint32_t offset = ins->access().offset(); + MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit()); + + const LAllocation* memoryBase = ins->memoryBase(); + const LAllocation* ptr = ins->ptr(); + + BaseIndex srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset); + + MOZ_ASSERT(ToRegister(memoryBase) == esi || ToRegister(memoryBase) == edi); + MOZ_ASSERT(ToRegister(ptr) == esi || ToRegister(ptr) == edi); + + Register64 value = ToRegister64(ins->value()); + + MOZ_ASSERT(value.low == ebx); + MOZ_ASSERT(value.high == ecx); + + Register64 output = ToOutRegister64(ins); + + MOZ_ASSERT(output.low == eax); + MOZ_ASSERT(output.high == edx); + + masm.Push(ecx); + masm.Push(ebx); + + Address valueAddr(esp, 0); + + // Here the `value` register acts as a temp, we'll restore it below. + masm.wasmAtomicFetchOp64(ins->access(), ins->operation(), valueAddr, srcAddr, + value, output); + + masm.Pop(ebx); + masm.Pop(ecx); +} + +namespace js { +namespace jit { + +class OutOfLineTruncate : public OutOfLineCodeBase { + LInstruction* ins_; + + public: + explicit OutOfLineTruncate(LInstruction* ins) : ins_(ins) { + MOZ_ASSERT(ins_->isTruncateDToInt32() || + ins_->isWasmBuiltinTruncateDToInt32()); + } + + void accept(CodeGeneratorX86* codegen) override { + codegen->visitOutOfLineTruncate(this); + } + + LAllocation* input() { return ins_->getOperand(0); } + LDefinition* output() { return ins_->getDef(0); } + LDefinition* tempFloat() { return ins_->getTemp(0); } + + wasm::BytecodeOffset bytecodeOffset() const { + if (ins_->isTruncateDToInt32()) { + return ins_->toTruncateDToInt32()->mir()->bytecodeOffset(); + } + + return ins_->toWasmBuiltinTruncateDToInt32()->mir()->bytecodeOffset(); + } +}; + +class OutOfLineTruncateFloat32 : public OutOfLineCodeBase { + LInstruction* ins_; + + public: + explicit OutOfLineTruncateFloat32(LInstruction* ins) : ins_(ins) { + MOZ_ASSERT(ins_->isTruncateFToInt32() || + ins_->isWasmBuiltinTruncateFToInt32()); + } + + void accept(CodeGeneratorX86* codegen) override { + codegen->visitOutOfLineTruncateFloat32(this); + } + + LAllocation* input() { return ins_->getOperand(0); } + LDefinition* output() { return ins_->getDef(0); } + LDefinition* tempFloat() { return ins_->getTemp(0); } + + wasm::BytecodeOffset bytecodeOffset() const { + if (ins_->isTruncateFToInt32()) { + return ins_->toTruncateDToInt32()->mir()->bytecodeOffset(); + } + + return ins_->toWasmBuiltinTruncateFToInt32()->mir()->bytecodeOffset(); + } +}; + +} // namespace jit +} // namespace js + +void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + Register output = ToRegister(ins->output()); + + OutOfLineTruncate* ool = new (alloc()) OutOfLineTruncate(ins); + addOutOfLineCode(ool, ins->mir()); + + masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitWasmBuiltinTruncateDToInt32( + LWasmBuiltinTruncateDToInt32* lir) { + FloatRegister input = ToFloatRegister(lir->getOperand(0)); + Register output = ToRegister(lir->getDef(0)); + + OutOfLineTruncate* ool = new (alloc()) OutOfLineTruncate(lir); + addOutOfLineCode(ool, lir->mir()); + + masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + Register output = ToRegister(ins->output()); + + OutOfLineTruncateFloat32* ool = new (alloc()) OutOfLineTruncateFloat32(ins); + addOutOfLineCode(ool, ins->mir()); + + masm.branchTruncateFloat32MaybeModUint32(input, output, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGenerator::visitWasmBuiltinTruncateFToInt32( + LWasmBuiltinTruncateFToInt32* lir) { + FloatRegister input = ToFloatRegister(lir->getOperand(0)); + Register output = ToRegister(lir->getDef(0)); + + OutOfLineTruncateFloat32* ool = new (alloc()) OutOfLineTruncateFloat32(lir); + addOutOfLineCode(ool, lir->mir()); + + masm.branchTruncateFloat32MaybeModUint32(input, output, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGeneratorX86::visitOutOfLineTruncate(OutOfLineTruncate* ool) { + FloatRegister input = ToFloatRegister(ool->input()); + Register output = ToRegister(ool->output()); + + Label fail; + + if (Assembler::HasSSE3()) { + Label failPopDouble; + // Push double. + masm.subl(Imm32(sizeof(double)), esp); + masm.storeDouble(input, Operand(esp, 0)); + + // Check exponent to avoid fp exceptions. + masm.branchDoubleNotInInt64Range(Address(esp, 0), output, &failPopDouble); + + // Load double, perform 64-bit truncation. + masm.truncateDoubleToInt64(Address(esp, 0), Address(esp, 0), output); + + // Load low word, pop double and jump back. + masm.load32(Address(esp, 0), output); + masm.addl(Imm32(sizeof(double)), esp); + masm.jump(ool->rejoin()); + + masm.bind(&failPopDouble); + masm.addl(Imm32(sizeof(double)), esp); + masm.jump(&fail); + } else { + FloatRegister temp = ToFloatRegister(ool->tempFloat()); + + // Try to convert doubles representing integers within 2^32 of a signed + // integer, by adding/subtracting 2^32 and then trying to convert to int32. + // This has to be an exact conversion, as otherwise the truncation works + // incorrectly on the modified value. + { + ScratchDoubleScope fpscratch(masm); + masm.zeroDouble(fpscratch); + masm.vucomisd(fpscratch, input); + masm.j(Assembler::Parity, &fail); + } + + { + Label positive; + masm.j(Assembler::Above, &positive); + + masm.loadConstantDouble(4294967296.0, temp); + Label skip; + masm.jmp(&skip); + + masm.bind(&positive); + masm.loadConstantDouble(-4294967296.0, temp); + masm.bind(&skip); + } + + masm.addDouble(input, temp); + masm.vcvttsd2si(temp, output); + ScratchDoubleScope fpscratch(masm); + masm.vcvtsi2sd(output, fpscratch, fpscratch); + + masm.vucomisd(fpscratch, temp); + masm.j(Assembler::Parity, &fail); + masm.j(Assembler::Equal, ool->rejoin()); + } + + masm.bind(&fail); + { + if (gen->compilingWasm()) { + masm.Push(InstanceReg); + } + int32_t framePushedAfterInstance = masm.framePushed(); + + saveVolatile(output); + + if (gen->compilingWasm()) { + masm.setupWasmABICall(); + masm.passABIArg(input, MoveOp::DOUBLE); + + int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + masm.callWithABI(ool->bytecodeOffset(), wasm::SymbolicAddress::ToInt32, + mozilla::Some(instanceOffset)); + } else { + using Fn = int32_t (*)(double); + masm.setupUnalignedABICall(output); + masm.passABIArg(input, MoveOp::DOUBLE); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + masm.storeCallInt32Result(output); + + restoreVolatile(output); + + if (gen->compilingWasm()) { + masm.Pop(InstanceReg); + } + } + + masm.jump(ool->rejoin()); +} + +void CodeGeneratorX86::visitOutOfLineTruncateFloat32( + OutOfLineTruncateFloat32* ool) { + FloatRegister input = ToFloatRegister(ool->input()); + Register output = ToRegister(ool->output()); + + Label fail; + + if (Assembler::HasSSE3()) { + Label failPopFloat; + + // Push float32, but subtracts 64 bits so that the value popped by fisttp + // fits + masm.subl(Imm32(sizeof(uint64_t)), esp); + masm.storeFloat32(input, Operand(esp, 0)); + + // Check exponent to avoid fp exceptions. + masm.branchFloat32NotInInt64Range(Address(esp, 0), output, &failPopFloat); + + // Load float, perform 32-bit truncation. + masm.truncateFloat32ToInt64(Address(esp, 0), Address(esp, 0), output); + + // Load low word, pop 64bits and jump back. + masm.load32(Address(esp, 0), output); + masm.addl(Imm32(sizeof(uint64_t)), esp); + masm.jump(ool->rejoin()); + + masm.bind(&failPopFloat); + masm.addl(Imm32(sizeof(uint64_t)), esp); + masm.jump(&fail); + } else { + FloatRegister temp = ToFloatRegister(ool->tempFloat()); + + // Try to convert float32 representing integers within 2^32 of a signed + // integer, by adding/subtracting 2^32 and then trying to convert to int32. + // This has to be an exact conversion, as otherwise the truncation works + // incorrectly on the modified value. + { + ScratchFloat32Scope fpscratch(masm); + masm.zeroFloat32(fpscratch); + masm.vucomiss(fpscratch, input); + masm.j(Assembler::Parity, &fail); + } + + { + Label positive; + masm.j(Assembler::Above, &positive); + + masm.loadConstantFloat32(4294967296.f, temp); + Label skip; + masm.jmp(&skip); + + masm.bind(&positive); + masm.loadConstantFloat32(-4294967296.f, temp); + masm.bind(&skip); + } + + masm.addFloat32(input, temp); + masm.vcvttss2si(temp, output); + ScratchFloat32Scope fpscratch(masm); + masm.vcvtsi2ss(output, fpscratch, fpscratch); + + masm.vucomiss(fpscratch, temp); + masm.j(Assembler::Parity, &fail); + masm.j(Assembler::Equal, ool->rejoin()); + } + + masm.bind(&fail); + { + if (gen->compilingWasm()) { + masm.Push(InstanceReg); + } + int32_t framePushedAfterInstance = masm.framePushed(); + + saveVolatile(output); + + masm.Push(input); + + if (gen->compilingWasm()) { + masm.setupWasmABICall(); + } else { + masm.setupUnalignedABICall(output); + } + + masm.vcvtss2sd(input, input, input); + masm.passABIArg(input.asDouble(), MoveOp::DOUBLE); + + if (gen->compilingWasm()) { + int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + masm.callWithABI(ool->bytecodeOffset(), wasm::SymbolicAddress::ToInt32, + mozilla::Some(instanceOffset)); + } else { + using Fn = int32_t (*)(double); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + + masm.storeCallInt32Result(output); + masm.Pop(input); + + restoreVolatile(output); + + if (gen->compilingWasm()) { + masm.Pop(InstanceReg); + } + } + + masm.jump(ool->rejoin()); +} + +void CodeGenerator::visitCompareI64(LCompareI64* lir) { + MCompare* mir = lir->mir(); + MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 || + mir->compareType() == MCompare::Compare_UInt64); + + const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs); + Register64 lhsRegs = ToRegister64(lhs); + Register output = ToRegister(lir->output()); + + bool isSigned = mir->compareType() == MCompare::Compare_Int64; + Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned); + Label done; + + masm.move32(Imm32(1), output); + + if (IsConstant(rhs)) { + Imm64 imm = Imm64(ToInt64(rhs)); + masm.branch64(condition, lhsRegs, imm, &done); + } else { + Register64 rhsRegs = ToRegister64(rhs); + masm.branch64(condition, lhsRegs, rhsRegs, &done); + } + + masm.xorl(output, output); + masm.bind(&done); +} + +void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) { + MCompare* mir = lir->cmpMir(); + MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 || + mir->compareType() == MCompare::Compare_UInt64); + + const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs); + const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs); + Register64 lhsRegs = ToRegister64(lhs); + + bool isSigned = mir->compareType() == MCompare::Compare_Int64; + Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned); + + Label* trueLabel = getJumpLabelForBranch(lir->ifTrue()); + Label* falseLabel = getJumpLabelForBranch(lir->ifFalse()); + + if (isNextBlock(lir->ifFalse()->lir())) { + falseLabel = nullptr; + } else if (isNextBlock(lir->ifTrue()->lir())) { + condition = Assembler::InvertCondition(condition); + trueLabel = falseLabel; + falseLabel = nullptr; + } + + if (IsConstant(rhs)) { + Imm64 imm = Imm64(ToInt64(rhs)); + masm.branch64(condition, lhsRegs, imm, trueLabel, falseLabel); + } else { + Register64 rhsRegs = ToRegister64(rhs); + masm.branch64(condition, lhsRegs, rhsRegs, trueLabel, falseLabel); + } +} + +void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) { + MOZ_ASSERT(gen->compilingWasm()); + MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) == + InstanceReg); + + masm.Push(InstanceReg); + int32_t framePushedAfterInstance = masm.framePushed(); + + Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs)); + Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs)); + Register64 output = ToOutRegister64(lir); + + MOZ_ASSERT(output == ReturnReg64); + + Label done; + + // Handle divide by zero. + if (lir->canBeDivideByZero()) { + Label nonZero; + // We can use InstanceReg as temp register because we preserved it + // before. + masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset()); + masm.bind(&nonZero); + } + + MDefinition* mir = lir->mir(); + + // Handle an integer overflow exception from INT64_MIN / -1. + if (lir->canBeNegativeOverflow()) { + Label notOverflow; + masm.branch64(Assembler::NotEqual, lhs, Imm64(INT64_MIN), ¬Overflow); + masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), ¬Overflow); + if (mir->isWasmBuiltinModI64()) { + masm.xor64(output, output); + } else { + masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset()); + } + masm.jump(&done); + masm.bind(¬Overflow); + } + + masm.setupWasmABICall(); + masm.passABIArg(lhs.high); + masm.passABIArg(lhs.low); + masm.passABIArg(rhs.high); + masm.passABIArg(rhs.low); + + int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + if (mir->isWasmBuiltinModI64()) { + masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::ModI64, + mozilla::Some(instanceOffset)); + } else { + masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::DivI64, + mozilla::Some(instanceOffset)); + } + + // output in edx:eax, move to output register. + masm.movl(edx, output.high); + MOZ_ASSERT(eax == output.low); + + masm.bind(&done); + masm.Pop(InstanceReg); +} + +void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) { + MOZ_ASSERT(gen->compilingWasm()); + MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) == + InstanceReg); + + masm.Push(InstanceReg); + int32_t framePushedAfterInstance = masm.framePushed(); + + Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs)); + Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs)); + Register64 output = ToOutRegister64(lir); + + MOZ_ASSERT(output == ReturnReg64); + + // Prevent divide by zero. + if (lir->canBeDivideByZero()) { + Label nonZero; + // We can use InstanceReg as temp register because we preserved it + // before. + masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset()); + masm.bind(&nonZero); + } + + masm.setupWasmABICall(); + masm.passABIArg(lhs.high); + masm.passABIArg(lhs.low); + masm.passABIArg(rhs.high); + masm.passABIArg(rhs.low); + + MDefinition* mir = lir->mir(); + int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + if (mir->isWasmBuiltinModI64()) { + masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UModI64, + mozilla::Some(instanceOffset)); + } else { + masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UDivI64, + mozilla::Some(instanceOffset)); + } + + // output in edx:eax, move to output register. + masm.movl(edx, output.high); + MOZ_ASSERT(eax == output.low); + + masm.Pop(InstanceReg); +} + +void CodeGeneratorX86::emitBigIntDiv(LBigIntDiv* ins, Register dividend, + Register divisor, Register output, + Label* fail) { + // Callers handle division by zero and integer overflow. + + MOZ_ASSERT(dividend == eax); + MOZ_ASSERT(output == edx); + + // Sign extend the lhs into rdx to make rdx:rax. + masm.cdq(); + + masm.idiv(divisor); + + // Create and return the result. + masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail); + masm.initializeBigInt(output, dividend); +} + +void CodeGeneratorX86::emitBigIntMod(LBigIntMod* ins, Register dividend, + Register divisor, Register output, + Label* fail) { + // Callers handle division by zero and integer overflow. + + MOZ_ASSERT(dividend == eax); + MOZ_ASSERT(output == edx); + + // Sign extend the lhs into rdx to make edx:eax. + masm.cdq(); + + masm.idiv(divisor); + + // Move the remainder from edx. + masm.movl(output, dividend); + + // Create and return the result. + masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail); + masm.initializeBigInt(output, dividend); +} + +void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Int64); + + Register cond = ToRegister(lir->condExpr()); + Register64 falseExpr = ToRegister64(lir->falseExpr()); + Register64 out = ToOutRegister64(lir); + + MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out, + "true expr is reused for input"); + + Label done; + masm.branchTest32(Assembler::NonZero, cond, cond, &done); + masm.movl(falseExpr.low, out.low); + masm.movl(falseExpr.high, out.high); + masm.bind(&done); +} + +// We expect to handle only the case where compare is {U,}Int32 and select is +// {U,}Int32. Some values may be stack allocated, and the "true" input is +// reused for the output. +void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) { + bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 || + ins->compareType() == MCompare::Compare_UInt32; + bool selIs32bit = ins->mir()->type() == MIRType::Int32; + + MOZ_RELEASE_ASSERT( + cmpIs32bit && selIs32bit, + "CodeGenerator::visitWasmCompareAndSelect: unexpected types"); + + Register trueExprAndDest = ToRegister(ins->output()); + MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest, + "true expr input is reused for output"); + + Assembler::Condition cond = Assembler::InvertCondition( + JSOpToCondition(ins->compareType(), ins->jsop())); + const LAllocation* rhs = ins->rightExpr(); + const LAllocation* falseExpr = ins->ifFalseExpr(); + Register lhs = ToRegister(ins->leftExpr()); + + if (rhs->isRegister()) { + if (falseExpr->isRegister()) { + masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr), + trueExprAndDest); + } else { + masm.cmp32Load32(cond, lhs, ToRegister(rhs), ToAddress(falseExpr), + trueExprAndDest); + } + } else { + if (falseExpr->isRegister()) { + masm.cmp32Move32(cond, lhs, ToAddress(rhs), ToRegister(falseExpr), + trueExprAndDest); + } else { + masm.cmp32Load32(cond, lhs, ToAddress(rhs), ToAddress(falseExpr), + trueExprAndDest); + } + } +} + +void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Double); + MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64); + Register64 input = ToRegister64(lir->getInt64Operand(0)); + + masm.Push(input.high); + masm.Push(input.low); + masm.vmovq(Operand(esp, 0), ToFloatRegister(lir->output())); + masm.freeStack(sizeof(uint64_t)); +} + +void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) { + MOZ_ASSERT(lir->mir()->type() == MIRType::Int64); + MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double); + Register64 output = ToOutRegister64(lir); + + masm.reserveStack(sizeof(uint64_t)); + masm.vmovq(ToFloatRegister(lir->input()), Operand(esp, 0)); + masm.Pop(output.low); + masm.Pop(output.high); +} + +void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) { + Register64 output = ToOutRegister64(lir); + Register input = ToRegister(lir->input()); + + if (lir->mir()->isUnsigned()) { + if (output.low != input) { + masm.movl(input, output.low); + } + masm.xorl(output.high, output.high); + } else { + MOZ_ASSERT(output.low == input); + MOZ_ASSERT(output.low == eax); + MOZ_ASSERT(output.high == edx); + masm.cdq(); + } +} + +void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) { +#ifdef DEBUG + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + MOZ_ASSERT(input.low == eax); + MOZ_ASSERT(output.low == eax); + MOZ_ASSERT(input.high == edx); + MOZ_ASSERT(output.high == edx); +#endif + switch (lir->mode()) { + case MSignExtendInt64::Byte: + masm.move8SignExtend(eax, eax); + break; + case MSignExtendInt64::Half: + masm.move16SignExtend(eax, eax); + break; + case MSignExtendInt64::Word: + break; + } + masm.cdq(); +} + +void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) { + const LInt64Allocation& input = lir->getInt64Operand(0); + Register output = ToRegister(lir->output()); + + if (lir->mir()->bottomHalf()) { + masm.movl(ToRegister(input.low()), output); + } else { + masm.movl(ToRegister(input.high()), output); + } +} + +void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index*) { + MOZ_CRASH("64-bit only"); +} + +void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) { + // Generates no code on this platform because we just return the low part of + // the input register pair. + MOZ_ASSERT(ToRegister(lir->input()) == ToRegister(lir->output())); +} + +void CodeGenerator::visitClzI64(LClzI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + + masm.clz64(input, output.low); + masm.xorl(output.high, output.high); +} + +void CodeGenerator::visitCtzI64(LCtzI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register64 output = ToOutRegister64(lir); + + masm.ctz64(input, output.low); + masm.xorl(output.high, output.high); +} + +void CodeGenerator::visitNotI64(LNotI64* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + Register output = ToRegister(lir->output()); + + if (input.high == output) { + masm.orl(input.low, output); + } else if (input.low == output) { + masm.orl(input.high, output); + } else { + masm.movl(input.high, output); + masm.orl(input.low, output); + } + + masm.cmpl(Imm32(0), output); + masm.emitSet(Assembler::Equal, output); +} + +void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) { + FloatRegister input = ToFloatRegister(lir->input()); + Register64 output = ToOutRegister64(lir); + + MWasmTruncateToInt64* mir = lir->mir(); + FloatRegister floatTemp = ToFloatRegister(lir->temp()); + + Label fail, convert; + + MOZ_ASSERT(mir->input()->type() == MIRType::Double || + mir->input()->type() == MIRType::Float32); + + auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output); + addOutOfLineCode(ool, mir); + + bool isSaturating = mir->isSaturating(); + if (mir->input()->type() == MIRType::Float32) { + if (mir->isUnsigned()) { + masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, + ool->entry(), ool->rejoin(), floatTemp); + } else { + masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, ool->entry(), + ool->rejoin(), floatTemp); + } + } else { + if (mir->isUnsigned()) { + masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, ool->entry(), + ool->rejoin(), floatTemp); + } else { + masm.wasmTruncateDoubleToInt64(input, output, isSaturating, ool->entry(), + ool->rejoin(), floatTemp); + } + } +} + +void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + FloatRegister output = ToFloatRegister(lir->output()); + Register temp = + lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp()); + + MIRType outputType = lir->mir()->type(); + MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32); + + if (outputType == MIRType::Double) { + if (lir->mir()->isUnsigned()) { + masm.convertUInt64ToDouble(input, output, temp); + } else { + masm.convertInt64ToDouble(input, output); + } + } else { + if (lir->mir()->isUnsigned()) { + masm.convertUInt64ToFloat32(input, output, temp); + } else { + masm.convertInt64ToFloat32(input, output); + } + } +} + +void CodeGenerator::visitBitNotI64(LBitNotI64* ins) { + const LInt64Allocation input = ins->getInt64Operand(0); + Register64 inputR = ToRegister64(input); + MOZ_ASSERT(inputR == ToOutRegister64(ins)); + masm.notl(inputR.high); + masm.notl(inputR.low); +} + +void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) { + Register64 input = ToRegister64(lir->getInt64Operand(0)); + + masm.testl(input.high, input.high); + jumpToBlock(lir->ifTrue(), Assembler::NonZero); + masm.testl(input.low, input.low); + emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse()); +} + +void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) { + // LBitAndAndBranch only represents single-word ANDs, hence it can't be + // 64-bit here. + MOZ_ASSERT(!baab->is64()); + Register regL = ToRegister(baab->left()); + if (baab->right()->isConstant()) { + masm.test32(regL, Imm32(ToInt32(baab->right()))); + } else { + masm.test32(regL, ToRegister(baab->right())); + } + emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse()); +} diff --git a/js/src/jit/x86/CodeGenerator-x86.h b/js/src/jit/x86/CodeGenerator-x86.h new file mode 100644 index 0000000000..4f92bf615f --- /dev/null +++ b/js/src/jit/x86/CodeGenerator-x86.h @@ -0,0 +1,49 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_CodeGenerator_x86_h +#define jit_x86_CodeGenerator_x86_h + +#include "jit/x86-shared/CodeGenerator-x86-shared.h" +#include "jit/x86/Assembler-x86.h" + +namespace js { +namespace jit { + +class OutOfLineTruncate; +class OutOfLineTruncateFloat32; + +class CodeGeneratorX86 : public CodeGeneratorX86Shared { + protected: + CodeGeneratorX86(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm); + + ValueOperand ToValue(LInstruction* ins, size_t pos); + ValueOperand ToTempValue(LInstruction* ins, size_t pos); + + void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor, + Register output, Label* fail); + void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor, + Register output, Label* fail); + + template + void emitWasmLoad(T* ins); + template + void emitWasmStore(T* ins); + template + void emitWasmStoreOrExchangeAtomicI64(T* ins, + const wasm::MemoryAccessDesc& access); + + public: + void visitOutOfLineTruncate(OutOfLineTruncate* ool); + void visitOutOfLineTruncateFloat32(OutOfLineTruncateFloat32* ool); +}; + +typedef CodeGeneratorX86 CodeGeneratorSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_CodeGenerator_x86_h */ diff --git a/js/src/jit/x86/LIR-x86.h b/js/src/jit/x86/LIR-x86.h new file mode 100644 index 0000000000..c7c9587e20 --- /dev/null +++ b/js/src/jit/x86/LIR-x86.h @@ -0,0 +1,308 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_LIR_x86_h +#define jit_x86_LIR_x86_h + +namespace js { +namespace jit { + +class LBoxFloatingPoint : public LInstructionHelper<2, 1, 2> { + MIRType type_; + + public: + LIR_HEADER(BoxFloatingPoint); + + LBoxFloatingPoint(const LAllocation& in, const LDefinition& temp, + const LDefinition& spectreTemp, MIRType type) + : LInstructionHelper(classOpcode), type_(type) { + MOZ_ASSERT(IsFloatingPointType(type)); + setOperand(0, in); + setTemp(0, temp); + setTemp(1, spectreTemp); + } + + const LDefinition* spectreTemp() { return getTemp(1); } + + MIRType type() const { return type_; } + const char* extraName() const { return StringFromMIRType(type_); } +}; + +class LUnbox : public LInstructionHelper<1, 2, 0> { + public: + LIR_HEADER(Unbox); + + LUnbox() : LInstructionHelper(classOpcode) {} + + MUnbox* mir() const { return mir_->toUnbox(); } + const LAllocation* payload() { return getOperand(0); } + const LAllocation* type() { return getOperand(1); } + const char* extraName() const { return StringFromMIRType(mir()->type()); } +}; + +class LUnboxFloatingPoint : public LInstructionHelper<1, 2, 0> { + MIRType type_; + + public: + LIR_HEADER(UnboxFloatingPoint); + + static const size_t Input = 0; + + LUnboxFloatingPoint(const LBoxAllocation& input, MIRType type) + : LInstructionHelper(classOpcode), type_(type) { + setBoxOperand(Input, input); + } + + MUnbox* mir() const { return mir_->toUnbox(); } + + MIRType type() const { return type_; } + const char* extraName() const { return StringFromMIRType(type_); } +}; + +// Convert a 32-bit unsigned integer to a double. +class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 1> { + public: + LIR_HEADER(WasmUint32ToDouble) + + LWasmUint32ToDouble(const LAllocation& input, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + } + const LDefinition* temp() { return getTemp(0); } +}; + +// Convert a 32-bit unsigned integer to a float32. +class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 1> { + public: + LIR_HEADER(WasmUint32ToFloat32) + + LWasmUint32ToFloat32(const LAllocation& input, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + } + const LDefinition* temp() { return getTemp(0); } +}; + +class LDivOrModI64 + : public LCallInstructionHelper { + public: + LIR_HEADER(DivOrModI64) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + static const size_t Instance = 2 * INT64_PIECES; + + LDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs, + const LAllocation& instance) + : LCallInstructionHelper(classOpcode) { + setInt64Operand(Lhs, lhs); + setInt64Operand(Rhs, rhs); + setOperand(Instance, instance); + } + + MDefinition* mir() const { + MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64()); + return mir_; + } + bool canBeDivideByZero() const { + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->canBeDivideByZero(); + } + return mir_->toWasmBuiltinDivI64()->canBeDivideByZero(); + } + bool canBeNegativeOverflow() const { + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->canBeNegativeDividend(); + } + return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow(); + } + wasm::BytecodeOffset bytecodeOffset() const { + MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64()); + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->bytecodeOffset(); + } + return mir_->toWasmBuiltinDivI64()->bytecodeOffset(); + } +}; + +class LUDivOrModI64 + : public LCallInstructionHelper { + public: + LIR_HEADER(UDivOrModI64) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + static const size_t Instance = 2 * INT64_PIECES; + + LUDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs, + const LAllocation& instance) + : LCallInstructionHelper(classOpcode) { + setInt64Operand(Lhs, lhs); + setInt64Operand(Rhs, rhs); + setOperand(Instance, instance); + } + + MDefinition* mir() const { + MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64()); + return mir_; + } + bool canBeDivideByZero() const { + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->canBeDivideByZero(); + } + return mir_->toWasmBuiltinDivI64()->canBeDivideByZero(); + } + bool canBeNegativeOverflow() const { + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->canBeNegativeDividend(); + } + return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow(); + } + wasm::BytecodeOffset bytecodeOffset() const { + MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64()); + if (mir_->isWasmBuiltinModI64()) { + return mir_->toWasmBuiltinModI64()->bytecodeOffset(); + } + return mir_->toWasmBuiltinDivI64()->bytecodeOffset(); + } +}; + +class LWasmTruncateToInt64 : public LInstructionHelper { + public: + LIR_HEADER(WasmTruncateToInt64); + + LWasmTruncateToInt64(const LAllocation& in, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, in); + setTemp(0, temp); + } + + MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); } + + const LDefinition* temp() { return getTemp(0); } +}; + +class LWasmAtomicLoadI64 : public LInstructionHelper { + public: + LIR_HEADER(WasmAtomicLoadI64); + + LWasmAtomicLoadI64(const LAllocation& memoryBase, const LAllocation& ptr, + const LDefinition& t1, const LDefinition& t2) + : LInstructionHelper(classOpcode) { + setOperand(0, memoryBase); + setOperand(1, ptr); + setTemp(0, t1); + setTemp(1, t2); + } + + MWasmLoad* mir() const { return mir_->toWasmLoad(); } + const LAllocation* memoryBase() { return getOperand(0); } + const LAllocation* ptr() { return getOperand(1); } + const LDefinition* t1() { return getTemp(0); } + const LDefinition* t2() { return getTemp(1); } +}; + +class LWasmAtomicStoreI64 : public LInstructionHelper<0, 2 + INT64_PIECES, 2> { + public: + LIR_HEADER(WasmAtomicStoreI64); + + LWasmAtomicStoreI64(const LAllocation& memoryBase, const LAllocation& ptr, + const LInt64Allocation& value, const LDefinition& t1, + const LDefinition& t2) + : LInstructionHelper(classOpcode) { + setOperand(0, memoryBase); + setOperand(1, ptr); + setInt64Operand(2, value); + setTemp(0, t1); + setTemp(1, t2); + } + + MWasmStore* mir() const { return mir_->toWasmStore(); } + const LAllocation* memoryBase() { return getOperand(0); } + const LAllocation* ptr() { return getOperand(1); } + const LInt64Allocation value() { return getInt64Operand(2); } + const LDefinition* t1() { return getTemp(0); } + const LDefinition* t2() { return getTemp(1); } +}; + +class LWasmCompareExchangeI64 + : public LInstructionHelper { + public: + LIR_HEADER(WasmCompareExchangeI64); + + LWasmCompareExchangeI64(const LAllocation& memoryBase, const LAllocation& ptr, + const LInt64Allocation& expected, + const LInt64Allocation& replacement) + : LInstructionHelper(classOpcode) { + setOperand(0, memoryBase); + setOperand(1, ptr); + setInt64Operand(2, expected); + setInt64Operand(2 + INT64_PIECES, replacement); + } + + MWasmCompareExchangeHeap* mir() const { + return mir_->toWasmCompareExchangeHeap(); + } + const LAllocation* memoryBase() { return getOperand(0); } + const LAllocation* ptr() { return getOperand(1); } + const LInt64Allocation expected() { return getInt64Operand(2); } + const LInt64Allocation replacement() { + return getInt64Operand(2 + INT64_PIECES); + } +}; + +class LWasmAtomicExchangeI64 + : public LInstructionHelper { + const wasm::MemoryAccessDesc& access_; + + public: + LIR_HEADER(WasmAtomicExchangeI64); + + LWasmAtomicExchangeI64(const LAllocation& memoryBase, const LAllocation& ptr, + const LInt64Allocation& value, + const wasm::MemoryAccessDesc& access) + : LInstructionHelper(classOpcode), access_(access) { + setOperand(0, memoryBase); + setOperand(1, ptr); + setInt64Operand(2, value); + } + + const LAllocation* memoryBase() { return getOperand(0); } + const LAllocation* ptr() { return getOperand(1); } + const LInt64Allocation value() { return getInt64Operand(2); } + const wasm::MemoryAccessDesc& access() { return access_; } +}; + +class LWasmAtomicBinopI64 + : public LInstructionHelper { + const wasm::MemoryAccessDesc& access_; + AtomicOp op_; + + public: + LIR_HEADER(WasmAtomicBinopI64); + + LWasmAtomicBinopI64(const LAllocation& memoryBase, const LAllocation& ptr, + const LInt64Allocation& value, + const wasm::MemoryAccessDesc& access, AtomicOp op) + : LInstructionHelper(classOpcode), access_(access), op_(op) { + setOperand(0, memoryBase); + setOperand(1, ptr); + setInt64Operand(2, value); + } + + const LAllocation* memoryBase() { return getOperand(0); } + const LAllocation* ptr() { return getOperand(1); } + const LInt64Allocation value() { return getInt64Operand(2); } + const wasm::MemoryAccessDesc& access() { return access_; } + AtomicOp operation() const { return op_; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_LIR_x86_h */ diff --git a/js/src/jit/x86/Lowering-x86.cpp b/js/src/jit/x86/Lowering-x86.cpp new file mode 100644 index 0000000000..968b5baf14 --- /dev/null +++ b/js/src/jit/x86/Lowering-x86.cpp @@ -0,0 +1,840 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86/Lowering-x86.h" + +#include "jit/Lowering.h" +#include "jit/MIR.h" +#include "jit/x86/Assembler-x86.h" + +#include "jit/shared/Lowering-shared-inl.h" + +using namespace js; +using namespace js::jit; + +LBoxAllocation LIRGeneratorX86::useBoxFixed(MDefinition* mir, Register reg1, + Register reg2, bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Value); + MOZ_ASSERT(reg1 != reg2); + + ensureDefined(mir); + return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart), + LUse(reg2, VirtualRegisterOfPayload(mir), useAtStart)); +} + +LAllocation LIRGeneratorX86::useByteOpRegister(MDefinition* mir) { + return useFixed(mir, eax); +} + +LAllocation LIRGeneratorX86::useByteOpRegisterAtStart(MDefinition* mir) { + return useFixedAtStart(mir, eax); +} + +LAllocation LIRGeneratorX86::useByteOpRegisterOrNonDoubleConstant( + MDefinition* mir) { + return useFixed(mir, eax); +} + +LDefinition LIRGeneratorX86::tempByteOpRegister() { return tempFixed(eax); } + +void LIRGenerator::visitBox(MBox* box) { + MDefinition* inner = box->getOperand(0); + + // If the box wrapped a double, it needs a new register. + if (IsFloatingPointType(inner->type())) { + LDefinition spectreTemp = + JitOptions.spectreValueMasking ? temp() : LDefinition::BogusTemp(); + defineBox(new (alloc()) LBoxFloatingPoint(useRegisterAtStart(inner), + tempCopy(inner, 0), spectreTemp, + inner->type()), + box); + return; + } + + if (box->canEmitAtUses()) { + emitAtUses(box); + return; + } + + if (inner->isConstant()) { + defineBox(new (alloc()) LValue(inner->toConstant()->toJSValue()), box); + return; + } + + LBox* lir = new (alloc()) LBox(use(inner), inner->type()); + + // Otherwise, we should not define a new register for the payload portion + // of the output, so bypass defineBox(). + uint32_t vreg = getVirtualRegister(); + + // Note that because we're using BogusTemp(), we do not change the type of + // the definition. We also do not define the first output as "TYPE", + // because it has no corresponding payload at (vreg + 1). Also note that + // although we copy the input's original type for the payload half of the + // definition, this is only for clarity. BogusTemp() definitions are + // ignored. + lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL)); + lir->setDef(1, LDefinition::BogusTemp()); + box->setVirtualRegister(vreg); + add(lir); +} + +void LIRGenerator::visitUnbox(MUnbox* unbox) { + MDefinition* inner = unbox->getOperand(0); + + // An unbox on x86 reads in a type tag (either in memory or a register) and + // a payload. Unlike most instructions consuming a box, we ask for the type + // second, so that the result can re-use the first input. + MOZ_ASSERT(inner->type() == MIRType::Value); + + ensureDefined(inner); + + if (IsFloatingPointType(unbox->type())) { + LUnboxFloatingPoint* lir = + new (alloc()) LUnboxFloatingPoint(useBox(inner), unbox->type()); + if (unbox->fallible()) { + assignSnapshot(lir, unbox->bailoutKind()); + } + define(lir, unbox); + return; + } + + // Swap the order we use the box pieces so we can re-use the payload register. + LUnbox* lir = new (alloc()) LUnbox; + bool reusePayloadReg = !JitOptions.spectreValueMasking || + unbox->type() == MIRType::Int32 || + unbox->type() == MIRType::Boolean; + if (reusePayloadReg) { + lir->setOperand(0, usePayloadInRegisterAtStart(inner)); + lir->setOperand(1, useType(inner, LUse::ANY)); + } else { + lir->setOperand(0, usePayload(inner, LUse::REGISTER)); + lir->setOperand(1, useType(inner, LUse::ANY)); + } + + if (unbox->fallible()) { + assignSnapshot(lir, unbox->bailoutKind()); + } + + // Types and payloads form two separate intervals. If the type becomes dead + // before the payload, it could be used as a Value without the type being + // recoverable. Unbox's purpose is to eagerly kill the definition of a type + // tag, so keeping both alive (for the purpose of gcmaps) is unappealing. + // Instead, we create a new virtual register. + if (reusePayloadReg) { + defineReuseInput(lir, unbox, 0); + } else { + define(lir, unbox); + } +} + +void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) { + MOZ_ASSERT(opd->type() == MIRType::Value); + + LReturn* ins = new (alloc()) LReturn(isGenerator); + ins->setOperand(0, LUse(JSReturnReg_Type)); + ins->setOperand(1, LUse(JSReturnReg_Data)); + fillBoxUses(ins, 0, opd); + add(ins); +} + +void LIRGeneratorX86::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + MDefinition* operand = phi->getOperand(inputPosition); + LPhi* type = block->getPhi(lirIndex + VREG_TYPE_OFFSET); + LPhi* payload = block->getPhi(lirIndex + VREG_DATA_OFFSET); + type->setOperand( + inputPosition, + LUse(operand->virtualRegister() + VREG_TYPE_OFFSET, LUse::ANY)); + payload->setOperand(inputPosition, + LUse(VirtualRegisterOfPayload(operand), LUse::ANY)); +} + +void LIRGeneratorX86::defineInt64Phi(MPhi* phi, size_t lirIndex) { + LPhi* low = current->getPhi(lirIndex + INT64LOW_INDEX); + LPhi* high = current->getPhi(lirIndex + INT64HIGH_INDEX); + + uint32_t lowVreg = getVirtualRegister(); + + phi->setVirtualRegister(lowVreg); + + uint32_t highVreg = getVirtualRegister(); + MOZ_ASSERT(lowVreg + INT64HIGH_INDEX == highVreg + INT64LOW_INDEX); + + low->setDef(0, LDefinition(lowVreg, LDefinition::INT32)); + high->setDef(0, LDefinition(highVreg, LDefinition::INT32)); + annotate(high); + annotate(low); +} + +void LIRGeneratorX86::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + MDefinition* operand = phi->getOperand(inputPosition); + LPhi* low = block->getPhi(lirIndex + INT64LOW_INDEX); + LPhi* high = block->getPhi(lirIndex + INT64HIGH_INDEX); + low->setOperand(inputPosition, + LUse(operand->virtualRegister() + INT64LOW_INDEX, LUse::ANY)); + high->setOperand( + inputPosition, + LUse(operand->virtualRegister() + INT64HIGH_INDEX, LUse::ANY)); +} + +void LIRGeneratorX86::lowerForALUInt64( + LInstructionHelper* ins, MDefinition* mir, + MDefinition* input) { + ins->setInt64Operand(0, useInt64RegisterAtStart(input)); + defineInt64ReuseInput(ins, mir, 0); +} + +void LIRGeneratorX86::lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs) { + ins->setInt64Operand(0, useInt64RegisterAtStart(lhs)); + ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs)); + defineInt64ReuseInput(ins, mir, 0); +} + +void LIRGeneratorX86::lowerForMulInt64(LMulI64* ins, MMul* mir, + MDefinition* lhs, MDefinition* rhs) { + bool needsTemp = true; + + if (rhs->isConstant()) { + int64_t constant = rhs->toConstant()->toInt64(); + int32_t shift = mozilla::FloorLog2(constant); + // See special cases in CodeGeneratorX86Shared::visitMulI64. + if (constant >= -1 && constant <= 2) { + needsTemp = false; + } + if (constant > 0 && int64_t(1) << shift == constant) { + needsTemp = false; + } + } + + // MulI64 on x86 needs output to be in edx, eax; + ins->setInt64Operand( + 0, useInt64Fixed(lhs, Register64(edx, eax), /*useAtStart = */ true)); + ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs)); + if (needsTemp) { + ins->setTemp(0, temp()); + } + + defineInt64Fixed(ins, mir, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); +} + +void LIRGenerator::visitCompareExchangeTypedArrayElement( + MCompareExchangeTypedArrayElement* ins) { + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + if (Scalar::isBigIntType(ins->arrayType())) { + LUse elements = useFixed(ins->elements(), esi); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + LUse oldval = useFixed(ins->oldval(), eax); + LUse newval = useFixed(ins->newval(), edx); + LDefinition temp = tempFixed(ebx); + + auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64( + elements, index, oldval, newval, temp); + defineFixed(lir, ins, LAllocation(AnyRegister(ecx))); + assignSafepoint(lir, ins); + return; + } + + lowerCompareExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ true); +} + +void LIRGenerator::visitAtomicExchangeTypedArrayElement( + MAtomicExchangeTypedArrayElement* ins) { + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + if (Scalar::isBigIntType(ins->arrayType())) { + LUse elements = useRegister(ins->elements()); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + LAllocation value = useFixed(ins->value(), edx); + LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx)); + + auto* lir = new (alloc()) + LAtomicExchangeTypedArrayElement64(elements, index, value, temp); + defineFixed(lir, ins, LAllocation(AnyRegister(eax))); + assignSafepoint(lir, ins); + return; + } + + lowerAtomicExchangeTypedArrayElement(ins, /*useI386ByteRegisters=*/true); +} + +void LIRGenerator::visitAtomicTypedArrayElementBinop( + MAtomicTypedArrayElementBinop* ins) { + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + if (Scalar::isBigIntType(ins->arrayType())) { + LUse elements = useRegister(ins->elements()); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + LAllocation value = useFixed(ins->value(), edx); + LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx)); + + // Case 1: the result of the operation is not used. + // + // We can omit allocating the result BigInt. + + if (ins->isForEffect()) { + LDefinition tempLow = tempFixed(eax); + + auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64( + elements, index, value, temp, tempLow); + add(lir, ins); + return; + } + + // Case 2: the result of the operation is used. + + auto* lir = new (alloc()) + LAtomicTypedArrayElementBinop64(elements, index, value, temp); + defineFixed(lir, ins, LAllocation(AnyRegister(eax))); + assignSafepoint(lir, ins); + return; + } + + lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ true); +} + +void LIRGeneratorX86::lowerAtomicLoad64(MLoadUnboxedScalar* ins) { + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->storageType()); + + auto* lir = new (alloc()) LAtomicLoad64(elements, index, tempFixed(ebx), + tempInt64Fixed(Register64(edx, eax))); + defineFixed(lir, ins, LAllocation(AnyRegister(ecx))); + assignSafepoint(lir, ins); +} + +void LIRGeneratorX86::lowerAtomicStore64(MStoreUnboxedScalar* ins) { + LUse elements = useRegister(ins->elements()); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->writeType()); + LAllocation value = useFixed(ins->value(), edx); + LInt64Definition temp1 = tempInt64Fixed(Register64(ecx, ebx)); + LDefinition temp2 = tempFixed(eax); + + add(new (alloc()) LAtomicStore64(elements, index, value, temp1, temp2), ins); +} + +void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + LWasmUint32ToDouble* lir = new (alloc()) + LWasmUint32ToDouble(useRegisterAtStart(ins->input()), temp()); + define(lir, ins); +} + +void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + LWasmUint32ToFloat32* lir = new (alloc()) + LWasmUint32ToFloat32(useRegisterAtStart(ins->input()), temp()); + define(lir, ins); +} + +// If the base is a constant, and it is zero or its offset is zero, then +// code generation will fold the values into the access. Allocate the +// pointer to a register only if that can't happen. + +static bool OptimizableConstantAccess(MDefinition* base, + const wasm::MemoryAccessDesc& access) { + MOZ_ASSERT(base->isConstant()); + MOZ_ASSERT(base->type() == MIRType::Int32); + + if (!(base->toConstant()->isInt32(0) || access.offset() == 0)) { + return false; + } + if (access.type() == Scalar::Int64) { + // For int64 accesses on 32-bit systems we will need to add another offset + // of 4 to access the high part of the value; make sure this does not + // overflow the value. + int32_t v; + if (base->toConstant()->isInt32(0)) { + v = access.offset(); + } else { + v = base->toConstant()->toInt32(); + } + return v <= int32_t(INT32_MAX - INT64HIGH_OFFSET); + } + return true; +} + +void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) { + auto* lir = new (alloc()) LWasmHeapBase(useRegisterAtStart(ins->instance())); + define(lir, ins); +} + +void LIRGenerator::visitWasmLoad(MWasmLoad* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* memoryBase = ins->memoryBase(); + MOZ_ASSERT(memoryBase->type() == MIRType::Pointer); + + if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) { + auto* lir = new (alloc()) + LWasmAtomicLoadI64(useRegister(memoryBase), useRegister(base), + tempFixed(ecx), tempFixed(ebx)); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + return; + } + + LAllocation baseAlloc; + if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) { + baseAlloc = ins->type() == MIRType::Int64 ? useRegister(base) + : useRegisterAtStart(base); + } + + if (ins->type() != MIRType::Int64) { + auto* lir = + new (alloc()) LWasmLoad(baseAlloc, useRegisterAtStart(memoryBase)); + define(lir, ins); + return; + } + + // "AtStart" register usage does not work for the 64-bit case because we + // clobber two registers for the result and may need two registers for a + // scaled address; we can't guarantee non-interference. + + auto* lir = new (alloc()) LWasmLoadI64(baseAlloc, useRegister(memoryBase)); + + Scalar::Type accessType = ins->access().type(); + if (accessType == Scalar::Int8 || accessType == Scalar::Int16 || + accessType == Scalar::Int32) { + // We use cdq to sign-extend the result and cdq demands these registers. + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + return; + } + + defineInt64(lir, ins); +} + +void LIRGenerator::visitWasmStore(MWasmStore* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* memoryBase = ins->memoryBase(); + MOZ_ASSERT(memoryBase->type() == MIRType::Pointer); + + if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) { + auto* lir = new (alloc()) + LWasmAtomicStoreI64(useRegister(memoryBase), useRegister(base), + useInt64Fixed(ins->value(), Register64(ecx, ebx)), + tempFixed(edx), tempFixed(eax)); + add(lir, ins); + return; + } + + LAllocation baseAlloc; + if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) { + baseAlloc = useRegisterAtStart(base); + } + + LAllocation valueAlloc; + switch (ins->access().type()) { + case Scalar::Int8: + case Scalar::Uint8: + // See comment for LIRGeneratorX86::useByteOpRegister. + valueAlloc = useFixed(ins->value(), eax); + break; + case Scalar::Int16: + case Scalar::Uint16: + case Scalar::Int32: + case Scalar::Uint32: + case Scalar::Float32: + case Scalar::Float64: + // For now, don't allow constant values. The immediate operand affects + // instruction layout which affects patching. + valueAlloc = useRegisterAtStart(ins->value()); + break; + case Scalar::Simd128: +#ifdef ENABLE_WASM_SIMD + valueAlloc = useRegisterAtStart(ins->value()); + break; +#else + MOZ_CRASH("unexpected array type"); +#endif + case Scalar::Int64: { + LInt64Allocation valueAlloc = useInt64RegisterAtStart(ins->value()); + auto* lir = new (alloc()) + LWasmStoreI64(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase)); + add(lir, ins); + return; + } + case Scalar::Uint8Clamped: + case Scalar::BigInt64: + case Scalar::BigUint64: + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected array type"); + } + + auto* lir = new (alloc()) + LWasmStore(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase)); + add(lir, ins); +} + +void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* memoryBase = ins->memoryBase(); + MOZ_ASSERT(memoryBase->type() == MIRType::Pointer); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmCompareExchangeI64( + useRegisterAtStart(memoryBase), useRegisterAtStart(base), + useInt64FixedAtStart(ins->oldValue(), Register64(edx, eax)), + useInt64FixedAtStart(ins->newValue(), Register64(ecx, ebx))); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + return; + } + + MOZ_ASSERT(ins->access().type() < Scalar::Float32); + + bool byteArray = byteSize(ins->access().type()) == 1; + + // Register allocation: + // + // The output may not be used, but eax will be clobbered regardless + // so pin the output to eax. + // + // oldval must be in a register. + // + // newval must be in a register. If the source is a byte array + // then newval must be a register that has a byte size: this must + // be ebx, ecx, or edx (eax is taken). + // + // Bug #1077036 describes some optimization opportunities. + + const LAllocation oldval = useRegister(ins->oldValue()); + const LAllocation newval = + byteArray ? useFixed(ins->newValue(), ebx) : useRegister(ins->newValue()); + + LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap( + useRegister(base), oldval, newval, useRegister(memoryBase)); + + lir->setAddrTemp(temp()); + defineFixed(lir, ins, LAllocation(AnyRegister(eax))); +} + +void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) { + MDefinition* memoryBase = ins->memoryBase(); + MOZ_ASSERT(memoryBase->type() == MIRType::Pointer); + + if (ins->access().type() == Scalar::Int64) { + MDefinition* base = ins->base(); + auto* lir = new (alloc()) LWasmAtomicExchangeI64( + useRegister(memoryBase), useRegister(base), + useInt64Fixed(ins->value(), Register64(ecx, ebx)), ins->access()); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + return; + } + + const LAllocation base = useRegister(ins->base()); + const LAllocation value = useRegister(ins->value()); + + LWasmAtomicExchangeHeap* lir = new (alloc()) + LWasmAtomicExchangeHeap(base, value, useRegister(memoryBase)); + + lir->setAddrTemp(temp()); + if (byteSize(ins->access().type()) == 1) { + defineFixed(lir, ins, LAllocation(AnyRegister(eax))); + } else { + define(lir, ins); + } +} + +void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* memoryBase = ins->memoryBase(); + MOZ_ASSERT(memoryBase->type() == MIRType::Pointer); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) + LWasmAtomicBinopI64(useRegister(memoryBase), useRegister(base), + useInt64Fixed(ins->value(), Register64(ecx, ebx)), + ins->access(), ins->operation()); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + return; + } + + MOZ_ASSERT(ins->access().type() < Scalar::Float32); + + bool byteArray = byteSize(ins->access().type()) == 1; + + // Case 1: the result of the operation is not used. + // + // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND, + // LOCK OR, or LOCK XOR. These can all take an immediate. + + if (!ins->hasUses()) { + LAllocation value; + if (byteArray && !ins->value()->isConstant()) { + value = useFixed(ins->value(), ebx); + } else { + value = useRegisterOrConstant(ins->value()); + } + LWasmAtomicBinopHeapForEffect* lir = + new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base), value, + LDefinition::BogusTemp(), + useRegister(memoryBase)); + lir->setAddrTemp(temp()); + add(lir, ins); + return; + } + + // Case 2: the result of the operation is used. + // + // For ADD and SUB we'll use XADD: + // + // movl value, output + // lock xaddl output, mem + // + // For the 8-bit variants XADD needs a byte register for the + // output only, we can still set up with movl; just pin the output + // to eax (or ebx / ecx / edx). + // + // For AND/OR/XOR we need to use a CMPXCHG loop: + // + // movl *mem, eax + // L: mov eax, temp + // andl value, temp + // lock cmpxchg temp, mem ; reads eax also + // jnz L + // ; result in eax + // + // Note the placement of L, cmpxchg will update eax with *mem if + // *mem does not have the expected value, so reloading it at the + // top of the loop would be redundant. + // + // We want to fix eax as the output. We also need a temp for + // the intermediate value. + // + // For the 8-bit variants the temp must have a byte register. + // + // There are optimization opportunities: + // - better 8-bit register allocation and instruction selection, Bug + // #1077036. + + bool bitOp = !(ins->operation() == AtomicFetchAddOp || + ins->operation() == AtomicFetchSubOp); + LDefinition tempDef = LDefinition::BogusTemp(); + LAllocation value; + + if (byteArray) { + value = useFixed(ins->value(), ebx); + if (bitOp) { + tempDef = tempFixed(ecx); + } + } else if (bitOp || ins->value()->isConstant()) { + value = useRegisterOrConstant(ins->value()); + if (bitOp) { + tempDef = temp(); + } + } else { + value = useRegisterAtStart(ins->value()); + } + + LWasmAtomicBinopHeap* lir = new (alloc()) + LWasmAtomicBinopHeap(useRegister(base), value, tempDef, + LDefinition::BogusTemp(), useRegister(memoryBase)); + + lir->setAddrTemp(temp()); + if (byteArray || bitOp) { + defineFixed(lir, ins, LAllocation(AnyRegister(eax))); + } else if (ins->value()->isConstant()) { + define(lir, ins); + } else { + defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp); + } +} + +void LIRGeneratorX86::lowerDivI64(MDiv* div) { + MOZ_CRASH("We use MWasmBuiltinModI64 instead."); +} + +void LIRGeneratorX86::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { + MOZ_ASSERT(div->lhs()->type() == div->rhs()->type()); + MOZ_ASSERT(IsNumberType(div->type())); + + MOZ_ASSERT(div->type() == MIRType::Int64); + + if (div->isUnsigned()) { + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)), + useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)), + useFixedAtStart(div->instance(), InstanceReg)); + defineReturn(lir, div); + return; + } + + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)), + useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)), + useFixedAtStart(div->instance(), InstanceReg)); + defineReturn(lir, div); +} + +void LIRGeneratorX86::lowerModI64(MMod* mod) { + MOZ_CRASH("We use MWasmBuiltinModI64 instead."); +} + +void LIRGeneratorX86::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { + MDefinition* lhs = mod->lhs(); + MDefinition* rhs = mod->rhs(); + MOZ_ASSERT(lhs->type() == rhs->type()); + MOZ_ASSERT(IsNumberType(mod->type())); + + MOZ_ASSERT(mod->type() == MIRType::Int64); + MOZ_ASSERT(mod->type() == MIRType::Int64); + + if (mod->isUnsigned()) { + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)), + useInt64FixedAtStart(rhs, Register64(ecx, edx)), + useFixedAtStart(mod->instance(), InstanceReg)); + defineReturn(lir, mod); + return; + } + + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)), + useInt64FixedAtStart(rhs, Register64(ecx, edx)), + useFixedAtStart(mod->instance(), InstanceReg)); + defineReturn(lir, mod); +} + +void LIRGeneratorX86::lowerUDivI64(MDiv* div) { + MOZ_CRASH("We use MWasmBuiltinDivI64 instead."); +} + +void LIRGeneratorX86::lowerUModI64(MMod* mod) { + MOZ_CRASH("We use MWasmBuiltinModI64 instead."); +} + +void LIRGeneratorX86::lowerBigIntDiv(MBigIntDiv* ins) { + auto* lir = new (alloc()) LBigIntDiv( + useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp()); + defineFixed(lir, ins, LAllocation(AnyRegister(edx))); + assignSafepoint(lir, ins); +} + +void LIRGeneratorX86::lowerBigIntMod(MBigIntMod* ins) { + auto* lir = new (alloc()) LBigIntMod( + useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp()); + defineFixed(lir, ins, LAllocation(AnyRegister(edx))); + assignSafepoint(lir, ins); +} + +void LIRGenerator::visitSubstr(MSubstr* ins) { + // Due to lack of registers on x86, we reuse the string register as + // temporary. As a result we only need two temporary registers and take a + // bogus temporary as fifth argument. + LSubstr* lir = new (alloc()) + LSubstr(useRegister(ins->string()), useRegister(ins->begin()), + useRegister(ins->length()), temp(), LDefinition::BogusTemp(), + tempByteOpRegister()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32); + + LDefinition temp = tempDouble(); + defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), temp), ins); +} + +void LIRGeneratorX86::lowerWasmBuiltinTruncateToInt64( + MWasmBuiltinTruncateToInt64* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} + +void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Int64); + MOZ_ASSERT(IsFloatingPointType(ins->type())); + + LDefinition maybeTemp = + (ins->isUnsigned() && + ((ins->type() == MIRType::Double && AssemblerX86Shared::HasSSE3()) || + ins->type() == MIRType::Float32)) + ? temp() + : LDefinition::BogusTemp(); + + define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp), + ins); +} + +void LIRGeneratorX86::lowerBuiltinInt64ToFloatingPoint( + MBuiltinInt64ToFloatingPoint* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} + +void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) { + if (ins->isUnsigned()) { + defineInt64(new (alloc()) + LExtendInt32ToInt64(useRegisterAtStart(ins->input())), + ins); + } else { + LExtendInt32ToInt64* lir = + new (alloc()) LExtendInt32ToInt64(useFixedAtStart(ins->input(), eax)); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); + } +} + +void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) { + // Here we'll end up using cdq which requires input and output in (edx,eax). + LSignExtendInt64* lir = new (alloc()) LSignExtendInt64( + useInt64FixedAtStart(ins->input(), Register64(edx, eax))); + defineInt64Fixed(lir, ins, + LInt64Allocation(LAllocation(AnyRegister(edx)), + LAllocation(AnyRegister(eax)))); +} + +// On x86 we specialize the only cases where compare is {U,}Int32 and select +// is {U,}Int32. +bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect( + MCompare::CompareType compTy, MIRType insTy) { + return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 || + compTy == MCompare::Compare_UInt32); +} + +void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins, + MDefinition* lhs, + MDefinition* rhs, + MCompare::CompareType compTy, + JSOp jsop) { + MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type())); + auto* lir = new (alloc()) LWasmCompareAndSelect( + useRegister(lhs), useAny(rhs), compTy, jsop, + useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr())); + defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex); +} diff --git a/js/src/jit/x86/Lowering-x86.h b/js/src/jit/x86/Lowering-x86.h new file mode 100644 index 0000000000..b82109981e --- /dev/null +++ b/js/src/jit/x86/Lowering-x86.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_Lowering_x86_h +#define jit_x86_Lowering_x86_h + +#include "jit/x86-shared/Lowering-x86-shared.h" + +namespace js { +namespace jit { + +class LIRGeneratorX86 : public LIRGeneratorX86Shared { + protected: + LIRGeneratorX86(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph) + : LIRGeneratorX86Shared(gen, graph, lirGraph) {} + + // Returns a box allocation with type set to reg1 and payload set to reg2. + LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2, + bool useAtStart = false); + + // It's a trap! On x86, the 1-byte store can only use one of + // {al,bl,cl,dl,ah,bh,ch,dh}. That means if the register allocator + // gives us one of {edi,esi,ebp,esp}, we're out of luck. (The formatter + // will assert on us.) Ideally, we'd just ask the register allocator to + // give us one of {al,bl,cl,dl}. For now, just useFixed(al). + LAllocation useByteOpRegister(MDefinition* mir); + LAllocation useByteOpRegisterAtStart(MDefinition* mir); + LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir); + LDefinition tempByteOpRegister(); + + inline LDefinition tempToUnbox() { return LDefinition::BogusTemp(); } + + bool needTempForPostBarrier() { return true; } + + void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, + size_t lirIndex); + + void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, + size_t lirIndex); + void defineInt64Phi(MPhi* phi, size_t lirIndex); + + void lowerForALUInt64(LInstructionHelper* ins, + MDefinition* mir, MDefinition* input); + void lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); + void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs, + MDefinition* rhs); + + void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins); + void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins); + void lowerDivI64(MDiv* div); + void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div); + void lowerModI64(MMod* mod); + void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod); + void lowerUDivI64(MDiv* div); + void lowerUModI64(MMod* mod); + + void lowerBigIntDiv(MBigIntDiv* ins); + void lowerBigIntMod(MBigIntMod* ins); + + void lowerAtomicLoad64(MLoadUnboxedScalar* ins); + void lowerAtomicStore64(MStoreUnboxedScalar* ins); + + void lowerPhi(MPhi* phi); + + public: + static bool allowTypedElementHoleCheck() { return true; } +}; + +typedef LIRGeneratorX86 LIRGeneratorSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_Lowering_x86_h */ diff --git a/js/src/jit/x86/MacroAssembler-x86-inl.h b/js/src/jit/x86/MacroAssembler-x86-inl.h new file mode 100644 index 0000000000..66050cc1b5 --- /dev/null +++ b/js/src/jit/x86/MacroAssembler-x86-inl.h @@ -0,0 +1,1386 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_MacroAssembler_x86_inl_h +#define jit_x86_MacroAssembler_x86_inl_h + +#include "jit/x86/MacroAssembler-x86.h" + +#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h" + +namespace js { +namespace jit { + +//{{{ check_macroassembler_style + +void MacroAssembler::move64(Imm64 imm, Register64 dest) { + move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low); + move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high); +} + +void MacroAssembler::move64(Register64 src, Register64 dest) { + movl(src.low, dest.low); + movl(src.high, dest.high); +} + +void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) { + ScratchDoubleScope scratch(*this); + + if (Assembler::HasSSE41()) { + vmovd(src, dest.low); + vpextrd(1, src, dest.high); + } else { + vmovd(src, dest.low); + moveDouble(src, scratch); + vpsrldq(Imm32(4), scratch, scratch); + vmovd(scratch, dest.high); + } +} + +void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) { + if (Assembler::HasSSE41()) { + vmovd(src.low, dest); + vpinsrd(1, src.high, dest, dest); + } else { + ScratchDoubleScope fpscratch(*this); + vmovd(src.low, dest); + vmovd(src.high, fpscratch); + vunpcklps(fpscratch, dest, dest); + } +} + +void MacroAssembler::move64To32(Register64 src, Register dest) { + if (src.low != dest) { + movl(src.low, dest); + } +} + +void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) { + if (src != dest.low) { + movl(src, dest.low); + } + movl(Imm32(0), dest.high); +} + +void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) { + move8SignExtend(src, dest.low); + if (dest.low == eax && dest.high == edx) { + masm.cdq(); + } else { + movl(dest.low, dest.high); + sarl(Imm32(31), dest.high); + } +} + +void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) { + move16SignExtend(src, dest.low); + if (dest.low == eax && dest.high == edx) { + masm.cdq(); + } else { + movl(dest.low, dest.high); + sarl(Imm32(31), dest.high); + } +} + +void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) { + if (src != dest.low) { + movl(src, dest.low); + } + if (dest.low == eax && dest.high == edx) { + masm.cdq(); + } else { + movl(dest.low, dest.high); + sarl(Imm32(31), dest.high); + } +} + +void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) { + movl(src, dest); +} + +void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) { + movl(src, dest); +} + +// =============================================================== +// Load instructions + +void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) { + load32(src, dest); +} + +// =============================================================== +// Logical functions + +void MacroAssembler::notPtr(Register reg) { notl(reg); } + +void MacroAssembler::andPtr(Register src, Register dest) { andl(src, dest); } + +void MacroAssembler::andPtr(Imm32 imm, Register dest) { andl(imm, dest); } + +void MacroAssembler::and64(Imm64 imm, Register64 dest) { + if (imm.low().value != int32_t(0xFFFFFFFF)) { + andl(imm.low(), dest.low); + } + if (imm.hi().value != int32_t(0xFFFFFFFF)) { + andl(imm.hi(), dest.high); + } +} + +void MacroAssembler::or64(Imm64 imm, Register64 dest) { + if (imm.low().value != 0) { + orl(imm.low(), dest.low); + } + if (imm.hi().value != 0) { + orl(imm.hi(), dest.high); + } +} + +void MacroAssembler::xor64(Imm64 imm, Register64 dest) { + if (imm.low().value != 0) { + xorl(imm.low(), dest.low); + } + if (imm.hi().value != 0) { + xorl(imm.hi(), dest.high); + } +} + +void MacroAssembler::orPtr(Register src, Register dest) { orl(src, dest); } + +void MacroAssembler::orPtr(Imm32 imm, Register dest) { orl(imm, dest); } + +void MacroAssembler::and64(Register64 src, Register64 dest) { + andl(src.low, dest.low); + andl(src.high, dest.high); +} + +void MacroAssembler::or64(Register64 src, Register64 dest) { + orl(src.low, dest.low); + orl(src.high, dest.high); +} + +void MacroAssembler::xor64(Register64 src, Register64 dest) { + xorl(src.low, dest.low); + xorl(src.high, dest.high); +} + +void MacroAssembler::xorPtr(Register src, Register dest) { xorl(src, dest); } + +void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorl(imm, dest); } + +// =============================================================== +// Swap instructions + +void MacroAssembler::byteSwap64(Register64 reg) { + bswapl(reg.low); + bswapl(reg.high); + xchgl(reg.low, reg.high); +} + +// =============================================================== +// Arithmetic functions + +void MacroAssembler::addPtr(Register src, Register dest) { addl(src, dest); } + +void MacroAssembler::addPtr(Imm32 imm, Register dest) { addl(imm, dest); } + +void MacroAssembler::addPtr(ImmWord imm, Register dest) { + addl(Imm32(imm.value), dest); +} + +void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { + addl(imm, Operand(dest)); +} + +void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) { + addl(imm, Operand(dest)); +} + +void MacroAssembler::addPtr(const Address& src, Register dest) { + addl(Operand(src), dest); +} + +void MacroAssembler::add64(Register64 src, Register64 dest) { + addl(src.low, dest.low); + adcl(src.high, dest.high); +} + +void MacroAssembler::add64(Imm32 imm, Register64 dest) { + addl(imm, dest.low); + adcl(Imm32(0), dest.high); +} + +void MacroAssembler::add64(Imm64 imm, Register64 dest) { + if (imm.low().value == 0) { + addl(imm.hi(), dest.high); + return; + } + addl(imm.low(), dest.low); + adcl(imm.hi(), dest.high); +} + +void MacroAssembler::addConstantDouble(double d, FloatRegister dest) { + Double* dbl = getDouble(d); + if (!dbl) { + return; + } + masm.vaddsd_mr(nullptr, dest.encoding(), dest.encoding()); + propagateOOM(dbl->uses.append(CodeOffset(masm.size()))); +} + +CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) { + moveStackPtrTo(dest); + addlWithPatch(Imm32(0), dest); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) { + patchAddl(offset, -imm.value); +} + +void MacroAssembler::subPtr(Register src, Register dest) { subl(src, dest); } + +void MacroAssembler::subPtr(Register src, const Address& dest) { + subl(src, Operand(dest)); +} + +void MacroAssembler::subPtr(Imm32 imm, Register dest) { subl(imm, dest); } + +void MacroAssembler::subPtr(const Address& addr, Register dest) { + subl(Operand(addr), dest); +} + +void MacroAssembler::sub64(Register64 src, Register64 dest) { + subl(src.low, dest.low); + sbbl(src.high, dest.high); +} + +void MacroAssembler::sub64(Imm64 imm, Register64 dest) { + if (imm.low().value == 0) { + subl(imm.hi(), dest.high); + return; + } + subl(imm.low(), dest.low); + sbbl(imm.hi(), dest.high); +} + +void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) { + // Preserve edx:eax, unless they're the destination register. + if (edx != dest) { + push(edx); + } + if (eax != dest) { + push(eax); + } + + if (src != eax) { + // Compute edx:eax := eax ∗ src + movl(imm, eax); + mull(src); + } else { + // Compute edx:eax := eax ∗ edx + movl(imm, edx); + mull(edx); + } + + // Move the high word from edx into |dest|. + if (edx != dest) { + movl(edx, dest); + } + + // Restore edx:eax. + if (eax != dest) { + pop(eax); + } + if (edx != dest) { + pop(edx); + } +} + +void MacroAssembler::mulPtr(Register rhs, Register srcDest) { + imull(rhs, srcDest); +} + +// Note: this function clobbers eax and edx. +void MacroAssembler::mul64(Imm64 imm, const Register64& dest) { + // LOW32 = LOW(LOW(dest) * LOW(imm)); + // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits] + // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits] + // + HIGH(LOW(dest) * LOW(imm)) [carry] + + MOZ_ASSERT(dest.low != eax && dest.low != edx); + MOZ_ASSERT(dest.high != eax && dest.high != edx); + + // HIGH(dest) = LOW(HIGH(dest) * LOW(imm)); + movl(Imm32(imm.value & 0xFFFFFFFFL), edx); + imull(edx, dest.high); + + // edx:eax = LOW(dest) * LOW(imm); + movl(Imm32(imm.value & 0xFFFFFFFFL), edx); + movl(dest.low, eax); + mull(edx); + + // HIGH(dest) += edx; + addl(edx, dest.high); + + // HIGH(dest) += LOW(LOW(dest) * HIGH(imm)); + if (((imm.value >> 32) & 0xFFFFFFFFL) == 5) { + leal(Operand(dest.low, dest.low, TimesFour), edx); + } else { + MOZ_CRASH("Unsupported imm"); + } + addl(edx, dest.high); + + // LOW(dest) = eax; + movl(eax, dest.low); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest, + const Register temp) { + // LOW32 = LOW(LOW(dest) * LOW(src)); (1) + // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2) + // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3) + // + HIGH(LOW(dest) * LOW(src)) [carry] (4) + + MOZ_ASSERT(dest == Register64(edx, eax)); + MOZ_ASSERT(temp != edx && temp != eax); + + movl(dest.low, temp); + + // Compute mul64 + imull(imm.low(), dest.high); // (2) + imull(imm.hi(), temp); // (3) + addl(dest.high, temp); + movl(imm.low(), dest.high); + mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax + // (dest_hi:dest_lo) + addl(temp, dest.high); +} + +void MacroAssembler::mul64(const Register64& src, const Register64& dest, + const Register temp) { + // LOW32 = LOW(LOW(dest) * LOW(src)); (1) + // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2) + // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3) + // + HIGH(LOW(dest) * LOW(src)) [carry] (4) + + MOZ_ASSERT(dest == Register64(edx, eax)); + MOZ_ASSERT(src != Register64(edx, eax) && src != Register64(eax, edx)); + + // Make sure the rhs.high isn't the dest.high register anymore. + // This saves us from doing other register moves. + movl(dest.low, temp); + + // Compute mul64 + imull(src.low, dest.high); // (2) + imull(src.high, temp); // (3) + addl(dest.high, temp); + movl(src.low, dest.high); + mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax + // (dest_hi:dest_lo) + addl(temp, dest.high); +} + +void MacroAssembler::mulBy3(Register src, Register dest) { + lea(Operand(src, src, TimesTwo), dest); +} + +void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, + FloatRegister dest) { + movl(imm, temp); + vmulsd(Operand(temp, 0), dest, dest); +} + +void MacroAssembler::inc64(AbsoluteAddress dest) { + addl(Imm32(1), Operand(dest)); + Label noOverflow; + j(NonZero, &noOverflow); + addl(Imm32(1), Operand(dest.offset(4))); + bind(&noOverflow); +} + +void MacroAssembler::neg64(Register64 reg) { + negl(reg.low); + adcl(Imm32(0), reg.high); + negl(reg.high); +} + +void MacroAssembler::negPtr(Register reg) { negl(reg); } + +// =============================================================== +// Shift functions + +void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 32); + shll(imm, dest); +} + +void MacroAssembler::lshiftPtr(Register shift, Register srcDest) { + if (HasBMI2()) { + shlxl(srcDest, shift, srcDest); + return; + } + MOZ_ASSERT(shift == ecx); + shll_cl(srcDest); +} + +void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + if (imm.value < 32) { + shldl(imm, dest.low, dest.high); + shll(imm, dest.low); + return; + } + + mov(dest.low, dest.high); + shll(Imm32(imm.value & 0x1f), dest.high); + xorl(dest.low, dest.low); +} + +void MacroAssembler::lshift64(Register shift, Register64 srcDest) { + MOZ_ASSERT(shift == ecx); + MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx); + + Label done; + + shldl_cl(srcDest.low, srcDest.high); + shll_cl(srcDest.low); + + testl(Imm32(0x20), ecx); + j(Condition::Equal, &done); + + // 32 - 63 bit shift + movl(srcDest.low, srcDest.high); + xorl(srcDest.low, srcDest.low); + + bind(&done); +} + +void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 32); + shrl(imm, dest); +} + +void MacroAssembler::rshiftPtr(Register shift, Register srcDest) { + if (HasBMI2()) { + shrxl(srcDest, shift, srcDest); + return; + } + MOZ_ASSERT(shift == ecx); + shrl_cl(srcDest); +} + +void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + if (imm.value < 32) { + shrdl(imm, dest.high, dest.low); + shrl(imm, dest.high); + return; + } + + movl(dest.high, dest.low); + shrl(Imm32(imm.value & 0x1f), dest.low); + xorl(dest.high, dest.high); +} + +void MacroAssembler::rshift64(Register shift, Register64 srcDest) { + MOZ_ASSERT(shift == ecx); + MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx); + + Label done; + + shrdl_cl(srcDest.high, srcDest.low); + shrl_cl(srcDest.high); + + testl(Imm32(0x20), ecx); + j(Condition::Equal, &done); + + // 32 - 63 bit shift + movl(srcDest.high, srcDest.low); + xorl(srcDest.high, srcDest.high); + + bind(&done); +} + +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 32); + sarl(imm, dest); +} + +void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + if (imm.value < 32) { + shrdl(imm, dest.high, dest.low); + sarl(imm, dest.high); + return; + } + + movl(dest.high, dest.low); + sarl(Imm32(imm.value & 0x1f), dest.low); + sarl(Imm32(0x1f), dest.high); +} + +void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) { + MOZ_ASSERT(shift == ecx); + MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx); + + Label done; + + shrdl_cl(srcDest.high, srcDest.low); + sarl_cl(srcDest.high); + + testl(Imm32(0x20), ecx); + j(Condition::Equal, &done); + + // 32 - 63 bit shift + movl(srcDest.high, srcDest.low); + sarl(Imm32(0x1f), srcDest.high); + + bind(&done); +} + +// =============================================================== +// Rotation functions + +void MacroAssembler::rotateLeft64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + MOZ_ASSERT(count == ecx, "defineFixed(ecx)"); + + Label done; + + movl(dest.high, temp); + shldl_cl(dest.low, dest.high); + shldl_cl(temp, dest.low); + + testl(Imm32(0x20), count); + j(Condition::Equal, &done); + xchgl(dest.high, dest.low); + + bind(&done); +} + +void MacroAssembler::rotateRight64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + MOZ_ASSERT(count == ecx, "defineFixed(ecx)"); + + Label done; + + movl(dest.high, temp); + shrdl_cl(dest.low, dest.high); + shrdl_cl(temp, dest.low); + + testl(Imm32(0x20), count); + j(Condition::Equal, &done); + xchgl(dest.high, dest.low); + + bind(&done); +} + +void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + + int32_t amount = count.value & 0x3f; + if ((amount & 0x1f) != 0) { + movl(dest.high, temp); + shldl(Imm32(amount & 0x1f), dest.low, dest.high); + shldl(Imm32(amount & 0x1f), temp, dest.low); + } + + if (!!(amount & 0x20)) { + xchgl(dest.high, dest.low); + } +} + +void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + + int32_t amount = count.value & 0x3f; + if ((amount & 0x1f) != 0) { + movl(dest.high, temp); + shrdl(Imm32(amount & 0x1f), dest.low, dest.high); + shrdl(Imm32(amount & 0x1f), temp, dest.low); + } + + if (!!(amount & 0x20)) { + xchgl(dest.high, dest.low); + } +} + +// =============================================================== +// Bit counting functions + +void MacroAssembler::clz64(Register64 src, Register dest) { + if (AssemblerX86Shared::HasLZCNT()) { + Label nonzero, zero; + + testl(src.high, src.high); + j(Assembler::Zero, &zero); + + lzcntl(src.high, dest); + jump(&nonzero); + + bind(&zero); + lzcntl(src.low, dest); + addl(Imm32(32), dest); + + bind(&nonzero); + return; + } + + // Because |dest| may be equal to |src.low|, we rely on BSR not modifying its + // output when the input is zero. AMD ISA documents BSR not modifying the + // output and current Intel CPUs follow AMD. + + Label nonzero, zero; + + bsrl(src.high, dest); + j(Assembler::Zero, &zero); + orl(Imm32(32), dest); + jump(&nonzero); + + bind(&zero); + bsrl(src.low, dest); + j(Assembler::NonZero, &nonzero); + movl(Imm32(0x7F), dest); + + bind(&nonzero); + xorl(Imm32(0x3F), dest); +} + +void MacroAssembler::ctz64(Register64 src, Register dest) { + if (AssemblerX86Shared::HasBMI1()) { + Label nonzero, zero; + + testl(src.low, src.low); + j(Assembler::Zero, &zero); + + tzcntl(src.low, dest); + jump(&nonzero); + + bind(&zero); + tzcntl(src.high, dest); + addl(Imm32(32), dest); + + bind(&nonzero); + return; + } + + // Because |dest| may be equal to |src.low|, we rely on BSF not modifying its + // output when the input is zero. AMD ISA documents BSF not modifying the + // output and current Intel CPUs follow AMD. + + Label done, nonzero; + + bsfl(src.low, dest); + j(Assembler::NonZero, &done); + bsfl(src.high, dest); + j(Assembler::NonZero, &nonzero); + movl(Imm32(64), dest); + jump(&done); + + bind(&nonzero); + orl(Imm32(32), dest); + + bind(&done); +} + +void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) { + // The tmp register is only needed if there is no native POPCNT. + + MOZ_ASSERT(src.low != tmp && src.high != tmp); + MOZ_ASSERT(dest.low != tmp && dest.high != tmp); + + if (dest.low != src.high) { + popcnt32(src.low, dest.low, tmp); + popcnt32(src.high, dest.high, tmp); + } else { + MOZ_ASSERT(dest.high != src.high); + popcnt32(src.low, dest.high, tmp); + popcnt32(src.high, dest.low, tmp); + } + addl(dest.high, dest.low); + xorl(dest.high, dest.high); +} + +// =============================================================== +// Condition functions + +void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs, + Register dest) { + Label success, done; + + branch64(cond, lhs, rhs, &success); + move32(Imm32(0), dest); + jump(&done); + bind(&success); + move32(Imm32(1), dest); + bind(&done); +} + +template +void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) { + cmpPtr(lhs, rhs); + emitSet(cond, dest); +} + +// =============================================================== +// Branch functions + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + cmp32(Operand(lhs), rhs); + j(cond, label); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + cmp32(Operand(lhs), rhs); + j(cond, label); +} + +void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs, + Imm32 rhs, Label* label) { + cmpl(rhs, lhs); + j(cond, label); +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, + Label* success, Label* fail) { + bool fallthrough = false; + Label fallthroughLabel; + + if (!fail) { + fail = &fallthroughLabel; + fallthrough = true; + } + + switch (cond) { + case Assembler::Equal: + branch32(Assembler::NotEqual, lhs.low, val.low(), fail); + branch32(Assembler::Equal, lhs.high, val.hi(), success); + if (!fallthrough) { + jump(fail); + } + break; + case Assembler::NotEqual: + branch32(Assembler::NotEqual, lhs.low, val.low(), success); + branch32(Assembler::NotEqual, lhs.high, val.hi(), success); + if (!fallthrough) { + jump(fail); + } + break; + case Assembler::LessThan: + case Assembler::LessThanOrEqual: + case Assembler::GreaterThan: + case Assembler::GreaterThanOrEqual: + case Assembler::Below: + case Assembler::BelowOrEqual: + case Assembler::Above: + case Assembler::AboveOrEqual: { + Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond); + Assembler::Condition cond2 = + Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond)); + Assembler::Condition cond3 = Assembler::UnsignedCondition(cond); + + cmp32(lhs.high, val.hi()); + j(cond1, success); + j(cond2, fail); + cmp32(lhs.low, val.low()); + j(cond3, success); + if (!fallthrough) { + jump(fail); + } + break; + } + default: + MOZ_CRASH("Condition code not supported"); + break; + } + + if (fallthrough) { + bind(fail); + } +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, + Label* success, Label* fail) { + bool fallthrough = false; + Label fallthroughLabel; + + if (!fail) { + fail = &fallthroughLabel; + fallthrough = true; + } + + switch (cond) { + case Assembler::Equal: + branch32(Assembler::NotEqual, lhs.low, rhs.low, fail); + branch32(Assembler::Equal, lhs.high, rhs.high, success); + if (!fallthrough) { + jump(fail); + } + break; + case Assembler::NotEqual: + branch32(Assembler::NotEqual, lhs.low, rhs.low, success); + branch32(Assembler::NotEqual, lhs.high, rhs.high, success); + if (!fallthrough) { + jump(fail); + } + break; + case Assembler::LessThan: + case Assembler::LessThanOrEqual: + case Assembler::GreaterThan: + case Assembler::GreaterThanOrEqual: + case Assembler::Below: + case Assembler::BelowOrEqual: + case Assembler::Above: + case Assembler::AboveOrEqual: { + Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond); + Assembler::Condition cond2 = + Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond)); + Assembler::Condition cond3 = Assembler::UnsignedCondition(cond); + + cmp32(lhs.high, rhs.high); + j(cond1, success); + j(cond2, fail); + cmp32(lhs.low, rhs.low); + j(cond3, success); + if (!fallthrough) { + jump(fail); + } + break; + } + default: + MOZ_CRASH("Condition code not supported"); + break; + } + + if (fallthrough) { + bind(fail); + } +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + + Label done; + + if (cond == Assembler::Equal) { + branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done); + } else { + branch32(Assembler::NotEqual, lhs, val.firstHalf(), label); + } + branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), + val.secondHalf(), label); + + bind(&done); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + Register64 rhs, Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + + Label done; + + if (cond == Assembler::Equal) { + branch32(Assembler::NotEqual, lhs, rhs.low, &done); + } else { + branch32(Assembler::NotEqual, lhs, rhs.low, label); + } + branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), rhs.high, + label); + + bind(&done); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + const Address& rhs, Register scratch, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + MOZ_ASSERT(lhs.base != scratch); + MOZ_ASSERT(rhs.base != scratch); + + Label done; + + load32(rhs, scratch); + if (cond == Assembler::Equal) { + branch32(Assembler::NotEqual, lhs, scratch, &done); + } else { + branch32(Assembler::NotEqual, lhs, scratch, label); + } + + load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch); + branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch, + label); + + bind(&done); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + branchPtrImpl(cond, lhs, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + ImmWord rhs, Label* label) { + branchPtrImpl(cond, lhs, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, + Register rhs, Label* label) { + cmpl(rhs, lhs); + j(cond, label); +} + +void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, + Register rhs, Label* label) { + branchPtr(cond, lhs, rhs, label); +} + +void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src, + Register dest, Label* fail) { + branchTruncateFloat32ToInt32(src, dest, fail); +} + +void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + branchTruncateFloat32ToInt32(src, dest, fail); +} + +void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src, + Register dest, Label* fail) { + vcvttss2si(src, dest); + + // vcvttss2si returns 0x80000000 on failure. Test for it by + // subtracting 1 and testing overflow (this permits the use of a + // smaller immediate field). + cmp32(dest, Imm32(1)); + j(Assembler::Overflow, fail); +} + +void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest, + Label* fail) { + branchTruncateDoubleToInt32(src, dest, fail); +} + +void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + // TODO: X64 supports supports integers up till 64bits. Here we only support + // 32bits, before failing. Implementing this for x86 might give a x86 kraken + // win. + branchTruncateDoubleToInt32(src, dest, fail); +} + +void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, + Register dest, Label* fail) { + vcvttsd2si(src, dest); + + // vcvttsd2si returns 0x80000000 on failure. Test for it by + // subtracting 1 and testing overflow (this permits the use of a + // smaller immediate field). + cmp32(dest, Imm32(1)); + j(Assembler::Overflow, fail); +} + +void MacroAssembler::branchAdd64(Condition cond, Imm64 imm, Register64 dest, + Label* label) { + add64(imm, dest); + j(cond, label); +} + +void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + test32(Operand(lhs), rhs); + j(cond, label); +} + +template +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, + Register64 rhs, Register temp, L label) { + if (cond == Assembler::Zero || cond == Assembler::NonZero) { + MOZ_ASSERT(lhs.low == rhs.low); + MOZ_ASSERT(lhs.high == rhs.high); + movl(lhs.low, temp); + orl(lhs.high, temp); + branchTestPtr(cond, temp, temp, label); + } else if (cond == Assembler::Signed || cond == Assembler::NotSigned) { + branchTest32(cond, lhs.high, rhs.high, label); + } else { + MOZ_CRASH("Unsupported condition"); + } +} + +void MacroAssembler::branchTestBooleanTruthy(bool truthy, + const ValueOperand& value, + Label* label) { + test32(value.payloadReg(), value.payloadReg()); + j(truthy ? NonZero : Zero, label); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, + JSWhyMagic why, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label notMagic; + if (cond == Assembler::Equal) { + branchTestMagic(Assembler::NotEqual, valaddr, ¬Magic); + } else { + branchTestMagic(Assembler::NotEqual, valaddr, label); + } + + branch32(cond, ToPayload(valaddr), Imm32(why), label); + bind(¬Magic); +} + +void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs, + const ValueOperand& rhs, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label notSameValue; + if (cond == Assembler::Equal) { + branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), ¬SameValue); + } else { + branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), label); + } + + branch32(cond, ToPayload(lhs), rhs.payloadReg(), label); + bind(¬SameValue); +} + +void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) { + jmp(Operand(addr)); +} + +void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + cmp32(lhs, rhs); + cmovCCl(cond, Operand(src), dest); +} + +void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, + const Address& src, Register dest) { + cmp32(lhs, rhs); + cmovCCl(cond, Operand(src), dest); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + cmp32Move32(cond, lhs, rhs, src, dest); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + cmp32Move32(cond, lhs, rhs, src, dest); +} + +void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr, + Imm32 mask, const Address& src, + Register dest) { + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero); + test32(addr, mask); + cmovCCl(cond, Operand(src), dest); +} + +void MacroAssembler::test32MovePtr(Condition cond, const Address& addr, + Imm32 mask, Register src, Register dest) { + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero); + test32(addr, mask); + cmovCCl(cond, Operand(src), dest); +} + +void MacroAssembler::spectreMovePtr(Condition cond, Register src, + Register dest) { + cmovCCl(cond, Operand(src), dest); +} + +void MacroAssembler::spectreBoundsCheck32(Register index, const Operand& length, + Register maybeScratch, + Label* failure) { + Label failurePopValue; + bool pushedValue = false; + if (JitOptions.spectreIndexMasking) { + if (maybeScratch == InvalidReg) { + push(Imm32(0)); + pushedValue = true; + } else { + move32(Imm32(0), maybeScratch); + } + } + + cmp32(index, length); + j(Assembler::AboveOrEqual, pushedValue ? &failurePopValue : failure); + + if (JitOptions.spectreIndexMasking) { + if (maybeScratch == InvalidReg) { + Label done; + cmovCCl(Assembler::AboveOrEqual, Operand(StackPointer, 0), index); + lea(Operand(StackPointer, sizeof(void*)), StackPointer); + jump(&done); + + bind(&failurePopValue); + lea(Operand(StackPointer, sizeof(void*)), StackPointer); + jump(failure); + + bind(&done); + } else { + cmovCCl(Assembler::AboveOrEqual, maybeScratch, index); + } + } +} + +void MacroAssembler::spectreBoundsCheck32(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(length != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + spectreBoundsCheck32(index, Operand(length), maybeScratch, failure); +} + +void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(index != length.base); + MOZ_ASSERT(length.base != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + spectreBoundsCheck32(index, Operand(length), maybeScratch, failure); +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length, + Register maybeScratch, + Label* failure) { + spectreBoundsCheck32(index, length, maybeScratch, failure); +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, + const Address& length, + Register maybeScratch, + Label* failure) { + spectreBoundsCheck32(index, length, maybeScratch, failure); +} + +// ======================================================================== +// SIMD + +void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src, + Register64 dest) { + if (lane == 0) { + vmovd(src, dest.low); + } else { + vpextrd(2 * lane, src, dest.low); + } + vpextrd(2 * lane + 1, src, dest.high); +} + +void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs, + FloatRegister lhsDest) { + vpinsrd(2 * lane, rhs.low, lhsDest, lhsDest); + vpinsrd(2 * lane + 1, rhs.high, lhsDest, lhsDest); +} + +void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs, + Register64 rhs, FloatRegister dest) { + vpinsrd(2 * lane, rhs.low, lhs, dest); + vpinsrd(2 * lane + 1, rhs.high, dest, dest); +} + +void MacroAssembler::splatX2(Register64 src, FloatRegister dest) { + vmovd(src.low, dest); + vpinsrd(1, src.high, dest, dest); + vpunpcklqdq(dest, dest, dest); +} + +// ======================================================================== +// Truncate floating point. + +void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest, + Register temp, + FloatRegister floatTemp) { + Label done; + + loadFloat32(src, floatTemp); + + truncateFloat32ToInt64(src, dest, temp); + + // For unsigned conversion the case of [INT64, UINT64] needs to get handle + // seperately. + load32(HighWord(dest), temp); + branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done); + + // Move the value inside INT64 range. + storeFloat32(floatTemp, dest); + loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp); + vaddss(Operand(dest), floatTemp, floatTemp); + storeFloat32(floatTemp, dest); + truncateFloat32ToInt64(dest, dest, temp); + + load32(HighWord(dest), temp); + orl(Imm32(0x80000000), temp); + store32(temp, HighWord(dest)); + + bind(&done); +} + +void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest, + Register temp, + FloatRegister floatTemp) { + Label done; + + loadDouble(src, floatTemp); + + truncateDoubleToInt64(src, dest, temp); + + // For unsigned conversion the case of [INT64, UINT64] needs to get handle + // seperately. + load32(HighWord(dest), temp); + branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done); + + // Move the value inside INT64 range. + storeDouble(floatTemp, dest); + loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp); + vaddsd(Operand(dest), floatTemp, floatTemp); + storeDouble(floatTemp, dest); + truncateDoubleToInt64(dest, dest, temp); + + load32(HighWord(dest), temp); + orl(Imm32(0x80000000), temp); + store32(temp, HighWord(dest)); + + bind(&done); +} + +template +void MacroAssemblerX86::fallibleUnboxPtrImpl(const T& src, Register dest, + JSValueType type, Label* fail) { + switch (type) { + case JSVAL_TYPE_OBJECT: + asMasm().branchTestObject(Assembler::NotEqual, src, fail); + break; + case JSVAL_TYPE_STRING: + asMasm().branchTestString(Assembler::NotEqual, src, fail); + break; + case JSVAL_TYPE_SYMBOL: + asMasm().branchTestSymbol(Assembler::NotEqual, src, fail); + break; + case JSVAL_TYPE_BIGINT: + asMasm().branchTestBigInt(Assembler::NotEqual, src, fail); + break; + default: + MOZ_CRASH("Unexpected type"); + } + unboxNonDouble(src, dest, type); +} + +void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(src, dest, type, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(src, dest, type, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(src, dest, type, fail); +} + +//}}} check_macroassembler_style +// =============================================================== + +// Note: this function clobbers the source register. +void MacroAssemblerX86::convertUInt32ToDouble(Register src, + FloatRegister dest) { + // src is [0, 2^32-1] + subl(Imm32(0x80000000), src); + + // Now src is [-2^31, 2^31-1] - int range, but not the same value. + convertInt32ToDouble(src, dest); + + // dest is now a double with the int range. + // correct the double value by adding 0x80000000. + asMasm().addConstantDouble(2147483648.0, dest); +} + +// Note: this function clobbers the source register. +void MacroAssemblerX86::convertUInt32ToFloat32(Register src, + FloatRegister dest) { + convertUInt32ToDouble(src, dest); + convertDoubleToFloat32(dest, dest); +} + +void MacroAssemblerX86::unboxValue(const ValueOperand& src, AnyRegister dest, + JSValueType) { + if (dest.isFloat()) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src.payloadReg(), dest.fpu()); + jump(&end); + bind(¬Int32); + unboxDouble(src, dest.fpu()); + bind(&end); + } else { + if (src.payloadReg() != dest.gpr()) { + movl(src.payloadReg(), dest.gpr()); + } + } +} + +template +void MacroAssemblerX86::loadInt32OrDouble(const T& src, FloatRegister dest) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(ToPayload(src), dest); + jump(&end); + bind(¬Int32); + loadDouble(src, dest); + bind(&end); +} + +template +void MacroAssemblerX86::loadUnboxedValue(const T& src, MIRType type, + AnyRegister dest) { + if (dest.isFloat()) { + loadInt32OrDouble(src, dest.fpu()); + } else { + movl(Operand(src), dest.gpr()); + } +} + +// If source is a double, load it into dest. If source is int32, +// convert it to double. Else, branch to failure. +void MacroAssemblerX86::ensureDouble(const ValueOperand& source, + FloatRegister dest, Label* failure) { + Label isDouble, done; + asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble); + asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure); + + convertInt32ToDouble(source.payloadReg(), dest); + jump(&done); + + bind(&isDouble); + unboxDouble(source, dest); + + bind(&done); +} + +} // namespace jit +} // namespace js + +#endif /* jit_x86_MacroAssembler_x86_inl_h */ diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp new file mode 100644 index 0000000000..c9d8acbd4d --- /dev/null +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -0,0 +1,1829 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86/MacroAssembler-x86.h" + +#include "mozilla/Alignment.h" +#include "mozilla/Casting.h" + +#include "jit/AtomicOp.h" +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/MacroAssembler.h" +#include "jit/MoveEmitter.h" +#include "util/Memory.h" +#include "vm/BigIntType.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" +#include "vm/StringType.h" + +#include "jit/MacroAssembler-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +void MacroAssemblerX86::loadConstantDouble(double d, FloatRegister dest) { + if (maybeInlineDouble(d, dest)) { + return; + } + Double* dbl = getDouble(d); + if (!dbl) { + return; + } + masm.vmovsd_mr(nullptr, dest.encoding()); + propagateOOM(dbl->uses.append(CodeOffset(masm.size()))); +} + +void MacroAssemblerX86::loadConstantFloat32(float f, FloatRegister dest) { + if (maybeInlineFloat(f, dest)) { + return; + } + Float* flt = getFloat(f); + if (!flt) { + return; + } + masm.vmovss_mr(nullptr, dest.encoding()); + propagateOOM(flt->uses.append(CodeOffset(masm.size()))); +} + +void MacroAssemblerX86::loadConstantSimd128Int(const SimdConstant& v, + FloatRegister dest) { + if (maybeInlineSimd128Int(v, dest)) { + return; + } + SimdData* i4 = getSimdData(v); + if (!i4) { + return; + } + masm.vmovdqa_mr(nullptr, dest.encoding()); + propagateOOM(i4->uses.append(CodeOffset(masm.size()))); +} + +void MacroAssemblerX86::loadConstantSimd128Float(const SimdConstant& v, + FloatRegister dest) { + if (maybeInlineSimd128Float(v, dest)) { + return; + } + SimdData* f4 = getSimdData(v); + if (!f4) { + return; + } + masm.vmovaps_mr(nullptr, dest.encoding()); + propagateOOM(f4->uses.append(CodeOffset(masm.size()))); +} + +void MacroAssemblerX86::vpPatchOpSimd128( + const SimdConstant& v, FloatRegister src, FloatRegister dest, + void (X86Encoding::BaseAssemblerX86::*op)( + const void* address, X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)) { + SimdData* val = getSimdData(v); + if (!val) { + return; + } + (masm.*op)(nullptr, src.encoding(), dest.encoding()); + propagateOOM(val->uses.append(CodeOffset(masm.size()))); +} + +void MacroAssemblerX86::vpPatchOpSimd128( + const SimdConstant& v, FloatRegister src, FloatRegister dest, + size_t (X86Encoding::BaseAssemblerX86::*op)( + const void* address, X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)) { + SimdData* val = getSimdData(v); + if (!val) { + return; + } + size_t patchOffsetFromEnd = + (masm.*op)(nullptr, src.encoding(), dest.encoding()); + propagateOOM(val->uses.append(CodeOffset(masm.size() - patchOffsetFromEnd))); +} + +void MacroAssemblerX86::vpaddbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddb_mr); +} + +void MacroAssemblerX86::vpaddwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddw_mr); +} + +void MacroAssemblerX86::vpadddSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddd_mr); +} + +void MacroAssemblerX86::vpaddqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddq_mr); +} + +void MacroAssemblerX86::vpsubbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubb_mr); +} + +void MacroAssemblerX86::vpsubwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubw_mr); +} + +void MacroAssemblerX86::vpsubdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubd_mr); +} + +void MacroAssemblerX86::vpsubqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubq_mr); +} + +void MacroAssemblerX86::vpmullwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmullw_mr); +} + +void MacroAssemblerX86::vpmulldSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmulld_mr); +} + +void MacroAssemblerX86::vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddsb_mr); +} + +void MacroAssemblerX86::vpaddusbSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddusb_mr); +} + +void MacroAssemblerX86::vpaddswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddsw_mr); +} + +void MacroAssemblerX86::vpadduswSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddusw_mr); +} + +void MacroAssemblerX86::vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubsb_mr); +} + +void MacroAssemblerX86::vpsubusbSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubusb_mr); +} + +void MacroAssemblerX86::vpsubswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubsw_mr); +} + +void MacroAssemblerX86::vpsubuswSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubusw_mr); +} + +void MacroAssemblerX86::vpminsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsb_mr); +} + +void MacroAssemblerX86::vpminubSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminub_mr); +} + +void MacroAssemblerX86::vpminswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsw_mr); +} + +void MacroAssemblerX86::vpminuwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminuw_mr); +} + +void MacroAssemblerX86::vpminsdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsd_mr); +} + +void MacroAssemblerX86::vpminudSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminud_mr); +} + +void MacroAssemblerX86::vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsb_mr); +} + +void MacroAssemblerX86::vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxub_mr); +} + +void MacroAssemblerX86::vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsw_mr); +} + +void MacroAssemblerX86::vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxuw_mr); +} + +void MacroAssemblerX86::vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsd_mr); +} + +void MacroAssemblerX86::vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxud_mr); +} + +void MacroAssemblerX86::vpandSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpand_mr); +} + +void MacroAssemblerX86::vpxorSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpxor_mr); +} + +void MacroAssemblerX86::vporSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpor_mr); +} + +void MacroAssemblerX86::vaddpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vaddps_mr); +} + +void MacroAssemblerX86::vaddpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vaddpd_mr); +} + +void MacroAssemblerX86::vsubpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vsubps_mr); +} + +void MacroAssemblerX86::vsubpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vsubpd_mr); +} + +void MacroAssemblerX86::vdivpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vdivps_mr); +} + +void MacroAssemblerX86::vdivpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vdivpd_mr); +} + +void MacroAssemblerX86::vmulpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vmulps_mr); +} + +void MacroAssemblerX86::vmulpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vmulpd_mr); +} + +void MacroAssemblerX86::vandpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vandpd_mr); +} + +void MacroAssemblerX86::vminpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vminpd_mr); +} + +void MacroAssemblerX86::vpacksswbSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpacksswb_mr); +} + +void MacroAssemblerX86::vpackuswbSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackuswb_mr); +} + +void MacroAssemblerX86::vpackssdwSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackssdw_mr); +} + +void MacroAssemblerX86::vpackusdwSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackusdw_mr); +} + +void MacroAssemblerX86::vpunpckldqSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpunpckldq_mr); +} + +void MacroAssemblerX86::vunpcklpsSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vunpcklps_mr); +} + +void MacroAssemblerX86::vpshufbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpshufb_mr); +} + +void MacroAssemblerX86::vptestSimd128(const SimdConstant& v, + FloatRegister lhs) { + vpPatchOpSimd128(v, lhs, &X86Encoding::BaseAssemblerX86::vptest_mr); +} + +void MacroAssemblerX86::vpmaddwdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaddwd_mr); +} + +void MacroAssemblerX86::vpcmpeqbSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqb_mr); +} + +void MacroAssemblerX86::vpcmpgtbSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtb_mr); +} + +void MacroAssemblerX86::vpcmpeqwSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqw_mr); +} + +void MacroAssemblerX86::vpcmpgtwSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtw_mr); +} + +void MacroAssemblerX86::vpcmpeqdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqd_mr); +} + +void MacroAssemblerX86::vpcmpgtdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtd_mr); +} + +void MacroAssemblerX86::vcmpeqpsSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpeqps_mr); +} + +void MacroAssemblerX86::vcmpneqpsSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpneqps_mr); +} + +void MacroAssemblerX86::vcmpltpsSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpltps_mr); +} + +void MacroAssemblerX86::vcmplepsSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpleps_mr); +} + +void MacroAssemblerX86::vcmpgepsSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpgeps_mr); +} + +void MacroAssemblerX86::vcmpeqpdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpeqpd_mr); +} + +void MacroAssemblerX86::vcmpneqpdSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpneqpd_mr); +} + +void MacroAssemblerX86::vcmpltpdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpltpd_mr); +} + +void MacroAssemblerX86::vcmplepdSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmplepd_mr); +} + +void MacroAssemblerX86::vpmaddubswSimd128(const SimdConstant& v, + FloatRegister lhs, + FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaddubsw_mr); +} + +void MacroAssemblerX86::vpmuludqSimd128(const SimdConstant& v, + FloatRegister lhs, FloatRegister dest) { + vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmuludq_mr); +} + +void MacroAssemblerX86::finish() { + // Last instruction may be an indirect jump so eagerly insert an undefined + // instruction byte to prevent processors from decoding data values into + // their pipelines. See Intel performance guides. + masm.ud2(); + + if (!doubles_.empty()) { + masm.haltingAlign(sizeof(double)); + } + for (const Double& d : doubles_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : d.uses) { + addCodeLabel(CodeLabel(use, cst)); + } + masm.doubleConstant(d.value); + if (!enoughMemory_) { + return; + } + } + + if (!floats_.empty()) { + masm.haltingAlign(sizeof(float)); + } + for (const Float& f : floats_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : f.uses) { + addCodeLabel(CodeLabel(use, cst)); + } + masm.floatConstant(f.value); + if (!enoughMemory_) { + return; + } + } + + // SIMD memory values must be suitably aligned. + if (!simds_.empty()) { + masm.haltingAlign(SimdMemoryAlignment); + } + for (const SimdData& v : simds_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : v.uses) { + addCodeLabel(CodeLabel(use, cst)); + } + masm.simd128Constant(v.value.bytes()); + if (!enoughMemory_) { + return; + } + } +} + +void MacroAssemblerX86::handleFailureWithHandlerTail(Label* profilerExitTail, + Label* bailoutTail) { + // Reserve space for exception information. + subl(Imm32(sizeof(ResumeFromException)), esp); + movl(esp, eax); + + // Call the handler. + using Fn = void (*)(ResumeFromException * rfe); + asMasm().setupUnalignedABICall(ecx); + asMasm().passABIArg(eax); + asMasm().callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + Label entryFrame; + Label catch_; + Label finally; + Label returnBaseline; + Label returnIon; + Label bailout; + Label wasm; + Label wasmCatch; + + loadPtr(Address(esp, ResumeFromException::offsetOfKind()), eax); + asMasm().branch32(Assembler::Equal, eax, + Imm32(ExceptionResumeKind::EntryFrame), &entryFrame); + asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Catch), + &catch_); + asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Finally), + &finally); + asMasm().branch32(Assembler::Equal, eax, + Imm32(ExceptionResumeKind::ForcedReturnBaseline), + &returnBaseline); + asMasm().branch32(Assembler::Equal, eax, + Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon); + asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Bailout), + &bailout); + asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Wasm), + &wasm); + asMasm().branch32(Assembler::Equal, eax, + Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch); + + breakpoint(); // Invalid kind. + + // No exception handler. Load the error value, restore state and return from + // the entry frame. + bind(&entryFrame); + asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + ret(); + + // If we found a catch handler, this must be a baseline frame. Restore state + // and jump to the catch block. + bind(&catch_); + loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + jmp(Operand(eax)); + + // If we found a finally block, this must be a baseline frame. Push two + // values expected by the finally block: the exception and BooleanValue(true). + bind(&finally); + ValueOperand exception = ValueOperand(ecx, edx); + loadValue(Address(esp, ResumeFromException::offsetOfException()), exception); + + loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + + pushValue(exception); + pushValue(BooleanValue(true)); + jmp(Operand(eax)); + + // Return BaselineFrame->returnValue() to the caller. + // Used in debug mode and for GeneratorReturn. + Label profilingInstrumentation; + bind(&returnBaseline); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + jump(&profilingInstrumentation); + + // Return the given value to the caller. + bind(&returnIon); + loadValue(Address(esp, ResumeFromException::offsetOfException()), + JSReturnOperand); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + + // If profiling is enabled, then update the lastProfilingFrame to refer to + // caller frame before returning. This code is shared by ForcedReturnIon + // and ForcedReturnBaseline. + bind(&profilingInstrumentation); + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled( + asMasm().runtime()->geckoProfiler().addressOfEnabled()); + asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + jump(profilerExitTail); + bind(&skipProfilingInstrumentation); + } + + movl(ebp, esp); + pop(ebp); + ret(); + + // If we are bailing out to baseline to handle an exception, jump to the + // bailout tail stub. Load 1 (true) in ReturnReg to indicate success. + bind(&bailout); + loadPtr(Address(esp, ResumeFromException::offsetOfBailoutInfo()), ecx); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + move32(Imm32(1), ReturnReg); + jump(bailoutTail); + + // If we are throwing and the innermost frame was a wasm frame, reset SP and + // FP; SP is pointing to the unwound return address to the wasm entry, so + // we can just ret(). + bind(&wasm); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + movePtr(ImmPtr((const void*)wasm::FailInstanceReg), InstanceReg); + masm.ret(); + + // Found a wasm catch handler, restore state and jump to it. + bind(&wasmCatch); + loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax); + loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp); + loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp); + jmp(Operand(eax)); +} + +void MacroAssemblerX86::profilerEnterFrame(Register framePtr, + Register scratch) { + asMasm().loadJSContext(scratch); + loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch); + storePtr(framePtr, + Address(scratch, JitActivation::offsetOfLastProfilingFrame())); + storePtr(ImmPtr(nullptr), + Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); +} + +void MacroAssemblerX86::profilerExitFrame() { + jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail()); +} + +Assembler::Condition MacroAssemblerX86::testStringTruthy( + bool truthy, const ValueOperand& value) { + Register string = value.payloadReg(); + cmp32(Operand(string, JSString::offsetOfLength()), Imm32(0)); + return truthy ? Assembler::NotEqual : Assembler::Equal; +} + +Assembler::Condition MacroAssemblerX86::testBigIntTruthy( + bool truthy, const ValueOperand& value) { + Register bi = value.payloadReg(); + cmp32(Operand(bi, JS::BigInt::offsetOfDigitLength()), Imm32(0)); + return truthy ? Assembler::NotEqual : Assembler::Equal; +} + +MacroAssembler& MacroAssemblerX86::asMasm() { + return *static_cast(this); +} + +const MacroAssembler& MacroAssemblerX86::asMasm() const { + return *static_cast(this); +} + +void MacroAssembler::subFromStackPtr(Imm32 imm32) { + if (imm32.value) { + // On windows, we cannot skip very far down the stack without touching the + // memory pages in-between. This is a corner-case code for situations where + // the Ion frame data for a piece of code is very large. To handle this + // special case, for frames over 4k in size we allocate memory on the stack + // incrementally, touching it as we go. + // + // When the amount is quite large, which it can be, we emit an actual loop, + // in order to keep the function prologue compact. Compactness is a + // requirement for eg Wasm's CodeRange data structure, which can encode only + // 8-bit offsets. + uint32_t amountLeft = imm32.value; + uint32_t fullPages = amountLeft / 4096; + if (fullPages <= 8) { + while (amountLeft > 4096) { + subl(Imm32(4096), StackPointer); + store32(Imm32(0), Address(StackPointer, 0)); + amountLeft -= 4096; + } + subl(Imm32(amountLeft), StackPointer); + } else { + // Save scratch register. + push(eax); + amountLeft -= 4; + fullPages = amountLeft / 4096; + + Label top; + move32(Imm32(fullPages), eax); + bind(&top); + subl(Imm32(4096), StackPointer); + store32(Imm32(0), Address(StackPointer, 0)); + subl(Imm32(1), eax); + j(Assembler::NonZero, &top); + amountLeft -= fullPages * 4096; + if (amountLeft) { + subl(Imm32(amountLeft), StackPointer); + } + + // Restore scratch register. + movl(Operand(StackPointer, uint32_t(imm32.value) - 4), eax); + } + } +} + +//{{{ check_macroassembler_style +// =============================================================== +// ABI function calls. + +void MacroAssembler::setupUnalignedABICall(Register scratch) { + setupNativeABICall(); + dynamicAlignment_ = true; + + movl(esp, scratch); + andl(Imm32(~(ABIStackAlignment - 1)), esp); + push(scratch); +} + +void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { + MOZ_ASSERT(inCall_); + uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); + + if (dynamicAlignment_) { + // sizeof(intptr_t) accounts for the saved stack pointer pushed by + // setupUnalignedABICall. + stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t), + ABIStackAlignment); + } else { + uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; + stackForCall += ComputeByteAlignment( + stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment); + } + + *stackAdjust = stackForCall; + reserveStack(stackForCall); + + // Position all arguments. + { + enoughMemory_ &= moveResolver_.resolve(); + if (!enoughMemory_) { + return; + } + + MoveEmitter emitter(*this); + emitter.emit(moveResolver_); + emitter.finish(); + } + + assertStackAlignment(ABIStackAlignment); +} + +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result, + bool callFromWasm) { + freeStack(stackAdjust); + + // Calls to native functions in wasm pass through a thunk which already + // fixes up the return value for us. + if (!callFromWasm) { + if (result == MoveOp::DOUBLE) { + reserveStack(sizeof(double)); + fstp(Operand(esp, 0)); + loadDouble(Operand(esp, 0), ReturnDoubleReg); + freeStack(sizeof(double)); + } else if (result == MoveOp::FLOAT32) { + reserveStack(sizeof(float)); + fstp32(Operand(esp, 0)); + loadFloat32(Operand(esp, 0), ReturnFloat32Reg); + freeStack(sizeof(float)); + } + } + + if (dynamicAlignment_) { + pop(esp); + } + +#ifdef DEBUG + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif +} + +void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) { + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(fun); + callWithABIPost(stackAdjust, result); +} + +void MacroAssembler::callWithABINoProfiler(const Address& fun, + MoveOp::Type result) { + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(fun); + callWithABIPost(stackAdjust, result); +} + +// =============================================================== +// Move instructions + +void MacroAssembler::moveValue(const TypedOrValueRegister& src, + const ValueOperand& dest) { + if (src.hasValue()) { + moveValue(src.valueReg(), dest); + return; + } + + MIRType type = src.type(); + AnyRegister reg = src.typedReg(); + + if (!IsFloatingPointType(type)) { + if (reg.gpr() != dest.payloadReg()) { + movl(reg.gpr(), dest.payloadReg()); + } + mov(ImmWord(MIRTypeToTag(type)), dest.typeReg()); + return; + } + + ScratchDoubleScope scratch(*this); + FloatRegister freg = reg.fpu(); + if (type == MIRType::Float32) { + convertFloat32ToDouble(freg, scratch); + freg = scratch; + } + boxDouble(freg, dest, scratch); +} + +void MacroAssembler::moveValue(const ValueOperand& src, + const ValueOperand& dest) { + Register s0 = src.typeReg(); + Register s1 = src.payloadReg(); + Register d0 = dest.typeReg(); + Register d1 = dest.payloadReg(); + + // Either one or both of the source registers could be the same as a + // destination register. + if (s1 == d0) { + if (s0 == d1) { + // If both are, this is just a swap of two registers. + xchgl(d0, d1); + return; + } + // If only one is, copy that source first. + std::swap(s0, s1); + std::swap(d0, d1); + } + + if (s0 != d0) { + movl(s0, d0); + } + if (s1 != d1) { + movl(s1, d1); + } +} + +void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { + movl(Imm32(src.toNunboxTag()), dest.typeReg()); + if (src.isGCThing()) { + movl(ImmGCPtr(src.toGCThing()), dest.payloadReg()); + } else { + movl(Imm32(src.toNunboxPayload()), dest.payloadReg()); + } +} + +// =============================================================== +// Branch functions + +void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { + if (ptr != buffer) { + movePtr(ptr, buffer); + } + andPtr(Imm32(~gc::ChunkMask), buffer); + loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer); +} + +void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, + Register temp, Label* label) { + MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86. + MOZ_ASSERT(ptr != temp); + movePtr(ptr, temp); + branchPtrInNurseryChunkImpl(cond, temp, label); +} + +void MacroAssembler::branchPtrInNurseryChunk(Condition cond, + const Address& address, + Register temp, Label* label) { + MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86. + loadPtr(address, temp); + branchPtrInNurseryChunkImpl(cond, temp, label); +} + +void MacroAssembler::branchPtrInNurseryChunkImpl(Condition cond, Register ptr, + Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + andPtr(Imm32(~gc::ChunkMask), ptr); + branchPtr(InvertCondition(cond), Address(ptr, gc::ChunkStoreBufferOffset), + ImmWord(0), label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + const Address& address, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestGCThing(Assembler::NotEqual, address, + cond == Assembler::Equal ? &done : label); + branchPtrInNurseryChunk(cond, ToPayload(address), temp, label); + + bind(&done); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + ValueOperand value, Register temp, + Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestGCThing(Assembler::NotEqual, value, + cond == Assembler::Equal ? &done : label); + branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label); + + bind(&done); +} + +void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, + const Value& rhs, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + if (rhs.isGCThing()) { + cmpPtr(lhs.payloadReg(), ImmGCPtr(rhs.toGCThing())); + } else { + cmpPtr(lhs.payloadReg(), ImmWord(rhs.toNunboxPayload())); + } + + if (cond == Equal) { + Label done; + j(NotEqual, &done); + { + cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag())); + j(Equal, label); + } + bind(&done); + } else { + j(NotEqual, label); + + cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag())); + j(NotEqual, label); + } +} + +// ======================================================================== +// Memory access primitives. +template +void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, const T& dest) { + MOZ_ASSERT(valueType < MIRType::Value); + + if (valueType == MIRType::Double) { + storeDouble(value.reg().typedReg().fpu(), dest); + return; + } + + // Store the type tag. + storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), Operand(dest)); + + // Store the payload. + if (value.constant()) { + storePayload(value.value(), Operand(dest)); + } else { + storePayload(value.reg().typedReg().gpr(), Operand(dest)); + } +} + +template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, + const Address& dest); +template void MacroAssembler::storeUnboxedValue( + const ConstantOrRegister& value, MIRType valueType, + const BaseObjectElementIndex& dest); + +// wasm specific methods, used in both the wasm baseline compiler and ion. + +void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, + Operand srcAddr, AnyRegister out) { + MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP || + srcAddr.kind() == Operand::MEM_SCALE); + + MOZ_ASSERT_IF( + access.isZeroExtendSimd128Load(), + access.type() == Scalar::Float32 || access.type() == Scalar::Float64); + MOZ_ASSERT_IF( + access.isSplatSimd128Load(), + access.type() == Scalar::Uint8 || access.type() == Scalar::Uint16 || + access.type() == Scalar::Float32 || access.type() == Scalar::Float64); + MOZ_ASSERT_IF(access.isWidenSimd128Load(), access.type() == Scalar::Float64); + + // NOTE: the generated code must match the assembly code in gen_load in + // GenerateAtomicOperations.py + memoryBarrierBefore(access.sync()); + + append(access, size()); + switch (access.type()) { + case Scalar::Int8: + movsbl(srcAddr, out.gpr()); + break; + case Scalar::Uint8: + if (access.isSplatSimd128Load()) { + vbroadcastb(srcAddr, out.fpu()); + } else { + movzbl(srcAddr, out.gpr()); + } + break; + case Scalar::Int16: + movswl(srcAddr, out.gpr()); + break; + case Scalar::Uint16: + if (access.isSplatSimd128Load()) { + vbroadcastw(srcAddr, out.fpu()); + } else { + movzwl(srcAddr, out.gpr()); + } + break; + case Scalar::Int32: + case Scalar::Uint32: + movl(srcAddr, out.gpr()); + break; + case Scalar::Float32: + if (access.isSplatSimd128Load()) { + vbroadcastss(srcAddr, out.fpu()); + } else { + // vmovss does the right thing also for access.isZeroExtendSimd128Load() + vmovss(srcAddr, out.fpu()); + } + break; + case Scalar::Float64: + if (access.isSplatSimd128Load()) { + vmovddup(srcAddr, out.fpu()); + } else if (access.isWidenSimd128Load()) { + switch (access.widenSimdOp()) { + case wasm::SimdOp::V128Load8x8S: + vpmovsxbw(srcAddr, out.fpu()); + break; + case wasm::SimdOp::V128Load8x8U: + vpmovzxbw(srcAddr, out.fpu()); + break; + case wasm::SimdOp::V128Load16x4S: + vpmovsxwd(srcAddr, out.fpu()); + break; + case wasm::SimdOp::V128Load16x4U: + vpmovzxwd(srcAddr, out.fpu()); + break; + case wasm::SimdOp::V128Load32x2S: + vpmovsxdq(srcAddr, out.fpu()); + break; + case wasm::SimdOp::V128Load32x2U: + vpmovzxdq(srcAddr, out.fpu()); + break; + default: + MOZ_CRASH("Unexpected widening op for wasmLoad"); + } + } else { + // vmovsd does the right thing also for access.isZeroExtendSimd128Load() + vmovsd(srcAddr, out.fpu()); + } + break; + case Scalar::Simd128: + vmovups(srcAddr, out.fpu()); + break; + case Scalar::Int64: + case Scalar::Uint8Clamped: + case Scalar::BigInt64: + case Scalar::BigUint64: + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected type"); + } + + memoryBarrierAfter(access.sync()); +} + +void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, + Operand srcAddr, Register64 out) { + // Atomic i64 load must use lock_cmpxchg8b. + MOZ_ASSERT_IF(access.isAtomic(), access.byteSize() <= 4); + MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP || + srcAddr.kind() == Operand::MEM_SCALE); + MOZ_ASSERT(!access.isZeroExtendSimd128Load()); // Use wasmLoad() + MOZ_ASSERT(!access.isSplatSimd128Load()); // Use wasmLoad() + MOZ_ASSERT(!access.isWidenSimd128Load()); // Use wasmLoad() + + memoryBarrierBefore(access.sync()); + + append(access, size()); + switch (access.type()) { + case Scalar::Int8: + MOZ_ASSERT(out == Register64(edx, eax)); + movsbl(srcAddr, out.low); + + cdq(); + break; + case Scalar::Uint8: + movzbl(srcAddr, out.low); + + xorl(out.high, out.high); + break; + case Scalar::Int16: + MOZ_ASSERT(out == Register64(edx, eax)); + movswl(srcAddr, out.low); + + cdq(); + break; + case Scalar::Uint16: + movzwl(srcAddr, out.low); + + xorl(out.high, out.high); + break; + case Scalar::Int32: + MOZ_ASSERT(out == Register64(edx, eax)); + movl(srcAddr, out.low); + + cdq(); + break; + case Scalar::Uint32: + movl(srcAddr, out.low); + + xorl(out.high, out.high); + break; + case Scalar::Int64: { + if (srcAddr.kind() == Operand::MEM_SCALE) { + MOZ_RELEASE_ASSERT(srcAddr.toBaseIndex().base != out.low && + srcAddr.toBaseIndex().index != out.low); + } + if (srcAddr.kind() == Operand::MEM_REG_DISP) { + MOZ_RELEASE_ASSERT(srcAddr.toAddress().base != out.low); + } + + movl(LowWord(srcAddr), out.low); + + append(access, size()); + movl(HighWord(srcAddr), out.high); + + break; + } + case Scalar::Float32: + case Scalar::Float64: + MOZ_CRASH("non-int64 loads should use load()"); + case Scalar::Simd128: + case Scalar::Uint8Clamped: + case Scalar::BigInt64: + case Scalar::BigUint64: + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected array type"); + } + + memoryBarrierAfter(access.sync()); +} + +void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, + AnyRegister value, Operand dstAddr) { + MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP || + dstAddr.kind() == Operand::MEM_SCALE); + + // NOTE: the generated code must match the assembly code in gen_store in + // GenerateAtomicOperations.py + memoryBarrierBefore(access.sync()); + + append(access, size()); + switch (access.type()) { + case Scalar::Int8: + case Scalar::Uint8Clamped: + case Scalar::Uint8: + movb(value.gpr(), dstAddr); + break; + case Scalar::Int16: + case Scalar::Uint16: + movw(value.gpr(), dstAddr); + break; + case Scalar::Int32: + case Scalar::Uint32: + movl(value.gpr(), dstAddr); + break; + case Scalar::Float32: + vmovss(value.fpu(), dstAddr); + break; + case Scalar::Float64: + vmovsd(value.fpu(), dstAddr); + break; + case Scalar::Simd128: + vmovups(value.fpu(), dstAddr); + break; + case Scalar::Int64: + MOZ_CRASH("Should be handled in storeI64."); + case Scalar::MaxTypedArrayViewType: + case Scalar::BigInt64: + case Scalar::BigUint64: + MOZ_CRASH("unexpected type"); + } + + memoryBarrierAfter(access.sync()); +} + +void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, + Register64 value, Operand dstAddr) { + // Atomic i64 store must use lock_cmpxchg8b. + MOZ_ASSERT(!access.isAtomic()); + MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP || + dstAddr.kind() == Operand::MEM_SCALE); + + // Store the high word first so as to hit guard-page-based OOB checks without + // writing partial data. + append(access, size()); + movl(value.high, HighWord(dstAddr)); + + append(access, size()); + movl(value.low, LowWord(dstAddr)); +} + +template +static void AtomicLoad64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, const T& address, + Register64 temp, Register64 output) { + MOZ_ASSERT(temp.low == ebx); + MOZ_ASSERT(temp.high == ecx); + MOZ_ASSERT(output.high == edx); + MOZ_ASSERT(output.low == eax); + + // In the event edx:eax matches what's in memory, ecx:ebx will be + // stored. The two pairs must therefore have the same values. + masm.movl(edx, ecx); + masm.movl(eax, ebx); + + if (access) { + masm.append(*access, masm.size()); + } + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); +} + +void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access, + const Address& mem, Register64 temp, + Register64 output) { + AtomicLoad64(*this, &access, mem, temp, output); +} + +void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register64 temp, + Register64 output) { + AtomicLoad64(*this, &access, mem, temp, output); +} + +template +static void CompareExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const T& mem, Register64 expected, + Register64 replacement, Register64 output) { + MOZ_ASSERT(expected == output); + MOZ_ASSERT(expected.high == edx); + MOZ_ASSERT(expected.low == eax); + MOZ_ASSERT(replacement.high == ecx); + MOZ_ASSERT(replacement.low == ebx); + + // NOTE: the generated code must match the assembly code in gen_cmpxchg in + // GenerateAtomicOperations.py + if (access) { + masm.append(*access, masm.size()); + } + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, + Register64 expected, + Register64 replacement, + Register64 output) { + CompareExchange64(*this, &access, mem, expected, replacement, output); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, + Register64 expected, + Register64 replacement, + Register64 output) { + CompareExchange64(*this, &access, mem, expected, replacement, output); +} + +template +static void AtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, const T& mem, + Register64 value, Register64 output) { + MOZ_ASSERT(value.low == ebx); + MOZ_ASSERT(value.high == ecx); + MOZ_ASSERT(output.high == edx); + MOZ_ASSERT(output.low == eax); + + // edx:eax has garbage initially, and that is the best we can do unless + // we can guess with high probability what's in memory. + + MOZ_ASSERT(mem.base != edx && mem.base != eax); + if constexpr (std::is_same_v) { + MOZ_ASSERT(mem.index != edx && mem.index != eax); + } else { + static_assert(std::is_same_v); + } + + Label again; + masm.bind(&again); + if (access) { + masm.append(*access, masm.size()); + } + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); + masm.j(MacroAssembler::NonZero, &again); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, &access, mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, + Register64 value, Register64 output) { + AtomicExchange64(*this, &access, mem, value, output); +} + +template +static void AtomicFetchOp64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, AtomicOp op, + const Address& value, const T& mem, Register64 temp, + Register64 output) { + // We don't have enough registers for all the operands on x86, so the rhs + // operand is in memory. + +#define ATOMIC_OP_BODY(OPERATE) \ + do { \ + MOZ_ASSERT(output.low == eax); \ + MOZ_ASSERT(output.high == edx); \ + MOZ_ASSERT(temp.low == ebx); \ + MOZ_ASSERT(temp.high == ecx); \ + if (access) { \ + masm.append(*access, masm.size()); \ + } \ + masm.load64(mem, output); \ + Label again; \ + masm.bind(&again); \ + masm.move64(output, temp); \ + masm.OPERATE(Operand(value), temp); \ + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); \ + masm.j(MacroAssembler::NonZero, &again); \ + } while (0) + + switch (op) { + case AtomicFetchAddOp: + ATOMIC_OP_BODY(add64FromMemory); + break; + case AtomicFetchSubOp: + ATOMIC_OP_BODY(sub64FromMemory); + break; + case AtomicFetchAndOp: + ATOMIC_OP_BODY(and64FromMemory); + break; + case AtomicFetchOrOp: + ATOMIC_OP_BODY(or64FromMemory); + break; + case AtomicFetchXorOp: + ATOMIC_OP_BODY(xor64FromMemory); + break; + default: + MOZ_CRASH(); + } + +#undef ATOMIC_OP_BODY +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, const Address& value, + const Address& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, op, value, mem, temp, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, const Address& value, + const BaseIndex& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, op, value, mem, temp, output); +} + +void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + Label done; + vcvttsd2si(input, output); + branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done); + + ScratchDoubleScope fpscratch(*this); + loadConstantDouble(double(int32_t(0x80000000)), fpscratch); + addDouble(input, fpscratch); + vcvttsd2si(fpscratch, output); + + branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry); + or32(Imm32(0x80000000), output); + + bind(&done); +} + +void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + Label done; + vcvttss2si(input, output); + branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done); + + ScratchFloat32Scope fpscratch(*this); + loadConstantFloat32(float(int32_t(0x80000000)), fpscratch); + addFloat32(input, fpscratch); + vcvttss2si(fpscratch, output); + + branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry); + or32(Imm32(0x80000000), output); + + bind(&done); +} + +void MacroAssembler::wasmTruncateDoubleToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) { + Label ok; + Register temp = output.high; + + reserveStack(2 * sizeof(int32_t)); + storeDouble(input, Operand(esp, 0)); + + truncateDoubleToInt64(Address(esp, 0), Address(esp, 0), temp); + load64(Address(esp, 0), output); + + cmpl(Imm32(0), Operand(esp, 0)); + j(Assembler::NotEqual, &ok); + + cmpl(Imm32(1), Operand(esp, 4)); + j(Assembler::Overflow, oolEntry); + + bind(&ok); + bind(oolRejoin); + + freeStack(2 * sizeof(int32_t)); +} + +void MacroAssembler::wasmTruncateFloat32ToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) { + Label ok; + Register temp = output.high; + + reserveStack(2 * sizeof(int32_t)); + storeFloat32(input, Operand(esp, 0)); + + truncateFloat32ToInt64(Address(esp, 0), Address(esp, 0), temp); + load64(Address(esp, 0), output); + + cmpl(Imm32(0), Operand(esp, 0)); + j(Assembler::NotEqual, &ok); + + cmpl(Imm32(1), Operand(esp, 4)); + j(Assembler::Overflow, oolEntry); + + bind(&ok); + bind(oolRejoin); + + freeStack(2 * sizeof(int32_t)); +} + +void MacroAssembler::wasmTruncateDoubleToUInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) { + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in uint64. + reserveStack(2 * sizeof(int32_t)); + storeDouble(input, Operand(esp, 0)); + branchDoubleNotInUInt64Range(Address(esp, 0), temp, &fail); + size_t stackBeforeBranch = framePushed(); + jump(&convert); + + bind(&fail); + freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + if (isSaturating) { + // The OOL path computes the right values. + setFramePushed(stackBeforeBranch); + } else { + // The OOL path just checks the input values. + bind(oolRejoin); + reserveStack(2 * sizeof(int32_t)); + storeDouble(input, Operand(esp, 0)); + } + + // Convert the double/float to uint64. + bind(&convert); + truncateDoubleToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg); + + // Load value into int64 register. + load64(Address(esp, 0), output); + freeStack(2 * sizeof(int32_t)); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateFloat32ToUInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) { + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in uint64. + reserveStack(2 * sizeof(int32_t)); + storeFloat32(input, Operand(esp, 0)); + branchFloat32NotInUInt64Range(Address(esp, 0), temp, &fail); + size_t stackBeforeBranch = framePushed(); + jump(&convert); + + bind(&fail); + freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + if (isSaturating) { + // The OOL path computes the right values. + setFramePushed(stackBeforeBranch); + } else { + // The OOL path just checks the input values. + bind(oolRejoin); + reserveStack(2 * sizeof(int32_t)); + storeFloat32(input, Operand(esp, 0)); + } + + // Convert the float to uint64. + bind(&convert); + truncateFloat32ToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg); + + // Load value into int64 register. + load64(Address(esp, 0), output); + freeStack(2 * sizeof(int32_t)); + + if (isSaturating) { + bind(oolRejoin); + } +} + +// ======================================================================== +// Primitive atomic operations. + +void MacroAssembler::atomicLoad64(const Synchronization&, const Address& mem, + Register64 temp, Register64 output) { + AtomicLoad64(*this, nullptr, mem, temp, output); +} + +void MacroAssembler::atomicLoad64(const Synchronization&, const BaseIndex& mem, + Register64 temp, Register64 output) { + AtomicLoad64(*this, nullptr, mem, temp, output); +} + +void MacroAssembler::atomicStore64(const Synchronization&, const Address& mem, + Register64 value, Register64 temp) { + AtomicExchange64(*this, nullptr, mem, value, temp); +} + +void MacroAssembler::atomicStore64(const Synchronization&, const BaseIndex& mem, + Register64 value, Register64 temp) { + AtomicExchange64(*this, nullptr, mem, value, temp); +} + +void MacroAssembler::compareExchange64(const Synchronization&, + const Address& mem, Register64 expected, + Register64 replacement, + Register64 output) { + CompareExchange64(*this, nullptr, mem, expected, replacement, output); +} + +void MacroAssembler::compareExchange64(const Synchronization&, + const BaseIndex& mem, + Register64 expected, + Register64 replacement, + Register64 output) { + CompareExchange64(*this, nullptr, mem, expected, replacement, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization&, + const Address& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, mem, value, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization&, + const BaseIndex& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, mem, value, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, + const Address& value, const Address& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, op, value, mem, temp, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, + const Address& value, const BaseIndex& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, op, value, mem, temp, output); +} + +// ======================================================================== +// Convert floating point. + +bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return HasSSE3(); } + +void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, + Register temp) { + // SUBPD needs SSE2, HADDPD needs SSE3. + if (!HasSSE3()) { + MOZ_ASSERT(temp == Register::Invalid()); + + // Zero the dest register to break dependencies, see convertInt32ToDouble. + zeroDouble(dest); + + Push(src.high); + Push(src.low); + fild(Operand(esp, 0)); + + Label notNegative; + branch32(Assembler::NotSigned, src.high, Imm32(0), ¬Negative); + double add_constant = 18446744073709551616.0; // 2^64 + store64(Imm64(mozilla::BitwiseCast(add_constant)), + Address(esp, 0)); + fld(Operand(esp, 0)); + faddp(); + bind(¬Negative); + + fstp(Operand(esp, 0)); + vmovsd(Address(esp, 0), dest); + freeStack(2 * sizeof(intptr_t)); + return; + } + + // Following operation uses entire 128-bit of dest XMM register. + // Currently higher 64-bit is free when we have access to lower 64-bit. + MOZ_ASSERT(dest.size() == 8); + FloatRegister dest128 = + FloatRegister(dest.encoding(), FloatRegisters::Simd128); + + // Assume that src is represented as following: + // src = 0x HHHHHHHH LLLLLLLL + + { + // Move src to dest (=dest128) and ScratchInt32x4Reg (=scratch): + // dest = 0x 00000000 00000000 00000000 LLLLLLLL + // scratch = 0x 00000000 00000000 00000000 HHHHHHHH + ScratchSimd128Scope scratch(*this); + vmovd(src.low, dest128); + vmovd(src.high, scratch); + + // Unpack and interleave dest and scratch to dest: + // dest = 0x 00000000 00000000 HHHHHHHH LLLLLLLL + vpunpckldq(scratch, dest128, dest128); + } + + // Unpack and interleave dest and a constant C1 to dest: + // C1 = 0x 00000000 00000000 45300000 43300000 + // dest = 0x 45300000 HHHHHHHH 43300000 LLLLLLLL + // here, each 64-bit part of dest represents following double: + // HI(dest) = 0x 1.00000HHHHHHHH * 2**84 == 2**84 + 0x HHHHHHHH 00000000 + // LO(dest) = 0x 1.00000LLLLLLLL * 2**52 == 2**52 + 0x 00000000 LLLLLLLL + // See convertUInt64ToDouble for the details. + static const int32_t CST1[4] = { + 0x43300000, + 0x45300000, + 0x0, + 0x0, + }; + + vpunpckldqSimd128(SimdConstant::CreateX4(CST1), dest128, dest128); + + // Subtract a constant C2 from dest, for each 64-bit part: + // C2 = 0x 45300000 00000000 43300000 00000000 + // here, each 64-bit part of C2 represents following double: + // HI(C2) = 0x 1.0000000000000 * 2**84 == 2**84 + // LO(C2) = 0x 1.0000000000000 * 2**52 == 2**52 + // after the operation each 64-bit part of dest represents following: + // HI(dest) = double(0x HHHHHHHH 00000000) + // LO(dest) = double(0x 00000000 LLLLLLLL) + static const int32_t CST2[4] = { + 0x0, + 0x43300000, + 0x0, + 0x45300000, + }; + + vsubpdSimd128(SimdConstant::CreateX4(CST2), dest128, dest128); + + // Add HI(dest) and LO(dest) in double and store it into LO(dest), + // LO(dest) = double(0x HHHHHHHH 00000000) + double(0x 00000000 LLLLLLLL) + // = double(0x HHHHHHHH LLLLLLLL) + // = double(src) + vhaddpd(dest128, dest128); +} + +void MacroAssembler::convertInt64ToDouble(Register64 input, + FloatRegister output) { + // Zero the output register to break dependencies, see convertInt32ToDouble. + zeroDouble(output); + + Push(input.high); + Push(input.low); + fild(Operand(esp, 0)); + + fstp(Operand(esp, 0)); + vmovsd(Address(esp, 0), output); + freeStack(2 * sizeof(intptr_t)); +} + +void MacroAssembler::convertUInt64ToFloat32(Register64 input, + FloatRegister output, + Register temp) { + // Zero the dest register to break dependencies, see convertInt32ToDouble. + zeroDouble(output); + + // Set the FPU precision to 80 bits. + reserveStack(2 * sizeof(intptr_t)); + fnstcw(Operand(esp, 0)); + load32(Operand(esp, 0), temp); + orl(Imm32(0x300), temp); + store32(temp, Operand(esp, sizeof(intptr_t))); + fldcw(Operand(esp, sizeof(intptr_t))); + + Push(input.high); + Push(input.low); + fild(Operand(esp, 0)); + + Label notNegative; + branch32(Assembler::NotSigned, input.high, Imm32(0), ¬Negative); + double add_constant = 18446744073709551616.0; // 2^64 + uint64_t add_constant_u64 = mozilla::BitwiseCast(add_constant); + store64(Imm64(add_constant_u64), Address(esp, 0)); + + fld(Operand(esp, 0)); + faddp(); + bind(¬Negative); + + fstp32(Operand(esp, 0)); + vmovss(Address(esp, 0), output); + freeStack(2 * sizeof(intptr_t)); + + // Restore FPU precision to the initial value. + fldcw(Operand(esp, 0)); + freeStack(2 * sizeof(intptr_t)); +} + +void MacroAssembler::convertInt64ToFloat32(Register64 input, + FloatRegister output) { + // Zero the output register to break dependencies, see convertInt32ToDouble. + zeroDouble(output); + + Push(input.high); + Push(input.low); + fild(Operand(esp, 0)); + + fstp32(Operand(esp, 0)); + vmovss(Address(esp, 0), output); + freeStack(2 * sizeof(intptr_t)); +} + +void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { + convertInt32ToDouble(src, dest); +} + +void MacroAssembler::PushBoxed(FloatRegister reg) { Push(reg); } + +CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { + return movWithPatch(ImmPtr(nullptr), dest); +} + +void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, + CodeLocationLabel target) { + PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr)); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Register64 boundsCheckLimit, Label* ok) { + Label notOk; + cmp32(index.high, Imm32(0)); + j(Assembler::NonZero, ¬Ok); + wasmBoundsCheck32(cond, index.low, boundsCheckLimit.low, ok); + bind(¬Ok); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Address boundsCheckLimit, Label* ok) { + Label notOk; + cmp32(index.high, Imm32(0)); + j(Assembler::NonZero, ¬Ok); + wasmBoundsCheck32(cond, index.low, boundsCheckLimit, ok); + bind(¬Ok); +} + +//}}} check_macroassembler_style diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h new file mode 100644 index 0000000000..6a99a44d04 --- /dev/null +++ b/js/src/jit/x86/MacroAssembler-x86.h @@ -0,0 +1,1149 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_MacroAssembler_x86_h +#define jit_x86_MacroAssembler_x86_h + +#include "jit/JitOptions.h" +#include "jit/MoveResolver.h" +#include "jit/x86-shared/MacroAssembler-x86-shared.h" +#include "js/HeapAPI.h" +#include "wasm/WasmBuiltins.h" + +namespace js { +namespace jit { + +// See documentation for ScratchTagScope and ScratchTagScopeRelease in +// MacroAssembler-x64.h. + +class ScratchTagScope { + const ValueOperand& v_; + + public: + ScratchTagScope(MacroAssembler&, const ValueOperand& v) : v_(v) {} + operator Register() { return v_.typeReg(); } + void release() {} + void reacquire() {} +}; + +class ScratchTagScopeRelease { + public: + explicit ScratchTagScopeRelease(ScratchTagScope*) {} +}; + +class MacroAssemblerX86 : public MacroAssemblerX86Shared { + private: + // Perform a downcast. Should be removed by Bug 996602. + MacroAssembler& asMasm(); + const MacroAssembler& asMasm() const; + + protected: + MoveResolver moveResolver_; + + private: + Operand payloadOfAfterStackPush(const Address& address) { + // If we are basing off %esp, the address will be invalid after the + // first push. + if (address.base == StackPointer) { + return Operand(address.base, address.offset + 4); + } + return payloadOf(address); + } + Operand payloadOfAfterStackPush(const BaseIndex& address) { + // If we are basing off %esp, the address will be invalid after the + // first push. + if (address.base == StackPointer) { + return Operand(address.base, address.index, address.scale, + address.offset + 4); + } + return payloadOf(address); + } + Operand payloadOf(const Address& address) { + return Operand(address.base, address.offset); + } + Operand payloadOf(const BaseIndex& address) { + return Operand(address.base, address.index, address.scale, address.offset); + } + Operand tagOf(const Address& address) { + return Operand(address.base, address.offset + 4); + } + Operand tagOf(const BaseIndex& address) { + return Operand(address.base, address.index, address.scale, + address.offset + 4); + } + + void setupABICall(uint32_t args); + + void vpPatchOpSimd128(const SimdConstant& v, FloatRegister reg, + void (X86Encoding::BaseAssemblerX86::*op)( + const void* address, + X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)) { + vpPatchOpSimd128(v, reg, reg, op); + } + + void vpPatchOpSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest, + void (X86Encoding::BaseAssemblerX86::*op)( + const void* address, + X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)); + + void vpPatchOpSimd128(const SimdConstant& v, FloatRegister reg, + size_t (X86Encoding::BaseAssemblerX86::*op)( + const void* address, + X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)) { + vpPatchOpSimd128(v, reg, reg, op); + } + + void vpPatchOpSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest, + size_t (X86Encoding::BaseAssemblerX86::*op)( + const void* address, + X86Encoding::XMMRegisterID srcId, + X86Encoding::XMMRegisterID destId)); + + public: + using MacroAssemblerX86Shared::call; + using MacroAssemblerX86Shared::load32; + using MacroAssemblerX86Shared::store16; + using MacroAssemblerX86Shared::store32; + + MacroAssemblerX86() {} + + // The buffer is about to be linked, make sure any constant pools or excess + // bookkeeping has been flushed to the instruction stream. + void finish(); + + ///////////////////////////////////////////////////////////////// + // X86-specific interface. + ///////////////////////////////////////////////////////////////// + + Operand ToPayload(Operand base) { return base; } + Address ToPayload(Address base) { return base; } + BaseIndex ToPayload(BaseIndex base) { return base; } + Operand ToType(Operand base) { + switch (base.kind()) { + case Operand::MEM_REG_DISP: + return Operand(Register::FromCode(base.base()), + base.disp() + sizeof(void*)); + + case Operand::MEM_SCALE: + return Operand(Register::FromCode(base.base()), + Register::FromCode(base.index()), base.scale(), + base.disp() + sizeof(void*)); + + default: + MOZ_CRASH("unexpected operand kind"); + } + } + Address ToType(Address base) { return ToType(Operand(base)).toAddress(); } + BaseIndex ToType(BaseIndex base) { + return ToType(Operand(base)).toBaseIndex(); + } + + template + void add64FromMemory(const T& address, Register64 dest) { + addl(Operand(LowWord(address)), dest.low); + adcl(Operand(HighWord(address)), dest.high); + } + template + void sub64FromMemory(const T& address, Register64 dest) { + subl(Operand(LowWord(address)), dest.low); + sbbl(Operand(HighWord(address)), dest.high); + } + template + void and64FromMemory(const T& address, Register64 dest) { + andl(Operand(LowWord(address)), dest.low); + andl(Operand(HighWord(address)), dest.high); + } + template + void or64FromMemory(const T& address, Register64 dest) { + orl(Operand(LowWord(address)), dest.low); + orl(Operand(HighWord(address)), dest.high); + } + template + void xor64FromMemory(const T& address, Register64 dest) { + xorl(Operand(LowWord(address)), dest.low); + xorl(Operand(HighWord(address)), dest.high); + } + + ///////////////////////////////////////////////////////////////// + // X86/X64-common interface. + ///////////////////////////////////////////////////////////////// + void storeValue(ValueOperand val, Operand dest) { + movl(val.payloadReg(), ToPayload(dest)); + movl(val.typeReg(), ToType(dest)); + } + void storeValue(ValueOperand val, const Address& dest) { + storeValue(val, Operand(dest)); + } + template + void storeValue(JSValueType type, Register reg, const T& dest) { + storeTypeTag(ImmTag(JSVAL_TYPE_TO_TAG(type)), Operand(dest)); + storePayload(reg, Operand(dest)); + } + template + void storeValue(const Value& val, const T& dest) { + storeTypeTag(ImmTag(val.toNunboxTag()), Operand(dest)); + storePayload(val, Operand(dest)); + } + void storeValue(ValueOperand val, BaseIndex dest) { + storeValue(val, Operand(dest)); + } + void storeValue(const Address& src, const Address& dest, Register temp) { + MOZ_ASSERT(src.base != temp); + MOZ_ASSERT(dest.base != temp); + + load32(ToType(src), temp); + store32(temp, ToType(dest)); + + load32(ToPayload(src), temp); + store32(temp, ToPayload(dest)); + } + void storePrivateValue(Register src, const Address& dest) { + store32(Imm32(0), ToType(dest)); + store32(src, ToPayload(dest)); + } + void storePrivateValue(ImmGCPtr imm, const Address& dest) { + store32(Imm32(0), ToType(dest)); + movl(imm, Operand(ToPayload(dest))); + } + void loadValue(Operand src, ValueOperand val) { + Operand payload = ToPayload(src); + Operand type = ToType(src); + + // Ensure that loading the payload does not erase the pointer to the + // Value in memory or the index. + Register baseReg = Register::FromCode(src.base()); + Register indexReg = (src.kind() == Operand::MEM_SCALE) + ? Register::FromCode(src.index()) + : InvalidReg; + + // If we have a BaseIndex that uses both result registers, first compute + // the address and then load the Value from there. + if ((baseReg == val.payloadReg() && indexReg == val.typeReg()) || + (baseReg == val.typeReg() && indexReg == val.payloadReg())) { + computeEffectiveAddress(src, val.scratchReg()); + loadValue(Address(val.scratchReg(), 0), val); + return; + } + + if (baseReg == val.payloadReg() || indexReg == val.payloadReg()) { + MOZ_ASSERT(baseReg != val.typeReg()); + MOZ_ASSERT(indexReg != val.typeReg()); + + movl(type, val.typeReg()); + movl(payload, val.payloadReg()); + } else { + MOZ_ASSERT(baseReg != val.payloadReg()); + MOZ_ASSERT(indexReg != val.payloadReg()); + + movl(payload, val.payloadReg()); + movl(type, val.typeReg()); + } + } + void loadValue(Address src, ValueOperand val) { + loadValue(Operand(src), val); + } + void loadValue(const BaseIndex& src, ValueOperand val) { + loadValue(Operand(src), val); + } + void loadUnalignedValue(const Address& src, ValueOperand dest) { + loadValue(src, dest); + } + void tagValue(JSValueType type, Register payload, ValueOperand dest) { + MOZ_ASSERT(dest.typeReg() != dest.payloadReg()); + if (payload != dest.payloadReg()) { + movl(payload, dest.payloadReg()); + } + movl(ImmType(type), dest.typeReg()); + } + void pushValue(ValueOperand val) { + push(val.typeReg()); + push(val.payloadReg()); + } + void popValue(ValueOperand val) { + pop(val.payloadReg()); + pop(val.typeReg()); + } + void pushValue(const Value& val) { + push(Imm32(val.toNunboxTag())); + if (val.isGCThing()) { + push(ImmGCPtr(val.toGCThing())); + } else { + push(Imm32(val.toNunboxPayload())); + } + } + void pushValue(JSValueType type, Register reg) { + push(ImmTag(JSVAL_TYPE_TO_TAG(type))); + push(reg); + } + void pushValue(const Address& addr) { + push(tagOf(addr)); + push(payloadOfAfterStackPush(addr)); + } + void pushValue(const BaseIndex& addr, Register scratch) { + push(tagOf(addr)); + push(payloadOfAfterStackPush(addr)); + } + void push64(Register64 src) { + push(src.high); + push(src.low); + } + void pop64(Register64 dest) { + pop(dest.low); + pop(dest.high); + } + void storePayload(const Value& val, Operand dest) { + if (val.isGCThing()) { + movl(ImmGCPtr(val.toGCThing()), ToPayload(dest)); + } else { + movl(Imm32(val.toNunboxPayload()), ToPayload(dest)); + } + } + void storePayload(Register src, Operand dest) { movl(src, ToPayload(dest)); } + void storeTypeTag(ImmTag tag, Operand dest) { movl(tag, ToType(dest)); } + + void movePtr(Register src, Register dest) { movl(src, dest); } + void movePtr(Register src, const Operand& dest) { movl(src, dest); } + + void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { + MOZ_ASSERT(value.typeReg() == tag); + } + + Condition testUndefined(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED)); + return cond; + } + Condition testBoolean(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN)); + return cond; + } + Condition testInt32(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_INT32)); + return cond; + } + Condition testDouble(Condition cond, Register tag) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + Condition actual = (cond == Equal) ? Below : AboveOrEqual; + cmp32(tag, ImmTag(JSVAL_TAG_CLEAR)); + return actual; + } + Condition testNull(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_NULL)); + return cond; + } + Condition testString(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_STRING)); + return cond; + } + Condition testSymbol(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL)); + return cond; + } + Condition testBigInt(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_BIGINT)); + return cond; + } + Condition testObject(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_OBJECT)); + return cond; + } + Condition testNumber(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JS::detail::ValueUpperInclNumberTag)); + return cond == Equal ? BelowOrEqual : Above; + } + Condition testGCThing(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag)); + return cond == Equal ? AboveOrEqual : Below; + } + Condition testGCThing(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JS::detail::ValueLowerInclGCThingTag)); + return cond == Equal ? AboveOrEqual : Below; + } + Condition testMagic(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_MAGIC)); + return cond; + } + Condition testMagic(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JSVAL_TAG_MAGIC)); + return cond; + } + Condition testMagic(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(ToType(operand), ImmTag(JSVAL_TAG_MAGIC)); + return cond; + } + Condition testPrimitive(Condition cond, Register tag) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag)); + return cond == Equal ? Below : AboveOrEqual; + } + Condition testError(Condition cond, Register tag) { + return testMagic(cond, tag); + } + Condition testBoolean(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(Operand(ToType(address)), ImmTag(JSVAL_TAG_BOOLEAN)); + return cond; + } + Condition testInt32(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(ToType(operand), ImmTag(JSVAL_TAG_INT32)); + return cond; + } + Condition testInt32(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + return testInt32(cond, Operand(address)); + } + Condition testObject(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(ToType(operand), ImmTag(JSVAL_TAG_OBJECT)); + return cond; + } + Condition testObject(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + return testObject(cond, Operand(address)); + } + Condition testDouble(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = (cond == Equal) ? Below : AboveOrEqual; + cmp32(ToType(operand), ImmTag(JSVAL_TAG_CLEAR)); + return actual; + } + Condition testDouble(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + return testDouble(cond, Operand(address)); + } + + Condition testUndefined(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(ToType(operand), ImmTag(JSVAL_TAG_UNDEFINED)); + return cond; + } + Condition testUndefined(Condition cond, const Address& addr) { + return testUndefined(cond, Operand(addr)); + } + Condition testNull(Condition cond, const Operand& operand) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(ToType(operand), ImmTag(JSVAL_TAG_NULL)); + return cond; + } + Condition testNull(Condition cond, const Address& addr) { + return testNull(cond, Operand(addr)); + } + + Condition testUndefined(Condition cond, const ValueOperand& value) { + return testUndefined(cond, value.typeReg()); + } + Condition testBoolean(Condition cond, const ValueOperand& value) { + return testBoolean(cond, value.typeReg()); + } + Condition testInt32(Condition cond, const ValueOperand& value) { + return testInt32(cond, value.typeReg()); + } + Condition testDouble(Condition cond, const ValueOperand& value) { + return testDouble(cond, value.typeReg()); + } + Condition testNull(Condition cond, const ValueOperand& value) { + return testNull(cond, value.typeReg()); + } + Condition testString(Condition cond, const ValueOperand& value) { + return testString(cond, value.typeReg()); + } + Condition testSymbol(Condition cond, const ValueOperand& value) { + return testSymbol(cond, value.typeReg()); + } + Condition testBigInt(Condition cond, const ValueOperand& value) { + return testBigInt(cond, value.typeReg()); + } + Condition testObject(Condition cond, const ValueOperand& value) { + return testObject(cond, value.typeReg()); + } + Condition testMagic(Condition cond, const ValueOperand& value) { + return testMagic(cond, value.typeReg()); + } + Condition testError(Condition cond, const ValueOperand& value) { + return testMagic(cond, value); + } + Condition testNumber(Condition cond, const ValueOperand& value) { + return testNumber(cond, value.typeReg()); + } + Condition testGCThing(Condition cond, const ValueOperand& value) { + return testGCThing(cond, value.typeReg()); + } + Condition testPrimitive(Condition cond, const ValueOperand& value) { + return testPrimitive(cond, value.typeReg()); + } + + Condition testUndefined(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_UNDEFINED)); + return cond; + } + Condition testNull(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_NULL)); + return cond; + } + Condition testBoolean(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_BOOLEAN)); + return cond; + } + Condition testString(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_STRING)); + return cond; + } + Condition testString(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_STRING)); + return cond; + } + Condition testSymbol(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_SYMBOL)); + return cond; + } + Condition testSymbol(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_SYMBOL)); + return cond; + } + Condition testBigInt(Condition cond, const Address& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_BIGINT)); + return cond; + } + Condition testBigInt(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_BIGINT)); + return cond; + } + Condition testInt32(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_INT32)); + return cond; + } + Condition testObject(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_OBJECT)); + return cond; + } + Condition testDouble(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = (cond == Equal) ? Below : AboveOrEqual; + cmp32(tagOf(address), ImmTag(JSVAL_TAG_CLEAR)); + return actual; + } + Condition testMagic(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JSVAL_TAG_MAGIC)); + return cond; + } + Condition testGCThing(Condition cond, const BaseIndex& address) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + cmp32(tagOf(address), ImmTag(JS::detail::ValueLowerInclGCThingTag)); + return cond == Equal ? AboveOrEqual : Below; + } + + void testNullSet(Condition cond, const ValueOperand& value, Register dest) { + cond = testNull(cond, value); + emitSet(cond, dest); + } + + void testObjectSet(Condition cond, const ValueOperand& value, Register dest) { + cond = testObject(cond, value); + emitSet(cond, dest); + } + + void testUndefinedSet(Condition cond, const ValueOperand& value, + Register dest) { + cond = testUndefined(cond, value); + emitSet(cond, dest); + } + + void cmpPtr(Register lhs, const ImmWord rhs) { cmpl(Imm32(rhs.value), lhs); } + void cmpPtr(Register lhs, const ImmPtr imm) { + cmpPtr(lhs, ImmWord(uintptr_t(imm.value))); + } + void cmpPtr(Register lhs, const ImmGCPtr rhs) { cmpl(rhs, lhs); } + void cmpPtr(const Operand& lhs, Imm32 rhs) { cmp32(lhs, rhs); } + void cmpPtr(const Operand& lhs, const ImmWord rhs) { + cmp32(lhs, Imm32(rhs.value)); + } + void cmpPtr(const Operand& lhs, const ImmPtr imm) { + cmpPtr(lhs, ImmWord(uintptr_t(imm.value))); + } + void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) { cmpl(rhs, lhs); } + void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); } + void cmpPtr(const Operand& lhs, Register rhs) { cmp32(lhs, rhs); } + void cmpPtr(const Address& lhs, const ImmWord rhs) { + cmpPtr(Operand(lhs), rhs); + } + void cmpPtr(const Address& lhs, const ImmPtr rhs) { + cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); + } + void cmpPtr(const Address& lhs, const ImmGCPtr rhs) { + cmpPtr(Operand(lhs), rhs); + } + void cmpPtr(Register lhs, Register rhs) { cmp32(lhs, rhs); } + void testPtr(Register lhs, Register rhs) { test32(lhs, rhs); } + void testPtr(Register lhs, Imm32 rhs) { test32(lhs, rhs); } + void testPtr(Register lhs, ImmWord rhs) { test32(lhs, Imm32(rhs.value)); } + void testPtr(const Operand& lhs, Imm32 rhs) { test32(lhs, rhs); } + void testPtr(const Operand& lhs, ImmWord rhs) { + test32(lhs, Imm32(rhs.value)); + } + + ///////////////////////////////////////////////////////////////// + // Common interface. + ///////////////////////////////////////////////////////////////// + + void movePtr(ImmWord imm, Register dest) { movl(Imm32(imm.value), dest); } + void movePtr(ImmPtr imm, Register dest) { movl(imm, dest); } + void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); } + void movePtr(ImmGCPtr imm, Register dest) { movl(imm, dest); } + void loadPtr(const Address& address, Register dest) { + movl(Operand(address), dest); + } + void loadPtr(const Operand& src, Register dest) { movl(src, dest); } + void loadPtr(const BaseIndex& src, Register dest) { + movl(Operand(src), dest); + } + void loadPtr(AbsoluteAddress address, Register dest) { + movl(Operand(address), dest); + } + void loadPrivate(const Address& src, Register dest) { + movl(payloadOf(src), dest); + } + void load32(AbsoluteAddress address, Register dest) { + movl(Operand(address), dest); + } + void load64(const Address& address, Register64 dest) { + bool highBeforeLow = address.base == dest.low; + if (highBeforeLow) { + movl(Operand(HighWord(address)), dest.high); + movl(Operand(LowWord(address)), dest.low); + } else { + movl(Operand(LowWord(address)), dest.low); + movl(Operand(HighWord(address)), dest.high); + } + } + void load64(const BaseIndex& address, Register64 dest) { + // If you run into this, relax your register allocation constraints. + MOZ_RELEASE_ASSERT( + !((address.base == dest.low || address.base == dest.high) && + (address.index == dest.low || address.index == dest.high))); + bool highBeforeLow = address.base == dest.low || address.index == dest.low; + if (highBeforeLow) { + movl(Operand(HighWord(address)), dest.high); + movl(Operand(LowWord(address)), dest.low); + } else { + movl(Operand(LowWord(address)), dest.low); + movl(Operand(HighWord(address)), dest.high); + } + } + template + void load64Unaligned(const T& address, Register64 dest) { + load64(address, dest); + } + template + void storePtr(ImmWord imm, T address) { + movl(Imm32(imm.value), Operand(address)); + } + template + void storePtr(ImmPtr imm, T address) { + storePtr(ImmWord(uintptr_t(imm.value)), address); + } + template + void storePtr(ImmGCPtr imm, T address) { + movl(imm, Operand(address)); + } + void storePtr(Register src, const Address& address) { + movl(src, Operand(address)); + } + void storePtr(Register src, const BaseIndex& address) { + movl(src, Operand(address)); + } + void storePtr(Register src, const Operand& dest) { movl(src, dest); } + void storePtr(Register src, AbsoluteAddress address) { + movl(src, Operand(address)); + } + void store32(Register src, AbsoluteAddress address) { + movl(src, Operand(address)); + } + void store16(Register src, AbsoluteAddress address) { + movw(src, Operand(address)); + } + template + void store64(Register64 src, const T& address) { + movl(src.low, Operand(LowWord(address))); + movl(src.high, Operand(HighWord(address))); + } + void store64(Imm64 imm, Address address) { + movl(imm.low(), Operand(LowWord(address))); + movl(imm.hi(), Operand(HighWord(address))); + } + template + void store64Unaligned(const S& src, const T& dest) { + store64(src, dest); + } + + void setStackArg(Register reg, uint32_t arg) { + movl(reg, Operand(esp, arg * sizeof(intptr_t))); + } + + void boxDouble(FloatRegister src, const ValueOperand& dest, + FloatRegister temp) { + if (Assembler::HasSSE41()) { + vmovd(src, dest.payloadReg()); + vpextrd(1, src, dest.typeReg()); + } else { + vmovd(src, dest.payloadReg()); + if (src != temp) { + moveDouble(src, temp); + } + vpsrldq(Imm32(4), temp, temp); + vmovd(temp, dest.typeReg()); + } + } + void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) { + if (src != dest.payloadReg()) { + movl(src, dest.payloadReg()); + } + movl(ImmType(type), dest.typeReg()); + } + + void unboxNonDouble(const ValueOperand& src, Register dest, JSValueType type, + Register scratch = InvalidReg) { + unboxNonDouble(Operand(src.typeReg()), Operand(src.payloadReg()), dest, + type, scratch); + } + void unboxNonDouble(const Operand& tag, const Operand& payload, Register dest, + JSValueType type, Register scratch = InvalidReg) { + auto movPayloadToDest = [&]() { + if (payload.kind() != Operand::REG || !payload.containsReg(dest)) { + movl(payload, dest); + } + }; + if (!JitOptions.spectreValueMasking) { + movPayloadToDest(); + return; + } + + // Spectre mitigation: We zero the payload if the tag does not match the + // expected type and if this is a pointer type. + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + movPayloadToDest(); + return; + } + + if (!tag.containsReg(dest) && !payload.containsReg(dest)) { + // We zero the destination register and move the payload into it if + // the tag corresponds to the given type. + xorl(dest, dest); + cmpl(Imm32(JSVAL_TYPE_TO_TAG(type)), tag); + cmovCCl(Condition::Equal, payload, dest); + return; + } + + if (scratch == InvalidReg || scratch == dest || tag.containsReg(scratch) || + payload.containsReg(scratch)) { + // UnboxedLayout::makeConstructorCode calls extractObject with a + // scratch register which aliases the tag register, thus we cannot + // assert the above condition. + scratch = InvalidReg; + } + + // The destination register aliases one of the operands. We create a + // zero value either in a scratch register or on the stack and use it + // to reset the destination register after reading both the tag and the + // payload. + Operand zero(Address(esp, 0)); + if (scratch == InvalidReg) { + push(Imm32(0)); + } else { + xorl(scratch, scratch); + zero = Operand(scratch); + } + cmpl(Imm32(JSVAL_TYPE_TO_TAG(type)), tag); + movPayloadToDest(); + cmovCCl(Condition::NotEqual, zero, dest); + if (scratch == InvalidReg) { + addl(Imm32(sizeof(void*)), esp); + } + } + void unboxNonDouble(const Address& src, Register dest, JSValueType type) { + unboxNonDouble(tagOf(src), payloadOf(src), dest, type); + } + void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) { + unboxNonDouble(tagOf(src), payloadOf(src), dest, type); + } + void unboxInt32(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_INT32); + } + void unboxInt32(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_INT32); + } + void unboxInt32(const BaseIndex& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_INT32); + } + void unboxBoolean(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN); + } + void unboxBoolean(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN); + } + void unboxBoolean(const BaseIndex& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN); + } + void unboxString(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); + } + void unboxString(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); + } + void unboxSymbol(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); + } + void unboxSymbol(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); + } + void unboxBigInt(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); + } + void unboxBigInt(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); + } + void unboxObject(const ValueOperand& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + void unboxObject(const Address& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + void unboxObject(const BaseIndex& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + template + void unboxObjectOrNull(const T& src, Register dest) { + // Due to Spectre mitigation logic (see Value.h), if the value is an Object + // then this yields the object; otherwise it yields zero (null), as desired. + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + template + void unboxDouble(const T& src, FloatRegister dest) { + loadDouble(Operand(src), dest); + } + void unboxDouble(const ValueOperand& src, FloatRegister dest) { + if (Assembler::HasSSE41()) { + vmovd(src.payloadReg(), dest); + vpinsrd(1, src.typeReg(), dest, dest); + } else { + ScratchDoubleScope fpscratch(asMasm()); + vmovd(src.payloadReg(), dest); + vmovd(src.typeReg(), fpscratch); + vunpcklps(fpscratch, dest, dest); + } + } + void unboxDouble(const Operand& payload, const Operand& type, + Register scratch, FloatRegister dest) { + if (Assembler::HasSSE41()) { + movl(payload, scratch); + vmovd(scratch, dest); + movl(type, scratch); + vpinsrd(1, scratch, dest, dest); + } else { + ScratchDoubleScope fpscratch(asMasm()); + movl(payload, scratch); + vmovd(scratch, dest); + movl(type, scratch); + vmovd(scratch, fpscratch); + vunpcklps(fpscratch, dest, dest); + } + } + inline void unboxValue(const ValueOperand& src, AnyRegister dest, + JSValueType type); + + // See comment in MacroAssembler-x64.h. + void unboxGCThingForGCBarrier(const Address& src, Register dest) { + movl(payloadOf(src), dest); + } + + void notBoolean(const ValueOperand& val) { xorl(Imm32(1), val.payloadReg()); } + + template + void fallibleUnboxPtrImpl(const T& src, Register dest, JSValueType type, + Label* fail); + + // Extended unboxing API. If the payload is already in a register, returns + // that register. Otherwise, provides a move to the given scratch register, + // and returns that. + [[nodiscard]] Register extractObject(const Address& address, Register dest) { + unboxObject(address, dest); + return dest; + } + [[nodiscard]] Register extractObject(const ValueOperand& value, + Register scratch) { + unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_OBJECT, scratch); + return value.payloadReg(); + } + [[nodiscard]] Register extractSymbol(const ValueOperand& value, + Register scratch) { + unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_SYMBOL, scratch); + return value.payloadReg(); + } + [[nodiscard]] Register extractInt32(const ValueOperand& value, + Register scratch) { + return value.payloadReg(); + } + [[nodiscard]] Register extractBoolean(const ValueOperand& value, + Register scratch) { + return value.payloadReg(); + } + [[nodiscard]] Register extractTag(const Address& address, Register scratch) { + movl(tagOf(address), scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const ValueOperand& value, + Register scratch) { + return value.typeReg(); + } + + void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true) { + convertDoubleToInt32(src, dest, fail, negativeZeroCheck); + } + + void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) { + convertInt32ToDouble(operand.payloadReg(), dest); + } + void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) { + convertInt32ToFloat32(operand.payloadReg(), dest); + } + void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) { + convertInt32ToDouble(operand.payloadReg(), dest); + } + void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) { + convertInt32ToFloat32(operand.payloadReg(), dest); + } + + void loadConstantDouble(double d, FloatRegister dest); + void loadConstantFloat32(float f, FloatRegister dest); + + void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest); + void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest); + void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpadddSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminubSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpminudSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpandSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpxorSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vporSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vandpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vminpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vptestSimd128(const SimdConstant& v, FloatRegister lhs); + void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs, + FloatRegister dest); + + Condition testInt32Truthy(bool truthy, const ValueOperand& operand) { + test32(operand.payloadReg(), operand.payloadReg()); + return truthy ? NonZero : Zero; + } + Condition testStringTruthy(bool truthy, const ValueOperand& value); + Condition testBigIntTruthy(bool truthy, const ValueOperand& value); + + template + inline void loadInt32OrDouble(const T& src, FloatRegister dest); + + template + inline void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest); + + template + void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes, + JSValueType) { + switch (nbytes) { + case 4: + storePtr(value.payloadReg(), address); + return; + case 1: + store8(value.payloadReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + // Note: this function clobbers the source register. + inline void convertUInt32ToDouble(Register src, FloatRegister dest); + + // Note: this function clobbers the source register. + inline void convertUInt32ToFloat32(Register src, FloatRegister dest); + + void incrementInt32Value(const Address& addr) { + addl(Imm32(1), payloadOf(addr)); + } + + inline void ensureDouble(const ValueOperand& source, FloatRegister dest, + Label* failure); + + public: + // Used from within an Exit frame to handle a pending exception. + void handleFailureWithHandlerTail(Label* profilerExitTail, + Label* bailoutTail); + + // Instrumentation for entering and leaving the profiler. + void profilerEnterFrame(Register framePtr, Register scratch); + void profilerExitFrame(); +}; + +typedef MacroAssemblerX86 MacroAssemblerSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_MacroAssembler_x86_h */ diff --git a/js/src/jit/x86/SharedICHelpers-x86-inl.h b/js/src/jit/x86/SharedICHelpers-x86-inl.h new file mode 100644 index 0000000000..2ab41c287a --- /dev/null +++ b/js/src/jit/x86/SharedICHelpers-x86-inl.h @@ -0,0 +1,77 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_SharedICHelpers_x86_inl_h +#define jit_x86_SharedICHelpers_x86_inl_h + +#include "jit/BaselineFrame.h" +#include "jit/SharedICHelpers.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm, + uint32_t argSize) { +#ifdef DEBUG + // We assume during this that R0 and R1 have been pushed. + // Store frame size without VMFunction arguments for debug assertions. + masm.movl(FramePointer, eax); + masm.subl(StackPointer, eax); + masm.subl(Imm32(argSize), eax); + Address frameSizeAddr(FramePointer, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(eax, frameSizeAddr); +#endif + + // Push frame descriptor and perform the tail call. + masm.pushFrameDescriptor(FrameType::BaselineJS); + masm.push(ICTailCallReg); + masm.jump(target); +} + +inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) { + masm.pushFrameDescriptor(FrameType::BaselineStub); + masm.call(target); +} + +inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) { +#ifdef DEBUG + // Compute frame size. Because the return address is still on the stack, + // this is: + // + // FramePointer + // - StackPointer + // - sizeof(return address) + + masm.movl(FramePointer, scratch); + masm.subl(StackPointer, scratch); + masm.subl(Imm32(sizeof(void*)), scratch); // Return address. + + Address frameSizeAddr(FramePointer, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(scratch, frameSizeAddr); +#endif + + // Push the return address that's currently on top of the stack. + masm.Push(Operand(StackPointer, 0)); + + // Replace the original return address with the frame descriptor. + masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)), + Address(StackPointer, sizeof(uintptr_t))); + + // Save old frame pointer, stack pointer and stub reg. + masm.Push(FramePointer); + masm.mov(StackPointer, FramePointer); + + masm.Push(ICStubReg); +} + +} // namespace jit +} // namespace js + +#endif /* jit_x86_SharedICHelpers_x86_inl_h */ diff --git a/js/src/jit/x86/SharedICHelpers-x86.h b/js/src/jit/x86/SharedICHelpers-x86.h new file mode 100644 index 0000000000..07d2ae8a6f --- /dev/null +++ b/js/src/jit/x86/SharedICHelpers-x86.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_SharedICHelpers_x86_h +#define jit_x86_SharedICHelpers_x86_h + +#include "jit/BaselineIC.h" +#include "jit/JitFrames.h" +#include "jit/MacroAssembler.h" +#include "jit/SharedICRegisters.h" + +namespace js { +namespace jit { + +// Distance from stack top to the top Value inside an IC stub (this is the +// return address). +static const size_t ICStackValueOffset = sizeof(void*); + +inline void EmitRestoreTailCallReg(MacroAssembler& masm) { + masm.Pop(ICTailCallReg); +} + +inline void EmitRepushTailCallReg(MacroAssembler& masm) { + masm.Push(ICTailCallReg); +} + +inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) { + // The stub pointer must already be in ICStubReg. + // Call the stubcode. + masm.call(Address(ICStubReg, ICStub::offsetOfStubCode())); + *callOffset = CodeOffset(masm.currentOffset()); +} + +inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ret(); } + +inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) { + Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP); + masm.loadPtr(stubAddr, ICStubReg); + + masm.mov(FramePointer, StackPointer); + masm.Pop(FramePointer); + + // The return address is on top of the stack, followed by the frame + // descriptor. Use a pop instruction to overwrite the frame descriptor + // with the return address. Note that pop increments the stack pointer + // before computing the address. + masm.Pop(Operand(StackPointer, 0)); +} + +template +inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr, + MIRType type) { + masm.guardedCallPreBarrier(addr, type); +} + +inline void EmitStubGuardFailure(MacroAssembler& masm) { + // Load next stub into ICStubReg + masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg); + + // Return address is already loaded, just jump to the next stubcode. + masm.jmp(Operand(ICStubReg, ICStub::offsetOfStubCode())); +} + +} // namespace jit +} // namespace js + +#endif /* jit_x86_SharedICHelpers_x86_h */ diff --git a/js/src/jit/x86/SharedICRegisters-x86.h b/js/src/jit/x86/SharedICRegisters-x86.h new file mode 100644 index 0000000000..44edb5288f --- /dev/null +++ b/js/src/jit/x86/SharedICRegisters-x86.h @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_SharedICRegisters_x86_h +#define jit_x86_SharedICRegisters_x86_h + +#include "jit/Registers.h" +#include "jit/RegisterSets.h" +#include "jit/x86/Assembler-x86.h" + +namespace js { +namespace jit { + +// ValueOperands R0, R1, and R2 +static constexpr ValueOperand R0(ecx, edx); +static constexpr ValueOperand R1(eax, ebx); +static constexpr ValueOperand R2(esi, edi); + +// ICTailCallReg and ICStubReg reuse +// registers from R2. +static constexpr Register ICTailCallReg = esi; +static constexpr Register ICStubReg = edi; + +// FloatReg0 must be equal to ReturnFloatReg. +static constexpr FloatRegister FloatReg0 = xmm0; +static constexpr FloatRegister FloatReg1 = xmm1; +static constexpr FloatRegister FloatReg2 = xmm2; +static constexpr FloatRegister FloatReg3 = xmm3; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_SharedICRegisters_x86_h */ diff --git a/js/src/jit/x86/Trampoline-x86.cpp b/js/src/jit/x86/Trampoline-x86.cpp new file mode 100644 index 0000000000..da459cdd3a --- /dev/null +++ b/js/src/jit/x86/Trampoline-x86.cpp @@ -0,0 +1,796 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/MathAlgorithms.h" + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineJIT.h" +#include "jit/CalleeToken.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/PerfSpewer.h" +#include "jit/VMFunctions.h" +#include "jit/x86/SharedICHelpers-x86.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" +#include "vm/Realm.h" +#ifdef MOZ_VTUNE +# include "vtune/VTuneWrapper.h" +#endif + +#include "jit/MacroAssembler-inl.h" +#include "vm/JSScript-inl.h" + +using mozilla::IsPowerOfTwo; + +using namespace js; +using namespace js::jit; + +// All registers to save and restore. This includes the stack pointer, since we +// use the ability to reference register values on the stack by index. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +enum EnterJitEbpArgumentOffset { + ARG_JITCODE = 2 * sizeof(void*), + ARG_ARGC = 3 * sizeof(void*), + ARG_ARGV = 4 * sizeof(void*), + ARG_STACKFRAME = 5 * sizeof(void*), + ARG_CALLEETOKEN = 6 * sizeof(void*), + ARG_SCOPECHAIN = 7 * sizeof(void*), + ARG_STACKVALUES = 8 * sizeof(void*), + ARG_RESULT = 9 * sizeof(void*) +}; + +// Generates a trampoline for calling Jit compiled code from a C++ function. +// The trampoline use the EnterJitCode signature, with the standard cdecl +// calling convention. +void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) { + AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT"); + + enterJITOffset_ = startTrampolineCode(masm); + + masm.assertStackAlignment(ABIStackAlignment, + -int32_t(sizeof(uintptr_t)) /* return address */); + + // Save old stack frame pointer, set new stack frame pointer. + masm.push(ebp); + masm.movl(esp, ebp); + + // Save non-volatile registers. These must be saved by the trampoline, + // rather than the JIT'd code, because they are scanned by the conservative + // scanner. + masm.push(ebx); + masm.push(esi); + masm.push(edi); + + // Load the number of values to be copied (argc) into eax + masm.loadPtr(Address(ebp, ARG_ARGC), eax); + + // If we are constructing, that also needs to include newTarget + { + Label noNewTarget; + masm.loadPtr(Address(ebp, ARG_CALLEETOKEN), edx); + masm.branchTest32(Assembler::Zero, edx, + Imm32(CalleeToken_FunctionConstructing), &noNewTarget); + + masm.addl(Imm32(1), eax); + + masm.bind(&noNewTarget); + } + + // eax <- 8*numValues, eax is now the offset betwen argv and the last value. + masm.shll(Imm32(3), eax); + + // Guarantee stack alignment of Jit frames. + // + // This code compensates for the offset created by the copy of the vector of + // arguments, such that the jit frame will be aligned once the return + // address is pushed on the stack. + // + // In the computation of the offset, we omit the size of the JitFrameLayout + // which is pushed on the stack, as the JitFrameLayout size is a multiple of + // the JitStackAlignment. + masm.movl(esp, ecx); + masm.subl(eax, ecx); + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + + // ecx = ecx & 15, holds alignment. + masm.andl(Imm32(JitStackAlignment - 1), ecx); + masm.subl(ecx, esp); + + /*************************************************************** + Loop over argv vector, push arguments onto stack in reverse order + ***************************************************************/ + + // ebx = argv --argv pointer is in ebp + 16 + masm.loadPtr(Address(ebp, ARG_ARGV), ebx); + + // eax = argv[8(argc)] --eax now points one value past the last argument + masm.addl(ebx, eax); + + // while (eax > ebx) --while still looping through arguments + { + Label header, footer; + masm.bind(&header); + + masm.cmp32(eax, ebx); + masm.j(Assembler::BelowOrEqual, &footer); + + // eax -= 8 --move to previous argument + masm.subl(Imm32(8), eax); + + // Push what eax points to on stack, a Value is 2 words + masm.push(Operand(eax, 4)); + masm.push(Operand(eax, 0)); + + masm.jmp(&header); + masm.bind(&footer); + } + + // Load the number of actual arguments. |result| is used to store the + // actual number of arguments without adding an extra argument to the enter + // JIT. + masm.mov(Operand(ebp, ARG_RESULT), eax); + masm.unboxInt32(Address(eax, 0x0), eax); + + // Push the callee token. + masm.push(Operand(ebp, ARG_CALLEETOKEN)); + + // Load the InterpreterFrame address into the OsrFrameReg. + // This address is also used for setting the constructing bit on all paths. + masm.loadPtr(Address(ebp, ARG_STACKFRAME), OsrFrameReg); + + // Push the descriptor. + masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, eax, eax); + + CodeLabel returnLabel; + Label oomReturnLabel; + { + // Handle Interpreter -> Baseline OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(ebp)); + regs.take(OsrFrameReg); + regs.take(ReturnReg); + + Register scratch = regs.takeAny(); + + Label notOsr; + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); + + Register numStackValues = regs.takeAny(); + masm.loadPtr(Address(ebp, ARG_STACKVALUES), numStackValues); + + Register jitcode = regs.takeAny(); + masm.loadPtr(Address(ebp, ARG_JITCODE), jitcode); + + // Push return address. + masm.mov(&returnLabel, scratch); + masm.push(scratch); + + // Frame prologue. + masm.push(ebp); + masm.mov(esp, ebp); + + // Reserve frame. + masm.subPtr(Imm32(BaselineFrame::Size()), esp); + + Register framePtrScratch = regs.takeAny(); + masm.touchFrameValues(numStackValues, scratch, framePtrScratch); + masm.mov(esp, framePtrScratch); + + // Reserve space for locals and stack values. + masm.mov(numStackValues, scratch); + masm.shll(Imm32(3), scratch); + masm.subPtr(scratch, esp); + + // Enter exit frame. + masm.pushFrameDescriptor(FrameType::BaselineJS); + masm.push(Imm32(0)); // Fake return address. + masm.push(FramePointer); + // No GC things to mark on the stack, push a bare token. + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare); + + masm.push(jitcode); + + using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame, + uint32_t numStackValues); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(framePtrScratch); // BaselineFrame + masm.passABIArg(OsrFrameReg); // InterpreterFrame + masm.passABIArg(numStackValues); + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + masm.pop(jitcode); + + MOZ_ASSERT(jitcode != ReturnReg); + + Label error; + masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), esp); + masm.branchIfFalseBool(ReturnReg, &error); + + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.profilerEnterFrame(ebp, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(jitcode); + + // OOM: frame epilogue, load error value, discard return address and return. + masm.bind(&error); + masm.mov(ebp, esp); + masm.pop(ebp); + masm.addPtr(Imm32(sizeof(uintptr_t)), esp); // Return address. + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.jump(&oomReturnLabel); + + masm.bind(¬Osr); + masm.loadPtr(Address(ebp, ARG_SCOPECHAIN), R1.scratchReg()); + } + + // The call will push the return address and frame pointer on the stack, thus + // we check that the stack would be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t)); + + /*************************************************************** + Call passed-in code, get return value and fill in the + passed in return value pointer + ***************************************************************/ + masm.call(Address(ebp, ARG_JITCODE)); + + { + // Interpreter -> Baseline OSR will return here. + masm.bind(&returnLabel); + masm.addCodeLabel(returnLabel); + masm.bind(&oomReturnLabel); + } + + // Restore the stack pointer so the stack looks like this: + // +20 ... arguments ... + // +16 + // +12 ebp <- %ebp pointing here. + // +8 ebx + // +4 esi + // +0 edi <- %esp pointing here. + masm.lea(Operand(ebp, -int32_t(3 * sizeof(void*))), esp); + + // Store the return value. + masm.loadPtr(Address(ebp, ARG_RESULT), eax); + masm.storeValue(JSReturnOperand, Operand(eax, 0)); + + /************************************************************** + Return stack and registers to correct state + **************************************************************/ + + // Restore non-volatile registers + masm.pop(edi); + masm.pop(esi); + masm.pop(ebx); + + // Restore old stack frame pointer + masm.pop(ebp); + masm.ret(); +} + +// static +mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> +JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) { + // Not supported, or not implemented yet. + // TODO: Implement along with the corresponding stack-walker changes, in + // coordination with the Gecko Profiler, see bug 1635987 and follow-ups. + return mozilla::Nothing{}; +} + +// Push AllRegs in a way that is compatible with RegisterDump, regardless of +// what PushRegsInMask might do to reduce the set size. +static void DumpAllRegs(MacroAssembler& masm) { +#ifdef ENABLE_WASM_SIMD + masm.PushRegsInMask(AllRegs); +#else + // When SIMD isn't supported, PushRegsInMask reduces the set of float + // registers to be double-sized, while the RegisterDump expects each of + // the float registers to have the maximal possible size + // (Simd128DataSize). To work around this, we just spill the double + // registers by hand here, using the register dump offset directly. + for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more(); + ++iter) { + masm.Push(*iter); + } + + masm.reserveStack(sizeof(RegisterDump::FPUArray)); + for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more(); + ++iter) { + FloatRegister reg = *iter; + Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes()); + masm.storeDouble(reg, spillAddress); + } +#endif +} + +void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator"); + + invalidatorOffset_ = startTrampolineCode(masm); + + // We do the minimum amount of work in assembly and shunt the rest + // off to InvalidationBailout. Assembly does: + // + // - Push the machine state onto the stack. + // - Call the InvalidationBailout routine with the stack pointer. + // - Now that the frame has been bailed out, convert the invalidated + // frame into an exit frame. + // - Do the normal check-return-code-and-thunk-to-the-interpreter dance. + + // Push registers such that we can access them from [base + code]. + DumpAllRegs(masm); + + masm.movl(esp, eax); // Argument to jit::InvalidationBailout. + + // Make space for InvalidationBailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movl(esp, ebx); + + using Fn = + bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupUnalignedABICall(edx); + masm.passABIArg(eax); + masm.passABIArg(ebx); + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(ecx); // Get bailoutInfo outparam. + + // Pop the machine state and the dead frame. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in ecx. + masm.jmp(bailoutTail); +} + +void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm, + ArgumentsRectifierKind kind) { + AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier"); + + switch (kind) { + case ArgumentsRectifierKind::Normal: + argumentsRectifierOffset_ = startTrampolineCode(masm); + break; + case ArgumentsRectifierKind::TrialInlining: + trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm); + break; + } + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- esp + + // Frame prologue. + // + // NOTE: if this changes, fix the Baseline bailout code too! + // See BaselineStackBuilder::calculatePrevFramePtr and + // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp). + masm.push(FramePointer); + masm.movl(esp, FramePointer); // Save %esp. + + // Load argc. + masm.loadNumActualArgs(FramePointer, esi); + + // Load the number of |undefined|s to push into %ecx. + masm.loadPtr(Address(ebp, RectifierFrameLayout::offsetOfCalleeToken()), eax); + masm.mov(eax, ecx); + masm.andl(Imm32(CalleeTokenMask), ecx); + masm.loadFunctionArgCount(ecx, ecx); + + // The frame pointer and its padding are pushed on the stack. + // Including |this|, there are (|nformals| + 1) arguments to push to the + // stack. Then we push a JitFrameLayout. We compute the padding expressed + // in the number of extra |undefined| values to push on the stack. + static_assert( + sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + static_assert( + JitStackAlignment % sizeof(Value) == 0, + "Ensure that we can pad the stack by pushing extra UndefinedValue"); + static_assert(IsPowerOfTwo(JitStackValueAlignment), + "must have power of two for masm.andl to do its job"); + + masm.addl( + Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */), + ecx); + + // Account for newTarget, if necessary. + static_assert( + CalleeToken_FunctionConstructing == 1, + "Ensure that we can use the constructing bit to count an extra push"); + masm.mov(eax, edx); + masm.andl(Imm32(CalleeToken_FunctionConstructing), edx); + masm.addl(edx, ecx); + + masm.andl(Imm32(~(JitStackValueAlignment - 1)), ecx); + masm.subl(esi, ecx); + masm.subl(Imm32(1), ecx); // For |this|. + + // Copy the number of actual arguments into edx. + masm.mov(esi, edx); + + masm.moveValue(UndefinedValue(), ValueOperand(ebx, edi)); + + // Caller: + // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] + // '-- #esi ---' + // + // Rectifier frame: + // [ebp'] <- ebp [padding] <- esp [undef] [undef] [arg2] [arg1] [this] + // '--- #ecx ----' '-- #esi ---' + // + // [ [argc] [callee] [descr] [raddr] ] + + // Push undefined. + { + Label undefLoopTop; + masm.bind(&undefLoopTop); + + masm.push(ebx); // type(undefined); + masm.push(edi); // payload(undefined); + masm.subl(Imm32(1), ecx); + masm.j(Assembler::NonZero, &undefLoopTop); + } + + // Get the topmost argument. + BaseIndex b(FramePointer, esi, TimesEight, sizeof(RectifierFrameLayout)); + masm.lea(Operand(b), ecx); + + // Push arguments, |nargs| + 1 times (to include |this|). + masm.addl(Imm32(1), esi); + { + Label copyLoopTop; + + masm.bind(©LoopTop); + masm.push(Operand(ecx, sizeof(Value) / 2)); + masm.push(Operand(ecx, 0x0)); + masm.subl(Imm32(sizeof(Value)), ecx); + masm.subl(Imm32(1), esi); + masm.j(Assembler::NonZero, ©LoopTop); + } + + { + Label notConstructing; + + masm.mov(eax, ebx); + masm.branchTest32(Assembler::Zero, ebx, + Imm32(CalleeToken_FunctionConstructing), + ¬Constructing); + + BaseValueIndex src(FramePointer, edx, + sizeof(RectifierFrameLayout) + sizeof(Value)); + + masm.andl(Imm32(CalleeTokenMask), ebx); + masm.loadFunctionArgCount(ebx, ebx); + + BaseValueIndex dst(esp, ebx, sizeof(Value)); + + ValueOperand newTarget(ecx, edi); + + masm.loadValue(src, newTarget); + masm.storeValue(newTarget, dst); + + masm.bind(¬Constructing); + } + + // Construct JitFrameLayout. + masm.push(eax); // callee token + masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, edx, edx); + + // Call the target function. + masm.andl(Imm32(CalleeTokenMask), eax); + switch (kind) { + case ArgumentsRectifierKind::Normal: + masm.loadJitCodeRaw(eax, eax); + argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(eax); + break; + case ArgumentsRectifierKind::TrialInlining: + Label noBaselineScript, done; + masm.loadBaselineJitCodeRaw(eax, ebx, &noBaselineScript); + masm.callJitNoProfiler(ebx); + masm.jump(&done); + + // See BaselineCacheIRCompiler::emitCallInlinedFunction. + masm.bind(&noBaselineScript); + masm.loadJitCodeRaw(eax, eax); + masm.callJitNoProfiler(eax); + masm.bind(&done); + break; + } + + masm.mov(FramePointer, StackPointer); + masm.pop(FramePointer); + masm.ret(); +} + +static void PushBailoutFrame(MacroAssembler& masm, Register spArg) { + // Push registers such that we can access them from [base + code]. + DumpAllRegs(masm); + + // The current stack pointer is the first argument to jit::Bailout. + masm.movl(esp, spArg); +} + +static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) { + PushBailoutFrame(masm, eax); + + // Make space for Bailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movl(esp, ebx); + + // Call the bailout function. + using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info); + masm.setupUnalignedABICall(ecx); + masm.passABIArg(eax); + masm.passABIArg(ebx); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(ecx); // Get the bailoutInfo outparam. + + // Remove both the bailout frame and the topmost Ion frame's stack. + masm.moveToStackPtr(FramePointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in ecx. + masm.jmp(bailoutTail); +} + +void JitRuntime::generateBailoutHandler(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler"); + + bailoutHandlerOffset_ = startTrampolineCode(masm); + + GenerateBailoutThunk(masm, bailoutTail); +} + +bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm, + const VMFunctionData& f, DynFn nativeFun, + uint32_t* wrapperOffset) { + AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper"); + + *wrapperOffset = startTrampolineCode(masm); + + // Avoid conflicts with argument registers while discarding the result after + // the function call. + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + static_assert( + (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set must be a superset of Volatile register set."); + + // The context is the first argument. + Register cxreg = regs.takeAny(); + + // Stack is: + // ... frame ... + // +8 [args] + // +4 descriptor + // +0 returnAddress + // + // Push the frame pointer to finish the exit frame, then link it up. + masm.Push(FramePointer); + masm.moveStackPtrTo(FramePointer); + masm.loadJSContext(cxreg); + masm.enterExitFrame(cxreg, regs.getAny(), &f); + + // Save the current stack pointer as the base for copying arguments. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + argsBase = regs.takeAny(); + masm.lea(Operand(esp, ExitFrameLayout::SizeWithFooter()), argsBase); + } + + // Reserve space for the outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = regs.takeAny(); + masm.Push(UndefinedValue()); + masm.movl(esp, outReg); + break; + + case Type_Handle: + outReg = regs.takeAny(); + masm.PushEmptyRooted(f.outParamRootType); + masm.movl(esp, outReg); + break; + + case Type_Int32: + case Type_Pointer: + case Type_Bool: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(int32_t)); + masm.movl(esp, outReg); + break; + + case Type_Double: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(double)); + masm.movl(esp, outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(cxreg); + + size_t argDisp = 0; + + // Copy arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + switch (f.argProperties(explicitArg)) { + case VMFunctionData::WordByValue: + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByValue: + // We don't pass doubles in float registers on x86, so no need + // to check for argPassedInFloatReg. + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + argDisp += sizeof(void*); + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::WordByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByRef: + masm.passABIArg( + MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress), + MoveOp::GENERAL); + argDisp += 2 * sizeof(void*); + break; + } + } + + // Copy the implicit outparam, if any. + if (outReg != InvalidReg) { + masm.passABIArg(outReg); + } + + masm.callWithABI(nativeFun, MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + // Test for failure. + switch (f.failType()) { + case Type_Cell: + masm.branchTestPtr(Assembler::Zero, eax, eax, masm.failureLabel()); + break; + case Type_Bool: + masm.testb(eax, eax); + masm.j(Assembler::Zero, masm.failureLabel()); + break; + case Type_Void: + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Value: + masm.Pop(JSReturnOperand); + break; + + case Type_Int32: + case Type_Pointer: + masm.Pop(ReturnReg); + break; + + case Type_Bool: + masm.Pop(ReturnReg); + masm.movzbl(ReturnReg, ReturnReg); + break; + + case Type_Double: + masm.Pop(ReturnDoubleReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + // Until C++ code is instrumented against Spectre, prevent speculative + // execution from returning any private data. + if (f.returnsData() && JitOptions.spectreJitToCxxCalls) { + masm.speculationBarrier(); + } + + // Pop ExitFooterFrame and the frame pointer. + masm.leaveExitFrame(0); + masm.pop(FramePointer); + + // Return. Subtract sizeof(void*) for the frame pointer. + masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + return true; +} + +uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type) { + AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier"); + + uint32_t offset = startTrampolineCode(masm); + + static_assert(PreBarrierReg == edx); + Register temp1 = eax; + Register temp2 = ebx; + Register temp3 = ecx; + masm.push(temp1); + masm.push(temp2); + masm.push(temp3); + + Label noBarrier; + masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3, + &noBarrier); + + // Call into C++ to mark this GC thing. + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + + LiveRegisterSet save; + save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileMask)); + masm.PushRegsInMask(save); + + masm.movl(ImmPtr(cx->runtime()), ecx); + + masm.setupUnalignedABICall(eax); + masm.passABIArg(ecx); + masm.passABIArg(edx); + masm.callWithABI(JitPreWriteBarrier(type)); + + masm.PopRegsInMask(save); + masm.ret(); + + masm.bind(&noBarrier); + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + masm.ret(); + + return offset; +} + +void JitRuntime::generateBailoutTailStub(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub"); + + masm.bind(bailoutTail); + masm.generateBailoutTail(edx, ecx); +} -- cgit v1.2.3