diff options
Diffstat (limited to 'js/src/jit/x64/MacroAssembler-x64-inl.h')
| -rw-r--r-- | js/src/jit/x64/MacroAssembler-x64-inl.h | 1099 |
1 files changed, 1099 insertions, 0 deletions
diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h new file mode 100644 index 0000000000..6869e6c4b6 --- /dev/null +++ b/js/src/jit/x64/MacroAssembler-x64-inl.h @@ -0,0 +1,1099 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x64_MacroAssembler_x64_inl_h +#define jit_x64_MacroAssembler_x64_inl_h + +#include "jit/x64/MacroAssembler-x64.h" + +#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h" + +namespace js { +namespace jit { + +//{{{ check_macroassembler_style +// =============================================================== + +void MacroAssembler::move64(Imm64 imm, Register64 dest) { + // Use mov instead of movq because it has special optimizations for imm == 0. + mov(ImmWord(imm.value), dest.reg); +} + +void MacroAssembler::move64(Register64 src, Register64 dest) { + movq(src.reg, dest.reg); +} + +void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) { + vmovq(src, dest.reg); +} + +void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) { + vmovq(src.reg, dest); +} + +void MacroAssembler::move64To32(Register64 src, Register dest) { + movl(src.reg, dest); +} + +void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) { + movl(src, dest.reg); +} + +void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) { + movsbq(Operand(src), dest.reg); +} + +void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) { + movswq(Operand(src), dest.reg); +} + +void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) { + movslq(src, dest.reg); +} + +void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) { + movslq(src, dest); +} + +void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) { + movl(src, dest); +} + +// =============================================================== +// Load instructions + +void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) { + movslq(Operand(src), dest); +} + +// =============================================================== +// Logical instructions + +void MacroAssembler::notPtr(Register reg) { notq(reg); } + +void MacroAssembler::andPtr(Register src, Register dest) { andq(src, dest); } + +void MacroAssembler::andPtr(Imm32 imm, Register dest) { andq(imm, dest); } + +void MacroAssembler::and64(Imm64 imm, Register64 dest) { + if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) { + andq(Imm32(imm.value), dest.reg); + } else { + ScratchRegisterScope scratch(*this); + movq(ImmWord(uintptr_t(imm.value)), scratch); + andq(scratch, dest.reg); + } +} + +void MacroAssembler::or64(Imm64 imm, Register64 dest) { + if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) { + orq(Imm32(imm.value), dest.reg); + } else { + ScratchRegisterScope scratch(*this); + movq(ImmWord(uintptr_t(imm.value)), scratch); + orq(scratch, dest.reg); + } +} + +void MacroAssembler::xor64(Imm64 imm, Register64 dest) { + if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) { + xorq(Imm32(imm.value), dest.reg); + } else { + ScratchRegisterScope scratch(*this); + movq(ImmWord(uintptr_t(imm.value)), scratch); + xorq(scratch, dest.reg); + } +} + +void MacroAssembler::orPtr(Register src, Register dest) { orq(src, dest); } + +void MacroAssembler::orPtr(Imm32 imm, Register dest) { orq(imm, dest); } + +void MacroAssembler::and64(Register64 src, Register64 dest) { + andq(src.reg, dest.reg); +} + +void MacroAssembler::or64(Register64 src, Register64 dest) { + orq(src.reg, dest.reg); +} + +void MacroAssembler::xor64(Register64 src, Register64 dest) { + xorq(src.reg, dest.reg); +} + +void MacroAssembler::xorPtr(Register src, Register dest) { xorq(src, dest); } + +void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorq(imm, dest); } + +void MacroAssembler::and64(const Operand& src, Register64 dest) { + andq(src, dest.reg); +} + +void MacroAssembler::or64(const Operand& src, Register64 dest) { + orq(src, dest.reg); +} + +void MacroAssembler::xor64(const Operand& src, Register64 dest) { + xorq(src, dest.reg); +} + +// =============================================================== +// Swap instructions + +void MacroAssembler::byteSwap64(Register64 reg) { bswapq(reg.reg); } + +// =============================================================== +// Arithmetic functions + +void MacroAssembler::addPtr(Register src, Register dest) { addq(src, dest); } + +void MacroAssembler::addPtr(Imm32 imm, Register dest) { addq(imm, dest); } + +void MacroAssembler::addPtr(ImmWord imm, Register dest) { + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(dest != scratch); + if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { + addq(Imm32((int32_t)imm.value), dest); + } else { + mov(imm, scratch); + addq(scratch, dest); + } +} + +void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { + addq(imm, Operand(dest)); +} + +void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) { + addq(imm, Operand(dest)); +} + +void MacroAssembler::addPtr(const Address& src, Register dest) { + addq(Operand(src), dest); +} + +void MacroAssembler::add64(const Operand& src, Register64 dest) { + addq(src, dest.reg); +} + +void MacroAssembler::add64(Register64 src, Register64 dest) { + addq(src.reg, dest.reg); +} + +void MacroAssembler::add64(Imm32 imm, Register64 dest) { addq(imm, dest.reg); } + +void MacroAssembler::add64(Imm64 imm, Register64 dest) { + addPtr(ImmWord(imm.value), dest.reg); +} + +CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) { + moveStackPtrTo(dest); + addqWithPatch(Imm32(0), dest); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) { + patchAddq(offset, -imm.value); +} + +void MacroAssembler::subPtr(Register src, Register dest) { subq(src, dest); } + +void MacroAssembler::subPtr(Register src, const Address& dest) { + subq(src, Operand(dest)); +} + +void MacroAssembler::subPtr(Imm32 imm, Register dest) { subq(imm, dest); } + +void MacroAssembler::subPtr(ImmWord imm, Register dest) { + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(dest != scratch); + if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { + subq(Imm32((int32_t)imm.value), dest); + } else { + mov(imm, scratch); + subq(scratch, dest); + } +} + +void MacroAssembler::subPtr(const Address& addr, Register dest) { + subq(Operand(addr), dest); +} + +void MacroAssembler::sub64(const Operand& src, Register64 dest) { + subq(src, dest.reg); +} + +void MacroAssembler::sub64(Register64 src, Register64 dest) { + subq(src.reg, dest.reg); +} + +void MacroAssembler::sub64(Imm64 imm, Register64 dest) { + subPtr(ImmWord(imm.value), dest.reg); +} + +void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) { + // To compute the unsigned multiplication using imulq, we have to ensure both + // operands don't have any bits set in the high word. + + if (imm.value >= 0) { + // Clear the high word of |src|. + movl(src, src); + + // |imm| and |src| are both positive, so directly perform imulq. + imulq(imm, src, dest); + } else { + // Store the low word of |src| into |dest|. + movl(src, dest); + + // Compute the unsigned value of |imm| before performing imulq. + movl(imm, ScratchReg); + imulq(ScratchReg, dest); + } + + // Move the high word into |dest|. + shrq(Imm32(32), dest); +} + +void MacroAssembler::mulPtr(Register rhs, Register srcDest) { + imulq(rhs, srcDest); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == InvalidReg); + mul64(imm, dest); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest) { + if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) { + imulq(Imm32((int32_t)imm.value), dest.reg, dest.reg); + } else { + movq(ImmWord(uintptr_t(imm.value)), ScratchReg); + imulq(ScratchReg, dest.reg); + } +} + +void MacroAssembler::mul64(const Register64& src, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == InvalidReg); + mul64(Operand(src.reg), dest); +} + +void MacroAssembler::mul64(const Operand& src, const Register64& dest) { + imulq(src, dest.reg); +} + +void MacroAssembler::mul64(const Operand& src, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == InvalidReg); + mul64(src, dest); +} + +void MacroAssembler::mulBy3(Register src, Register dest) { + lea(Operand(src, src, TimesTwo), dest); +} + +void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, + FloatRegister dest) { + movq(imm, ScratchReg); + vmulsd(Operand(ScratchReg, 0), dest, dest); +} + +void MacroAssembler::inc64(AbsoluteAddress dest) { + if (X86Encoding::IsAddressImmediate(dest.addr)) { + addPtr(Imm32(1), dest); + } else { + ScratchRegisterScope scratch(*this); + mov(ImmPtr(dest.addr), scratch); + addPtr(Imm32(1), Address(scratch, 0)); + } +} + +void MacroAssembler::neg64(Register64 reg) { negq(reg.reg); } + +void MacroAssembler::negPtr(Register reg) { negq(reg); } + +// =============================================================== +// Shift functions + +void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + shlq(imm, dest); +} + +void MacroAssembler::lshiftPtr(Register shift, Register srcDest) { + if (Assembler::HasBMI2()) { + shlxq(srcDest, shift, srcDest); + return; + } + MOZ_ASSERT(shift == rcx); + shlq_cl(srcDest); +} + +void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + lshiftPtr(imm, dest.reg); +} + +void MacroAssembler::lshift64(Register shift, Register64 srcDest) { + if (Assembler::HasBMI2()) { + shlxq(srcDest.reg, shift, srcDest.reg); + return; + } + MOZ_ASSERT(shift == rcx); + shlq_cl(srcDest.reg); +} + +void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + shrq(imm, dest); +} + +void MacroAssembler::rshiftPtr(Register shift, Register srcDest) { + if (Assembler::HasBMI2()) { + shrxq(srcDest, shift, srcDest); + return; + } + MOZ_ASSERT(shift == rcx); + shrq_cl(srcDest); +} + +void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { + rshiftPtr(imm, dest.reg); +} + +void MacroAssembler::rshift64(Register shift, Register64 srcDest) { + if (Assembler::HasBMI2()) { + shrxq(srcDest.reg, shift, srcDest.reg); + return; + } + MOZ_ASSERT(shift == rcx); + shrq_cl(srcDest.reg); +} + +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + sarq(imm, dest); +} + +void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + rshiftPtrArithmetic(imm, dest.reg); +} + +void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) { + if (Assembler::HasBMI2()) { + sarxq(srcDest.reg, shift, srcDest.reg); + return; + } + MOZ_ASSERT(shift == rcx); + sarq_cl(srcDest.reg); +} + +// =============================================================== +// Rotation functions + +void MacroAssembler::rotateLeft64(Register count, Register64 src, + Register64 dest) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + MOZ_ASSERT(count == ecx, "defineFixed(ecx)"); + + rolq_cl(dest.reg); +} + +void MacroAssembler::rotateLeft64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == InvalidReg); + rotateLeft64(count, src, dest); +} + +void MacroAssembler::rotateRight64(Register count, Register64 src, + Register64 dest) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + MOZ_ASSERT(count == ecx, "defineFixed(ecx)"); + + rorq_cl(dest.reg); +} + +void MacroAssembler::rotateRight64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == InvalidReg); + rotateRight64(count, src, dest); +} + +void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, + Register64 dest) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + rolq(count, dest.reg); +} + +void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == InvalidReg); + rotateLeft64(count, src, dest); +} + +void MacroAssembler::rotateRight64(Imm32 count, Register64 src, + Register64 dest) { + MOZ_ASSERT(src == dest, "defineReuseInput"); + rorq(count, dest.reg); +} + +void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == InvalidReg); + rotateRight64(count, src, dest); +} + +// =============================================================== +// Condition functions + +void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs, + Register dest) { + cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest); +} + +template <typename T1, typename T2> +void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) { + cmpPtr(lhs, rhs); + emitSet(cond, dest); +} + +// =============================================================== +// Bit counting functions + +void MacroAssembler::clz64(Register64 src, Register dest) { + if (AssemblerX86Shared::HasLZCNT()) { + lzcntq(src.reg, dest); + return; + } + + Label nonzero; + bsrq(src.reg, dest); + j(Assembler::NonZero, &nonzero); + movq(ImmWord(0x7F), dest); + bind(&nonzero); + xorq(Imm32(0x3F), dest); +} + +void MacroAssembler::ctz64(Register64 src, Register dest) { + if (AssemblerX86Shared::HasBMI1()) { + tzcntq(src.reg, dest); + return; + } + + Label nonzero; + bsfq(src.reg, dest); + j(Assembler::NonZero, &nonzero); + movq(ImmWord(64), dest); + bind(&nonzero); +} + +void MacroAssembler::popcnt64(Register64 src64, Register64 dest64, + Register tmp) { + Register src = src64.reg; + Register dest = dest64.reg; + + if (AssemblerX86Shared::HasPOPCNT()) { + MOZ_ASSERT(tmp == InvalidReg); + popcntq(src, dest); + return; + } + + if (src != dest) { + movq(src, dest); + } + + MOZ_ASSERT(tmp != dest); + + ScratchRegisterScope scratch(*this); + + // Equivalent to mozilla::CountPopulation32, adapted for 64 bits. + // x -= (x >> 1) & m1; + movq(src, tmp); + movq(ImmWord(0x5555555555555555), scratch); + shrq(Imm32(1), tmp); + andq(scratch, tmp); + subq(tmp, dest); + + // x = (x & m2) + ((x >> 2) & m2); + movq(dest, tmp); + movq(ImmWord(0x3333333333333333), scratch); + andq(scratch, dest); + shrq(Imm32(2), tmp); + andq(scratch, tmp); + addq(tmp, dest); + + // x = (x + (x >> 4)) & m4; + movq(dest, tmp); + movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch); + shrq(Imm32(4), tmp); + addq(tmp, dest); + andq(scratch, dest); + + // (x * h01) >> 56 + movq(ImmWord(0x0101010101010101), scratch); + imulq(scratch, dest); + shrq(Imm32(56), dest); +} + +// =============================================================== +// Branch functions + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + if (X86Encoding::IsAddressImmediate(lhs.addr)) { + branch32(cond, Operand(lhs), rhs, label); + } else { + ScratchRegisterScope scratch(*this); + mov(ImmPtr(lhs.addr), scratch); + branch32(cond, Address(scratch, 0), rhs, label); + } +} +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + if (X86Encoding::IsAddressImmediate(lhs.addr)) { + branch32(cond, Operand(lhs), rhs, label); + } else { + ScratchRegisterScope scratch(*this); + mov(ImmPtr(lhs.addr), scratch); + branch32(cond, Address(scratch, 0), rhs, label); + } +} + +void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs, + Imm32 rhs, Label* label) { + ScratchRegisterScope scratch(*this); + mov(lhs, scratch); + branch32(cond, Address(scratch, 0), rhs, label); +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, ImmWord(val.value), success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, rhs.reg, success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + + branchPtr(cond, lhs, ImmWord(val.value), label); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + Register64 rhs, Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + + branchPtr(cond, lhs, rhs.reg, label); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + const Address& rhs, Register scratch, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + MOZ_ASSERT(lhs.base != scratch); + MOZ_ASSERT(rhs.base != scratch); + + loadPtr(rhs, scratch); + branchPtr(cond, lhs, scratch, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(rhs != scratch); + if (X86Encoding::IsAddressImmediate(lhs.addr)) { + branchPtrImpl(cond, Operand(lhs), rhs, label); + } else { + mov(ImmPtr(lhs.addr), scratch); + branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label); + } +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + ImmWord rhs, Label* label) { + if (X86Encoding::IsAddressImmediate(lhs.addr)) { + branchPtrImpl(cond, Operand(lhs), rhs, label); + } else { + ScratchRegisterScope scratch(*this); + mov(ImmPtr(lhs.addr), scratch); + branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label); + } +} + +void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, + Register rhs, Label* label) { + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(rhs != scratch); + mov(lhs, scratch); + branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label); +} + +void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, + Register rhs, Label* label) { + branchPtr(cond, lhs, rhs, label); +} + +void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src, + Register dest, Label* fail) { + vcvttss2sq(src, dest); + + // Same trick as for Doubles + cmpPtr(dest, Imm32(1)); + j(Assembler::Overflow, fail); +} + +void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + branchTruncateFloat32ToPtr(src, dest, fail); + movl(dest, dest); // Zero upper 32-bits. +} + +void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src, + Register dest, Label* fail) { + branchTruncateFloat32ToPtr(src, dest, fail); + + // Check that the result is in the int32_t range. + ScratchRegisterScope scratch(*this); + move32To64SignExtend(dest, Register64(scratch)); + cmpPtr(dest, scratch); + j(Assembler::NotEqual, fail); + + movl(dest, dest); // Zero upper 32-bits. +} + +void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest, + Label* fail) { + vcvttsd2sq(src, dest); + + // vcvttsd2sq returns 0x8000000000000000 on failure. Test for it by + // subtracting 1 and testing overflow (this avoids the need to + // materialize that value in a register). + cmpPtr(dest, Imm32(1)); + j(Assembler::Overflow, fail); +} + +void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + branchTruncateDoubleToPtr(src, dest, fail); + movl(dest, dest); // Zero upper 32-bits. +} + +void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, + Register dest, Label* fail) { + branchTruncateDoubleToPtr(src, dest, fail); + + // Check that the result is in the int32_t range. + ScratchRegisterScope scratch(*this); + move32To64SignExtend(dest, Register64(scratch)); + cmpPtr(dest, scratch); + j(Assembler::NotEqual, fail); + + movl(dest, dest); // Zero upper 32-bits. +} + +void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + if (X86Encoding::IsAddressImmediate(lhs.addr)) { + test32(Operand(lhs), rhs); + } else { + ScratchRegisterScope scratch(*this); + mov(ImmPtr(lhs.addr), scratch); + test32(Operand(scratch, 0), rhs); + } + j(cond, label); +} + +template <class L> +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, + Register64 rhs, Register temp, L label) { + branchTestPtr(cond, lhs.reg, rhs.reg, label); +} + +void MacroAssembler::branchTestBooleanTruthy(bool truthy, + const ValueOperand& value, + Label* label) { + test32(value.valueReg(), value.valueReg()); + j(truthy ? NonZero : Zero, label); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, + JSWhyMagic why, Label* label) { + uint64_t magic = MagicValue(why).asRawBits(); + cmpPtr(valaddr, ImmWord(magic)); + j(cond, label); +} + +void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs, + const ValueOperand& rhs, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + branchPtr(cond, lhs, rhs.valueReg(), label); +} + +void MacroAssembler::branchToComputedAddress(const BaseIndex& address) { + jmp(Operand(address)); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + cmpPtr(lhs, rhs); + cmovCCq(cond, src, dest); +} + +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + cmpPtr(lhs, Operand(rhs)); + cmovCCq(cond, src, dest); +} + +void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + cmp32(lhs, rhs); + cmovCCq(cond, Operand(src), dest); +} + +void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, + const Address& src, Register dest) { + cmp32(lhs, rhs); + cmovCCq(cond, Operand(src), dest); +} + +void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr, + Imm32 mask, const Address& src, + Register dest) { + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero); + test32(addr, mask); + cmovCCq(cond, Operand(src), dest); +} + +void MacroAssembler::test32MovePtr(Condition cond, const Address& addr, + Imm32 mask, Register src, Register dest) { + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero); + test32(addr, mask); + cmovCCq(cond, Operand(src), dest); +} + +void MacroAssembler::spectreMovePtr(Condition cond, Register src, + Register dest) { + cmovCCq(cond, Operand(src), dest); +} + +void MacroAssembler::spectreBoundsCheck32(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(length != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(index != scratch); + MOZ_ASSERT(length != scratch); + + if (JitOptions.spectreIndexMasking) { + move32(Imm32(0), scratch); + } + + cmp32(index, length); + j(Assembler::AboveOrEqual, failure); + + if (JitOptions.spectreIndexMasking) { + cmovCCl(Assembler::AboveOrEqual, scratch, index); + } +} + +void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(index != length.base); + MOZ_ASSERT(length.base != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(index != scratch); + MOZ_ASSERT(length.base != scratch); + + if (JitOptions.spectreIndexMasking) { + move32(Imm32(0), scratch); + } + + cmp32(index, Operand(length)); + j(Assembler::AboveOrEqual, failure); + + if (JitOptions.spectreIndexMasking) { + cmovCCl(Assembler::AboveOrEqual, scratch, index); + } +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(length != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(index != scratch); + MOZ_ASSERT(length != scratch); + + if (JitOptions.spectreIndexMasking) { + movePtr(ImmWord(0), scratch); + } + + cmpPtr(index, length); + j(Assembler::AboveOrEqual, failure); + + if (JitOptions.spectreIndexMasking) { + cmovCCq(Assembler::AboveOrEqual, scratch, index); + } +} + +void MacroAssembler::spectreBoundsCheckPtr(Register index, + const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_ASSERT(index != length.base); + MOZ_ASSERT(length.base != maybeScratch); + MOZ_ASSERT(index != maybeScratch); + + ScratchRegisterScope scratch(*this); + MOZ_ASSERT(index != scratch); + MOZ_ASSERT(length.base != scratch); + + if (JitOptions.spectreIndexMasking) { + movePtr(ImmWord(0), scratch); + } + + cmpPtr(index, Operand(length)); + j(Assembler::AboveOrEqual, failure); + + if (JitOptions.spectreIndexMasking) { + cmovCCq(Assembler::AboveOrEqual, scratch, index); + } +} + +// ======================================================================== +// SIMD. + +// Extract lane as scalar + +void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src, + Register64 dest) { + if (lane == 0) { + vmovq(src, dest.reg); + } else { + vpextrq(lane, src, dest.reg); + } +} + +// Replace lane value + +void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs, + FloatRegister lhsDest) { + vpinsrq(lane, rhs.reg, lhsDest, lhsDest); +} + +void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs, + Register64 rhs, FloatRegister dest) { + vpinsrq(lane, rhs.reg, lhs, dest); +} + +// Splat + +void MacroAssembler::splatX2(Register64 src, FloatRegister dest) { + vmovq(src.reg, dest); + if (HasAVX2()) { + vbroadcastq(Operand(dest), dest); + } else { + vpunpcklqdq(dest, dest, dest); + } +} + +// ======================================================================== +// Truncate floating point. + +void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest, + Register temp, + FloatRegister floatTemp) { + Label done; + + loadFloat32(src, floatTemp); + + truncateFloat32ToInt64(src, dest, temp); + + // For unsigned conversion the case of [INT64, UINT64] needs to get handled + // separately. + loadPtr(dest, temp); + branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done); + + // Move the value inside INT64 range. + storeFloat32(floatTemp, dest); + loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp); + vaddss(Operand(dest), floatTemp, floatTemp); + storeFloat32(floatTemp, dest); + truncateFloat32ToInt64(dest, dest, temp); + + loadPtr(dest, temp); + or64(Imm64(0x8000000000000000), Register64(temp)); + storePtr(temp, dest); + + bind(&done); +} + +void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest, + Register temp, + FloatRegister floatTemp) { + Label done; + + loadDouble(src, floatTemp); + + truncateDoubleToInt64(src, dest, temp); + + // For unsigned conversion the case of [INT64, UINT64] needs to get handle + // seperately. + loadPtr(dest, temp); + branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done); + + // Move the value inside INT64 range. + storeDouble(floatTemp, dest); + loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp); + vaddsd(Operand(dest), floatTemp, floatTemp); + storeDouble(floatTemp, dest); + truncateDoubleToInt64(dest, dest, temp); + + loadPtr(dest, temp); + or64(Imm64(0x8000000000000000), Register64(temp)); + storePtr(temp, dest); + + bind(&done); +} + +void MacroAssemblerX64::fallibleUnboxPtrImpl(const Operand& src, Register dest, + JSValueType type, Label* fail) { + MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING || + type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT); + // dest := src XOR mask + // scratch := dest >> JSVAL_TAG_SHIFT + // fail if scratch != 0 + // + // Note: src and dest can be the same register. + ScratchRegisterScope scratch(asMasm()); + mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); + xorq(src, scratch); + mov(scratch, dest); + shrq(Imm32(JSVAL_TAG_SHIFT), scratch); + j(Assembler::NonZero, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(Operand(src.valueReg()), dest, type, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(Operand(src), dest, type, fail); +} + +void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest, + JSValueType type, Label* fail) { + fallibleUnboxPtrImpl(Operand(src), dest, type, fail); +} + +//}}} check_macroassembler_style +// =============================================================== + +void MacroAssemblerX64::incrementInt32Value(const Address& addr) { + asMasm().addPtr(Imm32(1), addr); +} + +void MacroAssemblerX64::unboxValue(const ValueOperand& src, AnyRegister dest, + JSValueType type) { + if (dest.isFloat()) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src.valueReg(), dest.fpu()); + jump(&end); + bind(¬Int32); + unboxDouble(src, dest.fpu()); + bind(&end); + } else { + unboxNonDouble(src, dest.gpr(), type); + } +} + +template <typename T> +void MacroAssemblerX64::loadInt32OrDouble(const T& src, FloatRegister dest) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src, dest); + jump(&end); + bind(¬Int32); + unboxDouble(src, dest); + bind(&end); +} + +// If source is a double, load it into dest. If source is int32, +// convert it to double. Else, branch to failure. +void MacroAssemblerX64::ensureDouble(const ValueOperand& source, + FloatRegister dest, Label* failure) { + Label isDouble, done; + { + ScratchTagScope tag(asMasm(), source); + splitTagForTest(source, tag); + asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble); + asMasm().branchTestInt32(Assembler::NotEqual, tag, failure); + } + + { + ScratchRegisterScope scratch(asMasm()); + unboxInt32(source, scratch); + convertInt32ToDouble(scratch, dest); + } + jump(&done); + + bind(&isDouble); + unboxDouble(source, dest); + + bind(&done); +} + +} // namespace jit +} // namespace js + +#endif /* jit_x64_MacroAssembler_x64_inl_h */ |