/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- * vim: set ts=8 sts=2 et sw=2 tw=80: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef jit_x64_MacroAssembler_x64_h #define jit_x64_MacroAssembler_x64_h #include "jit/x86-shared/MacroAssembler-x86-shared.h" #include "js/HeapAPI.h" #include "wasm/WasmBuiltins.h" namespace js { namespace jit { struct ImmShiftedTag : public ImmWord { explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {} explicit ImmShiftedTag(JSValueType type) : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {} }; struct ImmTag : public Imm32 { explicit ImmTag(JSValueTag tag) : Imm32(tag) {} }; // ScratchTagScope and ScratchTagScopeRelease are used to manage the tag // register for splitTagForTest(), which has different register management on // different platforms. On 64-bit platforms it requires a scratch register that // does not interfere with other operations; on 32-bit platforms it uses a // register that is already part of the Value. // // The ScratchTagScope RAII type acquires the appropriate register; a reference // to a variable of this type is then passed to splitTagForTest(). // // On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register // available in a dynamic scope during compilation. However it is important to // remember that that does not preserve the register value in any way, so this // RAII type should only be used along paths that eventually branch past further // uses of the extracted tag value. // // On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not // manage a register, it only aliases a register in the ValueOperand. class ScratchTagScope : public ScratchRegisterScope { public: ScratchTagScope(MacroAssembler& masm, const ValueOperand&) : ScratchRegisterScope(masm) {} }; class ScratchTagScopeRelease { ScratchTagScope* ts_; public: explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) { ts_->release(); } ~ScratchTagScopeRelease() { ts_->reacquire(); } }; class MacroAssemblerX64 : public MacroAssemblerX86Shared { private: // Perform a downcast. Should be removed by Bug 996602. MacroAssembler& asMasm(); const MacroAssembler& asMasm() const; void bindOffsets(const MacroAssemblerX86Shared::UsesVector&); void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg, JmpSrc (X86Encoding::BaseAssemblerX64::*op)( X86Encoding::XMMRegisterID id)); void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest, JmpSrc (X86Encoding::BaseAssemblerX64::*op)( X86Encoding::XMMRegisterID srcId, X86Encoding::XMMRegisterID destId)); public: using MacroAssemblerX86Shared::load32; using MacroAssemblerX86Shared::store16; using MacroAssemblerX86Shared::store32; MacroAssemblerX64() = default; // The buffer is about to be linked, make sure any constant pools or excess // bookkeeping has been flushed to the instruction stream. void finish(); ///////////////////////////////////////////////////////////////// // X64 helpers. ///////////////////////////////////////////////////////////////// void writeDataRelocation(const Value& val) { // Raw GC pointer relocations and Value relocations both end up in // Assembler::TraceDataRelocations. if (val.isGCThing()) { gc::Cell* cell = val.toGCThing(); if (cell && gc::IsInsideNursery(cell)) { embedsNurseryPointers_ = true; } dataRelocations_.writeUnsigned(masm.currentOffset()); } } // Refers to the upper 32 bits of a 64-bit Value operand. // On x86_64, the upper 32 bits do not necessarily only contain the type. Operand ToUpper32(Operand base) { switch (base.kind()) { case Operand::MEM_REG_DISP: return Operand(Register::FromCode(base.base()), base.disp() + 4); case Operand::MEM_SCALE: return Operand(Register::FromCode(base.base()), Register::FromCode(base.index()), base.scale(), base.disp() + 4); default: MOZ_CRASH("unexpected operand kind"); } } static inline Operand ToUpper32(const Address& address) { return Operand(address.base, address.offset + 4); } static inline Operand ToUpper32(const BaseIndex& address) { return Operand(address.base, address.index, address.scale, address.offset + 4); } uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); } JSValueShiftedTag GetShiftedTag(JSValueType type) { return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type); } ///////////////////////////////////////////////////////////////// // X86/X64-common interface. ///////////////////////////////////////////////////////////////// void storeValue(ValueOperand val, Operand dest) { movq(val.valueReg(), dest); } void storeValue(ValueOperand val, const Address& dest) { storeValue(val, Operand(dest)); } template void storeValue(JSValueType type, Register reg, const T& dest) { // Value types with 32-bit payloads can be emitted as two 32-bit moves. if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { movl(reg, Operand(dest)); movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest))); } else { ScratchRegisterScope scratch(asMasm()); boxValue(type, reg, scratch); movq(scratch, Operand(dest)); } } template void storeValue(const Value& val, const T& dest) { ScratchRegisterScope scratch(asMasm()); if (val.isGCThing()) { movWithPatch(ImmWord(val.asRawBits()), scratch); writeDataRelocation(val); } else { mov(ImmWord(val.asRawBits()), scratch); } movq(scratch, Operand(dest)); } void storeValue(ValueOperand val, BaseIndex dest) { storeValue(val, Operand(dest)); } void storeValue(const Address& src, const Address& dest, Register temp) { loadPtr(src, temp); storePtr(temp, dest); } void storePrivateValue(Register src, const Address& dest) { storePtr(src, dest); } void storePrivateValue(ImmGCPtr imm, const Address& dest) { storePtr(imm, dest); } void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); } void loadValue(Address src, ValueOperand val) { loadValue(Operand(src), val); } void loadValue(const BaseIndex& src, ValueOperand val) { loadValue(Operand(src), val); } void loadUnalignedValue(const Address& src, ValueOperand dest) { loadValue(src, dest); } void tagValue(JSValueType type, Register payload, ValueOperand dest) { ScratchRegisterScope scratch(asMasm()); MOZ_ASSERT(dest.valueReg() != scratch); if (payload != dest.valueReg()) { movq(payload, dest.valueReg()); } mov(ImmShiftedTag(type), scratch); orq(scratch, dest.valueReg()); } void pushValue(ValueOperand val) { push(val.valueReg()); } void popValue(ValueOperand val) { pop(val.valueReg()); } void pushValue(const Value& val) { if (val.isGCThing()) { ScratchRegisterScope scratch(asMasm()); movWithPatch(ImmWord(val.asRawBits()), scratch); writeDataRelocation(val); push(scratch); } else { push(ImmWord(val.asRawBits())); } } void pushValue(JSValueType type, Register reg) { ScratchRegisterScope scratch(asMasm()); boxValue(type, reg, scratch); push(scratch); } void pushValue(const Address& addr) { push(Operand(addr)); } void pushValue(const BaseIndex& addr, Register scratch) { push(Operand(addr)); } void boxValue(JSValueType type, Register src, Register dest); Condition testUndefined(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED)); return cond; } Condition testInt32(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_INT32)); return cond; } Condition testBoolean(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN)); return cond; } Condition testNull(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_NULL)); return cond; } Condition testString(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_STRING)); return cond; } Condition testSymbol(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL)); return cond; } Condition testBigInt(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_BIGINT)); return cond; } Condition testObject(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_OBJECT)); return cond; } Condition testDouble(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE)); return cond == Equal ? BelowOrEqual : Above; } Condition testNumber(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag)); return cond == Equal ? BelowOrEqual : Above; } Condition testGCThing(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag)); return cond == Equal ? AboveOrEqual : Below; } Condition testMagic(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JSVAL_TAG_MAGIC)); return cond; } Condition testError(Condition cond, Register tag) { return testMagic(cond, tag); } Condition testPrimitive(Condition cond, Register tag) { MOZ_ASSERT(cond == Equal || cond == NotEqual); cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag)); return cond == Equal ? Below : AboveOrEqual; } Condition testUndefined(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testUndefined(cond, scratch); } Condition testInt32(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testInt32(cond, scratch); } Condition testBoolean(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testBoolean(cond, scratch); } Condition testDouble(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testDouble(cond, scratch); } Condition testNumber(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testNumber(cond, scratch); } Condition testNull(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testNull(cond, scratch); } Condition testString(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testString(cond, scratch); } Condition testSymbol(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testSymbol(cond, scratch); } Condition testBigInt(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testBigInt(cond, scratch); } Condition testObject(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testObject(cond, scratch); } Condition testGCThing(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testGCThing(cond, scratch); } Condition testPrimitive(Condition cond, const ValueOperand& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testPrimitive(cond, scratch); } Condition testUndefined(Condition cond, const Address& src) { cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED)))); return cond; } Condition testInt32(Condition cond, const Address& src) { cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32)))); return cond; } Condition testBoolean(Condition cond, const Address& src) { cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN)))); return cond; } Condition testDouble(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testDouble(cond, scratch); } Condition testNumber(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testNumber(cond, scratch); } Condition testNull(Condition cond, const Address& src) { cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL)))); return cond; } Condition testString(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testString(cond, scratch); } Condition testSymbol(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testSymbol(cond, scratch); } Condition testBigInt(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testBigInt(cond, scratch); } Condition testObject(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testObject(cond, scratch); } Condition testPrimitive(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testPrimitive(cond, scratch); } Condition testGCThing(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testGCThing(cond, scratch); } Condition testMagic(Condition cond, const Address& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testMagic(cond, scratch); } Condition testUndefined(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testUndefined(cond, scratch); } Condition testNull(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testNull(cond, scratch); } Condition testBoolean(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testBoolean(cond, scratch); } Condition testString(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testString(cond, scratch); } Condition testSymbol(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testSymbol(cond, scratch); } Condition testBigInt(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testBigInt(cond, scratch); } Condition testInt32(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testInt32(cond, scratch); } Condition testObject(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testObject(cond, scratch); } Condition testDouble(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testDouble(cond, scratch); } Condition testMagic(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testMagic(cond, scratch); } Condition testGCThing(Condition cond, const BaseIndex& src) { ScratchRegisterScope scratch(asMasm()); splitTag(src, scratch); return testGCThing(cond, scratch); } Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) { uint64_t magic = MagicValue(why).asRawBits(); cmpPtr(src.valueReg(), ImmWord(magic)); return cond; } void cmpPtr(Register lhs, const ImmWord rhs) { ScratchRegisterScope scratch(asMasm()); MOZ_ASSERT(lhs != scratch); if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) { cmpPtr(lhs, Imm32(int32_t(rhs.value))); } else { movePtr(rhs, scratch); cmpPtr(lhs, scratch); } } void cmpPtr(Register lhs, const ImmPtr rhs) { cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); } void cmpPtr(Register lhs, const ImmGCPtr rhs) { ScratchRegisterScope scratch(asMasm()); MOZ_ASSERT(lhs != scratch); movePtr(rhs, scratch); cmpPtr(lhs, scratch); } void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); } void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) { ScratchRegisterScope scratch(asMasm()); MOZ_ASSERT(!lhs.containsReg(scratch)); movePtr(rhs, scratch); cmpPtr(lhs, scratch); } void cmpPtr(const Operand& lhs, const ImmWord rhs) { if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) { cmpPtr(lhs, Imm32((int32_t)rhs.value)); } else { ScratchRegisterScope scratch(asMasm()); movePtr(rhs, scratch); cmpPtr(lhs, scratch); } } void cmpPtr(const Operand& lhs, const ImmPtr rhs) { cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); } void cmpPtr(const Address& lhs, const ImmGCPtr rhs) { cmpPtr(Operand(lhs), rhs); } void cmpPtr(const Address& lhs, const ImmWord rhs) { cmpPtr(Operand(lhs), rhs); } void cmpPtr(const Address& lhs, const ImmPtr rhs) { cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); } void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); } void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); } void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); } void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); } void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); } void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); } void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); } void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); } void test64(Register lhs, Register rhs) { testq(rhs, lhs); } void test64(Register lhs, const Imm64 rhs) { if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) { testq(Imm32((int32_t)rhs.value), lhs); } else { ScratchRegisterScope scratch(asMasm()); movq(ImmWord(rhs.value), scratch); testq(scratch, lhs); } } // Compare-then-conditionally-move/load, for integer types template void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal, Register trueValAndDest); template void cmpMove(Condition cond, Register lhs, const Address& rhs, Register falseVal, Register trueValAndDest); template void cmpLoad(Condition cond, Register lhs, Register rhs, const Address& falseVal, Register trueValAndDest); template void cmpLoad(Condition cond, Register lhs, const Address& rhs, const Address& falseVal, Register trueValAndDest); ///////////////////////////////////////////////////////////////// // Common interface. ///////////////////////////////////////////////////////////////// void movePtr(Register src, Register dest) { movq(src, dest); } void movePtr(Register src, const Operand& dest) { movq(src, dest); } void movePtr(ImmWord imm, Register dest) { mov(imm, dest); } void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); } void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); } void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); } void loadPtr(AbsoluteAddress address, Register dest) { if (X86Encoding::IsAddressImmediate(address.addr)) { movq(Operand(address), dest); } else { ScratchRegisterScope scratch(asMasm()); mov(ImmPtr(address.addr), scratch); loadPtr(Address(scratch, 0x0), dest); } } void loadPtr(const Address& address, Register dest) { movq(Operand(address), dest); } void load64(const Address& address, Register dest) { movq(Operand(address), dest); } void loadPtr(const Operand& src, Register dest) { movq(src, dest); } void loadPtr(const BaseIndex& src, Register dest) { movq(Operand(src), dest); } void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); } void load32(AbsoluteAddress address, Register dest) { if (X86Encoding::IsAddressImmediate(address.addr)) { movl(Operand(address), dest); } else { ScratchRegisterScope scratch(asMasm()); mov(ImmPtr(address.addr), scratch); load32(Address(scratch, 0x0), dest); } } void load64(const Operand& address, Register64 dest) { movq(address, dest.reg); } void load64(const Address& address, Register64 dest) { movq(Operand(address), dest.reg); } void load64(const BaseIndex& address, Register64 dest) { movq(Operand(address), dest.reg); } template void load64Unaligned(const S& src, Register64 dest) { load64(src, dest); } template void storePtr(ImmWord imm, T address) { if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { movq(Imm32((int32_t)imm.value), Operand(address)); } else { ScratchRegisterScope scratch(asMasm()); mov(imm, scratch); movq(scratch, Operand(address)); } } template void storePtr(ImmPtr imm, T address) { storePtr(ImmWord(uintptr_t(imm.value)), address); } template void storePtr(ImmGCPtr imm, T address) { ScratchRegisterScope scratch(asMasm()); movq(imm, scratch); movq(scratch, Operand(address)); } void storePtr(Register src, const Address& address) { movq(src, Operand(address)); } void store64(Register src, const Address& address) { movq(src, Operand(address)); } void store64(Register64 src, const Operand& address) { movq(src.reg, address); } void storePtr(Register src, const BaseIndex& address) { movq(src, Operand(address)); } void storePtr(Register src, const Operand& dest) { movq(src, dest); } void storePtr(Register src, AbsoluteAddress address) { if (X86Encoding::IsAddressImmediate(address.addr)) { movq(src, Operand(address)); } else { ScratchRegisterScope scratch(asMasm()); mov(ImmPtr(address.addr), scratch); storePtr(src, Address(scratch, 0x0)); } } void store32(Register src, AbsoluteAddress address) { if (X86Encoding::IsAddressImmediate(address.addr)) { movl(src, Operand(address)); } else { ScratchRegisterScope scratch(asMasm()); mov(ImmPtr(address.addr), scratch); store32(src, Address(scratch, 0x0)); } } void store16(Register src, AbsoluteAddress address) { if (X86Encoding::IsAddressImmediate(address.addr)) { movw(src, Operand(address)); } else { ScratchRegisterScope scratch(asMasm()); mov(ImmPtr(address.addr), scratch); store16(src, Address(scratch, 0x0)); } } void store64(Register64 src, Address address) { storePtr(src.reg, address); } void store64(Register64 src, const BaseIndex& address) { storePtr(src.reg, address); } void store64(Imm64 imm, Address address) { storePtr(ImmWord(imm.value), address); } void store64(Imm64 imm, const BaseIndex& address) { storePtr(ImmWord(imm.value), address); } template void store64Unaligned(const S& src, const T& dest) { store64(src, dest); } void splitTag(Register src, Register dest) { if (src != dest) { movq(src, dest); } shrq(Imm32(JSVAL_TAG_SHIFT), dest); } void splitTag(const ValueOperand& operand, Register dest) { splitTag(operand.valueReg(), dest); } void splitTag(const Operand& operand, Register dest) { movq(operand, dest); shrq(Imm32(JSVAL_TAG_SHIFT), dest); } void splitTag(const Address& operand, Register dest) { splitTag(Operand(operand), dest); } void splitTag(const BaseIndex& operand, Register dest) { splitTag(Operand(operand), dest); } // Extracts the tag of a value and places it in tag. void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { splitTag(value, tag); } void cmpTag(const ValueOperand& operand, ImmTag tag) { ScratchTagScope reg(asMasm(), operand); splitTagForTest(operand, reg); cmp32(reg, tag); } Condition testMagic(Condition cond, const ValueOperand& src) { ScratchTagScope scratch(asMasm(), src); splitTagForTest(src, scratch); return testMagic(cond, scratch); } Condition testError(Condition cond, const ValueOperand& src) { return testMagic(cond, src); } void testNullSet(Condition cond, const ValueOperand& value, Register dest) { cond = testNull(cond, value); emitSet(cond, dest); } void testObjectSet(Condition cond, const ValueOperand& value, Register dest) { cond = testObject(cond, value); emitSet(cond, dest); } void testUndefinedSet(Condition cond, const ValueOperand& value, Register dest) { cond = testUndefined(cond, value); emitSet(cond, dest); } void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) { vmovq(src, dest.valueReg()); } void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) { MOZ_ASSERT(src != dest.valueReg()); boxValue(type, src, dest.valueReg()); } // Note that the |dest| register here may be ScratchReg, so we shouldn't // use it. void unboxInt32(const ValueOperand& src, Register dest) { movl(src.valueReg(), dest); } void unboxInt32(const Operand& src, Register dest) { movl(src, dest); } void unboxInt32(const Address& src, Register dest) { unboxInt32(Operand(src), dest); } void unboxInt32(const BaseIndex& src, Register dest) { unboxInt32(Operand(src), dest); } template void unboxDouble(const T& src, FloatRegister dest) { loadDouble(Operand(src), dest); } void unboxArgObjMagic(const ValueOperand& src, Register dest) { unboxArgObjMagic(Operand(src.valueReg()), dest); } void unboxArgObjMagic(const Operand& src, Register dest) { mov(ImmWord(0), dest); } void unboxArgObjMagic(const Address& src, Register dest) { unboxArgObjMagic(Operand(src), dest); } void unboxBoolean(const ValueOperand& src, Register dest) { movl(src.valueReg(), dest); } void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); } void unboxBoolean(const Address& src, Register dest) { unboxBoolean(Operand(src), dest); } void unboxBoolean(const BaseIndex& src, Register dest) { unboxBoolean(Operand(src), dest); } void unboxMagic(const ValueOperand& src, Register dest) { movl(src.valueReg(), dest); } void unboxDouble(const ValueOperand& src, FloatRegister dest) { vmovq(src.valueReg(), dest); } void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); } void unboxNonDouble(const ValueOperand& src, Register dest, JSValueType type) { MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { movl(src.valueReg(), dest); return; } if (src.valueReg() == dest) { ScratchRegisterScope scratch(asMasm()); mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); xorq(scratch, dest); } else { mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest); xorq(src.valueReg(), dest); } } void unboxNonDouble(const Operand& src, Register dest, JSValueType type) { MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { movl(src, dest); return; } // Explicitly permits |dest| to be used in |src|. ScratchRegisterScope scratch(asMasm()); MOZ_ASSERT(dest != scratch); if (src.containsReg(dest)) { mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); // If src is already a register, then src and dest are the same // thing and we don't need to move anything into dest. if (src.kind() != Operand::REG) { movq(src, dest); } xorq(scratch, dest); } else { mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest); xorq(src, dest); } } void unboxNonDouble(const Address& src, Register dest, JSValueType type) { unboxNonDouble(Operand(src), dest, type); } void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) { unboxNonDouble(Operand(src), dest, type); } void unboxString(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_STRING); } void unboxString(const Operand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_STRING); } void unboxString(const Address& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_STRING); } void unboxSymbol(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); } void unboxSymbol(const Operand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); } void unboxBigInt(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); } void unboxBigInt(const Operand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); } void unboxBigInt(const Address& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); } void unboxObject(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); } void unboxObject(const Operand& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); } void unboxObject(const Address& src, Register dest) { unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT); } void unboxObject(const BaseIndex& src, Register dest) { unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT); } template void unboxObjectOrNull(const T& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); ScratchRegisterScope scratch(asMasm()); mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch); andq(scratch, dest); } // This should only be used for GC barrier code, to unbox a GC thing Value. // It's fine there because we don't depend on the actual Value type (all Cells // are treated the same way). In almost all other cases this would be // Spectre-unsafe - use unboxNonDouble and friends instead. void unboxGCThingForGCBarrier(const Address& src, Register dest) { movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest); andq(Operand(src), dest); } void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) { MOZ_ASSERT(src.valueReg() != dest); movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest); andq(src.valueReg(), dest); } // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base. void getGCThingValueChunk(const Address& src, Register dest) { movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest); andq(Operand(src), dest); } void getGCThingValueChunk(const ValueOperand& src, Register dest) { MOZ_ASSERT(src.valueReg() != dest); movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest); andq(src.valueReg(), dest); } inline void fallibleUnboxPtrImpl(const Operand& src, Register dest, JSValueType type, Label* fail); // Extended unboxing API. If the payload is already in a register, returns // that register. Otherwise, provides a move to the given scratch register, // and returns that. [[nodiscard]] Register extractObject(const Address& address, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); unboxObject(address, scratch); return scratch; } [[nodiscard]] Register extractObject(const ValueOperand& value, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); unboxObject(value, scratch); return scratch; } [[nodiscard]] Register extractSymbol(const ValueOperand& value, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); unboxSymbol(value, scratch); return scratch; } [[nodiscard]] Register extractInt32(const ValueOperand& value, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); unboxInt32(value, scratch); return scratch; } [[nodiscard]] Register extractBoolean(const ValueOperand& value, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); unboxBoolean(value, scratch); return scratch; } [[nodiscard]] Register extractTag(const Address& address, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); loadPtr(address, scratch); splitTag(scratch, scratch); return scratch; } [[nodiscard]] Register extractTag(const ValueOperand& value, Register scratch) { MOZ_ASSERT(scratch != ScratchReg); splitTag(value, scratch); return scratch; } inline void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type); // These two functions use the low 32-bits of the full value register. void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) { convertInt32ToDouble(operand.valueReg(), dest); } void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) { convertInt32ToDouble(operand.valueReg(), dest); } void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) { convertInt32ToFloat32(operand.valueReg(), dest); } void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) { convertInt32ToFloat32(operand.valueReg(), dest); } void loadConstantDouble(double d, FloatRegister dest); void loadConstantFloat32(float f, FloatRegister dest); void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest); void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest); void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpadddSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminubSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpminudSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpandSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpxorSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vporSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vandpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vminpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vptestSimd128(const SimdConstant& v, FloatRegister lhs); void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs, FloatRegister dest); public: Condition testInt32Truthy(bool truthy, const ValueOperand& operand) { test32(operand.valueReg(), operand.valueReg()); return truthy ? NonZero : Zero; } Condition testStringTruthy(bool truthy, const ValueOperand& value); Condition testBigIntTruthy(bool truthy, const ValueOperand& value); template inline void loadInt32OrDouble(const T& src, FloatRegister dest); template void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) { if (dest.isFloat()) { loadInt32OrDouble(src, dest.fpu()); } else { unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type)); } } template void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes, JSValueType type) { switch (nbytes) { case 8: { ScratchRegisterScope scratch(asMasm()); unboxNonDouble(value, scratch, type); storePtr(scratch, address); if (type == JSVAL_TYPE_OBJECT) { // Ideally we would call unboxObjectOrNull, but we need an extra // scratch register for that. So unbox as object, then clear the // object-or-null bit. mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch); andq(scratch, Operand(address)); } return; } case 4: store32(value.valueReg(), address); return; case 1: store8(value.valueReg(), address); return; default: MOZ_CRASH("Bad payload width"); } } // Checks whether a double is representable as a 64-bit integer. If so, the // integer is written to the output register. Otherwise, a bailout is taken to // the given snapshot. This function overwrites the scratch float register. void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck = true); void convertUInt32ToDouble(Register src, FloatRegister dest) { // Zero the output register to break dependencies, see convertInt32ToDouble. zeroDouble(dest); vcvtsq2sd(src, dest, dest); } void convertUInt32ToFloat32(Register src, FloatRegister dest) { // Zero the output register to break dependencies, see convertInt32ToDouble. zeroDouble(dest); vcvtsq2ss(src, dest, dest); } inline void incrementInt32Value(const Address& addr); inline void ensureDouble(const ValueOperand& source, FloatRegister dest, Label* failure); public: void handleFailureWithHandlerTail(Label* profilerExitTail, Label* bailoutTail); // Instrumentation for entering and leaving the profiler. void profilerEnterFrame(Register framePtr, Register scratch); void profilerExitFrame(); }; using MacroAssemblerSpecific = MacroAssemblerX64; } // namespace jit } // namespace js #endif /* jit_x64_MacroAssembler_x64_h */