summaryrefslogtreecommitdiffstats
path: root/js/src/jit/x64/MacroAssembler-x64.h
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/x64/MacroAssembler-x64.h')
-rw-r--r--js/src/jit/x64/MacroAssembler-x64.h1245
1 files changed, 1245 insertions, 0 deletions
diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h
new file mode 100644
index 0000000000..12c8632dfa
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.h
@@ -0,0 +1,1245 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_h
+#define jit_x64_MacroAssembler_x64_h
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+#include "js/HeapAPI.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {}
+};
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
+};
+
+// ScratchTagScope and ScratchTagScopeRelease are used to manage the tag
+// register for splitTagForTest(), which has different register management on
+// different platforms. On 64-bit platforms it requires a scratch register that
+// does not interfere with other operations; on 32-bit platforms it uses a
+// register that is already part of the Value.
+//
+// The ScratchTagScope RAII type acquires the appropriate register; a reference
+// to a variable of this type is then passed to splitTagForTest().
+//
+// On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register
+// available in a dynamic scope during compilation. However it is important to
+// remember that that does not preserve the register value in any way, so this
+// RAII type should only be used along paths that eventually branch past further
+// uses of the extracted tag value.
+//
+// On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not
+// manage a register, it only aliases a register in the ValueOperand.
+
+class ScratchTagScope : public ScratchRegisterScope {
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : ScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerX64 : public MacroAssemblerX86Shared {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ void bindOffsets(const MacroAssemblerX86Shared::UsesVector&);
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID id));
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId));
+
+ public:
+ using MacroAssemblerX86Shared::load32;
+ using MacroAssemblerX86Shared::store16;
+ using MacroAssemblerX86Shared::store32;
+
+ MacroAssemblerX64() = default;
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ /////////////////////////////////////////////////////////////////
+ // X64 helpers.
+ /////////////////////////////////////////////////////////////////
+ void writeDataRelocation(const Value& val) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(masm.currentOffset());
+ }
+ }
+
+ // Refers to the upper 32 bits of a 64-bit Value operand.
+ // On x86_64, the upper 32 bits do not necessarily only contain the type.
+ Operand ToUpper32(Operand base) {
+ switch (base.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(Register::FromCode(base.base()), base.disp() + 4);
+
+ case Operand::MEM_SCALE:
+ return Operand(Register::FromCode(base.base()),
+ Register::FromCode(base.index()), base.scale(),
+ base.disp() + 4);
+
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ static inline Operand ToUpper32(const Address& address) {
+ return Operand(address.base, address.offset + 4);
+ }
+ static inline Operand ToUpper32(const BaseIndex& address) {
+ return Operand(address.base, address.index, address.scale,
+ address.offset + 4);
+ }
+
+ uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); }
+
+ JSValueShiftedTag GetShiftedTag(JSValueType type) {
+ return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+ }
+
+ /////////////////////////////////////////////////////////////////
+ // X86/X64-common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void storeValue(ValueOperand val, Operand dest) {
+ movq(val.valueReg(), dest);
+ }
+ void storeValue(ValueOperand val, const Address& dest) {
+ storeValue(val, Operand(dest));
+ }
+ template <typename T>
+ void storeValue(JSValueType type, Register reg, const T& dest) {
+ // Value types with 32-bit payloads can be emitted as two 32-bit moves.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(reg, Operand(dest));
+ movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest)));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ movq(scratch, Operand(dest));
+ }
+ }
+ template <typename T>
+ void storeValue(const Value& val, const T& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ if (val.isGCThing()) {
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ } else {
+ mov(ImmWord(val.asRawBits()), scratch);
+ }
+ movq(scratch, Operand(dest));
+ }
+ void storeValue(ValueOperand val, BaseIndex dest) {
+ storeValue(val, Operand(dest));
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+ void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); }
+ void loadValue(Address src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadValue(const BaseIndex& src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+ void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.valueReg() != scratch);
+ if (payload != dest.valueReg()) {
+ movq(payload, dest.valueReg());
+ }
+ mov(ImmShiftedTag(type), scratch);
+ orq(scratch, dest.valueReg());
+ }
+ void pushValue(ValueOperand val) { push(val.valueReg()); }
+ void popValue(ValueOperand val) { pop(val.valueReg()); }
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ ScratchRegisterScope scratch(asMasm());
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ push(scratch);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ push(scratch);
+ }
+ void pushValue(const Address& addr) { push(Operand(addr)); }
+
+ void pushValue(const BaseIndex& addr, Register scratch) {
+ push(Operand(addr));
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest);
+
+ Condition testUndefined(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testInt32(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testNull(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testString(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testObject(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testDouble(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testNumber(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testGCThing(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+
+ Condition testMagic(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testError(Condition cond, Register tag) {
+ return testMagic(cond, tag);
+ }
+ Condition testPrimitive(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ return cond == Equal ? Below : AboveOrEqual;
+ }
+
+ Condition testUndefined(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testString(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src),
+ Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
+ return cond;
+ }
+ Condition testDouble(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
+ return cond;
+ }
+ Condition testString(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testNull(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testString(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testObject(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+
+ Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(src.valueReg(), ImmWord(magic));
+ return cond;
+ }
+
+ void cmpPtr(Register lhs, const ImmWord rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) {
+ cmpPtr(lhs, Imm32(int32_t(rhs.value)));
+ } else {
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(Register lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(Register lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(!lhs.containsReg(scratch));
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(const Operand& lhs, const ImmWord rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ cmpPtr(lhs, Imm32((int32_t)rhs.value));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(const Operand& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmWord rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
+ void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); }
+ void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, const Imm64 rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ testq(Imm32((int32_t)rhs.value), lhs);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movq(ImmWord(rhs.value), scratch);
+ testq(scratch, lhs);
+ }
+ }
+
+ // Compare-then-conditionally-move/load, for integer types
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal,
+ Register trueValAndDest);
+
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, const Address& rhs,
+ Register falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, Register rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, const Address& rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void movePtr(Register src, Register dest) { movq(src, dest); }
+ void movePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void movePtr(ImmWord imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); }
+ void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); }
+ void loadPtr(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ loadPtr(Address(scratch, 0x0), dest);
+ }
+ }
+ FaultingCodeOffset loadPtr(const Address& address, Register dest) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(Operand(address), dest);
+ return fco;
+ }
+ void load64(const Address& address, Register dest) {
+ movq(Operand(address), dest);
+ }
+ void loadPtr(const Operand& src, Register dest) { movq(src, dest); }
+ FaultingCodeOffset loadPtr(const BaseIndex& src, Register dest) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(Operand(src), dest);
+ return fco;
+ }
+ void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); }
+ void load32(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ load32(Address(scratch, 0x0), dest);
+ }
+ }
+ void load64(const Operand& address, Register64 dest) {
+ movq(address, dest.reg);
+ }
+ FaultingCodeOffset load64(const Address& address, Register64 dest) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(Operand(address), dest.reg);
+ return fco;
+ }
+ FaultingCodeOffset load64(const BaseIndex& address, Register64 dest) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(Operand(address), dest.reg);
+ return fco;
+ }
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ load64(src, dest);
+ }
+ template <typename T>
+ void storePtr(ImmWord imm, T address) {
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ movq(Imm32((int32_t)imm.value), Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ }
+ template <typename T>
+ void storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+ }
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address) {
+ ScratchRegisterScope scratch(asMasm());
+ movq(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ FaultingCodeOffset storePtr(Register src, const Address& address) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(src, Operand(address));
+ return fco;
+ }
+ void store64(Register src, const Address& address) {
+ movq(src, Operand(address));
+ }
+ void store64(Register64 src, const Operand& address) {
+ movq(src.reg, address);
+ }
+ FaultingCodeOffset storePtr(Register src, const BaseIndex& address) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ movq(src, Operand(address));
+ return fco;
+ }
+ void storePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void storePtr(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ storePtr(src, Address(scratch, 0x0));
+ }
+ }
+ void store32(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store32(src, Address(scratch, 0x0));
+ }
+ }
+ void store16(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movw(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store16(src, Address(scratch, 0x0));
+ }
+ }
+ FaultingCodeOffset store64(Register64 src, Address address) {
+ FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+ storePtr(src.reg, address);
+ return fco;
+ }
+ FaultingCodeOffset store64(Register64 src, const BaseIndex& address) {
+ return storePtr(src.reg, address);
+ }
+ void store64(Imm64 imm, Address address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ store64(src, dest);
+ }
+
+ void splitTag(Register src, Register dest) {
+ if (src != dest) {
+ movq(src, dest);
+ }
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+ void splitTag(const Operand& operand, Register dest) {
+ movq(operand, dest);
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const Address& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+ void splitTag(const BaseIndex& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+
+ // Extracts the tag of a value and places it in tag.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+ void cmpTag(const ValueOperand& operand, ImmTag tag) {
+ ScratchTagScope reg(asMasm(), operand);
+ splitTagForTest(operand, reg);
+ cmp32(reg, tag);
+ }
+
+ Condition testMagic(Condition cond, const ValueOperand& src) {
+ ScratchTagScope scratch(asMasm(), src);
+ splitTagForTest(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testError(Condition cond, const ValueOperand& src) {
+ return testMagic(cond, src);
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
+ vmovq(src, dest.valueReg());
+ }
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+ MOZ_ASSERT(src != dest.valueReg());
+ boxValue(type, src, dest.valueReg());
+ }
+
+ // Note that the |dest| register here may be ScratchReg, so we shouldn't
+ // use it.
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxInt32(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxInt32(const Address& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ void unboxInt32(const BaseIndex& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ template <typename T>
+ void unboxDouble(const T& src, FloatRegister dest) {
+ loadDouble(Operand(src), dest);
+ }
+
+ void unboxArgObjMagic(const ValueOperand& src, Register dest) {
+ unboxArgObjMagic(Operand(src.valueReg()), dest);
+ }
+ void unboxArgObjMagic(const Operand& src, Register dest) {
+ mov(ImmWord(0), dest);
+ }
+ void unboxArgObjMagic(const Address& src, Register dest) {
+ unboxArgObjMagic(Operand(src), dest);
+ }
+
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxBoolean(const Address& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+ void unboxBoolean(const BaseIndex& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+
+ void unboxMagic(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+
+ void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+ vmovq(src.valueReg(), dest);
+ }
+
+ void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); }
+
+ void unboxNonDouble(const ValueOperand& src, Register dest,
+ JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src.valueReg(), dest);
+ return;
+ }
+ if (src.valueReg() == dest) {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src.valueReg(), dest);
+ }
+ }
+ void unboxNonDouble(const Operand& src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src, dest);
+ return;
+ }
+ // Explicitly permits |dest| to be used in |src|.
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest != scratch);
+ if (src.containsReg(dest)) {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ // If src is already a register, then src and dest are the same
+ // thing and we don't need to move anything into dest.
+ if (src.kind() != Operand::REG) {
+ movq(src, dest);
+ }
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src, dest);
+ }
+ }
+ void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+
+ void unboxString(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+
+ void unboxSymbol(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+
+ void unboxBigInt(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, dest);
+ }
+
+ // This should only be used for GC barrier code, to unbox a GC thing Value.
+ // It's fine there because we don't depend on the actual Value type (all Cells
+ // are treated the same way). In almost all other cases this would be
+ // Spectre-unsafe - use unboxNonDouble and friends instead.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(Operand(src), dest);
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
+ movq(ImmWord(wasm::AnyRef::GCThingMask), dest);
+ andq(Operand(src), dest);
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(Operand(src), dest);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ void getWasmAnyRefGCThingChunk(Register src, Register dest) {
+ MOZ_ASSERT(src != dest);
+ movq(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
+ andq(src, dest);
+ }
+
+ inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
+ JSValueType type, Label* fail);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(address, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ loadPtr(address, scratch);
+ splitTag(scratch, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type);
+
+ // These two functions use the low 32-bits of the full value register.
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+
+ void loadConstantDouble(double d, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
+ void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
+ void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
+ void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+
+ public:
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+ test32(operand.valueReg(), operand.valueReg());
+ return truthy ? NonZero : Zero;
+ }
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+ template <typename T>
+ inline void loadInt32OrDouble(const T& src, FloatRegister dest);
+
+ template <typename T>
+ void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(src, dest.fpu());
+ } else {
+ unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ ScratchRegisterScope scratch(asMasm());
+ unboxNonDouble(value, scratch, type);
+ storePtr(scratch, address);
+ if (type == JSVAL_TYPE_OBJECT) {
+ // Ideally we would call unboxObjectOrNull, but we need an extra
+ // scratch register for that. So unbox as object, then clear the
+ // object-or-null bit.
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, Operand(address));
+ }
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ // Checks whether a double is representable as a 64-bit integer. If so, the
+ // integer is written to the output register. Otherwise, a bailout is taken to
+ // the given snapshot. This function overwrites the scratch float register.
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertUInt32ToDouble(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2sd(src, dest, dest);
+ }
+
+ void convertUInt32ToFloat32(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2ss(src, dest, dest);
+ }
+
+ inline void incrementInt32Value(const Address& addr);
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ public:
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+using MacroAssemblerSpecific = MacroAssemblerX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_h */