summaryrefslogtreecommitdiffstats
path: root/js/src/jit/x64
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/x64')
-rw-r--r--js/src/jit/x64/Assembler-x64.cpp246
-rw-r--r--js/src/jit/x64/Assembler-x64.h1249
-rw-r--r--js/src/jit/x64/BaseAssembler-x64.h1373
-rw-r--r--js/src/jit/x64/CodeGenerator-x64.cpp984
-rw-r--r--js/src/jit/x64/CodeGenerator-x64.h41
-rw-r--r--js/src/jit/x64/LIR-x64.h170
-rw-r--r--js/src/jit/x64/Lowering-x64.cpp565
-rw-r--r--js/src/jit/x64/Lowering-x64.h70
-rw-r--r--js/src/jit/x64/MacroAssembler-x64-inl.h1099
-rw-r--r--js/src/jit/x64/MacroAssembler-x64.cpp1747
-rw-r--r--js/src/jit/x64/MacroAssembler-x64.h1218
-rw-r--r--js/src/jit/x64/SharedICHelpers-x64-inl.h80
-rw-r--r--js/src/jit/x64/SharedICHelpers-x64.h70
-rw-r--r--js/src/jit/x64/SharedICRegisters-x64.h33
-rw-r--r--js/src/jit/x64/Trampoline-x64.cpp888
15 files changed, 9833 insertions, 0 deletions
diff --git a/js/src/jit/x64/Assembler-x64.cpp b/js/src/jit/x64/Assembler-x64.cpp
new file mode 100644
index 0000000000..e00e34e830
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.cpp
@@ -0,0 +1,246 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Assembler-x64.h"
+
+#include "gc/Tracer.h"
+#include "util/Memory.h"
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+ :
+#if defined(XP_WIN)
+ regIndex_(0),
+ stackOffset_(ShadowStackSpace),
+#else
+ intRegIndex_(0),
+ floatRegIndex_(0),
+ stackOffset_(0),
+#endif
+ current_() {
+}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+#if defined(XP_WIN)
+ static_assert(NumIntArgRegs == NumFloatArgRegs);
+ if (regIndex_ == NumIntArgRegs) {
+ if (type == MIRType::Simd128) {
+ // On Win64, >64 bit args need to be passed by reference. However, wasm
+ // doesn't allow passing SIMD values to JS, so the only way to reach this
+ // is wasm to wasm calls. Ergo we can break the native ABI here and use
+ // the Wasm ABI instead.
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += Simd128DataSize;
+ } else {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ }
+ return current_;
+ }
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ current_ = ABIArg(IntArgRegs[regIndex_++]);
+ break;
+ case MIRType::Float32:
+ current_ = ABIArg(FloatArgRegs[regIndex_++].asSingle());
+ break;
+ case MIRType::Double:
+ current_ = ABIArg(FloatArgRegs[regIndex_++]);
+ break;
+ case MIRType::Simd128:
+ // On Win64, >64 bit args need to be passed by reference, but wasm
+ // doesn't allow passing SIMD values to FFIs. The only way to reach
+ // here is asm to asm calls, so we can break the ABI here.
+ current_ = ABIArg(FloatArgRegs[regIndex_++].asSimd128());
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+#else
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ current_ = ABIArg(IntArgRegs[intRegIndex_++]);
+ break;
+ case MIRType::Double:
+ case MIRType::Float32:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ if (type == MIRType::Float32) {
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSingle());
+ } else {
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++]);
+ }
+ break;
+ case MIRType::Simd128:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += Simd128DataSize;
+ break;
+ }
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSimd128());
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+#endif
+}
+
+void Assembler::addPendingJump(JmpSrc src, ImmPtr target,
+ RelocationKind reloc) {
+ MOZ_ASSERT(target.value != nullptr);
+
+ // Emit reloc before modifying the jump table, since it computes a 0-based
+ // index. This jump is not patchable at runtime.
+ if (reloc == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.offset());
+ }
+
+ static_assert(MaxCodeBytesPerProcess <= uint64_t(2) * 1024 * 1024 * 1024,
+ "Code depends on using int32_t for cross-JitCode jump offsets");
+
+ MOZ_ASSERT_IF(reloc == RelocationKind::JITCODE,
+ AddressIsInExecutableMemory(target.value));
+
+ RelativePatch patch(src.offset(), target.value, reloc);
+ if (reloc == RelocationKind::JITCODE ||
+ AddressIsInExecutableMemory(target.value)) {
+ enoughMemory_ &= codeJumps_.append(patch);
+ } else {
+ enoughMemory_ &= extendedJumps_.append(patch);
+ }
+}
+
+void Assembler::finish() {
+ if (oom()) {
+ return;
+ }
+
+ AutoCreatedBy acb(*this, "Assembler::finish");
+
+ if (!extendedJumps_.length()) {
+ // Since we may be folowed by non-executable data, eagerly insert an
+ // undefined instruction byte to prevent processors from decoding
+ // gibberish into their pipelines. See Intel performance guides.
+ masm.ud2();
+ return;
+ }
+
+ // Emit the jump table.
+ masm.haltingAlign(SizeOfJumpTableEntry);
+ extendedJumpTable_ = masm.size();
+
+ // Zero the extended jumps table.
+ for (size_t i = 0; i < extendedJumps_.length(); i++) {
+#ifdef DEBUG
+ size_t oldSize = masm.size();
+#endif
+ MOZ_ASSERT(hasCreator());
+ masm.jmp_rip(2);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 6);
+ // Following an indirect branch with ud2 hints to the hardware that
+ // there's no fall-through. This also aligns the 64-bit immediate.
+ masm.ud2();
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 8);
+ masm.immediate64(0);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfExtendedJump);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfJumpTableEntry);
+ }
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ AssemblerX86Shared::executableCopy(buffer);
+
+ for (RelativePatch& rp : codeJumps_) {
+ uint8_t* src = buffer + rp.offset;
+ MOZ_ASSERT(rp.target);
+
+ MOZ_RELEASE_ASSERT(X86Encoding::CanRelinkJump(src, rp.target));
+ X86Encoding::SetRel32(src, rp.target);
+ }
+
+ for (size_t i = 0; i < extendedJumps_.length(); i++) {
+ RelativePatch& rp = extendedJumps_[i];
+ uint8_t* src = buffer + rp.offset;
+ MOZ_ASSERT(rp.target);
+
+ if (X86Encoding::CanRelinkJump(src, rp.target)) {
+ X86Encoding::SetRel32(src, rp.target);
+ } else {
+ // An extended jump table must exist, and its offset must be in
+ // range.
+ MOZ_ASSERT(extendedJumpTable_);
+ MOZ_ASSERT((extendedJumpTable_ + i * SizeOfJumpTableEntry) <=
+ size() - SizeOfJumpTableEntry);
+
+ // Patch the jump to go to the extended jump entry.
+ uint8_t* entry = buffer + extendedJumpTable_ + i * SizeOfJumpTableEntry;
+ X86Encoding::SetRel32(src, entry);
+
+ // Now patch the pointer, note that we need to align it to
+ // *after* the extended jump, i.e. after the 64-bit immedate.
+ X86Encoding::SetPointer(entry + SizeOfExtendedJump, rp.target);
+ }
+ }
+}
+
+class RelocationIterator {
+ CompactBufferReader reader_;
+ uint32_t offset_ = 0;
+
+ public:
+ explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+ bool read() {
+ if (!reader_.more()) {
+ return false;
+ }
+ offset_ = reader_.readUnsigned();
+ return true;
+ }
+
+ uint32_t offset() const { return offset_; }
+};
+
+JitCode* Assembler::CodeFromJump(JitCode* code, uint8_t* jump) {
+ uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump);
+
+ MOZ_ASSERT(!code->containsNativePC(target),
+ "Extended jump table not used for cross-JitCode jumps");
+
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ RelocationIterator iter(reader);
+ while (iter.read()) {
+ JitCode* child = CodeFromJump(code, code->raw() + iter.offset());
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ MOZ_ASSERT(child == CodeFromJump(code, code->raw() + iter.offset()));
+ }
+}
diff --git a/js/src/jit/x64/Assembler-x64.h b/js/src/jit/x64/Assembler-x64.h
new file mode 100644
index 0000000000..2f44d9f3e4
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.h
@@ -0,0 +1,1249 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Assembler_x64_h
+#define jit_x64_Assembler_x64_h
+
+#include <iterator>
+
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register rax{X86Encoding::rax};
+static constexpr Register rbx{X86Encoding::rbx};
+static constexpr Register rcx{X86Encoding::rcx};
+static constexpr Register rdx{X86Encoding::rdx};
+static constexpr Register rsi{X86Encoding::rsi};
+static constexpr Register rdi{X86Encoding::rdi};
+static constexpr Register rbp{X86Encoding::rbp};
+static constexpr Register r8{X86Encoding::r8};
+static constexpr Register r9{X86Encoding::r9};
+static constexpr Register r10{X86Encoding::r10};
+static constexpr Register r11{X86Encoding::r11};
+static constexpr Register r12{X86Encoding::r12};
+static constexpr Register r13{X86Encoding::r13};
+static constexpr Register r14{X86Encoding::r14};
+static constexpr Register r15{X86Encoding::r15};
+static constexpr Register rsp{X86Encoding::rsp};
+
+static constexpr FloatRegister xmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister xmm1 =
+ FloatRegister(X86Encoding::xmm1, FloatRegisters::Double);
+static constexpr FloatRegister xmm2 =
+ FloatRegister(X86Encoding::xmm2, FloatRegisters::Double);
+static constexpr FloatRegister xmm3 =
+ FloatRegister(X86Encoding::xmm3, FloatRegisters::Double);
+static constexpr FloatRegister xmm4 =
+ FloatRegister(X86Encoding::xmm4, FloatRegisters::Double);
+static constexpr FloatRegister xmm5 =
+ FloatRegister(X86Encoding::xmm5, FloatRegisters::Double);
+static constexpr FloatRegister xmm6 =
+ FloatRegister(X86Encoding::xmm6, FloatRegisters::Double);
+static constexpr FloatRegister xmm7 =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Double);
+static constexpr FloatRegister xmm8 =
+ FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+static constexpr FloatRegister xmm9 =
+ FloatRegister(X86Encoding::xmm9, FloatRegisters::Double);
+static constexpr FloatRegister xmm10 =
+ FloatRegister(X86Encoding::xmm10, FloatRegisters::Double);
+static constexpr FloatRegister xmm11 =
+ FloatRegister(X86Encoding::xmm11, FloatRegisters::Double);
+static constexpr FloatRegister xmm12 =
+ FloatRegister(X86Encoding::xmm12, FloatRegisters::Double);
+static constexpr FloatRegister xmm13 =
+ FloatRegister(X86Encoding::xmm13, FloatRegisters::Double);
+static constexpr FloatRegister xmm14 =
+ FloatRegister(X86Encoding::xmm14, FloatRegisters::Double);
+static constexpr FloatRegister xmm15 =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+
+// Vector registers fixed for use with some instructions, e.g. PBLENDVB.
+static constexpr FloatRegister vmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+
+// X86-common synonyms.
+static constexpr Register eax = rax;
+static constexpr Register ebx = rbx;
+static constexpr Register ecx = rcx;
+static constexpr Register edx = rdx;
+static constexpr Register esi = rsi;
+static constexpr Register edi = rdi;
+static constexpr Register ebp = rbp;
+static constexpr Register esp = rsp;
+
+static constexpr Register InvalidReg{X86Encoding::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = FloatRegister();
+
+static constexpr Register StackPointer = rsp;
+static constexpr Register FramePointer = rbp;
+static constexpr Register JSReturnReg = rcx;
+// Avoid, except for assertions.
+static constexpr Register JSReturnReg_Type = JSReturnReg;
+static constexpr Register JSReturnReg_Data = JSReturnReg;
+
+static constexpr Register ScratchReg = r11;
+
+// Helper class for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of the scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchReg) {}
+};
+
+static constexpr Register ReturnReg = rax;
+static constexpr Register HeapReg = r15;
+static constexpr Register64 ReturnReg64(rax);
+static constexpr FloatRegister ReturnFloat32Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Single);
+static constexpr FloatRegister ReturnDoubleReg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister ReturnSimd128Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+static constexpr FloatRegister ScratchFloat32Reg_ =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Single);
+static constexpr FloatRegister ScratchDoubleReg_ =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+static constexpr FloatRegister ScratchSimd128Reg =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Simd128);
+
+// Avoid rbp, which is the FramePointer, which is unavailable in some modes.
+static constexpr Register CallTempReg0 = rax;
+static constexpr Register CallTempReg1 = rdi;
+static constexpr Register CallTempReg2 = rbx;
+static constexpr Register CallTempReg3 = rcx;
+static constexpr Register CallTempReg4 = rsi;
+static constexpr Register CallTempReg5 = rdx;
+
+// Different argument registers for WIN64
+#if defined(_WIN64)
+static constexpr Register IntArgReg0 = rcx;
+static constexpr Register IntArgReg1 = rdx;
+static constexpr Register IntArgReg2 = r8;
+static constexpr Register IntArgReg3 = r9;
+static constexpr uint32_t NumIntArgRegs = 4;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rcx, rdx, r8, r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rdi, rbx, rsi};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr uint32_t NumFloatArgRegs = 4;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {xmm0, xmm1,
+ xmm2, xmm3};
+#else
+static constexpr Register IntArgReg0 = rdi;
+static constexpr Register IntArgReg1 = rsi;
+static constexpr Register IntArgReg2 = rdx;
+static constexpr Register IntArgReg3 = rcx;
+static constexpr Register IntArgReg4 = r8;
+static constexpr Register IntArgReg5 = r9;
+static constexpr uint32_t NumIntArgRegs = 6;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rdi, rsi, rdx,
+ rcx, r8, r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rbx};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr FloatRegister FloatArgReg4 = xmm4;
+static constexpr FloatRegister FloatArgReg5 = xmm5;
+static constexpr FloatRegister FloatArgReg6 = xmm6;
+static constexpr FloatRegister FloatArgReg7 = xmm7;
+static constexpr uint32_t NumFloatArgRegs = 8;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {
+ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7};
+#endif
+
+// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use
+// JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registers used by RegExpExecTest stub (do not use ReturnReg).
+static constexpr Register RegExpExecTestRegExpReg = CallTempReg1;
+static constexpr Register RegExpExecTestStringReg = CallTempReg2;
+
+// Registers used by RegExpSearcher stub (do not use ReturnReg).
+static constexpr Register RegExpSearcherRegExpReg = CallTempReg1;
+static constexpr Register RegExpSearcherStringReg = CallTempReg2;
+static constexpr Register RegExpSearcherLastIndexReg = CallTempReg3;
+
+class ABIArgGenerator {
+#if defined(XP_WIN)
+ unsigned regIndex_;
+#else
+ unsigned intRegIndex_;
+ unsigned floatRegIndex_;
+#endif
+ uint32_t stackOffset_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+// These registers may be volatile or nonvolatile.
+// Avoid r11, which is the MacroAssembler's ScratchReg.
+static constexpr Register ABINonArgReg0 = rax;
+static constexpr Register ABINonArgReg1 = rbx;
+static constexpr Register ABINonArgReg2 = r10;
+static constexpr Register ABINonArgReg3 = r12;
+
+// This register may be volatile or nonvolatile. Avoid xmm15 which is the
+// ScratchDoubleReg.
+static constexpr FloatRegister ABINonArgDoubleReg =
+ FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = r10;
+static constexpr Register ABINonArgReturnReg1 = r12;
+static constexpr Register ABINonVolatileReg = r13;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = r10;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = r14;
+
+// Registers used for asm.js/wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = rbx;
+
+static constexpr Register OsrFrameReg = IntArgReg3;
+
+static constexpr Register PreBarrierReg = rdx;
+
+static constexpr Register InterpreterPCReg = r14;
+
+static constexpr uint32_t ABIStackAlignment = 16;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
+ "Stack alignment should be larger than any of the alignments "
+ "which are used for "
+ "spilled values. Thus it should be larger than the alignment "
+ "for SIMD accesses.");
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static constexpr uint32_t WasmTrapInstructionLength = 2;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesEight;
+
+} // namespace jit
+} // namespace js
+
+#include "jit/x86-shared/Assembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+// Return operand from a JS -> JS call.
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+class Assembler : public AssemblerX86Shared {
+ // x64 jumps may need extra bits of relocation, because a jump may extend
+ // beyond the signed 32-bit range. To account for this we add an extended
+ // jump table at the bottom of the instruction stream, and if a jump
+ // overflows its range, it will redirect here.
+ //
+ // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the
+ // hardware branch predictor that there is no fallthrough, followed by the
+ // eight bytes containing an immediate address. This comes out to 16 bytes.
+ // +1 byte for opcode
+ // +1 byte for mod r/m
+ // +4 bytes for rip-relative offset (2)
+ // +2 bytes for ud2 instruction
+ // +8 bytes for 64-bit address
+ //
+ static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8;
+ static const uint32_t SizeOfJumpTableEntry = 16;
+
+ // Two kinds of jumps on x64:
+ //
+ // * codeJumps_ tracks jumps with target within the executable code region
+ // for the process. These jumps don't need entries in the extended jump
+ // table because source and target must be within 2 GB of each other.
+ //
+ // * extendedJumps_ tracks jumps with target outside the executable code
+ // region. These jumps need entries in the extended jump table described
+ // above.
+ using PendingJumpVector = Vector<RelativePatch, 8, SystemAllocPolicy>;
+ PendingJumpVector codeJumps_;
+ PendingJumpVector extendedJumps_;
+
+ uint32_t extendedJumpTable_;
+
+ static JitCode* CodeFromJump(JitCode* code, uint8_t* jump);
+
+ private:
+ void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind reloc);
+
+ public:
+ using AssemblerX86Shared::j;
+ using AssemblerX86Shared::jmp;
+ using AssemblerX86Shared::pop;
+ using AssemblerX86Shared::push;
+ using AssemblerX86Shared::vmovq;
+
+ Assembler() : extendedJumpTable_(0) {}
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : codeJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+ for (auto& j : extendedJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ // Actual assembly emitting functions.
+
+ void push(const ImmGCPtr ptr) {
+ movq(ptr, ScratchReg);
+ push(ScratchReg);
+ }
+ void push(const ImmWord ptr) {
+ // We often end up with ImmWords that actually fit into int32.
+ // Be aware of the sign extension behavior.
+ if (ptr.value <= INT32_MAX) {
+ push(Imm32(ptr.value));
+ } else {
+ movq(ptr, ScratchReg);
+ push(ScratchReg);
+ }
+ }
+ void push(ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); }
+ void push(FloatRegister src) {
+ subq(Imm32(sizeof(double)), StackPointer);
+ vmovsd(src, Address(StackPointer, 0));
+ }
+ CodeOffset pushWithPatch(ImmWord word) {
+ CodeOffset label = movWithPatch(word, ScratchReg);
+ push(ScratchReg);
+ return label;
+ }
+
+ void pop(FloatRegister src) {
+ vmovsd(Address(StackPointer, 0), src);
+ addq(Imm32(sizeof(double)), StackPointer);
+ }
+
+ CodeOffset movWithPatch(ImmWord word, Register dest) {
+ masm.movq_i64r(word.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ // This is for patching during code generation, not after.
+ void patchAddq(CodeOffset offset, int32_t n) {
+ unsigned char* code = masm.data();
+ X86Encoding::SetInt32(code + offset.offset(), n);
+ }
+
+ // Load an ImmWord value into a register. Note that this instruction will
+ // attempt to optimize its immediate field size. When a full 64-bit
+ // immediate is needed for a relocation, use movWithPatch.
+ void movq(ImmWord word, Register dest) {
+ // Load a 64-bit immediate into a register. If the value falls into
+ // certain ranges, we can use specialized instructions which have
+ // smaller encodings.
+ if (word.value <= UINT32_MAX) {
+ // movl has a 32-bit unsigned (effectively) immediate field.
+ masm.movl_i32r((uint32_t)word.value, dest.encoding());
+ } else if ((intptr_t)word.value >= INT32_MIN &&
+ (intptr_t)word.value <= INT32_MAX) {
+ // movq has a 32-bit signed immediate field.
+ masm.movq_i32r((int32_t)(intptr_t)word.value, dest.encoding());
+ } else {
+ // Otherwise use movabs.
+ masm.movq_i64r(word.value, dest.encoding());
+ }
+ }
+ void movq(ImmPtr imm, Register dest) {
+ movq(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void movq(ImmGCPtr ptr, Register dest) {
+ masm.movq_i64r(uintptr_t(ptr.value), dest.encoding());
+ writeDataRelocation(ptr);
+ }
+ void movq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movq(Imm32 imm32, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movl_i32r(imm32.value, dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_i32m(imm32.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_i32m(imm32.value, dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovq(Register src, FloatRegister dest) {
+ masm.vmovq_rr(src.encoding(), dest.encoding());
+ }
+ void vmovq(FloatRegister src, Register dest) {
+ masm.vmovq_rr(src.encoding(), dest.encoding());
+ }
+ void movq(Register src, Register dest) {
+ masm.movq_rr(src.encoding(), dest.encoding());
+ }
+
+ void cmovCCq(Condition cond, const Operand& src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.cmovCCq_rr(cc, src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmovCCq_mr(cc, src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmovCCq_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmovCCq(Condition cond, Register src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ masm.cmovCCq_rr(cc, src.encoding(), dest.encoding());
+ }
+
+ void cmovzq(const Operand& src, Register dest) {
+ cmovCCq(Condition::Zero, src, dest);
+ }
+ void cmovnzq(const Operand& src, Register dest) {
+ cmovCCq(Condition::NonZero, src, dest);
+ }
+
+ template <typename T>
+ void lock_addq(T src, const Operand& op) {
+ masm.prefix_lock();
+ addq(src, op);
+ }
+ template <typename T>
+ void lock_subq(T src, const Operand& op) {
+ masm.prefix_lock();
+ subq(src, op);
+ }
+ template <typename T>
+ void lock_andq(T src, const Operand& op) {
+ masm.prefix_lock();
+ andq(src, op);
+ }
+ template <typename T>
+ void lock_orq(T src, const Operand& op) {
+ masm.prefix_lock();
+ orq(src, op);
+ }
+ template <typename T>
+ void lock_xorq(T src, const Operand& op) {
+ masm.prefix_lock();
+ xorq(src, op);
+ }
+
+ void lock_cmpxchgq(Register src, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchgq(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchgq(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void xchgq(Register src, Register dest) {
+ masm.xchgq_rr(src.encoding(), dest.encoding());
+ }
+
+ void xchgq(Register src, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xchgq_rm(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xchgq_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void lock_xaddq(Register srcdest, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base(),
+ mem.index(), mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movsbq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movsbq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movsbq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movsbq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movzbq(const Operand& src, Register dest) {
+ // movzbl zero-extends to 64 bits and is one byte smaller, so use that
+ // instead.
+ movzbl(src, dest);
+ }
+
+ void movswq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movswq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movswq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movswq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movzwq(const Operand& src, Register dest) {
+ // movzwl zero-extends to 64 bits and is one byte smaller, so use that
+ // instead.
+ movzwl(src, dest);
+ }
+
+ void movslq(Register src, Register dest) {
+ masm.movslq_rr(src.encoding(), dest.encoding());
+ }
+ void movslq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movslq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movslq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movslq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void andq(Register src, Register dest) {
+ masm.andq_rr(src.encoding(), dest.encoding());
+ }
+ void andq(Imm32 imm, Register dest) {
+ masm.andq_ir(imm.value, dest.encoding());
+ }
+ void andq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.andq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.andq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.andq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void addq(Imm32 imm, Register dest) {
+ masm.addq_ir(imm.value, dest.encoding());
+ }
+ CodeOffset addqWithPatch(Imm32 imm, Register dest) {
+ masm.addq_i32r(imm.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ void addq(Imm32 imm, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addq_ir(imm.value, dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_im(imm.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addq_im(imm.value, dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addq(Register src, Register dest) {
+ masm.addq_rr(src.encoding(), dest.encoding());
+ }
+ void addq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.addq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addq_mr(src.address(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void subq(Imm32 imm, Register dest) {
+ masm.subq_ir(imm.value, dest.encoding());
+ }
+ void subq(Register src, Register dest) {
+ masm.subq_rr(src.encoding(), dest.encoding());
+ }
+ void subq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.subq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.subq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.subq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void shlq(Imm32 imm, Register dest) {
+ masm.shlq_ir(imm.value, dest.encoding());
+ }
+ void shrq(Imm32 imm, Register dest) {
+ masm.shrq_ir(imm.value, dest.encoding());
+ }
+ void sarq(Imm32 imm, Register dest) {
+ masm.sarq_ir(imm.value, dest.encoding());
+ }
+ void shlq_cl(Register dest) { masm.shlq_CLr(dest.encoding()); }
+ void shrq_cl(Register dest) { masm.shrq_CLr(dest.encoding()); }
+ void sarq_cl(Register dest) { masm.sarq_CLr(dest.encoding()); }
+ void sarxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.sarxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shlxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shlxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shrxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shrxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void rolq(Imm32 imm, Register dest) {
+ masm.rolq_ir(imm.value, dest.encoding());
+ }
+ void rolq_cl(Register dest) { masm.rolq_CLr(dest.encoding()); }
+ void rorq(Imm32 imm, Register dest) {
+ masm.rorq_ir(imm.value, dest.encoding());
+ }
+ void rorq_cl(Register dest) { masm.rorq_CLr(dest.encoding()); }
+ void orq(Imm32 imm, Register dest) {
+ masm.orq_ir(imm.value, dest.encoding());
+ }
+ void orq(Register src, Register dest) {
+ masm.orq_rr(src.encoding(), dest.encoding());
+ }
+ void orq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.orq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.orq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.orq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorq(Register src, Register dest) {
+ masm.xorq_rr(src.encoding(), dest.encoding());
+ }
+ void xorq(Imm32 imm, Register dest) {
+ masm.xorq_ir(imm.value, dest.encoding());
+ }
+ void xorq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.xorq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.xorq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.xorq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void bsrq(const Register& src, const Register& dest) {
+ masm.bsrq_rr(src.encoding(), dest.encoding());
+ }
+ void bsfq(const Register& src, const Register& dest) {
+ masm.bsfq_rr(src.encoding(), dest.encoding());
+ }
+ void bswapq(const Register& reg) { masm.bswapq_r(reg.encoding()); }
+ void lzcntq(const Register& src, const Register& dest) {
+ masm.lzcntq_rr(src.encoding(), dest.encoding());
+ }
+ void tzcntq(const Register& src, const Register& dest) {
+ masm.tzcntq_rr(src.encoding(), dest.encoding());
+ }
+ void popcntq(const Register& src, const Register& dest) {
+ masm.popcntq_rr(src.encoding(), dest.encoding());
+ }
+
+ void imulq(Imm32 imm, Register src, Register dest) {
+ masm.imulq_ir(imm.value, src.encoding(), dest.encoding());
+ }
+ void imulq(Register src, Register dest) {
+ masm.imulq_rr(src.encoding(), dest.encoding());
+ }
+ void imulq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.imulq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.imulq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ MOZ_CRASH("NYI");
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void cqo() { masm.cqo(); }
+ void idivq(Register divisor) { masm.idivq_r(divisor.encoding()); }
+ void udivq(Register divisor) { masm.divq_r(divisor.encoding()); }
+
+ void vcvtsi2sdq(Register src, FloatRegister dest) {
+ masm.vcvtsi2sdq_rr(src.encoding(), dest.encoding());
+ }
+
+ void vpextrq(unsigned lane, FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpextrq_irr(lane, src.encoding(), dest.encoding());
+ }
+
+ void vpinsrq(unsigned lane, Register src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpinsrq_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void negq(Register reg) { masm.negq_r(reg.encoding()); }
+
+ void notq(Register reg) { masm.notq_r(reg.encoding()); }
+
+ void mov(ImmWord word, Register dest) {
+ // Use xor for setting registers to zero, as it is specially optimized
+ // for this purpose on modern hardware. Note that it does clobber FLAGS
+ // though. Use xorl instead of xorq since they are functionally
+ // equivalent (32-bit instructions zero-extend their results to 64 bits)
+ // and xorl has a smaller encoding.
+ if (word.value == 0) {
+ xorl(dest, dest);
+ } else {
+ movq(word, dest);
+ }
+ }
+ void mov(ImmPtr imm, Register dest) { movq(imm, dest); }
+ void mov(wasm::SymbolicAddress imm, Register dest) {
+ masm.movq_i64r(-1, dest.encoding());
+ append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm));
+ }
+ void mov(const Operand& src, Register dest) { movq(src, dest); }
+ void mov(Register src, const Operand& dest) { movq(src, dest); }
+ void mov(Imm32 imm32, const Operand& dest) { movq(imm32, dest); }
+ void mov(Register src, Register dest) { movq(src, dest); }
+ void mov(CodeLabel* label, Register dest) {
+ masm.movq_i64r(/* placeholder */ 0, dest.encoding());
+ label->patchAt()->bind(masm.size());
+ }
+ void xchg(Register src, Register dest) { xchgq(src, dest); }
+
+ void lea(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.leaq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.leaq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexepcted operand kind");
+ }
+ }
+
+ void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); }
+ void cmovzPtr(const Operand& src, Register dest) { return cmovzq(src, dest); }
+
+ CodeOffset loadRipRelativeInt32(Register dest) {
+ return CodeOffset(masm.movl_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeInt64(Register dest) {
+ return CodeOffset(masm.movq_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeDouble(FloatRegister dest) {
+ return CodeOffset(masm.vmovsd_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeFloat32(FloatRegister dest) {
+ return CodeOffset(masm.vmovss_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeInt32x4(FloatRegister dest) {
+ return CodeOffset(masm.vmovdqa_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeFloat32x4(FloatRegister dest) {
+ return CodeOffset(masm.vmovaps_ripr(dest.encoding()).offset());
+ }
+ CodeOffset leaRipRelative(Register dest) {
+ return CodeOffset(masm.leaq_rip(dest.encoding()).offset());
+ }
+
+ void cmpq(Register rhs, Register lhs) {
+ masm.cmpq_rr(rhs.encoding(), lhs.encoding());
+ }
+ void cmpq(Register rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_rr(rhs.encoding(), lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpq_rm(rhs.encoding(), lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpq(Imm32 rhs, Register lhs) {
+ masm.cmpq_ir(rhs.value, lhs.encoding());
+ }
+ void cmpq(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_im(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpq_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpq_im(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpq(const Operand& rhs, Register lhs) {
+ switch (rhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_rr(rhs.reg(), lhs.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_mr(rhs.disp(), rhs.base(), lhs.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void testq(Imm32 rhs, Register lhs) {
+ masm.testq_ir(rhs.value, lhs.encoding());
+ }
+ void testq(Register rhs, Register lhs) {
+ masm.testq_rr(rhs.encoding(), lhs.encoding());
+ }
+ void testq(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.testq_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.testq_i32m(rhs.value, lhs.disp(), lhs.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) {
+ MOZ_ASSERT(hasCreator());
+ JmpSrc src = masm.jmp();
+ addPendingJump(src, target, reloc);
+ }
+ void j(Condition cond, ImmPtr target,
+ RelocationKind reloc = RelocationKind::HARDCODED) {
+ JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond));
+ addPendingJump(src, target, reloc);
+ }
+
+ void jmp(JitCode* target) {
+ jmp(ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void j(Condition cond, JitCode* target) {
+ j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(JitCode* target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+ void call(ImmPtr target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, target, RelocationKind::HARDCODED);
+ }
+
+ // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled) {
+ CodeOffset offset(size());
+ JmpSrc src = enabled ? masm.call() : masm.cmp_eax();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr));
+ return offset;
+ }
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Size of a call instruction.
+ return 5;
+ }
+
+ // Do not mask shared implementations.
+ using AssemblerX86Shared::call;
+
+ void vcvttsd2sq(FloatRegister src, Register dest) {
+ masm.vcvttsd2sq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvttss2sq(FloatRegister src, Register dest) {
+ masm.vcvttss2sq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtsq2sd(Register src1, FloatRegister src0, FloatRegister dest) {
+ masm.vcvtsq2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vcvtsq2ss(Register src1, FloatRegister src0, FloatRegister dest) {
+ masm.vcvtsq2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+};
+
+static inline bool GetIntArgReg(uint32_t intArg, uint32_t floatArg,
+ Register* out) {
+#if defined(_WIN64)
+ uint32_t arg = intArg + floatArg;
+#else
+ uint32_t arg = intArg;
+#endif
+ if (arg >= NumIntArgRegs) {
+ return false;
+ }
+ *out = IntArgRegs[arg];
+ return true;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+#if defined(_WIN64)
+ uint32_t arg = usedIntArgs + usedFloatArgs;
+#else
+ uint32_t arg = usedIntArgs;
+#endif
+ arg -= NumIntArgRegs;
+ if (arg >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[arg];
+ return true;
+}
+
+static inline bool GetFloatArgReg(uint32_t intArg, uint32_t floatArg,
+ FloatRegister* out) {
+#if defined(_WIN64)
+ uint32_t arg = intArg + floatArg;
+#else
+ uint32_t arg = floatArg;
+#endif
+ if (floatArg >= NumFloatArgRegs) {
+ return false;
+ }
+ *out = FloatArgRegs[arg];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_Assembler_x64_h */
diff --git a/js/src/jit/x64/BaseAssembler-x64.h b/js/src/jit/x64/BaseAssembler-x64.h
new file mode 100644
index 0000000000..f5a9bb99f9
--- /dev/null
+++ b/js/src/jit/x64/BaseAssembler-x64.h
@@ -0,0 +1,1373 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_BaseAssembler_x64_h
+#define jit_x64_BaseAssembler_x64_h
+
+#include "jit/x86-shared/BaseAssembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+class BaseAssemblerX64 : public BaseAssembler {
+ public:
+ // Arithmetic operations:
+
+ void addq_rr(RegisterID src, RegisterID dst) {
+ spew("addq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, src, dst);
+ }
+
+ void addq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("addq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, dst);
+ }
+
+ void addq_mr(const void* addr, RegisterID dst) {
+ spew("addq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, addr, dst);
+ }
+
+ void addq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("addq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, index, scale, dst);
+ }
+
+ void addq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("addq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, src);
+ }
+
+ void addq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("addq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, index, scale, src);
+ }
+
+ void addq_ir(int32_t imm, RegisterID dst) {
+ spew("addq $%d, %s", imm, GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addq_i32r(int32_t imm, RegisterID dst) {
+ // 32-bit immediate always, for patching.
+ spew("addq $0x%04x, %s", uint32_t(imm), GPReg64Name(dst));
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+
+ void addq_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("addq $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addq_im(int32_t imm, const void* addr) {
+ spew("addq $%d, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void andq_rr(RegisterID src, RegisterID dst) {
+ spew("andq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, src, dst);
+ }
+
+ void andq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("andq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, dst);
+ }
+
+ void andq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("andq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, index, scale, dst);
+ }
+
+ void andq_mr(const void* addr, RegisterID dst) {
+ spew("andq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, addr, dst);
+ }
+
+ void andq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("andq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, src);
+ }
+
+ void andq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("andq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, index, scale, src);
+ }
+
+ void orq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("orq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, offset, base, dst);
+ }
+
+ void orq_mr(const void* addr, RegisterID dst) {
+ spew("orq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, addr, dst);
+ }
+
+ void orq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("orq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, src);
+ }
+
+ void orq_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, index, scale, src);
+ }
+
+ void xorq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("xorq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, dst);
+ }
+
+ void xorq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("xorq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, index, scale, dst);
+ }
+
+ void xorq_mr(const void* addr, RegisterID dst) {
+ spew("xorq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, addr, dst);
+ }
+
+ void xorq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xorq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, src);
+ }
+
+ void xorq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xorq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, index, scale, src);
+ }
+
+ void bswapq_r(RegisterID dst) {
+ spew("bswapq %s", GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSWAP, dst);
+ }
+
+ void bsrq_rr(RegisterID src, RegisterID dst) {
+ spew("bsrq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSR_GvEv, src, dst);
+ }
+
+ void bsfq_rr(RegisterID src, RegisterID dst) {
+ spew("bsfq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSF_GvEv, src, dst);
+ }
+
+ void lzcntq_rr(RegisterID src, RegisterID dst) {
+ spew("lzcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_LZCNT_GvEv, src, dst);
+ }
+
+ void tzcntq_rr(RegisterID src, RegisterID dst) {
+ spew("tzcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_TZCNT_GvEv, src, dst);
+ }
+
+ void popcntq_rr(RegisterID src, RegisterID dst) {
+ spew("popcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_POPCNT_GvEv, src, dst);
+ }
+
+ void andq_ir(int32_t imm, RegisterID dst) {
+ spew("andq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_AND_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_AND);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void negq_r(RegisterID dst) {
+ spew("negq %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NEG);
+ }
+
+ void orq_rr(RegisterID src, RegisterID dst) {
+ spew("orq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, src, dst);
+ }
+
+ void orq_ir(int32_t imm, RegisterID dst) {
+ spew("orq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_OR_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_OR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void notq_r(RegisterID dst) {
+ spew("notq %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NOT);
+ }
+
+ void subq_rr(RegisterID src, RegisterID dst) {
+ spew("subq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, src, dst);
+ }
+
+ void subq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("subq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, src);
+ }
+
+ void subq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("subq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, index, scale, src);
+ }
+
+ void subq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("subq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, offset, base, dst);
+ }
+
+ void subq_mr(const void* addr, RegisterID dst) {
+ spew("subq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, addr, dst);
+ }
+
+ void subq_ir(int32_t imm, RegisterID dst) {
+ spew("subq $%d, %s", imm, GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_SUB_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_SUB);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void xorq_rr(RegisterID src, RegisterID dst) {
+ spew("xorq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, src, dst);
+ }
+
+ void xorq_ir(int32_t imm, RegisterID dst) {
+ spew("xorq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_XOR_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_XOR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void sarq_CLr(RegisterID dst) {
+ spew("sarq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SAR);
+ }
+
+ void shlq_CLr(RegisterID dst) {
+ spew("shlq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHL);
+ }
+
+ void shrq_CLr(RegisterID dst) {
+ spew("shrq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHR);
+ }
+
+ void sarq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("sarq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SAR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SAR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shlq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("shlq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHL);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shrq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("shrq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void rolq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("rolq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROL);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rolq_CLr(RegisterID dst) {
+ spew("rolq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROL);
+ }
+
+ void rorq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("rorq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rorq_CLr(RegisterID dst) {
+ spew("rorq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROR);
+ }
+
+ void imulq_rr(RegisterID src, RegisterID dst) {
+ spew("imulq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_IMUL_GvEv, src, dst);
+ }
+
+ void imulq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("imulq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_IMUL_GvEv, offset, base, dst);
+ }
+
+ void imulq_ir(int32_t value, RegisterID src, RegisterID dst) {
+ spew("imulq $%d, %s, %s", value, GPReg64Name(src), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(value)) {
+ m_formatter.oneByteOp64(OP_IMUL_GvEvIb, src, dst);
+ m_formatter.immediate8s(value);
+ } else {
+ m_formatter.oneByteOp64(OP_IMUL_GvEvIz, src, dst);
+ m_formatter.immediate32(value);
+ }
+ }
+
+ void cqo() {
+ spew("cqo ");
+ m_formatter.oneByteOp64(OP_CDQ);
+ }
+
+ void idivq_r(RegisterID divisor) {
+ spew("idivq %s", GPReg64Name(divisor));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_IDIV);
+ }
+
+ void divq_r(RegisterID divisor) {
+ spew("divq %s", GPReg64Name(divisor));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_DIV);
+ }
+
+ // Comparisons:
+
+ void cmpq_rr(RegisterID rhs, RegisterID lhs) {
+ spew("cmpq %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_CMP_GvEv, rhs, lhs);
+ }
+
+ void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base) {
+ spew("cmpq %s, " MEM_ob, GPReg64Name(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, rhs);
+ }
+
+ void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpq %s, " MEM_obs, GPReg64Name(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, index, scale, rhs);
+ }
+
+ void cmpq_mr(int32_t offset, RegisterID base, RegisterID lhs) {
+ spew("cmpq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_CMP_GvEv, offset, base, lhs);
+ }
+
+ void cmpq_ir(int32_t rhs, RegisterID lhs) {
+ if (rhs == 0) {
+ testq_rr(lhs, lhs);
+ return;
+ }
+
+ spew("cmpq $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, lhs, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ if (lhs == rax) {
+ m_formatter.oneByteOp64(OP_CMP_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+ }
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpq_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpq $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+ ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpq_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("cmpq $0x%x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+ void cmpq_im(int32_t rhs, const void* addr) {
+ spew("cmpq $0x%" PRIx64 ", %p", uint64_t(rhs), addr);
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+ void cmpq_rm(RegisterID rhs, const void* addr) {
+ spew("cmpq %s, %p", GPReg64Name(rhs), addr);
+ m_formatter.oneByteOp64(OP_CMP_EvGv, addr, rhs);
+ }
+
+ void testq_rr(RegisterID rhs, RegisterID lhs) {
+ spew("testq %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_TEST_EvGv, lhs, rhs);
+ }
+
+ void testq_ir(int32_t rhs, RegisterID lhs) {
+ // If the mask fits in a 32-bit immediate, we can use testl with a
+ // 32-bit subreg.
+ if (CAN_ZERO_EXTEND_32_64(rhs)) {
+ testl_ir(rhs, lhs);
+ return;
+ }
+ spew("testq $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp64(OP_TEST_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, lhs, GROUP3_OP_TEST);
+ }
+ m_formatter.immediate32(rhs);
+ }
+
+ void testq_i32m(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("testq $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+ ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ void testq_i32m(int32_t rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("testq $0x%4x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, index, scale,
+ GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ // Various move ops:
+
+ void cmovCCq_rr(Condition cond, RegisterID src, RegisterID dst) {
+ spew("cmov%s %s, %s", CCName(cond), GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), src, dst);
+ }
+ void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID dst) {
+ spew("cmov%s " MEM_ob ", %s", CCName(cond), ADDR_ob(offset, base),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, dst);
+ }
+ void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID dst) {
+ spew("cmov%s " MEM_obs ", %s", CCName(cond),
+ ADDR_obs(offset, base, index, scale), GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, index, scale,
+ dst);
+ }
+
+ void cmpxchgq(RegisterID src, int32_t offset, RegisterID base) {
+ spew("cmpxchgq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, src);
+ }
+
+ void cmpxchgq(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpxchgq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
+ }
+
+ void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base) {
+ spew("lock xaddq %s, " MEM_ob, GPReg64Name(srcdest), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, srcdest);
+ }
+
+ void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("lock xaddq %s, " MEM_obs, GPReg64Name(srcdest),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, index, scale, srcdest);
+ }
+
+ void xchgq_rr(RegisterID src, RegisterID dst) {
+ spew("xchgq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, src, dst);
+ }
+ void xchgq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xchgq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, src);
+ }
+ void xchgq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xchgq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, index, scale, src);
+ }
+
+ void movq_rr(RegisterID src, RegisterID dst) {
+ spew("movq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_EvGv, dst, src);
+ }
+
+ void movq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movq_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movq %s, " MEM_o32b, GPReg64Name(src), ADDR_o32b(offset, base));
+ m_formatter.oneByteOp64_disp32(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("movq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, index, scale, src);
+ }
+
+ void movq_rm(RegisterID src, const void* addr) {
+ if (src == rax && !IsAddressImmediate(addr)) {
+ movq_EAXm(addr);
+ return;
+ }
+
+ spew("movq %s, %p", GPReg64Name(src), addr);
+ m_formatter.oneByteOp64(OP_MOV_EvGv, addr, src);
+ }
+
+ void movq_mEAX(const void* addr) {
+ if (IsAddressImmediate(addr)) {
+ movq_mr(addr, rax);
+ return;
+ }
+
+ spew("movq %p, %%rax", addr);
+ m_formatter.oneByteOp64(OP_MOV_EAXOv);
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+ }
+
+ void movq_EAXm(const void* addr) {
+ if (IsAddressImmediate(addr)) {
+ movq_rm(rax, addr);
+ return;
+ }
+
+ spew("movq %%rax, %p", addr);
+ m_formatter.oneByteOp64(OP_MOV_OvEAX);
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+ }
+
+ void movq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movq_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movq " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64_disp32(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, index, scale, dst);
+ }
+
+ void movq_mr(const void* addr, RegisterID dst) {
+ if (dst == rax && !IsAddressImmediate(addr)) {
+ movq_mEAX(addr);
+ return;
+ }
+
+ spew("movq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, addr, dst);
+ }
+
+ void leaq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("leaq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_LEA, offset, base, index, scale, dst);
+ }
+
+ void movq_i32m(int32_t imm, int32_t offset, RegisterID base) {
+ spew("movq $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+ void movq_i32m(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("movq $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, index, scale,
+ GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+ void movq_i32m(int32_t imm, const void* addr) {
+ spew("movq $%d, %p", imm, addr);
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, addr, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ // Note that this instruction sign-extends its 32-bit immediate field to 64
+ // bits and loads the 64-bit value into a 64-bit register.
+ //
+ // Note also that this is similar to the movl_i32r instruction, except that
+ // movl_i32r *zero*-extends its 32-bit immediate, and it has smaller code
+ // size, so it's preferred for values which could use either.
+ void movq_i32r(int32_t imm, RegisterID dst) {
+ spew("movq $%d, %s", imm, GPRegName(dst));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, dst, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ void movq_i64r(int64_t imm, RegisterID dst) {
+ spew("movabsq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_EAXIv, dst);
+ m_formatter.immediate64(imm);
+ }
+
+ void movsbq_rr(RegisterID src, RegisterID dst) {
+ spew("movsbq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, src, dst);
+ }
+ void movsbq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movsbq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, dst);
+ }
+ void movsbq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movsbq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, index, scale, dst);
+ }
+
+ void movswq_rr(RegisterID src, RegisterID dst) {
+ spew("movswq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, src, dst);
+ }
+ void movswq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movswq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, dst);
+ }
+ void movswq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movswq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, index, scale, dst);
+ }
+
+ void movslq_rr(RegisterID src, RegisterID dst) {
+ spew("movslq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, src, dst);
+ }
+ void movslq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movslq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, dst);
+ }
+ void movslq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movslq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, index, scale, dst);
+ }
+
+ [[nodiscard]] JmpSrc movl_ripr(RegisterID dst) {
+ m_formatter.oneByteRipOp(OP_MOV_GvEv, 0, (RegisterID)dst);
+ JmpSrc label(m_formatter.size());
+ spew("movl " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPReg32Name(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movl_rrip(RegisterID src) {
+ m_formatter.oneByteRipOp(OP_MOV_EvGv, 0, (RegisterID)src);
+ JmpSrc label(m_formatter.size());
+ spew("movl %s, " MEM_o32r "", GPReg32Name(src),
+ ADDR_o32r(label.offset()));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movq_ripr(RegisterID dst) {
+ m_formatter.oneByteRipOp64(OP_MOV_GvEv, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("movq " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movq_rrip(RegisterID src) {
+ m_formatter.oneByteRipOp64(OP_MOV_EvGv, 0, (RegisterID)src);
+ JmpSrc label(m_formatter.size());
+ spew("movq %s, " MEM_o32r "", GPRegName(src),
+ ADDR_o32r(label.offset()));
+ return label;
+ }
+
+ void leaq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("leaq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_LEA, offset, base, dst);
+ }
+
+ [[nodiscard]] JmpSrc leaq_rip(RegisterID dst) {
+ m_formatter.oneByteRipOp64(OP_LEA, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("leaq " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPRegName(dst));
+ return label;
+ }
+
+ // Flow control:
+
+ void jmp_rip(int ripOffset) {
+ // rip-relative addressing.
+ spew("jmp *%d(%%rip)", ripOffset);
+ m_formatter.oneByteRipOp(OP_GROUP5_Ev, ripOffset, GROUP5_OP_JMPN);
+ }
+
+ void immediate64(int64_t imm) {
+ spew(".quad %lld", (long long)imm);
+ m_formatter.immediate64(imm);
+ }
+
+ // SSE operations:
+
+ void vcvtsq2sd_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+ void vcvtsq2ss_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+
+ void vcvtsi2sdq_rr(RegisterID src, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2sdq", VEX_SD, OP2_CVTSI2SD_VsdEd, src,
+ invalid_xmm, dst);
+ }
+
+ void vcvttsd2sq_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt64("vcvttsd2si", VEX_SD, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vcvttss2sq_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt64("vcvttss2si", VEX_SS, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vmovq_rr(XMMRegisterID src, RegisterID dst) {
+ // While this is called "vmovq", it actually uses the vmovd encoding
+ // with a REX prefix modifying it to be 64-bit.
+ twoByteOpSimdInt64("vmovq", VEX_PD, OP2_MOVD_EdVd, (XMMRegisterID)dst,
+ (RegisterID)src);
+ }
+
+ void vpextrq_irr(unsigned lane, XMMRegisterID src, RegisterID dst) {
+ MOZ_ASSERT(lane < 2);
+ threeByteOpImmSimdInt64("vpextrq", VEX_PD, OP3_PEXTRQ_EvVdqIb, ESCAPE_3A,
+ lane, src, dst);
+ }
+
+ void vpinsrq_irr(unsigned lane, RegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 2);
+ threeByteOpImmInt64Simd("vpinsrq", VEX_PD, OP3_PINSRQ_VdqEvIb, ESCAPE_3A,
+ lane, src1, src0, dst);
+ }
+
+ void vmovq_rr(RegisterID src, XMMRegisterID dst) {
+ // While this is called "vmovq", it actually uses the vmovd encoding
+ // with a REX prefix modifying it to be 64-bit.
+ twoByteOpInt64Simd("vmovq", VEX_PD, OP2_MOVD_VdEd, src, invalid_xmm, dst);
+ }
+
+ [[nodiscard]] JmpSrc vmovsd_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovss_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovaps_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovdqa_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, dst);
+ }
+
+ [[nodiscard]] JmpSrc vpaddb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddq", VEX_PD, OP2_PADDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubq", VEX_PD, OP2_PSUBQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmullw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmulld_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminsb", VEX_PD, OP3_PMINSB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpminub", VEX_PD, OP2_PMINUB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpminsw", VEX_PD, OP2_PMINSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminuw", VEX_PD, OP3_PMINUW_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminsd", VEX_PD, OP3_PMINSD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminud", VEX_PD, OP3_PMINUD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxsb", VEX_PD, OP3_PMAXSB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaxub", VEX_PD, OP2_PMAXUB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaxsw", VEX_PD, OP2_PMAXSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxuw", VEX_PD, OP3_PMAXUW_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxsd", VEX_PD, OP3_PMAXSD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxud", VEX_PD, OP3_PMAXUD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpand_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpxor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vaddps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vaddpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vaddpd", VEX_PD, OP2_ADDPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vsubps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vsubpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vsubpd", VEX_PD, OP2_SUBPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vdivps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vdivpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vdivpd", VEX_PD, OP2_DIVPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vmulps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vmulpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmulpd", VEX_PD, OP2_MULPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vandpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vandpd", VEX_PD, OP2_ANDPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vminpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vminpd", VEX_PD, OP2_MINPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpacksswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpacksswb", VEX_PD, OP2_PACKSSWB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackuswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpackuswb", VEX_PD, OP2_PACKUSWB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackssdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpackssdw", VEX_PD, OP2_PACKSSDW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackusdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpackusdw", VEX_PD, OP3_PACKUSDW_VdqWdq,
+ ESCAPE_38, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpunpckldq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, src,
+ dst);
+ }
+ [[nodiscard]] JmpSrc vunpcklps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vptest_ripr(XMMRegisterID lhs) {
+ return threeByteRipOpSimd("vptest", VEX_PD, OP3_PTEST_VdVd, ESCAPE_38, lhs);
+ }
+ [[nodiscard]] JmpSrc vpshufb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpshufb", VEX_PD, OP3_PSHUFB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaddwd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaddwd", VEX_PD, OP2_PMADDWD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpeqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_EQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpneqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_NEQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpltps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_LT, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpleps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_LE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpgeps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_GE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpeqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_EQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpneqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_NEQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpltpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_LT, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmplepd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_LE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaddubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaddubsw", VEX_PD, OP3_PMADDUBSW_VdqWdq,
+ ESCAPE_38, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmuludq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, src, dst);
+ }
+
+ // BMI instructions:
+
+ void sarxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("sarxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_SS /* = F3 */, OP3_SARX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shlxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shlxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_PD /* = 66 */, OP3_SHLX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shrxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shrxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_SD /* = F2 */, OP3_SHRX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ private:
+ [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode,
+ XMMRegisterID reg) {
+ MOZ_ASSERT(!IsXMMReversedOperands(opcode));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, reg);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s " MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(reg));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s, " MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+ XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc twoByteRipOpImmSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, dst);
+ m_formatter.immediate8u(imm);
+ JmpSrc label(m_formatter.size(),
+ /* bytes trailing the patch field = */ 1);
+ spew("%-11s$0x%x, " MEM_o32r ", %s", legacySSEOpName(name), imm,
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+ m_formatter.immediate8u(imm);
+ JmpSrc label(m_formatter.size(),
+ /* bytes trailing the patch field = */ 1);
+ spew("%-11s$0x%x, " MEM_o32r ", %s, %s", name, imm,
+ ADDR_o32r(label.offset()), XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ void twoByteOpInt64Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, RegisterID rm,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(dst),
+ GPRegName(rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(rm),
+ XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp64(opcode, rm, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, XMMRegName(dst), GPRegName(rm));
+ } else {
+ spew("%-11s%s, %s", name, GPRegName(rm), XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s%s, %s, %s", name, GPRegName(rm), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex64(ty, opcode, rm, src0, dst);
+ }
+
+ void twoByteOpSimdInt64(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, XMMRegisterID rm,
+ RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(dst),
+ XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", legacySSEOpName(name),
+ XMMRegName((XMMRegisterID)dst), GPRegName((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ GPRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp64(opcode, (RegisterID)rm, dst);
+ return;
+ }
+
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, GPRegName(dst), XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", name, XMMRegName((XMMRegisterID)dst),
+ GPRegName((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", name, XMMRegName(rm), GPRegName(dst));
+ }
+ m_formatter.twoByteOpVex64(ty, opcode, (RegisterID)rm, invalid_xmm,
+ (XMMRegisterID)dst);
+ }
+
+ [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape,
+ XMMRegisterID dst) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape,
+ XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm);
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.threeByteRipOpVex(ty, opcode, escape, 0, src0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+ XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ void threeByteOpImmSimdInt64(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, XMMRegisterID src,
+ RegisterID dst) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(dst),
+ XMMRegName(src));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp64(opcode, escape, dst, (RegisterID)src);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmInt64Simd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, RegisterID src1,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(src1),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp64(opcode, escape, src1, (RegisterID)dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ MOZ_ASSERT(src0 != invalid_xmm);
+ spew("%-11s$0x%x, %s, %s, %s", name, imm, GPReg64Name(src1),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.threeByteOpVex64(ty, opcode, escape, src1, src0,
+ (RegisterID)dst);
+ m_formatter.immediate8u(imm);
+ }
+};
+
+using BaseAssemblerSpecific = BaseAssemblerX64;
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_BaseAssembler_x64_h */
diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp
new file mode 100644
index 0000000000..9bd7e9b253
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.cpp
@@ -0,0 +1,984 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/CodeGenerator-x64.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+
+CodeGeneratorX64::CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorX86Shared(gen, graph, masm) {}
+
+ValueOperand CodeGeneratorX64::ToValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorX64::ToTempValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+Operand CodeGeneratorX64::ToOperand64(const LInt64Allocation& a64) {
+ const LAllocation& a = a64.value();
+ MOZ_ASSERT(!a.isFloatReg());
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ ValueOperand result = ToOutValue(value);
+ masm.moveValue(value->value(), result);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LAllocation* in = box->getOperand(0);
+ ValueOperand result = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+
+ if (JitOptions.spectreValueMasking && IsFloatingPointType(box->type())) {
+ ScratchRegisterScope scratch(masm);
+ masm.movePtr(ImmWord(JSVAL_SHIFTED_TAG_MAX_DOUBLE), scratch);
+ masm.cmpPtrMovePtr(Assembler::Below, scratch, result.valueReg(), scratch,
+ result.valueReg());
+ }
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ MUnbox* mir = unbox->mir();
+
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ // Infallible unbox.
+
+ Operand input = ToOperand(unbox->getOperand(LUnbox::Input));
+
+#ifdef DEBUG
+ // Assert the types match.
+ JSValueTag tag = MIRTypeToTag(mir->type());
+ Label ok;
+ masm.splitTag(input, ScratchReg);
+ masm.branch32(Assembler::Equal, ScratchReg, Imm32(tag), &ok);
+ masm.assumeUnreachable("Infallible unbox type mismatch");
+ masm.bind(&ok);
+#endif
+
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(input, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(input, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(input, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(input, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(input, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(input, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ masm.cmpPtr(lhsReg, ToOperand64(rhs));
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ masm.emitSet(JSOpToCondition(lir->jsop(), isSigned), output);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ masm.cmpPtr(lhsReg, ToOperand64(rhs));
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ emitBranch(JSOpToCondition(lir->jsop(), isSigned), lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+ Register regL = ToRegister(baab->left());
+ if (baab->is64()) {
+ if (baab->right()->isConstant()) {
+ masm.test64(regL, Imm64(ToInt64(baab->right())));
+ } else {
+ masm.test64(regL, ToRegister(baab->right()));
+ }
+ } else {
+ if (baab->right()->isConstant()) {
+ masm.test32(regL, Imm32(ToInt32(baab->right())));
+ } else {
+ masm.test32(regL, ToRegister(baab->right()));
+ }
+ }
+ emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+ MOZ_ASSERT(rhs != rdx);
+ MOZ_ASSERT_IF(output == rax, ToRegister(lir->remainder()) == rdx);
+ MOZ_ASSERT_IF(output == rdx, ToRegister(lir->remainder()) == rax);
+
+ Label done;
+
+ // Put the lhs in rax.
+ if (lhs != rax) {
+ masm.mov(lhs, rax);
+ }
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
+ if (lir->mir()->isMod()) {
+ masm.xorl(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+ masm.idivq(rhs);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+
+ DebugOnly<Register> output = ToRegister(lir->output());
+ MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+ MOZ_ASSERT(rhs != rdx);
+ MOZ_ASSERT_IF(output.value == rax, ToRegister(lir->remainder()) == rdx);
+ MOZ_ASSERT_IF(output.value == rdx, ToRegister(lir->remainder()) == rax);
+
+ // Put the lhs in rax.
+ if (lhs != rax) {
+ masm.mov(lhs, rax);
+ }
+
+ Label done;
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Zero extend the lhs into rdx to make (rdx:rax).
+ masm.xorl(rdx, rdx);
+ masm.udivq(rhs);
+
+ masm.bind(&done);
+}
+
+void CodeGeneratorX64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == rax);
+ MOZ_ASSERT(output == rdx);
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+
+ masm.idivq(divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorX64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == rax);
+ MOZ_ASSERT(output == rdx);
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+
+ masm.idivq(divisor);
+
+ // Move the remainder from rdx.
+ masm.movq(output, dividend);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Load();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Store();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ MOZ_ASSERT(temp1.reg == rax);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, temp2);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp1, out, temp2.reg);
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp1, out, temp2);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ Register64 fetchTemp = Register64(out);
+ Register64 fetchOut = temp2;
+ Register createTemp = temp1.reg;
+
+ // Add and Sub don't need |fetchTemp| and can save a `mov` when the value and
+ // output register are equal to each other.
+ if (atomicOp == AtomicFetchAddOp || atomicOp == AtomicFetchSubOp) {
+ fetchTemp = Register64::Invalid();
+ fetchOut = temp1;
+ createTemp = temp2.reg;
+ } else {
+ MOZ_ASSERT(temp2.reg == rax);
+ }
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ fetchTemp, fetchOut);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ fetchTemp, fetchOut);
+ }
+
+ emitCreateBigInt(lir, arrayType, fetchOut, out, createTemp);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+ }
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+
+ Operand falseExpr = ToOperandOrRegister64(lir->falseExpr());
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ masm.test32(cond, cond);
+ masm.cmovzq(falseExpr, out.reg);
+}
+
+// We expect to handle only the cases: compare is {U,}Int{32,64}, and select
+// is {U,}Int{32,64}, independently. Some values may be stack allocated, and
+// the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool cmpIs64bit = ins->compareType() == MCompare::Compare_Int64 ||
+ ins->compareType() == MCompare::Compare_UInt64;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+ bool selIs64bit = ins->mir()->type() == MIRType::Int64;
+
+ // Throw out unhandled cases
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit != cmpIs64bit && selIs32bit != selIs64bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ using C = Assembler::Condition;
+ using R = Register;
+ using A = const Address&;
+
+ // Identify macroassembler methods to generate instructions, based on the
+ // type of the comparison and the select. This avoids having to duplicate
+ // the code-generation tree below 4 times. These assignments to
+ // `cmpMove_CRRRR` et al are unambiguous as a result of the combination of
+ // the template parameters and the 5 argument types ((C, R, R, R, R) etc).
+ void (MacroAssembler::*cmpMove_CRRRR)(C, R, R, R, R) = nullptr;
+ void (MacroAssembler::*cmpMove_CRARR)(C, R, A, R, R) = nullptr;
+ void (MacroAssembler::*cmpLoad_CRRAR)(C, R, R, A, R) = nullptr;
+ void (MacroAssembler::*cmpLoad_CRAAR)(C, R, A, A, R) = nullptr;
+
+ if (cmpIs32bit) {
+ if (selIs32bit) {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 32>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 32>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+ } else {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 64>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 64>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+ }
+ } else {
+ if (selIs32bit) {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 32>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 32>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+ } else {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 64>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 64>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+ }
+ }
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ // We generate one of four cmp+cmov pairings, depending on whether one of
+ // the cmp args and one of the cmov args is in memory or a register.
+ if (rhs->isRegister()) {
+ if (falseExpr->isRegister()) {
+ (masm.*cmpMove_CRRRR)(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ (masm.*cmpLoad_CRRAR)(cond, lhs, ToRegister(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ } else {
+ if (falseExpr->isRegister()) {
+ (masm.*cmpMove_CRARR)(cond, lhs, ToAddress(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ (masm.*cmpLoad_CRAAR)(cond, lhs, ToAddress(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ masm.vmovq(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ masm.vmovq(ToFloatRegister(lir->input()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access,
+ const LAllocation* value, Operand dstAddr) {
+ if (value->isConstant()) {
+ masm.memoryBarrierBefore(access.sync());
+
+ const MConstant* mir = value->toConstant();
+ Imm32 cst =
+ Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64());
+
+ masm.append(access, masm.size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ masm.movb(cst, dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.movw(cst, dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.movl(cst, dstAddr);
+ break;
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ masm.memoryBarrierAfter(access.sync());
+ } else {
+ masm.wasmStore(access, ToAnyRegister(value), dstAddr);
+ }
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorX64::emitWasmLoad(T* ins) {
+ const MWasmLoad* mir = ins->mir();
+
+ uint32_t offset = mir->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+ // true 64-bit value.
+ const LAllocation* ptr = ins->ptr();
+ Operand srcAddr = ptr->isBogus()
+ ? Operand(HeapReg, offset)
+ : Operand(HeapReg, ToRegister(ptr), TimesOne, offset);
+
+ if (mir->type() == MIRType::Int64) {
+ masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins));
+ } else {
+ masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); }
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); }
+
+template <typename T>
+void CodeGeneratorX64::emitWasmStore(T* ins) {
+ const MWasmStore* mir = ins->mir();
+ const wasm::MemoryAccessDesc& access = mir->access();
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* value = ins->getOperand(ins->ValueIndex);
+ const LAllocation* ptr = ins->ptr();
+ Operand dstAddr = ptr->isBogus()
+ ? Operand(HeapReg, offset)
+ : Operand(HeapReg, ToRegister(ptr), TimesOne, offset);
+
+ wasmStore(access, value, dstAddr);
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); }
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) {
+ MOZ_CRASH("Unused on this platform");
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ masm.wasmCompareExchange64(mir->access(), srcAddr, Register64(oldval),
+ Register64(newval), ToOutRegister64(ins));
+ } else {
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval,
+ ToRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ masm.wasmAtomicExchange64(mir->access(), srcAddr, Register64(value),
+ ToOutRegister64(ins));
+ } else {
+ masm.wasmAtomicExchange(mir->access(), srcAddr, value,
+ ToRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ const LAllocation* value = ins->value();
+ Register temp =
+ ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
+ Register output = ToRegister(ins->output());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ if (accessType == Scalar::Uint32) {
+ accessType = Scalar::Int32;
+ }
+
+ AtomicOp op = mir->operation();
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ Register64 val = Register64(ToRegister(value));
+ Register64 out = Register64(output);
+ Register64 tmp = Register64(temp);
+ masm.wasmAtomicFetchOp64(mir->access(), op, val, srcAddr, tmp, out);
+ } else if (value->isConstant()) {
+ masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), srcAddr,
+ temp, output);
+ } else {
+ masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr, temp,
+ output);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ const LAllocation* value = ins->value();
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ AtomicOp op = mir->operation();
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ Register64 val = Register64(ToRegister(value));
+ masm.wasmAtomicEffectOp64(mir->access(), op, val, srcAddr);
+ } else if (value->isConstant()) {
+ Imm32 c(0);
+ if (value->toConstant()->type() == MIRType::Int64) {
+ c = Imm32(ToInt64(value));
+ } else {
+ c = Imm32(ToInt32(value));
+ }
+ masm.wasmAtomicEffectOp(mir->access(), op, c, srcAddr, InvalidReg);
+ } else {
+ masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
+ InvalidReg);
+ }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ // On x64, branchTruncateDouble uses vcvttsd2sq. Unlike the x86
+ // implementation, this should handle most doubles and we can just
+ // call a stub if it fails.
+ emitTruncateDouble(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ emitTruncateDouble(input, output, lir->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ emitTruncateFloat32(input, output, lir->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ // On x64, branchTruncateFloat32 uses vcvttss2sq. Unlike the x86
+ // implementation, this should handle most floats and we can just
+ // call a stub if it fails.
+ emitTruncateFloat32(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ masm.movl(ToOperand(input), output);
+ } else {
+ MOZ_CRASH("Not implemented.");
+ }
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->isUnsigned()) {
+ masm.movl(ToOperand(input), output);
+ } else {
+ masm.movslq(ToOperand(input), output);
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+ switch (ins->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.movsbq(Operand(input.reg), output.reg);
+ break;
+ case MSignExtendInt64::Half:
+ masm.movswq(Operand(input.reg), output.reg);
+ break;
+ case MSignExtendInt64::Word:
+ masm.movslq(Operand(input.reg), output.reg);
+ break;
+ }
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType inputType = mir->input()->type();
+
+ MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ FloatRegister temp =
+ mir->isUnsigned() ? ToFloatRegister(lir->temp()) : InvalidFloatReg;
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+ if (inputType == MIRType::Double) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MInt64ToFloatingPoint* mir = lir->mir();
+ bool isUnsigned = mir->isUnsigned();
+
+ MIRType outputType = mir->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+ MOZ_ASSERT(isUnsigned == !lir->getTemp(0)->isBogusTemp());
+
+ if (outputType == MIRType::Double) {
+ if (isUnsigned) {
+ masm.convertUInt64ToDouble(input, output, ToRegister(lir->getTemp(0)));
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (isUnsigned) {
+ masm.convertUInt64ToFloat32(input, output, ToRegister(lir->getTemp(0)));
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ masm.cmpq(Imm32(0), ToRegister(lir->input()));
+ masm.emitSet(Assembler::Equal, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.clz64(input, output.reg);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.ctz64(input, output.reg);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+ Register inputR = ToRegister(input);
+ MOZ_ASSERT(inputR == ToRegister(ins->output()));
+ masm.notq(inputR);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register input = ToRegister(lir->input());
+ masm.testq(input, input);
+ emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse());
+}
diff --git a/js/src/jit/x64/CodeGenerator-x64.h b/js/src/jit/x64/CodeGenerator-x64.h
new file mode 100644
index 0000000000..c3359d0190
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_CodeGenerator_x64_h
+#define jit_x64_CodeGenerator_x64_h
+
+#include "jit/x86-shared/CodeGenerator-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorX64 : public CodeGeneratorX86Shared {
+ protected:
+ CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ Operand ToOperand64(const LInt64Allocation& a);
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ void wasmStore(const wasm::MemoryAccessDesc& access, const LAllocation* value,
+ Operand dstAddr);
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+};
+
+using CodeGeneratorSpecific = CodeGeneratorX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_CodeGenerator_x64_h */
diff --git a/js/src/jit/x64/LIR-x64.h b/js/src/jit/x64/LIR-x64.h
new file mode 100644
index 0000000000..efaedc4499
--- /dev/null
+++ b/js/src/jit/x64/LIR-x64.h
@@ -0,0 +1,170 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_LIR_x64_h
+#define jit_x64_LIR_x64_h
+
+namespace js {
+namespace jit {
+
+// Given an untyped input, guards on whether it's a specific type and returns
+// the unboxed payload.
+class LUnboxBase : public LInstructionHelper<1, 1, 0> {
+ public:
+ LUnboxBase(LNode::Opcode op, const LAllocation& input)
+ : LInstructionHelper<1, 1, 0>(op) {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+};
+
+class LUnbox : public LUnboxBase {
+ public:
+ LIR_HEADER(Unbox)
+
+ explicit LUnbox(const LAllocation& input) : LUnboxBase(classOpcode, input) {}
+
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnboxBase {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint)
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnboxBase(classOpcode, input), type_(type) {}
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ explicit LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ explicit LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+// This class performs a simple x86 'div', yielding either a quotient or
+// remainder depending on whether this instruction is defined to output
+// rax (quotient) or rdx (remainder).
+class LUDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ LWasmTruncateToInt64(const LAllocation& in, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, temp);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_LIR_x64_h */
diff --git a/js/src/jit/x64/Lowering-x64.cpp b/js/src/jit/x64/Lowering-x64.cpp
new file mode 100644
index 0000000000..d2e328930c
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.cpp
@@ -0,0 +1,565 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Lowering-x64.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/x64/Assembler-x64.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+LBoxAllocation LIRGeneratorX64::useBoxFixed(MDefinition* mir, Register reg1,
+ Register, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorX64::tempByteOpRegister() { return temp(); }
+
+LDefinition LIRGeneratorX64::tempToUnbox() { return temp(); }
+
+void LIRGeneratorX64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ // X64 doesn't need a temp for 64bit multiplication.
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+ LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* box = unbox->getOperand(0);
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnboxBase* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new (alloc())
+ LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ lir = new (alloc()) LUnbox(useAtStart(box));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ define(lir, unbox);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+void LIRGeneratorX64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGeneratorX64::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ defineTypedPhi(phi, lirIndex);
+}
+
+void LIRGeneratorX64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LUse oldval = useRegister(ins->oldval());
+ LUse newval = useRegister(ins->newval());
+ LInt64Definition temp1 = tempInt64Fixed(Register64(rax));
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerCompareExchangeTypedArrayElement(ins,
+ /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useRegister(ins->value());
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useRegister(ins->value());
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ LInt64Definition temp = tempInt64();
+
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD.
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop with rax as a temp register.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2;
+ if (bitOp) {
+ temp2 = tempInt64Fixed(Register64(rax));
+ } else {
+ temp2 = tempInt64();
+ }
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGeneratorX64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ // 'base' is a GPR but may be of either type. If it is 32-bit it is
+ // zero-extended and can act as 64-bit.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->type() != MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmLoad(useRegisterOrZeroAtStart(base));
+ define(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmLoadI64(useRegisterOrZeroAtStart(base));
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+ LAllocation valueAlloc;
+ switch (ins->access().type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ valueAlloc = useRegisterOrConstantAtStart(value);
+ break;
+ case Scalar::Int64:
+ // No way to encode an int64-to-memory move on x64.
+ if (value->isConstant() && value->type() != MIRType::Int64) {
+ valueAlloc = useOrConstantAtStart(value);
+ } else {
+ valueAlloc = useRegisterAtStart(value);
+ }
+ break;
+ case Scalar::Float32:
+ case Scalar::Float64:
+ valueAlloc = useRegisterAtStart(value);
+ break;
+ case Scalar::Simd128:
+#ifdef ENABLE_WASM_SIMD
+ valueAlloc = useRegisterAtStart(value);
+ break;
+#else
+ MOZ_CRASH("unexpected array type");
+#endif
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Uint8Clamped:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ LAllocation baseAlloc = useRegisterOrZeroAtStart(base);
+ auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // The output may not be used but will be clobbered regardless, so
+ // pin the output to eax.
+ //
+ // The input values must both be in registers.
+
+ const LAllocation oldval = useRegister(ins->oldValue());
+ const LAllocation newval = useRegister(ins->newValue());
+
+ LWasmCompareExchangeHeap* lir =
+ new (alloc()) LWasmCompareExchangeHeap(useRegister(base), oldval, newval);
+
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32 ||
+ ins->base()->type() == MIRType::Int64);
+
+ const LAllocation base = useRegister(ins->base());
+ const LAllocation value = useRegister(ins->value());
+
+ // The output may not be used but will be clobbered regardless,
+ // so ignore the case where we're not using the value and just
+ // use the output register as a temp.
+
+ LWasmAtomicExchangeHeap* lir =
+ new (alloc()) LWasmAtomicExchangeHeap(base, value);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // No support for 64-bit operations with constants at the masm level.
+
+ bool canTakeConstant = ins->access().type() != Scalar::Int64;
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
+ // LOCK OR, or LOCK XOR.
+
+ if (!ins->hasUses()) {
+ LAllocation value = canTakeConstant ? useRegisterOrConstant(ins->value())
+ : useRegister(ins->value());
+ LWasmAtomicBinopHeapForEffect* lir =
+ new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base), value);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD with word and byte ops as
+ // appropriate. Any output register can be used and if value is a
+ // register it's best if it's the same as output:
+ //
+ // movl value, output ; if value != output
+ // lock xaddl output, mem
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop, and the output is
+ // always in rax:
+ //
+ // movl *mem, rax
+ // L: mov rax, temp
+ // andl value, temp
+ // lock cmpxchg temp, mem ; reads rax also
+ // jnz L
+ // ; result in rax
+ //
+ // Note the placement of L, cmpxchg will update rax with *mem if
+ // *mem does not have the expected value, so reloading it at the
+ // top of the loop would be redundant.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+ bool reuseInput = false;
+ LAllocation value;
+
+ if (bitOp || ins->value()->isConstant()) {
+ value = canTakeConstant ? useRegisterOrConstant(ins->value())
+ : useRegister(ins->value());
+ } else {
+ reuseInput = true;
+ value = useRegisterAtStart(ins->value());
+ }
+
+ auto* lir = new (alloc()) LWasmAtomicBinopHeap(
+ useRegister(base), value, bitOp ? temp() : LDefinition::BogusTemp());
+
+ if (reuseInput) {
+ defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp);
+ } else if (bitOp) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(rax)));
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+ defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorX64::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+ defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorX64::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+ defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+ defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ LDefinition maybeTemp =
+ ins->isUnsigned() ? tempDouble() : LDefinition::BogusTemp();
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ LDefinition maybeTemp = ins->isUnsigned() ? temp() : LDefinition::BogusTemp();
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX64::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ defineInt64(new (alloc()) LExtendInt32ToInt64(useAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+// On x64 we specialize the cases: compare is {U,}Int{32,64}, and select is
+// {U,}Int{32,64}, independently.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return (insTy == MIRType::Int32 || insTy == MIRType::Int64) &&
+ (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32 ||
+ compTy == MCompare::Compare_Int64 ||
+ compTy == MCompare::Compare_UInt64);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useAny(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
diff --git a/js/src/jit/x64/Lowering-x64.h b/js/src/jit/x64/Lowering-x64.h
new file mode 100644
index 0000000000..1c34ea8693
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Lowering_x64_h
+#define jit_x64_Lowering_x64_h
+
+#include "jit/x86-shared/Lowering-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorX64 : public LIRGeneratorX86Shared {
+ protected:
+ LIRGeneratorX64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorX86Shared(gen, graph, lirGraph) {}
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+ // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register,
+ bool useAtStart = false);
+
+ // x86 has constraints on what registers can be formatted for 1-byte
+ // stores and loads; on x64 all registers are okay.
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ LDefinition tempToUnbox();
+
+ bool needTempForPostBarrier() { return true; }
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+};
+
+using LIRGeneratorSpecific = LIRGeneratorX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_Lowering_x64_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h
new file mode 100644
index 0000000000..6869e6c4b6
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64-inl.h
@@ -0,0 +1,1099 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_inl_h
+#define jit_x64_MacroAssembler_x64_inl_h
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+// ===============================================================
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ // Use mov instead of movq because it has special optimizations for imm == 0.
+ mov(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ movq(src.reg, dest.reg);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ vmovq(src, dest.reg);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ vmovq(src.reg, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ movl(src.reg, dest);
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ movl(src, dest.reg);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ movsbq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ movswq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ movslq(src, dest.reg);
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ movslq(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ movl(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ movslq(Operand(src), dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::notPtr(Register reg) { notq(reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { andq(src, dest); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { andq(imm, dest); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ andq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ andq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ orq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ orq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ xorq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ xorq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { orq(src, dest); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { orq(imm, dest); }
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ andq(src.reg, dest.reg);
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ orq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ xorq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { xorq(src, dest); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorq(imm, dest); }
+
+void MacroAssembler::and64(const Operand& src, Register64 dest) {
+ andq(src, dest.reg);
+}
+
+void MacroAssembler::or64(const Operand& src, Register64 dest) {
+ orq(src, dest.reg);
+}
+
+void MacroAssembler::xor64(const Operand& src, Register64 dest) {
+ xorq(src, dest.reg);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg) { bswapq(reg.reg); }
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) { addq(src, dest); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { addq(imm, dest); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(dest != scratch);
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ addq(Imm32((int32_t)imm.value), dest);
+ } else {
+ mov(imm, scratch);
+ addq(scratch, dest);
+ }
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
+ addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ addq(Operand(src), dest);
+}
+
+void MacroAssembler::add64(const Operand& src, Register64 dest) {
+ addq(src, dest.reg);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addq(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) { addq(imm, dest.reg); }
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ addPtr(ImmWord(imm.value), dest.reg);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ moveStackPtrTo(dest);
+ addqWithPatch(Imm32(0), dest);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ patchAddq(offset, -imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { subq(src, dest); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ subq(src, Operand(dest));
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) { subq(imm, dest); }
+
+void MacroAssembler::subPtr(ImmWord imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(dest != scratch);
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ subq(Imm32((int32_t)imm.value), dest);
+ } else {
+ mov(imm, scratch);
+ subq(scratch, dest);
+ }
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ subq(Operand(addr), dest);
+}
+
+void MacroAssembler::sub64(const Operand& src, Register64 dest) {
+ subq(src, dest.reg);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ subq(src.reg, dest.reg);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ subPtr(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ // To compute the unsigned multiplication using imulq, we have to ensure both
+ // operands don't have any bits set in the high word.
+
+ if (imm.value >= 0) {
+ // Clear the high word of |src|.
+ movl(src, src);
+
+ // |imm| and |src| are both positive, so directly perform imulq.
+ imulq(imm, src, dest);
+ } else {
+ // Store the low word of |src| into |dest|.
+ movl(src, dest);
+
+ // Compute the unsigned value of |imm| before performing imulq.
+ movl(imm, ScratchReg);
+ imulq(ScratchReg, dest);
+ }
+
+ // Move the high word into |dest|.
+ shrq(Imm32(32), dest);
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ imulq(rhs, srcDest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(imm, dest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ imulq(Imm32((int32_t)imm.value), dest.reg, dest.reg);
+ } else {
+ movq(ImmWord(uintptr_t(imm.value)), ScratchReg);
+ imulq(ScratchReg, dest.reg);
+ }
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(Operand(src.reg), dest);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest) {
+ imulq(src, dest.reg);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(src, dest);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ lea(Operand(src, src, TimesTwo), dest);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ movq(imm, ScratchReg);
+ vmulsd(Operand(ScratchReg, 0), dest, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ if (X86Encoding::IsAddressImmediate(dest.addr)) {
+ addPtr(Imm32(1), dest);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(dest.addr), scratch);
+ addPtr(Imm32(1), Address(scratch, 0));
+ }
+}
+
+void MacroAssembler::neg64(Register64 reg) { negq(reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) { negq(reg); }
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ shlq(imm, dest);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
+ if (Assembler::HasBMI2()) {
+ shlxq(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shlq_cl(srcDest);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ lshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ shlxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shlq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ shrq(imm, dest);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
+ if (Assembler::HasBMI2()) {
+ shrxq(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shrq_cl(srcDest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ rshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ shrxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shrq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ sarq(imm, dest);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ rshiftPtrArithmetic(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ sarxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ sarq_cl(srcDest.reg);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ rolq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ rorq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateRight64(count, src, dest);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ rolq(count, dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ rorq(count, dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateRight64(count, src, dest);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmpPtr(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasLZCNT()) {
+ lzcntq(src.reg, dest);
+ return;
+ }
+
+ Label nonzero;
+ bsrq(src.reg, dest);
+ j(Assembler::NonZero, &nonzero);
+ movq(ImmWord(0x7F), dest);
+ bind(&nonzero);
+ xorq(Imm32(0x3F), dest);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasBMI1()) {
+ tzcntq(src.reg, dest);
+ return;
+ }
+
+ Label nonzero;
+ bsfq(src.reg, dest);
+ j(Assembler::NonZero, &nonzero);
+ movq(ImmWord(64), dest);
+ bind(&nonzero);
+}
+
+void MacroAssembler::popcnt64(Register64 src64, Register64 dest64,
+ Register tmp) {
+ Register src = src64.reg;
+ Register dest = dest64.reg;
+
+ if (AssemblerX86Shared::HasPOPCNT()) {
+ MOZ_ASSERT(tmp == InvalidReg);
+ popcntq(src, dest);
+ return;
+ }
+
+ if (src != dest) {
+ movq(src, dest);
+ }
+
+ MOZ_ASSERT(tmp != dest);
+
+ ScratchRegisterScope scratch(*this);
+
+ // Equivalent to mozilla::CountPopulation32, adapted for 64 bits.
+ // x -= (x >> 1) & m1;
+ movq(src, tmp);
+ movq(ImmWord(0x5555555555555555), scratch);
+ shrq(Imm32(1), tmp);
+ andq(scratch, tmp);
+ subq(tmp, dest);
+
+ // x = (x & m2) + ((x >> 2) & m2);
+ movq(dest, tmp);
+ movq(ImmWord(0x3333333333333333), scratch);
+ andq(scratch, dest);
+ shrq(Imm32(2), tmp);
+ andq(scratch, tmp);
+ addq(tmp, dest);
+
+ // x = (x + (x >> 4)) & m4;
+ movq(dest, tmp);
+ movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch);
+ shrq(Imm32(4), tmp);
+ addq(tmp, dest);
+ andq(scratch, dest);
+
+ // (x * h01) >> 56
+ movq(ImmWord(0x0101010101010101), scratch);
+ imulq(scratch, dest);
+ shrq(Imm32(56), dest);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branch32(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+ }
+}
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branch32(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+ }
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ mov(lhs, scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, rhs.reg, success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(rhs != scratch);
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branchPtrImpl(cond, Operand(lhs), rhs, label);
+ } else {
+ mov(ImmPtr(lhs.addr), scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branchPtrImpl(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(rhs != scratch);
+ mov(lhs, scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
+ Register dest, Label* fail) {
+ vcvttss2sq(src, dest);
+
+ // Same trick as for Doubles
+ cmpPtr(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateFloat32ToPtr(src, dest, fail);
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ branchTruncateFloat32ToPtr(src, dest, fail);
+
+ // Check that the result is in the int32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32To64SignExtend(dest, Register64(scratch));
+ cmpPtr(dest, scratch);
+ j(Assembler::NotEqual, fail);
+
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail) {
+ vcvttsd2sq(src, dest);
+
+ // vcvttsd2sq returns 0x8000000000000000 on failure. Test for it by
+ // subtracting 1 and testing overflow (this avoids the need to
+ // materialize that value in a register).
+ cmpPtr(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateDoubleToPtr(src, dest, fail);
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ branchTruncateDoubleToPtr(src, dest, fail);
+
+ // Check that the result is in the int32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32To64SignExtend(dest, Register64(scratch));
+ cmpPtr(dest, scratch);
+ j(Assembler::NotEqual, fail);
+
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ test32(Operand(lhs), rhs);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ test32(Operand(scratch, 0), rhs);
+ }
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ test32(value.valueReg(), value.valueReg());
+ j(truthy ? NonZero : Zero, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(valaddr, ImmWord(magic));
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& address) {
+ jmp(Operand(address));
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmpPtr(lhs, rhs);
+ cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmpPtr(lhs, Operand(rhs));
+ cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ move32(Imm32(0), scratch);
+ }
+
+ cmp32(index, length);
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length.base != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ move32(Imm32(0), scratch);
+ }
+
+ cmp32(index, Operand(length));
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ movePtr(ImmWord(0), scratch);
+ }
+
+ cmpPtr(index, length);
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length.base != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ movePtr(ImmWord(0), scratch);
+ }
+
+ cmpPtr(index, Operand(length));
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+// ========================================================================
+// SIMD.
+
+// Extract lane as scalar
+
+void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
+ Register64 dest) {
+ if (lane == 0) {
+ vmovq(src, dest.reg);
+ } else {
+ vpextrq(lane, src, dest.reg);
+ }
+}
+
+// Replace lane value
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
+ FloatRegister lhsDest) {
+ vpinsrq(lane, rhs.reg, lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
+ Register64 rhs, FloatRegister dest) {
+ vpinsrq(lane, rhs.reg, lhs, dest);
+}
+
+// Splat
+
+void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
+ vmovq(src.reg, dest);
+ if (HasAVX2()) {
+ vbroadcastq(Operand(dest), dest);
+ } else {
+ vpunpcklqdq(dest, dest, dest);
+ }
+}
+
+// ========================================================================
+// Truncate floating point.
+
+void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadFloat32(src, floatTemp);
+
+ truncateFloat32ToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handled
+ // separately.
+ loadPtr(dest, temp);
+ branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeFloat32(floatTemp, dest);
+ loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddss(Operand(dest), floatTemp, floatTemp);
+ storeFloat32(floatTemp, dest);
+ truncateFloat32ToInt64(dest, dest, temp);
+
+ loadPtr(dest, temp);
+ or64(Imm64(0x8000000000000000), Register64(temp));
+ storePtr(temp, dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadDouble(src, floatTemp);
+
+ truncateDoubleToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handle
+ // seperately.
+ loadPtr(dest, temp);
+ branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeDouble(floatTemp, dest);
+ loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddsd(Operand(dest), floatTemp, floatTemp);
+ storeDouble(floatTemp, dest);
+ truncateDoubleToInt64(dest, dest, temp);
+
+ loadPtr(dest, temp);
+ or64(Imm64(0x8000000000000000), Register64(temp));
+ storePtr(temp, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerX64::fallibleUnboxPtrImpl(const Operand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // scratch := dest >> JSVAL_TAG_SHIFT
+ // fail if scratch != 0
+ //
+ // Note: src and dest can be the same register.
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ xorq(src, scratch);
+ mov(scratch, dest);
+ shrq(Imm32(JSVAL_TAG_SHIFT), scratch);
+ j(Assembler::NonZero, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src.valueReg()), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssemblerX64::incrementInt32Value(const Address& addr) {
+ asMasm().addPtr(Imm32(1), addr);
+}
+
+void MacroAssemblerX64::unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+template <typename T>
+void MacroAssemblerX64::loadInt32OrDouble(const T& src, FloatRegister dest) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src, dest);
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest);
+ bind(&end);
+}
+
+// If source is a double, load it into dest. If source is int32,
+// convert it to double. Else, branch to failure.
+void MacroAssemblerX64::ensureDouble(const ValueOperand& source,
+ FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ {
+ ScratchRegisterScope scratch(asMasm());
+ unboxInt32(source, scratch);
+ convertInt32ToDouble(scratch, dest);
+ }
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_inl_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp
new file mode 100644
index 0000000000..c6ab1fe935
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.cpp
@@ -0,0 +1,1747 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MacroAssemblerX64::loadConstantDouble(double d, FloatRegister dest) {
+ if (maybeInlineDouble(d, dest)) {
+ return;
+ }
+ Double* dbl = getDouble(d);
+ if (!dbl) {
+ return;
+ }
+ // The constants will be stored in a pool appended to the text (see
+ // finish()), so they will always be a fixed distance from the
+ // instructions which reference them. This allows the instructions to use
+ // PC-relative addressing. Use "jump" label support code, because we need
+ // the same PC-relative address patching that jumps use.
+ JmpSrc j = masm.vmovsd_ripr(dest.encoding());
+ propagateOOM(dbl->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantFloat32(float f, FloatRegister dest) {
+ if (maybeInlineFloat(f, dest)) {
+ return;
+ }
+ Float* flt = getFloat(f);
+ if (!flt) {
+ return;
+ }
+ // See comment in loadConstantDouble
+ JmpSrc j = masm.vmovss_ripr(dest.encoding());
+ propagateOOM(flt->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+ const SimdConstant& v, FloatRegister reg,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID id)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ JmpSrc j = (masm.*op)(reg.encoding());
+ propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+ const SimdConstant& v, FloatRegister src, FloatRegister dest,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID srcId, X86Encoding::XMMRegisterID destId)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ JmpSrc j = (masm.*op)(src.encoding(), dest.encoding());
+ propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantSimd128Int(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Int(v, dest)) {
+ return;
+ }
+ vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovdqa_ripr);
+}
+
+void MacroAssemblerX64::loadConstantSimd128Float(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Float(v, dest)) {
+ return;
+ }
+ vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovaps_ripr);
+}
+
+void MacroAssemblerX64::vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddb_ripr);
+}
+
+void MacroAssemblerX64::vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddw_ripr);
+}
+
+void MacroAssemblerX64::vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddd_ripr);
+}
+
+void MacroAssemblerX64::vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddq_ripr);
+}
+
+void MacroAssemblerX64::vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubb_ripr);
+}
+
+void MacroAssemblerX64::vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubw_ripr);
+}
+
+void MacroAssemblerX64::vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubd_ripr);
+}
+
+void MacroAssemblerX64::vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubq_ripr);
+}
+
+void MacroAssemblerX64::vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmullw_ripr);
+}
+
+void MacroAssemblerX64::vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmulld_ripr);
+}
+
+void MacroAssemblerX64::vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsb_ripr);
+}
+
+void MacroAssemblerX64::vpaddusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusb_ripr);
+}
+
+void MacroAssemblerX64::vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsw_ripr);
+}
+
+void MacroAssemblerX64::vpadduswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusw_ripr);
+}
+
+void MacroAssemblerX64::vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsb_ripr);
+}
+
+void MacroAssemblerX64::vpsubusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusb_ripr);
+}
+
+void MacroAssemblerX64::vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsw_ripr);
+}
+
+void MacroAssemblerX64::vpsubuswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusw_ripr);
+}
+
+void MacroAssemblerX64::vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsb_ripr);
+}
+
+void MacroAssemblerX64::vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminub_ripr);
+}
+
+void MacroAssemblerX64::vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsw_ripr);
+}
+
+void MacroAssemblerX64::vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminuw_ripr);
+}
+
+void MacroAssemblerX64::vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsd_ripr);
+}
+
+void MacroAssemblerX64::vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminud_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsb_ripr);
+}
+
+void MacroAssemblerX64::vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxub_ripr);
+}
+
+void MacroAssemblerX64::vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxuw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsd_ripr);
+}
+
+void MacroAssemblerX64::vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxud_ripr);
+}
+
+void MacroAssemblerX64::vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpand_ripr);
+}
+
+void MacroAssemblerX64::vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpxor_ripr);
+}
+
+void MacroAssemblerX64::vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpor_ripr);
+}
+
+void MacroAssemblerX64::vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddps_ripr);
+}
+
+void MacroAssemblerX64::vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddpd_ripr);
+}
+
+void MacroAssemblerX64::vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubps_ripr);
+}
+
+void MacroAssemblerX64::vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubpd_ripr);
+}
+
+void MacroAssemblerX64::vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivps_ripr);
+}
+
+void MacroAssemblerX64::vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivpd_ripr);
+}
+
+void MacroAssemblerX64::vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulps_ripr);
+}
+
+void MacroAssemblerX64::vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulpd_ripr);
+}
+
+void MacroAssemblerX64::vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vandpd_ripr);
+}
+
+void MacroAssemblerX64::vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vminpd_ripr);
+}
+
+void MacroAssemblerX64::vpacksswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpacksswb_ripr);
+}
+
+void MacroAssemblerX64::vpackuswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackuswb_ripr);
+}
+
+void MacroAssemblerX64::vpackssdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackssdw_ripr);
+}
+
+void MacroAssemblerX64::vpackusdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackusdw_ripr);
+}
+
+void MacroAssemblerX64::vpunpckldqSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest,
+ &X86Encoding::BaseAssemblerX64::vpunpckldq_ripr);
+}
+
+void MacroAssemblerX64::vunpcklpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vunpcklps_ripr);
+}
+
+void MacroAssemblerX64::vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpshufb_ripr);
+}
+
+void MacroAssemblerX64::vptestSimd128(const SimdConstant& v,
+ FloatRegister lhs) {
+ vpRiprOpSimd128(v, lhs, &X86Encoding::BaseAssemblerX64::vptest_ripr);
+}
+
+void MacroAssemblerX64::vpmaddwdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaddwd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtd_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltps_ripr);
+}
+
+void MacroAssemblerX64::vcmplepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpleps_ripr);
+}
+
+void MacroAssemblerX64::vcmpgepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpgeps_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltpd_ripr);
+}
+
+void MacroAssemblerX64::vcmplepdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmplepd_ripr);
+}
+
+void MacroAssemblerX64::vpmaddubswSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest,
+ &X86Encoding::BaseAssemblerX64::vpmaddubsw_ripr);
+}
+
+void MacroAssemblerX64::vpmuludqSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmuludq_ripr);
+}
+
+void MacroAssemblerX64::bindOffsets(
+ const MacroAssemblerX86Shared::UsesVector& uses) {
+ for (JmpSrc src : uses) {
+ JmpDst dst(currentOffset());
+ // Using linkJump here is safe, as explained in the comment in
+ // loadConstantDouble.
+ masm.linkJump(src, dst);
+ }
+}
+
+void MacroAssemblerX64::finish() {
+ if (!doubles_.empty()) {
+ masm.haltingAlign(sizeof(double));
+ }
+ for (const Double& d : doubles_) {
+ bindOffsets(d.uses);
+ masm.doubleConstant(d.value);
+ }
+
+ if (!floats_.empty()) {
+ masm.haltingAlign(sizeof(float));
+ }
+ for (const Float& f : floats_) {
+ bindOffsets(f.uses);
+ masm.floatConstant(f.value);
+ }
+
+ // SIMD memory values must be suitably aligned.
+ if (!simds_.empty()) {
+ masm.haltingAlign(SimdMemoryAlignment);
+ }
+ for (const SimdData& v : simds_) {
+ bindOffsets(v.uses);
+ masm.simd128Constant(v.value.bytes());
+ }
+
+ MacroAssemblerX86Shared::finish();
+}
+
+void MacroAssemblerX64::boxValue(JSValueType type, Register src,
+ Register dest) {
+ MOZ_ASSERT(src != dest);
+
+ JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+#ifdef DEBUG
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ Label upper32BitsZeroed;
+ movePtr(ImmWord(UINT32_MAX), dest);
+ asMasm().branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed);
+ breakpoint();
+ bind(&upper32BitsZeroed);
+ }
+#endif
+ mov(ImmShiftedTag(tag), dest);
+ orq(src, dest);
+}
+
+void MacroAssemblerX64::handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail) {
+ // Reserve space for exception information.
+ subq(Imm32(sizeof(ResumeFromException)), rsp);
+ movq(rsp, rax);
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(rcx);
+ asMasm().passABIArg(rax);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ load32(Address(rsp, ResumeFromException::offsetOfKind()), rax);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ ret();
+
+ // If we found a catch handler, this must be a baseline frame. Restore state
+ // and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ jmp(Operand(rax));
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(rcx);
+ loadValue(Address(esp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jmp(Operand(rax));
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ loadValue(Address(rbp, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jmp(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(rsp, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ movq(rbp, rsp);
+ pop(rbp);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to the
+ // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfBailoutInfo()), r9);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ move32(Imm32(1), ReturnReg);
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ movePtr(ImmPtr((const void*)wasm::FailInstanceReg), InstanceReg);
+ masm.ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ jmp(Operand(rax));
+}
+
+void MacroAssemblerX64::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerX64::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+Assembler::Condition MacroAssemblerX64::testStringTruthy(
+ bool truthy, const ValueOperand& value) {
+ ScratchRegisterScope scratch(asMasm());
+ unboxString(value, scratch);
+ cmp32(Operand(scratch, JSString::offsetOfLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+Assembler::Condition MacroAssemblerX64::testBigIntTruthy(
+ bool truthy, const ValueOperand& value) {
+ ScratchRegisterScope scratch(asMasm());
+ unboxBigInt(value, scratch);
+ cmp32(Operand(scratch, JS::BigInt::offsetOfDigitLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+MacroAssembler& MacroAssemblerX64::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerX64::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ // On windows, we cannot skip very far down the stack without touching the
+ // memory pages in-between. This is a corner-case code for situations where
+ // the Ion frame data for a piece of code is very large. To handle this
+ // special case, for frames over 4k in size we allocate memory on the stack
+ // incrementally, touching it as we go.
+ //
+ // When the amount is quite large, which it can be, we emit an actual loop,
+ // in order to keep the function prologue compact. Compactness is a
+ // requirement for eg Wasm's CodeRange data structure, which can encode only
+ // 8-bit offsets.
+ uint32_t amountLeft = imm32.value;
+ uint32_t fullPages = amountLeft / 4096;
+ if (fullPages <= 8) {
+ while (amountLeft > 4096) {
+ subq(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ amountLeft -= 4096;
+ }
+ subq(Imm32(amountLeft), StackPointer);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ Label top;
+ move32(Imm32(fullPages), scratch);
+ bind(&top);
+ subq(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ subl(Imm32(1), scratch);
+ j(Assembler::NonZero, &top);
+ amountLeft -= fullPages * 4096;
+ if (amountLeft) {
+ subq(Imm32(amountLeft), StackPointer);
+ }
+ }
+ }
+}
+
+void MacroAssemblerX64::convertDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ // Check for -0.0
+ if (negativeZeroCheck) {
+ branchNegativeZero(src, dest, fail);
+ }
+
+ ScratchDoubleScope scratch(asMasm());
+ vcvttsd2sq(src, dest);
+ asMasm().convertInt64ToDouble(Register64(dest), scratch);
+ vucomisd(scratch, src);
+ j(Assembler::Parity, fail);
+ j(Assembler::NotEqual, fail);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ movq(rsp, scratch);
+ andq(Imm32(~(ABIStackAlignment - 1)), rsp);
+ push(scratch);
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ if (dynamicAlignment_) {
+ // sizeof(intptr_t) accounts for the saved stack pointer pushed by
+ // setupUnalignedABICall.
+ stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
+ ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool cleanupArg) {
+ freeStack(stackAdjust);
+ if (dynamicAlignment_) {
+ pop(rsp);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+static bool IsIntArgReg(Register reg) {
+ for (uint32_t i = 0; i < NumIntArgRegs; i++) {
+ if (IntArgRegs[i] == reg) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ if (IsIntArgReg(fun)) {
+ // Callee register may be clobbered for an argument. Move the callee to
+ // r10, a volatile, non-argument register.
+ propagateOOM(moveResolver_.addMove(MoveOperand(fun), MoveOperand(r10),
+ MoveOp::GENERAL));
+ fun = r10;
+ }
+
+ MOZ_ASSERT(!IsIntArgReg(fun));
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(fun);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ Address safeFun = fun;
+ if (IsIntArgReg(safeFun.base)) {
+ // Callee register may be clobbered for an argument. Move the callee to
+ // r10, a volatile, non-argument register.
+ propagateOOM(moveResolver_.addMove(MoveOperand(fun.base), MoveOperand(r10),
+ MoveOp::GENERAL));
+ safeFun.base = r10;
+ }
+
+ MOZ_ASSERT(!IsIntArgReg(safeFun.base));
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(safeFun);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxValue(ValueTypeFromMIRType(type), reg.gpr(), dest.valueReg());
+ return;
+ }
+
+ ScratchDoubleScope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, freg);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ if (src == dest) {
+ return;
+ }
+ movq(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+ writeDataRelocation(src);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ if (ptr != buffer) {
+ movePtr(ptr, buffer);
+ }
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), buffer);
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != scratch);
+
+ movePtr(ptr, scratch);
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), scratch);
+ branchPtr(InvertCondition(cond), Address(scratch, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+ const T& value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ getGCThingValueChunk(value, temp);
+ branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs.valueReg() != scratch);
+ moveValue(rhs, ValueOperand(scratch));
+ cmpPtr(lhs.valueReg(), scratch);
+ j(cond, label);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ subq(Imm32(sizeof(double)), StackPointer);
+ boxDouble(reg, Address(StackPointer, 0));
+ adjustFrame(sizeof(double));
+}
+
+// ========================================================================
+// wasm support
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, AnyRegister out) {
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ MOZ_ASSERT_IF(
+ access.isZeroExtendSimd128Load(),
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(
+ access.isSplatSimd128Load(),
+ access.type() == Scalar::Uint8 || access.type() == Scalar::Uint16 ||
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(access.isWidenSimd128Load(), access.type() == Scalar::Float64);
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ movsbl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint8:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastb(srcAddr, out.fpu());
+ } else {
+ movzbl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int16:
+ movswl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint16:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastw(srcAddr, out.fpu());
+ } else {
+ movzwl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(srcAddr, out.gpr());
+ break;
+ case Scalar::Float32:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastss(srcAddr, out.fpu());
+ } else {
+ // vmovss does the right thing also for access.isZeroExtendSimd128Load()
+ vmovss(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Float64:
+ if (access.isSplatSimd128Load()) {
+ vmovddup(srcAddr, out.fpu());
+ } else if (access.isWidenSimd128Load()) {
+ switch (access.widenSimdOp()) {
+ case wasm::SimdOp::V128Load8x8S:
+ vpmovsxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ vpmovzxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ vpmovsxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ vpmovzxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ vpmovsxdq(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ vpmovzxdq(srcAddr, out.fpu());
+ break;
+ default:
+ MOZ_CRASH("Unexpected widening op for wasmLoad");
+ }
+ } else {
+ // vmovsd does the right thing also for access.isZeroExtendSimd128Load()
+ vmovsd(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Simd128:
+ MacroAssemblerX64::loadUnalignedSimd128(srcAddr, out.fpu());
+ break;
+ case Scalar::Int64:
+ MOZ_CRASH("int64 loads must use load64");
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Uint8Clamped:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected scalar type for wasmLoad");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, Register64 out) {
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ movsbq(srcAddr, out.reg);
+ break;
+ case Scalar::Uint8:
+ movzbq(srcAddr, out.reg);
+ break;
+ case Scalar::Int16:
+ movswq(srcAddr, out.reg);
+ break;
+ case Scalar::Uint16:
+ movzwq(srcAddr, out.reg);
+ break;
+ case Scalar::Int32:
+ movslq(srcAddr, out.reg);
+ break;
+ // Int32 to int64 moves zero-extend by default.
+ case Scalar::Uint32:
+ movl(srcAddr, out.reg);
+ break;
+ case Scalar::Int64:
+ movq(srcAddr, out.reg);
+ break;
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::Simd128:
+ MOZ_CRASH("float loads must use wasmLoad");
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected scalar type for wasmLoadI64");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Operand dstAddr) {
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, masm.size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ movb(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ movw(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int64:
+ movq(value.gpr(), dstAddr);
+ break;
+ case Scalar::Float32:
+ storeUncanonicalizedFloat32(value.fpu(), dstAddr);
+ break;
+ case Scalar::Float64:
+ storeUncanonicalizedDouble(value.fpu(), dstAddr);
+ break;
+ case Scalar::Simd128:
+ MacroAssemblerX64::storeUnalignedSimd128(value.fpu(), dstAddr);
+ break;
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttsd2sq(input, output);
+
+ // Check that the result is in the uint32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32(Imm32(0xffffffff), scratch);
+ cmpq(scratch, output);
+ j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttss2sq(input, output);
+
+ // Check that the result is in the uint32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32(Imm32(0xffffffff), scratch);
+ cmpq(scratch, output);
+ j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ vcvttsd2sq(input, output.reg);
+ cmpq(Imm32(1), output.reg);
+ j(Assembler::Overflow, oolEntry);
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ vcvttss2sq(input, output.reg);
+ cmpq(Imm32(1), output.reg);
+ j(Assembler::Overflow, oolEntry);
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ // If the input < INT64_MAX, vcvttsd2sq will do the right thing, so
+ // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+ // and then add INT64_MAX to the result.
+
+ Label isLarge;
+
+ ScratchDoubleScope scratch(*this);
+ loadConstantDouble(double(0x8000000000000000), scratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+ vcvttsd2sq(input, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ jump(oolRejoin);
+
+ bind(&isLarge);
+
+ moveDouble(input, tempReg);
+ vsubsd(scratch, tempReg, tempReg);
+ vcvttsd2sq(tempReg, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ or64(Imm64(0x8000000000000000), output);
+
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ // If the input < INT64_MAX, vcvttss2sq will do the right thing, so
+ // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+ // and then add INT64_MAX to the result.
+
+ Label isLarge;
+
+ ScratchFloat32Scope scratch(*this);
+ loadConstantFloat32(float(0x8000000000000000), scratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+ vcvttss2sq(input, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ jump(oolRejoin);
+
+ bind(&isLarge);
+
+ moveFloat32(input, tempReg);
+ vsubss(scratch, tempReg, tempReg);
+ vcvttss2sq(tempReg, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ or64(Imm64(0x8000000000000000), output);
+
+ bind(oolRejoin);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+ move32To64ZeroExtend(r, Register64(r));
+}
+
+// ========================================================================
+// Convert floating point.
+
+void MacroAssembler::convertInt64ToDouble(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ vcvtsq2sd(input.reg, output, output);
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroFloat32(output);
+
+ vcvtsq2ss(input.reg, output, output);
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return true; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 input,
+ FloatRegister output,
+ Register temp) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ // If the input's sign bit is not set we use vcvtsq2sd directly.
+ // Else, we divide by 2 and keep the LSB, convert to double, and multiply
+ // the result by 2.
+ Label done;
+ Label isSigned;
+
+ testq(input.reg, input.reg);
+ j(Assembler::Signed, &isSigned);
+ vcvtsq2sd(input.reg, output, output);
+ jump(&done);
+
+ bind(&isSigned);
+
+ ScratchRegisterScope scratch(*this);
+ mov(input.reg, scratch);
+ mov(input.reg, temp);
+ shrq(Imm32(1), scratch);
+ andq(Imm32(1), temp);
+ orq(temp, scratch);
+
+ vcvtsq2sd(scratch, output, output);
+ vaddsd(output, output, output);
+
+ bind(&done);
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 input,
+ FloatRegister output,
+ Register temp) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroFloat32(output);
+
+ // See comment in convertUInt64ToDouble.
+ Label done;
+ Label isSigned;
+
+ testq(input.reg, input.reg);
+ j(Assembler::Signed, &isSigned);
+ vcvtsq2ss(input.reg, output, output);
+ jump(&done);
+
+ bind(&isSigned);
+
+ ScratchRegisterScope scratch(*this);
+ mov(input.reg, scratch);
+ mov(input.reg, temp);
+ shrq(Imm32(1), scratch);
+ andq(Imm32(1), temp);
+ orq(temp, scratch);
+
+ vcvtsq2ss(scratch, output, output);
+ vaddss(output, output, output);
+
+ bind(&done);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt64ToDouble(Register64(src), dest);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ append(access, size());
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ append(access, size());
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ append(access, masm.size());
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ append(access, masm.size());
+ xchgq(output.reg, Operand(mem));
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, AtomicOp op,
+ Register value, const T& mem, Register temp,
+ Register output) {
+ // NOTE: the generated code must match the assembly code in gen_fetchop in
+ // GenerateAtomicOperations.py
+ if (op == AtomicFetchAddOp) {
+ if (value != output) {
+ masm.movq(value, output);
+ }
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_xaddq(output, Operand(mem));
+ } else if (op == AtomicFetchSubOp) {
+ if (value != output) {
+ masm.movq(value, output);
+ }
+ masm.negq(output);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_xaddq(output, Operand(mem));
+ } else {
+ Label again;
+ MOZ_ASSERT(output == rax);
+ MOZ_ASSERT(value != output);
+ MOZ_ASSERT(value != temp);
+ MOZ_ASSERT(temp != output);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.movq(Operand(mem), rax);
+ masm.bind(&again);
+ masm.movq(rax, temp);
+ switch (op) {
+ case AtomicFetchAndOp:
+ masm.andq(value, temp);
+ break;
+ case AtomicFetchOrOp:
+ masm.orq(value, temp);
+ break;
+ case AtomicFetchXorOp:
+ masm.xorq(value, temp);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ masm.lock_cmpxchgq(temp, Operand(mem));
+ masm.j(MacroAssembler::NonZero, &again);
+ }
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+template <typename T>
+static void AtomicEffectOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, AtomicOp op,
+ Register value, const T& mem) {
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.lock_addq(value, Operand(mem));
+ break;
+ case AtomicFetchSubOp:
+ masm.lock_subq(value, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ masm.lock_andq(value, Operand(mem));
+ break;
+ case AtomicFetchOrOp:
+ masm.lock_orq(value, Operand(mem));
+ break;
+ case AtomicFetchXorOp:
+ masm.lock_xorq(value, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void MacroAssembler::wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem) {
+ AtomicEffectOp64(*this, &access, op, value.reg, mem);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const Address& mem, Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ // NOTE: the generated code must match the assembly code in gen_exchange in
+ // GenerateAtomicOperations.py
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem) {
+ AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem) {
+ AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return leaRipRelative(dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ ptrdiff_t off = target - loc;
+ MOZ_ASSERT(off > ptrdiff_t(INT32_MIN));
+ MOZ_ASSERT(off < ptrdiff_t(INT32_MAX));
+ PatchWrite_Imm32(loc, Imm32(off));
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ cmpPtr(index.reg, boundsCheckLimit.reg);
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(cond, Operand(boundsCheckLimit.reg), index.reg);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ cmpPtr(index.reg, Operand(boundsCheckLimit));
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(cond, Operand(boundsCheckLimit), index.reg);
+ }
+}
+
+// ========================================================================
+// Integer compare-then-conditionally-load/move operations.
+
+// cmpMove, Cond-Reg-Reg-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs, Register rhs,
+ Register falseVal, Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, rhs);
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, rhs);
+ }
+ if constexpr (MoveSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(MoveSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+
+// cmpMove, Cond-Reg-Addr-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs,
+ const Address& rhs, Register falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, Operand(rhs));
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, Operand(rhs));
+ }
+ if constexpr (MoveSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(MoveSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Reg-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs, Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, rhs);
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, rhs);
+ }
+ if constexpr (LoadSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(LoadSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Addr-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs,
+ const Address& rhs, const Address& falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, Operand(rhs));
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, Operand(rhs));
+ }
+ if constexpr (LoadSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(LoadSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h
new file mode 100644
index 0000000000..80e2dfed28
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.h
@@ -0,0 +1,1218 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_h
+#define jit_x64_MacroAssembler_x64_h
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+#include "js/HeapAPI.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {}
+};
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
+};
+
+// ScratchTagScope and ScratchTagScopeRelease are used to manage the tag
+// register for splitTagForTest(), which has different register management on
+// different platforms. On 64-bit platforms it requires a scratch register that
+// does not interfere with other operations; on 32-bit platforms it uses a
+// register that is already part of the Value.
+//
+// The ScratchTagScope RAII type acquires the appropriate register; a reference
+// to a variable of this type is then passed to splitTagForTest().
+//
+// On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register
+// available in a dynamic scope during compilation. However it is important to
+// remember that that does not preserve the register value in any way, so this
+// RAII type should only be used along paths that eventually branch past further
+// uses of the extracted tag value.
+//
+// On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not
+// manage a register, it only aliases a register in the ValueOperand.
+
+class ScratchTagScope : public ScratchRegisterScope {
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : ScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerX64 : public MacroAssemblerX86Shared {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ void bindOffsets(const MacroAssemblerX86Shared::UsesVector&);
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID id));
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId));
+
+ public:
+ using MacroAssemblerX86Shared::load32;
+ using MacroAssemblerX86Shared::store16;
+ using MacroAssemblerX86Shared::store32;
+
+ MacroAssemblerX64() = default;
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ /////////////////////////////////////////////////////////////////
+ // X64 helpers.
+ /////////////////////////////////////////////////////////////////
+ void writeDataRelocation(const Value& val) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(masm.currentOffset());
+ }
+ }
+
+ // Refers to the upper 32 bits of a 64-bit Value operand.
+ // On x86_64, the upper 32 bits do not necessarily only contain the type.
+ Operand ToUpper32(Operand base) {
+ switch (base.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(Register::FromCode(base.base()), base.disp() + 4);
+
+ case Operand::MEM_SCALE:
+ return Operand(Register::FromCode(base.base()),
+ Register::FromCode(base.index()), base.scale(),
+ base.disp() + 4);
+
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ static inline Operand ToUpper32(const Address& address) {
+ return Operand(address.base, address.offset + 4);
+ }
+ static inline Operand ToUpper32(const BaseIndex& address) {
+ return Operand(address.base, address.index, address.scale,
+ address.offset + 4);
+ }
+
+ uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); }
+
+ JSValueShiftedTag GetShiftedTag(JSValueType type) {
+ return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+ }
+
+ /////////////////////////////////////////////////////////////////
+ // X86/X64-common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void storeValue(ValueOperand val, Operand dest) {
+ movq(val.valueReg(), dest);
+ }
+ void storeValue(ValueOperand val, const Address& dest) {
+ storeValue(val, Operand(dest));
+ }
+ template <typename T>
+ void storeValue(JSValueType type, Register reg, const T& dest) {
+ // Value types with 32-bit payloads can be emitted as two 32-bit moves.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(reg, Operand(dest));
+ movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest)));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ movq(scratch, Operand(dest));
+ }
+ }
+ template <typename T>
+ void storeValue(const Value& val, const T& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ if (val.isGCThing()) {
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ } else {
+ mov(ImmWord(val.asRawBits()), scratch);
+ }
+ movq(scratch, Operand(dest));
+ }
+ void storeValue(ValueOperand val, BaseIndex dest) {
+ storeValue(val, Operand(dest));
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+ void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); }
+ void loadValue(Address src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadValue(const BaseIndex& src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+ void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.valueReg() != scratch);
+ if (payload != dest.valueReg()) {
+ movq(payload, dest.valueReg());
+ }
+ mov(ImmShiftedTag(type), scratch);
+ orq(scratch, dest.valueReg());
+ }
+ void pushValue(ValueOperand val) { push(val.valueReg()); }
+ void popValue(ValueOperand val) { pop(val.valueReg()); }
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ ScratchRegisterScope scratch(asMasm());
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ push(scratch);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ push(scratch);
+ }
+ void pushValue(const Address& addr) { push(Operand(addr)); }
+
+ void pushValue(const BaseIndex& addr, Register scratch) {
+ push(Operand(addr));
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest);
+
+ Condition testUndefined(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testInt32(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testNull(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testString(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testObject(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testDouble(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testNumber(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testGCThing(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+
+ Condition testMagic(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testError(Condition cond, Register tag) {
+ return testMagic(cond, tag);
+ }
+ Condition testPrimitive(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ return cond == Equal ? Below : AboveOrEqual;
+ }
+
+ Condition testUndefined(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testString(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src),
+ Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
+ return cond;
+ }
+ Condition testDouble(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
+ return cond;
+ }
+ Condition testString(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testNull(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testString(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testObject(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+
+ Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(src.valueReg(), ImmWord(magic));
+ return cond;
+ }
+
+ void cmpPtr(Register lhs, const ImmWord rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) {
+ cmpPtr(lhs, Imm32(int32_t(rhs.value)));
+ } else {
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(Register lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(Register lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(!lhs.containsReg(scratch));
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(const Operand& lhs, const ImmWord rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ cmpPtr(lhs, Imm32((int32_t)rhs.value));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(const Operand& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmWord rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
+ void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); }
+ void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, const Imm64 rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ testq(Imm32((int32_t)rhs.value), lhs);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movq(ImmWord(rhs.value), scratch);
+ testq(scratch, lhs);
+ }
+ }
+
+ // Compare-then-conditionally-move/load, for integer types
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal,
+ Register trueValAndDest);
+
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, const Address& rhs,
+ Register falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, Register rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, const Address& rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void movePtr(Register src, Register dest) { movq(src, dest); }
+ void movePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void movePtr(ImmWord imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); }
+ void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); }
+ void loadPtr(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ loadPtr(Address(scratch, 0x0), dest);
+ }
+ }
+ void loadPtr(const Address& address, Register dest) {
+ movq(Operand(address), dest);
+ }
+ void load64(const Address& address, Register dest) {
+ movq(Operand(address), dest);
+ }
+ void loadPtr(const Operand& src, Register dest) { movq(src, dest); }
+ void loadPtr(const BaseIndex& src, Register dest) {
+ movq(Operand(src), dest);
+ }
+ void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); }
+ void load32(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ load32(Address(scratch, 0x0), dest);
+ }
+ }
+ void load64(const Operand& address, Register64 dest) {
+ movq(address, dest.reg);
+ }
+ void load64(const Address& address, Register64 dest) {
+ movq(Operand(address), dest.reg);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ movq(Operand(address), dest.reg);
+ }
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ load64(src, dest);
+ }
+ template <typename T>
+ void storePtr(ImmWord imm, T address) {
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ movq(Imm32((int32_t)imm.value), Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ }
+ template <typename T>
+ void storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+ }
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address) {
+ ScratchRegisterScope scratch(asMasm());
+ movq(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ void storePtr(Register src, const Address& address) {
+ movq(src, Operand(address));
+ }
+ void store64(Register src, const Address& address) {
+ movq(src, Operand(address));
+ }
+ void store64(Register64 src, const Operand& address) {
+ movq(src.reg, address);
+ }
+ void storePtr(Register src, const BaseIndex& address) {
+ movq(src, Operand(address));
+ }
+ void storePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void storePtr(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ storePtr(src, Address(scratch, 0x0));
+ }
+ }
+ void store32(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store32(src, Address(scratch, 0x0));
+ }
+ }
+ void store16(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movw(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store16(src, Address(scratch, 0x0));
+ }
+ }
+ void store64(Register64 src, Address address) { storePtr(src.reg, address); }
+ void store64(Register64 src, const BaseIndex& address) {
+ storePtr(src.reg, address);
+ }
+ void store64(Imm64 imm, Address address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ store64(src, dest);
+ }
+
+ void splitTag(Register src, Register dest) {
+ if (src != dest) {
+ movq(src, dest);
+ }
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+ void splitTag(const Operand& operand, Register dest) {
+ movq(operand, dest);
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const Address& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+ void splitTag(const BaseIndex& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+
+ // Extracts the tag of a value and places it in tag.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+ void cmpTag(const ValueOperand& operand, ImmTag tag) {
+ ScratchTagScope reg(asMasm(), operand);
+ splitTagForTest(operand, reg);
+ cmp32(reg, tag);
+ }
+
+ Condition testMagic(Condition cond, const ValueOperand& src) {
+ ScratchTagScope scratch(asMasm(), src);
+ splitTagForTest(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testError(Condition cond, const ValueOperand& src) {
+ return testMagic(cond, src);
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
+ vmovq(src, dest.valueReg());
+ }
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+ MOZ_ASSERT(src != dest.valueReg());
+ boxValue(type, src, dest.valueReg());
+ }
+
+ // Note that the |dest| register here may be ScratchReg, so we shouldn't
+ // use it.
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxInt32(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxInt32(const Address& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ void unboxInt32(const BaseIndex& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ template <typename T>
+ void unboxDouble(const T& src, FloatRegister dest) {
+ loadDouble(Operand(src), dest);
+ }
+
+ void unboxArgObjMagic(const ValueOperand& src, Register dest) {
+ unboxArgObjMagic(Operand(src.valueReg()), dest);
+ }
+ void unboxArgObjMagic(const Operand& src, Register dest) {
+ mov(ImmWord(0), dest);
+ }
+ void unboxArgObjMagic(const Address& src, Register dest) {
+ unboxArgObjMagic(Operand(src), dest);
+ }
+
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxBoolean(const Address& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+ void unboxBoolean(const BaseIndex& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+
+ void unboxMagic(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+
+ void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+ vmovq(src.valueReg(), dest);
+ }
+
+ void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); }
+
+ void unboxNonDouble(const ValueOperand& src, Register dest,
+ JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src.valueReg(), dest);
+ return;
+ }
+ if (src.valueReg() == dest) {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src.valueReg(), dest);
+ }
+ }
+ void unboxNonDouble(const Operand& src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src, dest);
+ return;
+ }
+ // Explicitly permits |dest| to be used in |src|.
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest != scratch);
+ if (src.containsReg(dest)) {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ // If src is already a register, then src and dest are the same
+ // thing and we don't need to move anything into dest.
+ if (src.kind() != Operand::REG) {
+ movq(src, dest);
+ }
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src, dest);
+ }
+ }
+ void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+
+ void unboxString(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+
+ void unboxSymbol(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+
+ void unboxBigInt(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, dest);
+ }
+
+ // This should only be used for GC barrier code, to unbox a GC thing Value.
+ // It's fine there because we don't depend on the actual Value type (all Cells
+ // are treated the same way). In almost all other cases this would be
+ // Spectre-unsafe - use unboxNonDouble and friends instead.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(Operand(src), dest);
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(Operand(src), dest);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
+ JSValueType type, Label* fail);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(address, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ loadPtr(address, scratch);
+ splitTag(scratch, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type);
+
+ // These two functions use the low 32-bits of the full value register.
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+
+ void loadConstantDouble(double d, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
+ void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
+ void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
+ void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+
+ public:
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+ test32(operand.valueReg(), operand.valueReg());
+ return truthy ? NonZero : Zero;
+ }
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+ template <typename T>
+ inline void loadInt32OrDouble(const T& src, FloatRegister dest);
+
+ template <typename T>
+ void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(src, dest.fpu());
+ } else {
+ unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ ScratchRegisterScope scratch(asMasm());
+ unboxNonDouble(value, scratch, type);
+ storePtr(scratch, address);
+ if (type == JSVAL_TYPE_OBJECT) {
+ // Ideally we would call unboxObjectOrNull, but we need an extra
+ // scratch register for that. So unbox as object, then clear the
+ // object-or-null bit.
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, Operand(address));
+ }
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ // Checks whether a double is representable as a 64-bit integer. If so, the
+ // integer is written to the output register. Otherwise, a bailout is taken to
+ // the given snapshot. This function overwrites the scratch float register.
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertUInt32ToDouble(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2sd(src, dest, dest);
+ }
+
+ void convertUInt32ToFloat32(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2ss(src, dest, dest);
+ }
+
+ inline void incrementInt32Value(const Address& addr);
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ public:
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+using MacroAssemblerSpecific = MacroAssemblerX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64-inl.h b/js/src/jit/x64/SharedICHelpers-x64-inl.h
new file mode 100644
index 0000000000..c40eb45c74
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64-inl.h
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_inl_h
+#define jit_x64_SharedICHelpers_x64_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ ScratchRegisterScope scratch(masm);
+
+ // We can assume during this that R0 and R1 have been pushed.
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.movq(FramePointer, scratch);
+ masm.subq(StackPointer, scratch);
+ masm.subq(Imm32(argSize), scratch);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(ICTailCallReg);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register) {
+#ifdef DEBUG
+ // Compute frame size. Because the return address is still on the stack,
+ // this is:
+ //
+ // FramePointer
+ // - StackPointer
+ // - sizeof(return address)
+
+ ScratchRegisterScope scratch(masm);
+ masm.movq(FramePointer, scratch);
+ masm.subq(StackPointer, scratch);
+ masm.subq(Imm32(sizeof(void*)), scratch); // Return address.
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push the return address that's currently on top of the stack.
+ masm.Push(Operand(StackPointer, 0));
+
+ // Replace the original return address with the frame descriptor.
+ masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, sizeof(uintptr_t)));
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ masm.Push(ICStubReg);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_inl_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64.h b/js/src/jit/x64/SharedICHelpers-x64.h
new file mode 100644
index 0000000000..8233db5735
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_h
+#define jit_x64_SharedICHelpers_x64_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from Stack top to the top Value inside an IC stub (this is the
+// return address).
+static const size_t ICStackValueOffset = sizeof(void*);
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ masm.Pop(ICTailCallReg);
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ masm.Push(ICTailCallReg);
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Call the stubcode.
+ masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ret(); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+ masm.loadPtr(stubAddr, ICStubReg);
+
+ masm.mov(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // The return address is on top of the stack, followed by the frame
+ // descriptor. Use a pop instruction to overwrite the frame descriptor
+ // with the return address. Note that pop increments the stack pointer
+ // before computing the address.
+ masm.Pop(Operand(StackPointer, 0));
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ masm.guardedCallPreBarrier(addr, type);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ masm.jmp(Operand(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_h */
diff --git a/js/src/jit/x64/SharedICRegisters-x64.h b/js/src/jit/x64/SharedICRegisters-x64.h
new file mode 100644
index 0000000000..8e52e5f3c9
--- /dev/null
+++ b/js/src/jit/x64/SharedICRegisters-x64.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICRegisters_x64_h
+#define jit_x64_SharedICRegisters_x64_h
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/x64/Assembler-x64.h"
+
+namespace js {
+namespace jit {
+
+static constexpr ValueOperand R0(rcx);
+static constexpr ValueOperand R1(rbx);
+static constexpr ValueOperand R2(rax);
+
+static constexpr Register ICTailCallReg = rsi;
+static constexpr Register ICStubReg = rdi;
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = xmm0;
+static constexpr FloatRegister FloatReg1 = xmm1;
+static constexpr FloatRegister FloatReg2 = xmm2;
+static constexpr FloatRegister FloatReg3 = xmm3;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICRegisters_x64_h */
diff --git a/js/src/jit/x64/Trampoline-x64.cpp b/js/src/jit/x64/Trampoline-x64.cpp
new file mode 100644
index 0000000000..dcc9291299
--- /dev/null
+++ b/js/src/jit/x64/Trampoline-x64.cpp
@@ -0,0 +1,888 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "jit/x64/SharedICRegisters-x64.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::IsPowerOfTwo;
+
+// This struct reflects the contents of the stack entry.
+// Given a `CommonFrameLayout* frame`:
+// - `frame->prevType()` should be `FrameType::CppToJSJit`.
+// - Then EnterJITStackEntry starts at:
+// frame->callerFramePtr() + EnterJITStackEntry::offsetFromFP()
+// (the offset is negative, so this subtracts from the frame pointer)
+struct EnterJITStackEntry {
+ // Offset from frame pointer to EnterJITStackEntry*.
+ static constexpr int32_t offsetFromFP() {
+ return -int32_t(offsetof(EnterJITStackEntry, rbp));
+ }
+
+ void* result;
+
+#if defined(_WIN64)
+ struct XMM {
+ using XMM128 = char[16];
+ XMM128 xmm6;
+ XMM128 xmm7;
+ XMM128 xmm8;
+ XMM128 xmm9;
+ XMM128 xmm10;
+ XMM128 xmm11;
+ XMM128 xmm12;
+ XMM128 xmm13;
+ XMM128 xmm14;
+ XMM128 xmm15;
+ } xmm;
+
+ // 16-byte aligment for xmm registers above.
+ uint64_t xmmPadding;
+
+ void* rsi;
+ void* rdi;
+#endif
+
+ void* r15;
+ void* r14;
+ void* r13;
+ void* r12;
+ void* rbx;
+ void* rbp;
+
+ // Pushed by CALL.
+ void* rip;
+};
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard x64 fastcall
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ masm.assertStackAlignment(ABIStackAlignment,
+ -int32_t(sizeof(uintptr_t)) /* return address */);
+
+ const Register reg_code = IntArgReg0;
+ const Register reg_argc = IntArgReg1;
+ const Register reg_argv = IntArgReg2;
+ static_assert(OsrFrameReg == IntArgReg3);
+
+#if defined(_WIN64)
+ const Address token = Address(rbp, 16 + ShadowStackSpace);
+ const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace);
+ const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace);
+ const Operand result = Operand(rbp, 40 + ShadowStackSpace);
+#else
+ const Register token = IntArgReg4;
+ const Register scopeChain = IntArgReg5;
+ const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace);
+ const Operand result = Operand(rbp, 24 + ShadowStackSpace);
+#endif
+
+ // Note: the stack pushes below must match the fields in EnterJITStackEntry.
+
+ // Save old stack frame pointer, set new stack frame pointer.
+ masm.push(rbp);
+ masm.mov(rsp, rbp);
+
+ // Save non-volatile registers. These must be saved by the trampoline, rather
+ // than by the JIT'd code, because they are scanned by the conservative
+ // scanner.
+ masm.push(rbx);
+ masm.push(r12);
+ masm.push(r13);
+ masm.push(r14);
+ masm.push(r15);
+#if defined(_WIN64)
+ masm.push(rdi);
+ masm.push(rsi);
+
+ // 16-byte aligment for vmovdqa
+ masm.subq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+ masm.vmovdqa(xmm6, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)));
+ masm.vmovdqa(xmm7, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)));
+ masm.vmovdqa(xmm8, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)));
+ masm.vmovdqa(xmm9, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)));
+ masm.vmovdqa(xmm10, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)));
+ masm.vmovdqa(xmm11, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)));
+ masm.vmovdqa(xmm12, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)));
+ masm.vmovdqa(xmm13, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)));
+ masm.vmovdqa(xmm14, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)));
+ masm.vmovdqa(xmm15, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)));
+#endif
+
+ // Save arguments passed in registers needed after function call.
+ masm.push(result);
+
+ // End of pushes reflected in EnterJITStackEntry, i.e. EnterJITStackEntry
+ // starts at this rsp.
+
+ // Remember number of bytes occupied by argument vector
+ masm.mov(reg_argc, r13);
+
+ // if we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, token,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.addq(Imm32(1), r13);
+
+ masm.bind(&noNewTarget);
+ }
+
+ masm.shll(Imm32(3), r13); // r13 = argc * sizeof(Value)
+ static_assert(sizeof(Value) == 1 << 3, "Constant is baked in assembly code");
+
+ // Guarantee stack alignment of Jit frames.
+ //
+ // This code compensates for the offset created by the copy of the vector of
+ // arguments, such that the jit frame will be aligned once the return
+ // address is pushed on the stack.
+ //
+ // In the computation of the offset, we omit the size of the JitFrameLayout
+ // which is pushed on the stack, as the JitFrameLayout size is a multiple of
+ // the JitStackAlignment.
+ masm.mov(rsp, r12);
+ masm.subq(r13, r12);
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ masm.andl(Imm32(JitStackAlignment - 1), r12);
+ masm.subq(r12, rsp);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // r13 still stores the number of bytes in the argument vector.
+ masm.addq(reg_argv, r13); // r13 points above last argument or newTarget
+
+ // while r13 > rdx, push arguments.
+ {
+ Label header, footer;
+ masm.bind(&header);
+
+ masm.cmpPtr(r13, reg_argv);
+ masm.j(AssemblerX86Shared::BelowOrEqual, &footer);
+
+ masm.subq(Imm32(8), r13);
+ masm.push(Operand(r13, 0));
+ masm.jmp(&header);
+
+ masm.bind(&footer);
+ }
+
+ // Load the number of actual arguments. |result| is used to store the
+ // actual number of arguments without adding an extra argument to the enter
+ // JIT.
+ masm.movq(result, reg_argc);
+ masm.unboxInt32(Operand(reg_argc, 0), reg_argc);
+
+ // Push the callee token.
+ masm.push(token);
+
+ // Push the descriptor.
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, reg_argc, reg_argc);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(rbp));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+
+ Register scratch = regs.takeAny();
+
+ Label notOsr;
+ masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+ Register numStackValues = regs.takeAny();
+ masm.movq(numStackValuesAddr, numStackValues);
+
+ // Push return address
+ masm.mov(&returnLabel, scratch);
+ masm.push(scratch);
+
+ // Frame prologue.
+ masm.push(rbp);
+ masm.mov(rsp, rbp);
+
+ // Reserve frame.
+ masm.subPtr(Imm32(BaselineFrame::Size()), rsp);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+ masm.mov(rsp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ Register valuesSize = regs.takeAny();
+ masm.mov(numStackValues, valuesSize);
+ masm.shll(Imm32(3), valuesSize);
+ masm.subPtr(valuesSize, rsp);
+
+ // Enter exit frame.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(Imm32(0)); // Fake return address.
+ masm.push(FramePointer);
+ // No GC things to mark, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ regs.add(valuesSize);
+
+ masm.push(reg_code);
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.pop(reg_code);
+
+ MOZ_ASSERT(reg_code != ReturnReg);
+
+ Label error;
+ masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp);
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(rbp, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(reg_code);
+
+ // OOM: frame epilogue, load error value, discard return address and return.
+ masm.bind(&error);
+ masm.mov(rbp, rsp);
+ masm.pop(rbp);
+ masm.addPtr(Imm32(sizeof(uintptr_t)), rsp); // Return address.
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ masm.movq(scopeChain, R1.scratchReg());
+ }
+
+ // The call will push the return address and frame pointer on the stack, thus
+ // we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call function.
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Discard arguments and padding. Set rsp to the address of the
+ // EnterJITStackEntry on the stack.
+ masm.lea(Operand(rbp, EnterJITStackEntry::offsetFromFP()), rsp);
+
+ /*****************************************************************
+ Place return value where it belongs, pop all saved registers
+ *****************************************************************/
+ masm.pop(r12); // vp
+ masm.storeValue(JSReturnOperand, Operand(r12, 0));
+
+ // Restore non-volatile registers.
+#if defined(_WIN64)
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)), xmm6);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)), xmm7);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)), xmm8);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)), xmm9);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)), xmm10);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)), xmm11);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)), xmm12);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)), xmm13);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)), xmm14);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)), xmm15);
+
+ masm.addq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+ masm.pop(rsi);
+ masm.pop(rdi);
+#endif
+ masm.pop(r15);
+ masm.pop(r14);
+ masm.pop(r13);
+ masm.pop(r12);
+ masm.pop(rbx);
+
+ // Restore frame pointer and return.
+ masm.pop(rbp);
+ masm.ret();
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ if (frameStackAddress->prevType() != FrameType::CppToJSJit) {
+ // This is not a CppToJSJit frame, there are no C++ registers here.
+ return mozilla::Nothing{};
+ }
+
+ // Compute pointer to start of EnterJITStackEntry on the stack.
+ uint8_t* fp = frameStackAddress->callerFramePtr();
+ auto* enterJITStackEntry = reinterpret_cast<EnterJITStackEntry*>(
+ fp + EnterJITStackEntry::offsetFromFP());
+
+ // Extract native function call registers.
+ ::JS::ProfilingFrameIterator::RegisterState registerState;
+ registerState.fp = enterJITStackEntry->rbp;
+ registerState.pc = enterJITStackEntry->rip;
+ // sp should be inside the caller's frame, so set sp to the value of the stack
+ // pointer before the call to the EnterJit trampoline.
+ registerState.sp = &enterJITStackEntry->rip + 1;
+ // No lr in this world.
+ registerState.lr = nullptr;
+ return mozilla::Some(registerState);
+}
+
+// Push AllRegs in a way that is compatible with RegisterDump, regardless of
+// what PushRegsInMask might do to reduce the set size.
+static void DumpAllRegs(MacroAssembler& masm) {
+#ifdef ENABLE_WASM_SIMD
+ masm.PushRegsInMask(AllRegs);
+#else
+ // When SIMD isn't supported, PushRegsInMask reduces the set of float
+ // registers to be double-sized, while the RegisterDump expects each of
+ // the float registers to have the maximal possible size
+ // (Simd128DataSize). To work around this, we just spill the double
+ // registers by hand here, using the register dump offset directly.
+ for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more();
+ ++iter) {
+ masm.Push(*iter);
+ }
+
+ masm.reserveStack(sizeof(RegisterDump::FPUArray));
+ for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more();
+ ++iter) {
+ FloatRegister reg = *iter;
+ Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes());
+ masm.storeDouble(reg, spillAddress);
+ }
+#endif
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ // See explanatory comment in x86's JitRuntime::generateInvalidator.
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ masm.movq(rsp, rax); // Argument to jit::InvalidationBailout.
+
+ // Make space for InvalidationBailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movq(rsp, rbx);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(rdx);
+ masm.passABIArg(rax);
+ masm.passABIArg(rbx);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r9); // Get the bailoutInfo outparam.
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ // Do not erase the frame pointer in this function.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- rsp
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.movq(rsp, FramePointer);
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, r8);
+
+ // Load |nformals| into %rcx.
+ masm.loadPtr(Address(rbp, RectifierFrameLayout::offsetOfCalleeToken()), rax);
+ masm.mov(rax, rcx);
+ masm.andq(Imm32(uint32_t(CalleeTokenMask)), rcx);
+ masm.loadFunctionArgCount(rcx, rcx);
+
+ // Stash another copy in r11, since we are going to do destructive operations
+ // on rcx
+ masm.mov(rcx, r11);
+
+ static_assert(
+ CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count the value");
+ masm.mov(rax, rdx);
+ masm.andq(Imm32(uint32_t(CalleeToken_FunctionConstructing)), rdx);
+
+ // Including |this|, and |new.target|, there are (|nformals| + 1 +
+ // isConstructing) arguments to push to the stack. Then we push a
+ // JitFrameLayout. We compute the padding expressed in the number of extra
+ // |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+ static_assert(IsPowerOfTwo(JitStackValueAlignment),
+ "must have power of two for masm.andl to do its job");
+
+ masm.addl(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ rcx);
+ masm.addl(rdx, rcx);
+ masm.andl(Imm32(~(JitStackValueAlignment - 1)), rcx);
+
+ // Load the number of |undefined|s to push into %rcx. Subtract 1 for |this|.
+ masm.subl(r8, rcx);
+ masm.subl(Imm32(1), rcx);
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- rsp
+ // '--- #r8 ---'
+ //
+ // Rectifier frame:
+ // [rbp'] [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+ // [descr] [raddr] ]
+ // '------- #rcx --------' '--- #r8 ---'
+
+ // Copy the number of actual arguments into rdx.
+ masm.mov(r8, rdx);
+
+ masm.moveValue(UndefinedValue(), ValueOperand(r10));
+
+ // Push undefined. (including the padding)
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+
+ masm.push(r10);
+ masm.subl(Imm32(1), rcx);
+ masm.j(Assembler::NonZero, &undefLoopTop);
+ }
+
+ // Get the topmost argument.
+ static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+
+ // Get the topmost argument.
+ BaseIndex b(FramePointer, r8, TimesEight, sizeof(RectifierFrameLayout));
+ masm.lea(Operand(b), rcx);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ masm.addl(Imm32(1), r8);
+ {
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.push(Operand(rcx, 0x0));
+ masm.subq(Imm32(sizeof(Value)), rcx);
+ masm.subl(Imm32(1), r8);
+ masm.j(Assembler::NonZero, &copyLoopTop);
+ }
+
+ // if constructing, copy newTarget
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, rax,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // thisFrame[numFormals] = prevFrame[argc]
+ ValueOperand newTarget(r10);
+
+ // Load vp[argc]. Add sizeof(Value) for |this|.
+ BaseIndex newTargetSrc(FramePointer, rdx, TimesEight,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+ masm.loadValue(newTargetSrc, newTarget);
+
+ // Again, 1 for |this|
+ BaseIndex newTargetDest(rsp, r11, TimesEight, sizeof(Value));
+ masm.storeValue(newTarget, newTargetDest);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ //
+ //
+ // Rectifier frame:
+ // [rbp'] <- rbp [undef] [undef] [undef] [arg2] [arg1] [this] <- rsp [ [argc]
+ // [callee] [descr] [raddr] ]
+ //
+
+ // Construct JitFrameLayout.
+ masm.push(rax); // callee token
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, rdx, rdx);
+
+ // Call the target function.
+ masm.andq(Imm32(uint32_t(CalleeTokenMask)), rax);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(rax, rax);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(rax);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(rax, rbx, &noBaselineScript);
+ masm.callJitNoProfiler(rbx);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(rax, rax);
+ masm.callJitNoProfiler(rax);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ // Get the stack pointer into a register, pre-alignment.
+ masm.movq(rsp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, r8);
+
+ // Make space for Bailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movq(rsp, r9);
+
+ // Call the bailout function.
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(rax);
+ masm.passABIArg(r8);
+ masm.passABIArg(r9);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r9); // Get the bailoutInfo outparam.
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set must be a superset of Volatile register set");
+
+ // The context is the first argument.
+ Register cxreg = IntArgReg0;
+ regs.take(cxreg);
+
+ // Stack is:
+ // ... frame ...
+ // +12 [args]
+ // +8 descriptor
+ // +0 returnAddress
+ //
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the current stack pointer as the base for copying arguments.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = r10;
+ regs.take(argsBase);
+ masm.lea(Operand(rsp, ExitFrameLayout::SizeWithFooter()), argsBase);
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(Value));
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Int32:
+ case Type_Bool:
+ case Type_Pointer:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(uintptr_t));
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.movq(esp, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ if (f.argPassedInFloatReg(explicitArg)) {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ } else {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ }
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::Kind::EffectiveAddress),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH("NYI: x64 callVM should not be used with 128bits values.");
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (outReg != InvalidReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel());
+ break;
+ case Type_Bool:
+ masm.testb(rax, rax);
+ masm.j(Assembler::Zero, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(esp, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ masm.load32(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Double:
+ masm.loadDouble(Address(esp, 0), ReturnDoubleReg);
+ masm.freeStack(sizeof(double));
+ break;
+
+ case Type_Pointer:
+ masm.loadPtr(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(0);
+ masm.pop(FramePointer);
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ static_assert(PreBarrierReg == rdx);
+ Register temp1 = rax;
+ Register temp2 = rbx;
+ Register temp3 = rcx;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet regs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ masm.PushRegsInMask(regs);
+
+ masm.mov(ImmPtr(cx->runtime()), rcx);
+
+ masm.setupUnalignedABICall(rax);
+ masm.passABIArg(rcx);
+ masm.passABIArg(rdx);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ masm.PopRegsInMask(regs);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.ret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(rdx, r9);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-09 20:45:43 +0000