Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

15 files changed, 9891 insertions, 0 deletions

diff --git a/js/src/jit/x64/Assembler-x64.cpp b/js/src/jit/x64/Assembler-x64.cpp
new file mode 100644
index 0000000000..82e6efba85
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.cpp
@@ -0,0 +1,246 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Assembler-x64.h"
+
+#include "gc/Tracer.h"
+#include "util/Memory.h"
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+    :
+#if defined(XP_WIN)
+      regIndex_(0),
+      stackOffset_(ShadowStackSpace)
+#else
+      intRegIndex_(0),
+      floatRegIndex_(0),
+      stackOffset_(0)
+#endif
+{
+}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+#if defined(XP_WIN)
+  static_assert(NumIntArgRegs == NumFloatArgRegs);
+  if (regIndex_ == NumIntArgRegs) {
+    if (type == MIRType::Simd128) {
+      // On Win64, >64 bit args need to be passed by reference.  However, wasm
+      // doesn't allow passing SIMD values to JS, so the only way to reach this
+      // is wasm to wasm calls.  Ergo we can break the native ABI here and use
+      // the Wasm ABI instead.
+      stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+      current_ = ABIArg(stackOffset_);
+      stackOffset_ += Simd128DataSize;
+    } else {
+      current_ = ABIArg(stackOffset_);
+      stackOffset_ += sizeof(uint64_t);
+    }
+    return current_;
+  }
+  switch (type) {
+    case MIRType::Int32:
+    case MIRType::Int64:
+    case MIRType::Pointer:
+    case MIRType::WasmAnyRef:
+    case MIRType::StackResults:
+      current_ = ABIArg(IntArgRegs[regIndex_++]);
+      break;
+    case MIRType::Float32:
+      current_ = ABIArg(FloatArgRegs[regIndex_++].asSingle());
+      break;
+    case MIRType::Double:
+      current_ = ABIArg(FloatArgRegs[regIndex_++]);
+      break;
+    case MIRType::Simd128:
+      // On Win64, >64 bit args need to be passed by reference, but wasm
+      // doesn't allow passing SIMD values to FFIs. The only way to reach
+      // here is asm to asm calls, so we can break the ABI here.
+      current_ = ABIArg(FloatArgRegs[regIndex_++].asSimd128());
+      break;
+    default:
+      MOZ_CRASH("Unexpected argument type");
+  }
+  return current_;
+#else
+  switch (type) {
+    case MIRType::Int32:
+    case MIRType::Int64:
+    case MIRType::Pointer:
+    case MIRType::WasmAnyRef:
+    case MIRType::StackResults:
+      if (intRegIndex_ == NumIntArgRegs) {
+        current_ = ABIArg(stackOffset_);
+        stackOffset_ += sizeof(uint64_t);
+        break;
+      }
+      current_ = ABIArg(IntArgRegs[intRegIndex_++]);
+      break;
+    case MIRType::Double:
+    case MIRType::Float32:
+      if (floatRegIndex_ == NumFloatArgRegs) {
+        current_ = ABIArg(stackOffset_);
+        stackOffset_ += sizeof(uint64_t);
+        break;
+      }
+      if (type == MIRType::Float32) {
+        current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSingle());
+      } else {
+        current_ = ABIArg(FloatArgRegs[floatRegIndex_++]);
+      }
+      break;
+    case MIRType::Simd128:
+      if (floatRegIndex_ == NumFloatArgRegs) {
+        stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+        current_ = ABIArg(stackOffset_);
+        stackOffset_ += Simd128DataSize;
+        break;
+      }
+      current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSimd128());
+      break;
+    default:
+      MOZ_CRASH("Unexpected argument type");
+  }
+  return current_;
+#endif
+}
+
+void Assembler::addPendingJump(JmpSrc src, ImmPtr target,
+                               RelocationKind reloc) {
+  MOZ_ASSERT(target.value != nullptr);
+
+  // Emit reloc before modifying the jump table, since it computes a 0-based
+  // index. This jump is not patchable at runtime.
+  if (reloc == RelocationKind::JITCODE) {
+    jumpRelocations_.writeUnsigned(src.offset());
+  }
+
+  static_assert(MaxCodeBytesPerProcess <= uint64_t(2) * 1024 * 1024 * 1024,
+                "Code depends on using int32_t for cross-JitCode jump offsets");
+
+  MOZ_ASSERT_IF(reloc == RelocationKind::JITCODE,
+                AddressIsInExecutableMemory(target.value));
+
+  RelativePatch patch(src.offset(), target.value, reloc);
+  if (reloc == RelocationKind::JITCODE ||
+      AddressIsInExecutableMemory(target.value)) {
+    enoughMemory_ &= codeJumps_.append(patch);
+  } else {
+    enoughMemory_ &= extendedJumps_.append(patch);
+  }
+}
+
+void Assembler::finish() {
+  if (oom()) {
+    return;
+  }
+
+  AutoCreatedBy acb(*this, "Assembler::finish");
+
+  if (!extendedJumps_.length()) {
+    // Since we may be folowed by non-executable data, eagerly insert an
+    // undefined instruction byte to prevent processors from decoding
+    // gibberish into their pipelines. See Intel performance guides.
+    masm.ud2();
+    return;
+  }
+
+  // Emit the jump table.
+  masm.haltingAlign(SizeOfJumpTableEntry);
+  extendedJumpTable_ = masm.size();
+
+  // Zero the extended jumps table.
+  for (size_t i = 0; i < extendedJumps_.length(); i++) {
+#ifdef DEBUG
+    size_t oldSize = masm.size();
+#endif
+    MOZ_ASSERT(hasCreator());
+    masm.jmp_rip(2);
+    MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 6);
+    // Following an indirect branch with ud2 hints to the hardware that
+    // there's no fall-through. This also aligns the 64-bit immediate.
+    masm.ud2();
+    MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 8);
+    masm.immediate64(0);
+    MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfExtendedJump);
+    MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfJumpTableEntry);
+  }
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+  AssemblerX86Shared::executableCopy(buffer);
+
+  for (RelativePatch& rp : codeJumps_) {
+    uint8_t* src = buffer + rp.offset;
+    MOZ_ASSERT(rp.target);
+
+    MOZ_RELEASE_ASSERT(X86Encoding::CanRelinkJump(src, rp.target));
+    X86Encoding::SetRel32(src, rp.target);
+  }
+
+  for (size_t i = 0; i < extendedJumps_.length(); i++) {
+    RelativePatch& rp = extendedJumps_[i];
+    uint8_t* src = buffer + rp.offset;
+    MOZ_ASSERT(rp.target);
+
+    if (X86Encoding::CanRelinkJump(src, rp.target)) {
+      X86Encoding::SetRel32(src, rp.target);
+    } else {
+      // An extended jump table must exist, and its offset must be in
+      // range.
+      MOZ_ASSERT(extendedJumpTable_);
+      MOZ_ASSERT((extendedJumpTable_ + i * SizeOfJumpTableEntry) <=
+                 size() - SizeOfJumpTableEntry);
+
+      // Patch the jump to go to the extended jump entry.
+      uint8_t* entry = buffer + extendedJumpTable_ + i * SizeOfJumpTableEntry;
+      X86Encoding::SetRel32(src, entry);
+
+      // Now patch the pointer, note that we need to align it to
+      // *after* the extended jump, i.e. after the 64-bit immedate.
+      X86Encoding::SetPointer(entry + SizeOfExtendedJump, rp.target);
+    }
+  }
+}
+
+class RelocationIterator {
+  CompactBufferReader reader_;
+  uint32_t offset_ = 0;
+
+ public:
+  explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+  bool read() {
+    if (!reader_.more()) {
+      return false;
+    }
+    offset_ = reader_.readUnsigned();
+    return true;
+  }
+
+  uint32_t offset() const { return offset_; }
+};
+
+JitCode* Assembler::CodeFromJump(JitCode* code, uint8_t* jump) {
+  uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump);
+
+  MOZ_ASSERT(!code->containsNativePC(target),
+             "Extended jump table not used for cross-JitCode jumps");
+
+  return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+                                     CompactBufferReader& reader) {
+  RelocationIterator iter(reader);
+  while (iter.read()) {
+    JitCode* child = CodeFromJump(code, code->raw() + iter.offset());
+    TraceManuallyBarrieredEdge(trc, &child, "rel32");
+    MOZ_ASSERT(child == CodeFromJump(code, code->raw() + iter.offset()));
+  }
+}
diff --git a/js/src/jit/x64/Assembler-x64.h b/js/src/jit/x64/Assembler-x64.h
new file mode 100644
index 0000000000..b1020350d5
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.h
@@ -0,0 +1,1254 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Assembler_x64_h
+#define jit_x64_Assembler_x64_h
+
+#include <iterator>
+
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register rax{X86Encoding::rax};
+static constexpr Register rbx{X86Encoding::rbx};
+static constexpr Register rcx{X86Encoding::rcx};
+static constexpr Register rdx{X86Encoding::rdx};
+static constexpr Register rsi{X86Encoding::rsi};
+static constexpr Register rdi{X86Encoding::rdi};
+static constexpr Register rbp{X86Encoding::rbp};
+static constexpr Register r8{X86Encoding::r8};
+static constexpr Register r9{X86Encoding::r9};
+static constexpr Register r10{X86Encoding::r10};
+static constexpr Register r11{X86Encoding::r11};
+static constexpr Register r12{X86Encoding::r12};
+static constexpr Register r13{X86Encoding::r13};
+static constexpr Register r14{X86Encoding::r14};
+static constexpr Register r15{X86Encoding::r15};
+static constexpr Register rsp{X86Encoding::rsp};
+
+static constexpr FloatRegister xmm0 =
+    FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister xmm1 =
+    FloatRegister(X86Encoding::xmm1, FloatRegisters::Double);
+static constexpr FloatRegister xmm2 =
+    FloatRegister(X86Encoding::xmm2, FloatRegisters::Double);
+static constexpr FloatRegister xmm3 =
+    FloatRegister(X86Encoding::xmm3, FloatRegisters::Double);
+static constexpr FloatRegister xmm4 =
+    FloatRegister(X86Encoding::xmm4, FloatRegisters::Double);
+static constexpr FloatRegister xmm5 =
+    FloatRegister(X86Encoding::xmm5, FloatRegisters::Double);
+static constexpr FloatRegister xmm6 =
+    FloatRegister(X86Encoding::xmm6, FloatRegisters::Double);
+static constexpr FloatRegister xmm7 =
+    FloatRegister(X86Encoding::xmm7, FloatRegisters::Double);
+static constexpr FloatRegister xmm8 =
+    FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+static constexpr FloatRegister xmm9 =
+    FloatRegister(X86Encoding::xmm9, FloatRegisters::Double);
+static constexpr FloatRegister xmm10 =
+    FloatRegister(X86Encoding::xmm10, FloatRegisters::Double);
+static constexpr FloatRegister xmm11 =
+    FloatRegister(X86Encoding::xmm11, FloatRegisters::Double);
+static constexpr FloatRegister xmm12 =
+    FloatRegister(X86Encoding::xmm12, FloatRegisters::Double);
+static constexpr FloatRegister xmm13 =
+    FloatRegister(X86Encoding::xmm13, FloatRegisters::Double);
+static constexpr FloatRegister xmm14 =
+    FloatRegister(X86Encoding::xmm14, FloatRegisters::Double);
+static constexpr FloatRegister xmm15 =
+    FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+
+// Vector registers fixed for use with some instructions, e.g. PBLENDVB.
+static constexpr FloatRegister vmm0 =
+    FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+
+// X86-common synonyms.
+static constexpr Register eax = rax;
+static constexpr Register ebx = rbx;
+static constexpr Register ecx = rcx;
+static constexpr Register edx = rdx;
+static constexpr Register esi = rsi;
+static constexpr Register edi = rdi;
+static constexpr Register ebp = rbp;
+static constexpr Register esp = rsp;
+
+static constexpr Register InvalidReg{X86Encoding::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = FloatRegister();
+
+static constexpr Register StackPointer = rsp;
+static constexpr Register FramePointer = rbp;
+static constexpr Register JSReturnReg = rcx;
+// Avoid, except for assertions.
+static constexpr Register JSReturnReg_Type = JSReturnReg;
+static constexpr Register JSReturnReg_Data = JSReturnReg;
+
+static constexpr Register ScratchReg = r11;
+
+// Helper class for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of the scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+  explicit ScratchRegisterScope(MacroAssembler& masm)
+      : AutoRegisterScope(masm, ScratchReg) {}
+};
+
+static constexpr Register ReturnReg = rax;
+static constexpr Register HeapReg = r15;
+static constexpr Register64 ReturnReg64(rax);
+static constexpr FloatRegister ReturnFloat32Reg =
+    FloatRegister(X86Encoding::xmm0, FloatRegisters::Single);
+static constexpr FloatRegister ReturnDoubleReg =
+    FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister ReturnSimd128Reg =
+    FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+static constexpr FloatRegister ScratchFloat32Reg_ =
+    FloatRegister(X86Encoding::xmm15, FloatRegisters::Single);
+static constexpr FloatRegister ScratchDoubleReg_ =
+    FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+static constexpr FloatRegister ScratchSimd128Reg =
+    FloatRegister(X86Encoding::xmm15, FloatRegisters::Simd128);
+
+// Avoid rbp, which is the FramePointer, which is unavailable in some modes.
+static constexpr Register CallTempReg0 = rax;
+static constexpr Register CallTempReg1 = rdi;
+static constexpr Register CallTempReg2 = rbx;
+static constexpr Register CallTempReg3 = rcx;
+static constexpr Register CallTempReg4 = rsi;
+static constexpr Register CallTempReg5 = rdx;
+
+// Different argument registers for WIN64
+#if defined(_WIN64)
+static constexpr Register IntArgReg0 = rcx;
+static constexpr Register IntArgReg1 = rdx;
+static constexpr Register IntArgReg2 = r8;
+static constexpr Register IntArgReg3 = r9;
+static constexpr uint32_t NumIntArgRegs = 4;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rcx, rdx, r8, r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rdi, rbx, rsi};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr uint32_t NumFloatArgRegs = 4;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {xmm0, xmm1,
+                                                                xmm2, xmm3};
+#else
+static constexpr Register IntArgReg0 = rdi;
+static constexpr Register IntArgReg1 = rsi;
+static constexpr Register IntArgReg2 = rdx;
+static constexpr Register IntArgReg3 = rcx;
+static constexpr Register IntArgReg4 = r8;
+static constexpr Register IntArgReg5 = r9;
+static constexpr uint32_t NumIntArgRegs = 6;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rdi, rsi, rdx,
+                                                       rcx, r8,  r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rbx};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr FloatRegister FloatArgReg4 = xmm4;
+static constexpr FloatRegister FloatArgReg5 = xmm5;
+static constexpr FloatRegister FloatArgReg6 = xmm6;
+static constexpr FloatRegister FloatArgReg7 = xmm7;
+static constexpr uint32_t NumFloatArgRegs = 8;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {
+    xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7};
+#endif
+
+// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use
+// JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registers used by RegExpExecTest stub (do not use ReturnReg).
+static constexpr Register RegExpExecTestRegExpReg = CallTempReg1;
+static constexpr Register RegExpExecTestStringReg = CallTempReg2;
+
+// Registers used by RegExpSearcher stub (do not use ReturnReg).
+static constexpr Register RegExpSearcherRegExpReg = CallTempReg1;
+static constexpr Register RegExpSearcherStringReg = CallTempReg2;
+static constexpr Register RegExpSearcherLastIndexReg = CallTempReg3;
+
+class ABIArgGenerator {
+#if defined(XP_WIN)
+  unsigned regIndex_;
+#else
+  unsigned intRegIndex_;
+  unsigned floatRegIndex_;
+#endif
+  uint32_t stackOffset_;
+  ABIArg current_;
+
+ public:
+  ABIArgGenerator();
+  ABIArg next(MIRType argType);
+  ABIArg& current() { return current_; }
+  uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+  void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+// These registers may be volatile or nonvolatile.
+// Avoid r11, which is the MacroAssembler's ScratchReg.
+static constexpr Register ABINonArgReg0 = rax;
+static constexpr Register ABINonArgReg1 = rbx;
+static constexpr Register ABINonArgReg2 = r10;
+static constexpr Register ABINonArgReg3 = r12;
+
+// This register may be volatile or nonvolatile. Avoid xmm15 which is the
+// ScratchDoubleReg.
+static constexpr FloatRegister ABINonArgDoubleReg =
+    FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = r10;
+static constexpr Register ABINonArgReturnReg1 = r12;
+static constexpr Register ABINonVolatileReg = r13;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = r10;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = r14;
+
+// Registers used for asm.js/wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Registers used for wasm tail calls operations.
+static constexpr Register WasmTailCallInstanceScratchReg = ABINonArgReg1;
+static constexpr Register WasmTailCallRAScratchReg = ABINonArgReg2;
+static constexpr Register WasmTailCallFPScratchReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code.  This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = rbx;
+
+static constexpr Register OsrFrameReg = IntArgReg3;
+
+static constexpr Register PreBarrierReg = rdx;
+
+static constexpr Register InterpreterPCReg = r14;
+
+static constexpr uint32_t ABIStackAlignment = 16;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+    JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+                  JitStackValueAlignment >= 1,
+              "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+              "Code alignment should be larger than any of the alignments "
+              "which are used for "
+              "the constant sections of the code buffer.  Thus it should be "
+              "larger than the "
+              "alignment for SIMD constants.");
+
+static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
+              "Stack alignment should be larger than any of the alignments "
+              "which are used for "
+              "spilled values.  Thus it should be larger than the alignment "
+              "for SIMD accesses.");
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static constexpr uint32_t WasmTrapInstructionLength = 2;
+
+// See comments in wasm::GenerateFunctionPrologue.  The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesEight;
+
+}  // namespace jit
+}  // namespace js
+
+#include "jit/x86-shared/Assembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+// Return operand from a JS -> JS call.
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+class Assembler : public AssemblerX86Shared {
+  // x64 jumps may need extra bits of relocation, because a jump may extend
+  // beyond the signed 32-bit range. To account for this we add an extended
+  // jump table at the bottom of the instruction stream, and if a jump
+  // overflows its range, it will redirect here.
+  //
+  // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the
+  // hardware branch predictor that there is no fallthrough, followed by the
+  // eight bytes containing an immediate address. This comes out to 16 bytes.
+  //    +1 byte for opcode
+  //    +1 byte for mod r/m
+  //    +4 bytes for rip-relative offset (2)
+  //    +2 bytes for ud2 instruction
+  //    +8 bytes for 64-bit address
+  //
+  static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8;
+  static const uint32_t SizeOfJumpTableEntry = 16;
+
+  // Two kinds of jumps on x64:
+  //
+  // * codeJumps_ tracks jumps with target within the executable code region
+  //   for the process. These jumps don't need entries in the extended jump
+  //   table because source and target must be within 2 GB of each other.
+  //
+  // * extendedJumps_ tracks jumps with target outside the executable code
+  //   region. These jumps need entries in the extended jump table described
+  //   above.
+  using PendingJumpVector = Vector<RelativePatch, 8, SystemAllocPolicy>;
+  PendingJumpVector codeJumps_;
+  PendingJumpVector extendedJumps_;
+
+  uint32_t extendedJumpTable_;
+
+  static JitCode* CodeFromJump(JitCode* code, uint8_t* jump);
+
+ private:
+  void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind reloc);
+
+ public:
+  using AssemblerX86Shared::j;
+  using AssemblerX86Shared::jmp;
+  using AssemblerX86Shared::pop;
+  using AssemblerX86Shared::push;
+  using AssemblerX86Shared::vmovq;
+
+  Assembler() : extendedJumpTable_(0) {}
+
+  static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+                                   CompactBufferReader& reader);
+
+  // The buffer is about to be linked, make sure any constant pools or excess
+  // bookkeeping has been flushed to the instruction stream.
+  void finish();
+
+  // Copy the assembly code to the given buffer, and perform any pending
+  // relocations relying on the target address.
+  void executableCopy(uint8_t* buffer);
+
+  void assertNoGCThings() const {
+#ifdef DEBUG
+    MOZ_ASSERT(dataRelocations_.length() == 0);
+    for (auto& j : codeJumps_) {
+      MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+    }
+    for (auto& j : extendedJumps_) {
+      MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+    }
+#endif
+  }
+
+  // Actual assembly emitting functions.
+
+  void push(const ImmGCPtr ptr) {
+    movq(ptr, ScratchReg);
+    push(ScratchReg);
+  }
+  void push(const ImmWord ptr) {
+    // We often end up with ImmWords that actually fit into int32.
+    // Be aware of the sign extension behavior.
+    if (ptr.value <= INT32_MAX) {
+      push(Imm32(ptr.value));
+    } else {
+      movq(ptr, ScratchReg);
+      push(ScratchReg);
+    }
+  }
+  void push(ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); }
+  void push(FloatRegister src) {
+    subq(Imm32(sizeof(double)), StackPointer);
+    vmovsd(src, Address(StackPointer, 0));
+  }
+  CodeOffset pushWithPatch(ImmWord word) {
+    CodeOffset label = movWithPatch(word, ScratchReg);
+    push(ScratchReg);
+    return label;
+  }
+
+  void pop(FloatRegister src) {
+    vmovsd(Address(StackPointer, 0), src);
+    addq(Imm32(sizeof(double)), StackPointer);
+  }
+
+  CodeOffset movWithPatch(ImmWord word, Register dest) {
+    masm.movq_i64r(word.value, dest.encoding());
+    return CodeOffset(masm.currentOffset());
+  }
+  CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+    return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+  }
+
+  // This is for patching during code generation, not after.
+  void patchAddq(CodeOffset offset, int32_t n) {
+    unsigned char* code = masm.data();
+    X86Encoding::SetInt32(code + offset.offset(), n);
+  }
+
+  // Load an ImmWord value into a register. Note that this instruction will
+  // attempt to optimize its immediate field size. When a full 64-bit
+  // immediate is needed for a relocation, use movWithPatch.
+  void movq(ImmWord word, Register dest) {
+    // Load a 64-bit immediate into a register. If the value falls into
+    // certain ranges, we can use specialized instructions which have
+    // smaller encodings.
+    if (word.value <= UINT32_MAX) {
+      // movl has a 32-bit unsigned (effectively) immediate field.
+      masm.movl_i32r((uint32_t)word.value, dest.encoding());
+    } else if ((intptr_t)word.value >= INT32_MIN &&
+               (intptr_t)word.value <= INT32_MAX) {
+      // movq has a 32-bit signed immediate field.
+      masm.movq_i32r((int32_t)(intptr_t)word.value, dest.encoding());
+    } else {
+      // Otherwise use movabs.
+      masm.movq_i64r(word.value, dest.encoding());
+    }
+  }
+  void movq(ImmPtr imm, Register dest) {
+    movq(ImmWord(uintptr_t(imm.value)), dest);
+  }
+  void movq(ImmGCPtr ptr, Register dest) {
+    masm.movq_i64r(uintptr_t(ptr.value), dest.encoding());
+    writeDataRelocation(ptr);
+  }
+  void movq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.movq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                     dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.movq_mr(src.address(), dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void movq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.movq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                     dest.scale());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.movq_rm(src.encoding(), dest.address());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void movq(Imm32 imm32, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.movl_i32r(imm32.value, dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movq_i32m(imm32.value, dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movq_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
+                       dest.scale());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.movq_i32m(imm32.value, dest.address());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void vmovq(Register src, FloatRegister dest) {
+    masm.vmovq_rr(src.encoding(), dest.encoding());
+  }
+  void vmovq(FloatRegister src, Register dest) {
+    masm.vmovq_rr(src.encoding(), dest.encoding());
+  }
+  void movq(Register src, Register dest) {
+    masm.movq_rr(src.encoding(), dest.encoding());
+  }
+
+  void cmovCCq(Condition cond, const Operand& src, Register dest) {
+    X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.cmovCCq_rr(cc, src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.cmovCCq_mr(cc, src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.cmovCCq_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
+                        dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void cmovCCq(Condition cond, Register src, Register dest) {
+    X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+    masm.cmovCCq_rr(cc, src.encoding(), dest.encoding());
+  }
+
+  void cmovzq(const Operand& src, Register dest) {
+    cmovCCq(Condition::Zero, src, dest);
+  }
+  void cmovnzq(const Operand& src, Register dest) {
+    cmovCCq(Condition::NonZero, src, dest);
+  }
+
+  template <typename T>
+  void lock_addq(T src, const Operand& op) {
+    masm.prefix_lock();
+    addq(src, op);
+  }
+  template <typename T>
+  void lock_subq(T src, const Operand& op) {
+    masm.prefix_lock();
+    subq(src, op);
+  }
+  template <typename T>
+  void lock_andq(T src, const Operand& op) {
+    masm.prefix_lock();
+    andq(src, op);
+  }
+  template <typename T>
+  void lock_orq(T src, const Operand& op) {
+    masm.prefix_lock();
+    orq(src, op);
+  }
+  template <typename T>
+  void lock_xorq(T src, const Operand& op) {
+    masm.prefix_lock();
+    xorq(src, op);
+  }
+
+  void lock_cmpxchgq(Register src, const Operand& mem) {
+    masm.prefix_lock();
+    switch (mem.kind()) {
+      case Operand::MEM_REG_DISP:
+        masm.cmpxchgq(src.encoding(), mem.disp(), mem.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.cmpxchgq(src.encoding(), mem.disp(), mem.base(), mem.index(),
+                      mem.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void xchgq(Register src, Register dest) {
+    masm.xchgq_rr(src.encoding(), dest.encoding());
+  }
+
+  void xchgq(Register src, const Operand& mem) {
+    switch (mem.kind()) {
+      case Operand::MEM_REG_DISP:
+        masm.xchgq_rm(src.encoding(), mem.disp(), mem.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.xchgq_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+                      mem.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void lock_xaddq(Register srcdest, const Operand& mem) {
+    switch (mem.kind()) {
+      case Operand::MEM_REG_DISP:
+        masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base(),
+                           mem.index(), mem.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void movsbq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.movsbq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movsbq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movsbq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                       dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void movzbq(const Operand& src, Register dest) {
+    // movzbl zero-extends to 64 bits and is one byte smaller, so use that
+    // instead.
+    movzbl(src, dest);
+  }
+
+  void movswq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.movswq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movswq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movswq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                       dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void movzwq(const Operand& src, Register dest) {
+    // movzwl zero-extends to 64 bits and is one byte smaller, so use that
+    // instead.
+    movzwl(src, dest);
+  }
+
+  void movslq(Register src, Register dest) {
+    masm.movslq_rr(src.encoding(), dest.encoding());
+  }
+  void movslq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.movslq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.movslq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.movslq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                       dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void andq(Register src, Register dest) {
+    masm.andq_rr(src.encoding(), dest.encoding());
+  }
+  void andq(Imm32 imm, Register dest) {
+    masm.andq_ir(imm.value, dest.encoding());
+  }
+  void andq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.andq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.andq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.andq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                     dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.andq_mr(src.address(), dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void andq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.andq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.andq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.andq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                     dest.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void addq(Imm32 imm, Register dest) {
+    masm.addq_ir(imm.value, dest.encoding());
+  }
+  CodeOffset addqWithPatch(Imm32 imm, Register dest) {
+    masm.addq_i32r(imm.value, dest.encoding());
+    return CodeOffset(masm.currentOffset());
+  }
+  void addq(Imm32 imm, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.addq_ir(imm.value, dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.addq_im(imm.value, dest.disp(), dest.base());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.addq_im(imm.value, dest.address());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void addq(Register src, Register dest) {
+    masm.addq_rr(src.encoding(), dest.encoding());
+  }
+  void addq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.addq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.addq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.addq_mr(src.address(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.addq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                     dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void addq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.addq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.addq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.addq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                     dest.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void subq(Imm32 imm, Register dest) {
+    masm.subq_ir(imm.value, dest.encoding());
+  }
+  void subq(Register src, Register dest) {
+    masm.subq_rr(src.encoding(), dest.encoding());
+  }
+  void subq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.subq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.subq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.subq_mr(src.address(), dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void subq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.subq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.subq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.subq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                     dest.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void shlq(Imm32 imm, Register dest) {
+    masm.shlq_ir(imm.value, dest.encoding());
+  }
+  void shrq(Imm32 imm, Register dest) {
+    masm.shrq_ir(imm.value, dest.encoding());
+  }
+  void sarq(Imm32 imm, Register dest) {
+    masm.sarq_ir(imm.value, dest.encoding());
+  }
+  void shlq_cl(Register dest) { masm.shlq_CLr(dest.encoding()); }
+  void shrq_cl(Register dest) { masm.shrq_CLr(dest.encoding()); }
+  void sarq_cl(Register dest) { masm.sarq_CLr(dest.encoding()); }
+  void sarxq(Register src, Register shift, Register dest) {
+    MOZ_ASSERT(HasBMI2());
+    masm.sarxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+  }
+  void shlxq(Register src, Register shift, Register dest) {
+    MOZ_ASSERT(HasBMI2());
+    masm.shlxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+  }
+  void shrxq(Register src, Register shift, Register dest) {
+    MOZ_ASSERT(HasBMI2());
+    masm.shrxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+  }
+  void rolq(Imm32 imm, Register dest) {
+    masm.rolq_ir(imm.value, dest.encoding());
+  }
+  void rolq_cl(Register dest) { masm.rolq_CLr(dest.encoding()); }
+  void rorq(Imm32 imm, Register dest) {
+    masm.rorq_ir(imm.value, dest.encoding());
+  }
+  void rorq_cl(Register dest) { masm.rorq_CLr(dest.encoding()); }
+  void orq(Imm32 imm, Register dest) {
+    masm.orq_ir(imm.value, dest.encoding());
+  }
+  void orq(Register src, Register dest) {
+    masm.orq_rr(src.encoding(), dest.encoding());
+  }
+  void orq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.orq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.orq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.orq_mr(src.address(), dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void orq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.orq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.orq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.orq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                    dest.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void xorq(Register src, Register dest) {
+    masm.xorq_rr(src.encoding(), dest.encoding());
+  }
+  void xorq(Imm32 imm, Register dest) {
+    masm.xorq_ir(imm.value, dest.encoding());
+  }
+  void xorq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.xorq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.xorq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.xorq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                     dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.xorq_mr(src.address(), dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void xorq(Register src, const Operand& dest) {
+    switch (dest.kind()) {
+      case Operand::REG:
+        masm.xorq_rr(src.encoding(), dest.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.xorq_rm(src.encoding(), dest.disp(), dest.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.xorq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+                     dest.scale());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void bsrq(const Register& src, const Register& dest) {
+    masm.bsrq_rr(src.encoding(), dest.encoding());
+  }
+  void bsfq(const Register& src, const Register& dest) {
+    masm.bsfq_rr(src.encoding(), dest.encoding());
+  }
+  void bswapq(const Register& reg) { masm.bswapq_r(reg.encoding()); }
+  void lzcntq(const Register& src, const Register& dest) {
+    masm.lzcntq_rr(src.encoding(), dest.encoding());
+  }
+  void tzcntq(const Register& src, const Register& dest) {
+    masm.tzcntq_rr(src.encoding(), dest.encoding());
+  }
+  void popcntq(const Register& src, const Register& dest) {
+    masm.popcntq_rr(src.encoding(), dest.encoding());
+  }
+
+  void imulq(Imm32 imm, Register src, Register dest) {
+    masm.imulq_ir(imm.value, src.encoding(), dest.encoding());
+  }
+  void imulq(Register src, Register dest) {
+    masm.imulq_rr(src.encoding(), dest.encoding());
+  }
+  void imulq(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::REG:
+        masm.imulq_rr(src.reg(), dest.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.imulq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_ADDRESS32:
+        MOZ_CRASH("NYI");
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void cqo() { masm.cqo(); }
+  void idivq(Register divisor) { masm.idivq_r(divisor.encoding()); }
+  void udivq(Register divisor) { masm.divq_r(divisor.encoding()); }
+
+  void vcvtsi2sdq(Register src, FloatRegister dest) {
+    masm.vcvtsi2sdq_rr(src.encoding(), dest.encoding());
+  }
+
+  void vpextrq(unsigned lane, FloatRegister src, Register dest) {
+    MOZ_ASSERT(HasSSE41());
+    masm.vpextrq_irr(lane, src.encoding(), dest.encoding());
+  }
+
+  void vpinsrq(unsigned lane, Register src1, FloatRegister src0,
+               FloatRegister dest) {
+    MOZ_ASSERT(HasSSE41());
+    masm.vpinsrq_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
+  }
+
+  void negq(Register reg) { masm.negq_r(reg.encoding()); }
+
+  void notq(Register reg) { masm.notq_r(reg.encoding()); }
+
+  void mov(ImmWord word, Register dest) {
+    // Use xor for setting registers to zero, as it is specially optimized
+    // for this purpose on modern hardware. Note that it does clobber FLAGS
+    // though. Use xorl instead of xorq since they are functionally
+    // equivalent (32-bit instructions zero-extend their results to 64 bits)
+    // and xorl has a smaller encoding.
+    if (word.value == 0) {
+      xorl(dest, dest);
+    } else {
+      movq(word, dest);
+    }
+  }
+  void mov(ImmPtr imm, Register dest) { movq(imm, dest); }
+  void mov(wasm::SymbolicAddress imm, Register dest) {
+    masm.movq_i64r(-1, dest.encoding());
+    append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm));
+  }
+  void mov(const Operand& src, Register dest) { movq(src, dest); }
+  void mov(Register src, const Operand& dest) { movq(src, dest); }
+  void mov(Imm32 imm32, const Operand& dest) { movq(imm32, dest); }
+  void mov(Register src, Register dest) { movq(src, dest); }
+  void mov(CodeLabel* label, Register dest) {
+    masm.movq_i64r(/* placeholder */ 0, dest.encoding());
+    label->patchAt()->bind(masm.size());
+  }
+  void xchg(Register src, Register dest) { xchgq(src, dest); }
+
+  void lea(const Operand& src, Register dest) {
+    switch (src.kind()) {
+      case Operand::MEM_REG_DISP:
+        masm.leaq_mr(src.disp(), src.base(), dest.encoding());
+        break;
+      case Operand::MEM_SCALE:
+        masm.leaq_mr(src.disp(), src.base(), src.index(), src.scale(),
+                     dest.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexepcted operand kind");
+    }
+  }
+
+  void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); }
+  void cmovzPtr(const Operand& src, Register dest) { return cmovzq(src, dest); }
+
+  CodeOffset loadRipRelativeInt32(Register dest) {
+    return CodeOffset(masm.movl_ripr(dest.encoding()).offset());
+  }
+  CodeOffset loadRipRelativeInt64(Register dest) {
+    return CodeOffset(masm.movq_ripr(dest.encoding()).offset());
+  }
+  CodeOffset loadRipRelativeDouble(FloatRegister dest) {
+    return CodeOffset(masm.vmovsd_ripr(dest.encoding()).offset());
+  }
+  CodeOffset loadRipRelativeFloat32(FloatRegister dest) {
+    return CodeOffset(masm.vmovss_ripr(dest.encoding()).offset());
+  }
+  CodeOffset loadRipRelativeInt32x4(FloatRegister dest) {
+    return CodeOffset(masm.vmovdqa_ripr(dest.encoding()).offset());
+  }
+  CodeOffset loadRipRelativeFloat32x4(FloatRegister dest) {
+    return CodeOffset(masm.vmovaps_ripr(dest.encoding()).offset());
+  }
+  CodeOffset leaRipRelative(Register dest) {
+    return CodeOffset(masm.leaq_rip(dest.encoding()).offset());
+  }
+
+  void cmpq(Register rhs, Register lhs) {
+    masm.cmpq_rr(rhs.encoding(), lhs.encoding());
+  }
+  void cmpq(Register rhs, const Operand& lhs) {
+    switch (lhs.kind()) {
+      case Operand::REG:
+        masm.cmpq_rr(rhs.encoding(), lhs.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
+                     lhs.scale());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.cmpq_rm(rhs.encoding(), lhs.address());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void cmpq(Imm32 rhs, Register lhs) {
+    masm.cmpq_ir(rhs.value, lhs.encoding());
+  }
+  void cmpq(Imm32 rhs, const Operand& lhs) {
+    switch (lhs.kind()) {
+      case Operand::REG:
+        masm.cmpq_ir(rhs.value, lhs.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.cmpq_im(rhs.value, lhs.disp(), lhs.base());
+        break;
+      case Operand::MEM_SCALE:
+        masm.cmpq_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+                     lhs.scale());
+        break;
+      case Operand::MEM_ADDRESS32:
+        masm.cmpq_im(rhs.value, lhs.address());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  void cmpq(const Operand& rhs, Register lhs) {
+    switch (rhs.kind()) {
+      case Operand::REG:
+        masm.cmpq_rr(rhs.reg(), lhs.encoding());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.cmpq_mr(rhs.disp(), rhs.base(), lhs.encoding());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+
+  void testq(Imm32 rhs, Register lhs) {
+    masm.testq_ir(rhs.value, lhs.encoding());
+  }
+  void testq(Register rhs, Register lhs) {
+    masm.testq_rr(rhs.encoding(), lhs.encoding());
+  }
+  void testq(Imm32 rhs, const Operand& lhs) {
+    switch (lhs.kind()) {
+      case Operand::REG:
+        masm.testq_ir(rhs.value, lhs.reg());
+        break;
+      case Operand::MEM_REG_DISP:
+        masm.testq_i32m(rhs.value, lhs.disp(), lhs.base());
+        break;
+      default:
+        MOZ_CRASH("unexpected operand kind");
+        break;
+    }
+  }
+
+  void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) {
+    MOZ_ASSERT(hasCreator());
+    JmpSrc src = masm.jmp();
+    addPendingJump(src, target, reloc);
+  }
+  void j(Condition cond, ImmPtr target,
+         RelocationKind reloc = RelocationKind::HARDCODED) {
+    JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond));
+    addPendingJump(src, target, reloc);
+  }
+
+  void jmp(JitCode* target) {
+    jmp(ImmPtr(target->raw()), RelocationKind::JITCODE);
+  }
+  void j(Condition cond, JitCode* target) {
+    j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE);
+  }
+  void call(JitCode* target) {
+    JmpSrc src = masm.call();
+    addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+  }
+  void call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+  void call(ImmPtr target) {
+    JmpSrc src = masm.call();
+    addPendingJump(src, target, RelocationKind::HARDCODED);
+  }
+
+  // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch
+  // this instruction.
+  CodeOffset toggledCall(JitCode* target, bool enabled) {
+    CodeOffset offset(size());
+    JmpSrc src = enabled ? masm.call() : masm.cmp_eax();
+    addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+    MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr));
+    return offset;
+  }
+
+  static size_t ToggledCallSize(uint8_t* code) {
+    // Size of a call instruction.
+    return 5;
+  }
+
+  // Do not mask shared implementations.
+  using AssemblerX86Shared::call;
+
+  void vcvttsd2sq(FloatRegister src, Register dest) {
+    masm.vcvttsd2sq_rr(src.encoding(), dest.encoding());
+  }
+  void vcvttss2sq(FloatRegister src, Register dest) {
+    masm.vcvttss2sq_rr(src.encoding(), dest.encoding());
+  }
+  void vcvtsq2sd(Register src1, FloatRegister src0, FloatRegister dest) {
+    masm.vcvtsq2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+  }
+  void vcvtsq2ss(Register src1, FloatRegister src0, FloatRegister dest) {
+    masm.vcvtsq2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+  }
+};
+
+static inline bool GetIntArgReg(uint32_t intArg, uint32_t floatArg,
+                                Register* out) {
+#if defined(_WIN64)
+  uint32_t arg = intArg + floatArg;
+#else
+  uint32_t arg = intArg;
+#endif
+  if (arg >= NumIntArgRegs) {
+    return false;
+  }
+  *out = IntArgRegs[arg];
+  return true;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument.  This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+                                       uint32_t usedFloatArgs, Register* out) {
+  if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
+    return true;
+  }
+  // Unfortunately, we have to assume things about the point at which
+  // GetIntArgReg returns false, because we need to know how many registers it
+  // can allocate.
+#if defined(_WIN64)
+  uint32_t arg = usedIntArgs + usedFloatArgs;
+#else
+  uint32_t arg = usedIntArgs;
+#endif
+  arg -= NumIntArgRegs;
+  if (arg >= NumCallTempNonArgRegs) {
+    return false;
+  }
+  *out = CallTempNonArgRegs[arg];
+  return true;
+}
+
+static inline bool GetFloatArgReg(uint32_t intArg, uint32_t floatArg,
+                                  FloatRegister* out) {
+#if defined(_WIN64)
+  uint32_t arg = intArg + floatArg;
+#else
+  uint32_t arg = floatArg;
+#endif
+  if (floatArg >= NumFloatArgRegs) {
+    return false;
+  }
+  *out = FloatArgRegs[arg];
+  return true;
+}
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_Assembler_x64_h */
diff --git a/js/src/jit/x64/BaseAssembler-x64.h b/js/src/jit/x64/BaseAssembler-x64.h
new file mode 100644
index 0000000000..f5a9bb99f9
--- /dev/null
+++ b/js/src/jit/x64/BaseAssembler-x64.h
@@ -0,0 +1,1373 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_BaseAssembler_x64_h
+#define jit_x64_BaseAssembler_x64_h
+
+#include "jit/x86-shared/BaseAssembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+class BaseAssemblerX64 : public BaseAssembler {
+ public:
+  // Arithmetic operations:
+
+  void addq_rr(RegisterID src, RegisterID dst) {
+    spew("addq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_ADD_GvEv, src, dst);
+  }
+
+  void addq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("addq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, dst);
+  }
+
+  void addq_mr(const void* addr, RegisterID dst) {
+    spew("addq       %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_ADD_GvEv, addr, dst);
+  }
+
+  void addq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+               RegisterID dst) {
+    spew("addq       " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, index, scale, dst);
+  }
+
+  void addq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("addq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, src);
+  }
+
+  void addq_rm(RegisterID src, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("addq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, index, scale, src);
+  }
+
+  void addq_ir(int32_t imm, RegisterID dst) {
+    spew("addq       $%d, %s", imm, GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_ADD);
+      m_formatter.immediate8s(imm);
+    } else {
+      if (dst == rax) {
+        m_formatter.oneByteOp64(OP_ADD_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+      }
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void addq_i32r(int32_t imm, RegisterID dst) {
+    // 32-bit immediate always, for patching.
+    spew("addq       $0x%04x, %s", uint32_t(imm), GPReg64Name(dst));
+    if (dst == rax) {
+      m_formatter.oneByteOp64(OP_ADD_EAXIv);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+    }
+    m_formatter.immediate32(imm);
+  }
+
+  void addq_im(int32_t imm, int32_t offset, RegisterID base) {
+    spew("addq       $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_ADD);
+      m_formatter.immediate8s(imm);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_ADD);
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void addq_im(int32_t imm, const void* addr) {
+    spew("addq       $%d, %p", imm, addr);
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
+      m_formatter.immediate8s(imm);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void andq_rr(RegisterID src, RegisterID dst) {
+    spew("andq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_AND_GvEv, src, dst);
+  }
+
+  void andq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("andq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, dst);
+  }
+
+  void andq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+               RegisterID dst) {
+    spew("andq       " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, index, scale, dst);
+  }
+
+  void andq_mr(const void* addr, RegisterID dst) {
+    spew("andq       %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_AND_GvEv, addr, dst);
+  }
+
+  void andq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("andq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, src);
+  }
+
+  void andq_rm(RegisterID src, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("andq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, index, scale, src);
+  }
+
+  void orq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("orq        " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_OR_GvEv, offset, base, dst);
+  }
+
+  void orq_mr(const void* addr, RegisterID dst) {
+    spew("orq        %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_OR_GvEv, addr, dst);
+  }
+
+  void orq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("orq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, src);
+  }
+
+  void orq_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+              int scale) {
+    spew("orq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, index, scale, src);
+  }
+
+  void xorq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("xorq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, dst);
+  }
+
+  void xorq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+               RegisterID dst) {
+    spew("xorq       " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, index, scale, dst);
+  }
+
+  void xorq_mr(const void* addr, RegisterID dst) {
+    spew("xorq       %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_XOR_GvEv, addr, dst);
+  }
+
+  void xorq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("xorq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, src);
+  }
+
+  void xorq_rm(RegisterID src, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("xorq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, index, scale, src);
+  }
+
+  void bswapq_r(RegisterID dst) {
+    spew("bswapq     %s", GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_BSWAP, dst);
+  }
+
+  void bsrq_rr(RegisterID src, RegisterID dst) {
+    spew("bsrq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_BSR_GvEv, src, dst);
+  }
+
+  void bsfq_rr(RegisterID src, RegisterID dst) {
+    spew("bsfq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_BSF_GvEv, src, dst);
+  }
+
+  void lzcntq_rr(RegisterID src, RegisterID dst) {
+    spew("lzcntq     %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.legacySSEPrefix(VEX_SS);
+    m_formatter.twoByteOp64(OP2_LZCNT_GvEv, src, dst);
+  }
+
+  void tzcntq_rr(RegisterID src, RegisterID dst) {
+    spew("tzcntq     %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.legacySSEPrefix(VEX_SS);
+    m_formatter.twoByteOp64(OP2_TZCNT_GvEv, src, dst);
+  }
+
+  void popcntq_rr(RegisterID src, RegisterID dst) {
+    spew("popcntq    %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.legacySSEPrefix(VEX_SS);
+    m_formatter.twoByteOp64(OP2_POPCNT_GvEv, src, dst);
+  }
+
+  void andq_ir(int32_t imm, RegisterID dst) {
+    spew("andq       $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
+      m_formatter.immediate8s(imm);
+    } else {
+      if (dst == rax) {
+        m_formatter.oneByteOp64(OP_AND_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_AND);
+      }
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void negq_r(RegisterID dst) {
+    spew("negq       %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NEG);
+  }
+
+  void orq_rr(RegisterID src, RegisterID dst) {
+    spew("orq        %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_OR_GvEv, src, dst);
+  }
+
+  void orq_ir(int32_t imm, RegisterID dst) {
+    spew("orq        $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
+      m_formatter.immediate8s(imm);
+    } else {
+      if (dst == rax) {
+        m_formatter.oneByteOp64(OP_OR_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_OR);
+      }
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void notq_r(RegisterID dst) {
+    spew("notq       %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NOT);
+  }
+
+  void subq_rr(RegisterID src, RegisterID dst) {
+    spew("subq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_SUB_GvEv, src, dst);
+  }
+
+  void subq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("subq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, src);
+  }
+
+  void subq_rm(RegisterID src, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("subq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, index, scale, src);
+  }
+
+  void subq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("subq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_SUB_GvEv, offset, base, dst);
+  }
+
+  void subq_mr(const void* addr, RegisterID dst) {
+    spew("subq       %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_SUB_GvEv, addr, dst);
+  }
+
+  void subq_ir(int32_t imm, RegisterID dst) {
+    spew("subq       $%d, %s", imm, GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
+      m_formatter.immediate8s(imm);
+    } else {
+      if (dst == rax) {
+        m_formatter.oneByteOp64(OP_SUB_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_SUB);
+      }
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void xorq_rr(RegisterID src, RegisterID dst) {
+    spew("xorq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_XOR_GvEv, src, dst);
+  }
+
+  void xorq_ir(int32_t imm, RegisterID dst) {
+    spew("xorq       $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(imm)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_XOR);
+      m_formatter.immediate8s(imm);
+    } else {
+      if (dst == rax) {
+        m_formatter.oneByteOp64(OP_XOR_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_XOR);
+      }
+      m_formatter.immediate32(imm);
+    }
+  }
+
+  void sarq_CLr(RegisterID dst) {
+    spew("sarq       %%cl, %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SAR);
+  }
+
+  void shlq_CLr(RegisterID dst) {
+    spew("shlq       %%cl, %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHL);
+  }
+
+  void shrq_CLr(RegisterID dst) {
+    spew("shrq       %%cl, %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHR);
+  }
+
+  void sarq_ir(int32_t imm, RegisterID dst) {
+    MOZ_ASSERT(imm < 64);
+    spew("sarq       $%d, %s", imm, GPReg64Name(dst));
+    if (imm == 1) {
+      m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SAR);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SAR);
+      m_formatter.immediate8u(imm);
+    }
+  }
+
+  void shlq_ir(int32_t imm, RegisterID dst) {
+    MOZ_ASSERT(imm < 64);
+    spew("shlq       $%d, %s", imm, GPReg64Name(dst));
+    if (imm == 1) {
+      m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHL);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHL);
+      m_formatter.immediate8u(imm);
+    }
+  }
+
+  void shrq_ir(int32_t imm, RegisterID dst) {
+    MOZ_ASSERT(imm < 64);
+    spew("shrq       $%d, %s", imm, GPReg64Name(dst));
+    if (imm == 1) {
+      m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHR);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHR);
+      m_formatter.immediate8u(imm);
+    }
+  }
+
+  void rolq_ir(int32_t imm, RegisterID dst) {
+    MOZ_ASSERT(imm < 64);
+    spew("rolq       $%d, %s", imm, GPReg64Name(dst));
+    if (imm == 1) {
+      m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROL);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROL);
+      m_formatter.immediate8u(imm);
+    }
+  }
+  void rolq_CLr(RegisterID dst) {
+    spew("rolq       %%cl, %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROL);
+  }
+
+  void rorq_ir(int32_t imm, RegisterID dst) {
+    MOZ_ASSERT(imm < 64);
+    spew("rorq       $%d, %s", imm, GPReg64Name(dst));
+    if (imm == 1) {
+      m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROR);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROR);
+      m_formatter.immediate8u(imm);
+    }
+  }
+  void rorq_CLr(RegisterID dst) {
+    spew("rorq       %%cl, %s", GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROR);
+  }
+
+  void imulq_rr(RegisterID src, RegisterID dst) {
+    spew("imulq      %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_IMUL_GvEv, src, dst);
+  }
+
+  void imulq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("imulq      " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_IMUL_GvEv, offset, base, dst);
+  }
+
+  void imulq_ir(int32_t value, RegisterID src, RegisterID dst) {
+    spew("imulq      $%d, %s, %s", value, GPReg64Name(src), GPReg64Name(dst));
+    if (CAN_SIGN_EXTEND_8_32(value)) {
+      m_formatter.oneByteOp64(OP_IMUL_GvEvIb, src, dst);
+      m_formatter.immediate8s(value);
+    } else {
+      m_formatter.oneByteOp64(OP_IMUL_GvEvIz, src, dst);
+      m_formatter.immediate32(value);
+    }
+  }
+
+  void cqo() {
+    spew("cqo        ");
+    m_formatter.oneByteOp64(OP_CDQ);
+  }
+
+  void idivq_r(RegisterID divisor) {
+    spew("idivq      %s", GPReg64Name(divisor));
+    m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_IDIV);
+  }
+
+  void divq_r(RegisterID divisor) {
+    spew("divq       %s", GPReg64Name(divisor));
+    m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_DIV);
+  }
+
+  // Comparisons:
+
+  void cmpq_rr(RegisterID rhs, RegisterID lhs) {
+    spew("cmpq       %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+    m_formatter.oneByteOp64(OP_CMP_GvEv, rhs, lhs);
+  }
+
+  void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base) {
+    spew("cmpq       %s, " MEM_ob, GPReg64Name(rhs), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, rhs);
+  }
+
+  void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("cmpq       %s, " MEM_obs, GPReg64Name(rhs),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, index, scale, rhs);
+  }
+
+  void cmpq_mr(int32_t offset, RegisterID base, RegisterID lhs) {
+    spew("cmpq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(lhs));
+    m_formatter.oneByteOp64(OP_CMP_GvEv, offset, base, lhs);
+  }
+
+  void cmpq_ir(int32_t rhs, RegisterID lhs) {
+    if (rhs == 0) {
+      testq_rr(lhs, lhs);
+      return;
+    }
+
+    spew("cmpq       $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+    if (CAN_SIGN_EXTEND_8_32(rhs)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, lhs, GROUP1_OP_CMP);
+      m_formatter.immediate8s(rhs);
+    } else {
+      if (lhs == rax) {
+        m_formatter.oneByteOp64(OP_CMP_EAXIv);
+      } else {
+        m_formatter.oneByteOp64(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+      }
+      m_formatter.immediate32(rhs);
+    }
+  }
+
+  void cmpq_im(int32_t rhs, int32_t offset, RegisterID base) {
+    spew("cmpq       $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+         ADDR_ob(offset, base));
+    if (CAN_SIGN_EXTEND_8_32(rhs)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+      m_formatter.immediate8s(rhs);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+      m_formatter.immediate32(rhs);
+    }
+  }
+
+  void cmpq_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+               int scale) {
+    spew("cmpq       $0x%x, " MEM_obs, uint32_t(rhs),
+         ADDR_obs(offset, base, index, scale));
+    if (CAN_SIGN_EXTEND_8_32(rhs)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, index, scale,
+                              GROUP1_OP_CMP);
+      m_formatter.immediate8s(rhs);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, index, scale,
+                              GROUP1_OP_CMP);
+      m_formatter.immediate32(rhs);
+    }
+  }
+  void cmpq_im(int32_t rhs, const void* addr) {
+    spew("cmpq       $0x%" PRIx64 ", %p", uint64_t(rhs), addr);
+    if (CAN_SIGN_EXTEND_8_32(rhs)) {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+      m_formatter.immediate8s(rhs);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+      m_formatter.immediate32(rhs);
+    }
+  }
+  void cmpq_rm(RegisterID rhs, const void* addr) {
+    spew("cmpq       %s, %p", GPReg64Name(rhs), addr);
+    m_formatter.oneByteOp64(OP_CMP_EvGv, addr, rhs);
+  }
+
+  void testq_rr(RegisterID rhs, RegisterID lhs) {
+    spew("testq      %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+    m_formatter.oneByteOp64(OP_TEST_EvGv, lhs, rhs);
+  }
+
+  void testq_ir(int32_t rhs, RegisterID lhs) {
+    // If the mask fits in a 32-bit immediate, we can use testl with a
+    // 32-bit subreg.
+    if (CAN_ZERO_EXTEND_32_64(rhs)) {
+      testl_ir(rhs, lhs);
+      return;
+    }
+    spew("testq      $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+    if (lhs == rax) {
+      m_formatter.oneByteOp64(OP_TEST_EAXIv);
+    } else {
+      m_formatter.oneByteOp64(OP_GROUP3_EvIz, lhs, GROUP3_OP_TEST);
+    }
+    m_formatter.immediate32(rhs);
+  }
+
+  void testq_i32m(int32_t rhs, int32_t offset, RegisterID base) {
+    spew("testq      $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+         ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, GROUP3_OP_TEST);
+    m_formatter.immediate32(rhs);
+  }
+
+  void testq_i32m(int32_t rhs, int32_t offset, RegisterID base,
+                  RegisterID index, int scale) {
+    spew("testq      $0x%4x, " MEM_obs, uint32_t(rhs),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, index, scale,
+                            GROUP3_OP_TEST);
+    m_formatter.immediate32(rhs);
+  }
+
+  // Various move ops:
+
+  void cmovCCq_rr(Condition cond, RegisterID src, RegisterID dst) {
+    spew("cmov%s     %s, %s", CCName(cond), GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(cmovccOpcode(cond), src, dst);
+  }
+  void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+                  RegisterID dst) {
+    spew("cmov%s     " MEM_ob ", %s", CCName(cond), ADDR_ob(offset, base),
+         GPReg64Name(dst));
+    m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, dst);
+  }
+  void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+                  RegisterID index, int scale, RegisterID dst) {
+    spew("cmov%s     " MEM_obs ", %s", CCName(cond),
+         ADDR_obs(offset, base, index, scale), GPReg64Name(dst));
+    m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, index, scale,
+                            dst);
+  }
+
+  void cmpxchgq(RegisterID src, int32_t offset, RegisterID base) {
+    spew("cmpxchgq   %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, src);
+  }
+
+  void cmpxchgq(RegisterID src, int32_t offset, RegisterID base,
+                RegisterID index, int scale) {
+    spew("cmpxchgq   %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
+  }
+
+  void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base) {
+    spew("lock xaddq %s, " MEM_ob, GPReg64Name(srcdest), ADDR_ob(offset, base));
+    m_formatter.oneByteOp(PRE_LOCK);
+    m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, srcdest);
+  }
+
+  void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base,
+                     RegisterID index, int scale) {
+    spew("lock xaddq %s, " MEM_obs, GPReg64Name(srcdest),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp(PRE_LOCK);
+    m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, index, scale, srcdest);
+  }
+
+  void xchgq_rr(RegisterID src, RegisterID dst) {
+    spew("xchgq      %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_XCHG_GvEv, src, dst);
+  }
+  void xchgq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("xchgq      %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, src);
+  }
+  void xchgq_rm(RegisterID src, int32_t offset, RegisterID base,
+                RegisterID index, int scale) {
+    spew("xchgq      %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, index, scale, src);
+  }
+
+  void movq_rr(RegisterID src, RegisterID dst) {
+    spew("movq       %s, %s", GPReg64Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOV_EvGv, dst, src);
+  }
+
+  void movq_rm(RegisterID src, int32_t offset, RegisterID base) {
+    spew("movq       %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, src);
+  }
+
+  void movq_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+    spew("movq       %s, " MEM_o32b, GPReg64Name(src), ADDR_o32b(offset, base));
+    m_formatter.oneByteOp64_disp32(OP_MOV_EvGv, offset, base, src);
+  }
+
+  void movq_rm(RegisterID src, int32_t offset, RegisterID base,
+               RegisterID index, int scale) {
+    spew("movq       %s, " MEM_obs, GPReg64Name(src),
+         ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, index, scale, src);
+  }
+
+  void movq_rm(RegisterID src, const void* addr) {
+    if (src == rax && !IsAddressImmediate(addr)) {
+      movq_EAXm(addr);
+      return;
+    }
+
+    spew("movq       %s, %p", GPReg64Name(src), addr);
+    m_formatter.oneByteOp64(OP_MOV_EvGv, addr, src);
+  }
+
+  void movq_mEAX(const void* addr) {
+    if (IsAddressImmediate(addr)) {
+      movq_mr(addr, rax);
+      return;
+    }
+
+    spew("movq       %p, %%rax", addr);
+    m_formatter.oneByteOp64(OP_MOV_EAXOv);
+    m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+  }
+
+  void movq_EAXm(const void* addr) {
+    if (IsAddressImmediate(addr)) {
+      movq_rm(rax, addr);
+      return;
+    }
+
+    spew("movq       %%rax, %p", addr);
+    m_formatter.oneByteOp64(OP_MOV_OvEAX);
+    m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+  }
+
+  void movq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("movq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, dst);
+  }
+
+  void movq_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("movq       " MEM_o32b ", %s", ADDR_o32b(offset, base),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64_disp32(OP_MOV_GvEv, offset, base, dst);
+  }
+
+  void movq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+               RegisterID dst) {
+    spew("movq       " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, index, scale, dst);
+  }
+
+  void movq_mr(const void* addr, RegisterID dst) {
+    if (dst == rax && !IsAddressImmediate(addr)) {
+      movq_mEAX(addr);
+      return;
+    }
+
+    spew("movq       %p, %s", addr, GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOV_GvEv, addr, dst);
+  }
+
+  void leaq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+               RegisterID dst) {
+    spew("leaq       " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_LEA, offset, base, index, scale, dst);
+  }
+
+  void movq_i32m(int32_t imm, int32_t offset, RegisterID base) {
+    spew("movq       $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+    m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
+    m_formatter.immediate32(imm);
+  }
+  void movq_i32m(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+                 int scale) {
+    spew("movq       $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+    m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, index, scale,
+                            GROUP11_MOV);
+    m_formatter.immediate32(imm);
+  }
+  void movq_i32m(int32_t imm, const void* addr) {
+    spew("movq       $%d, %p", imm, addr);
+    m_formatter.oneByteOp64(OP_GROUP11_EvIz, addr, GROUP11_MOV);
+    m_formatter.immediate32(imm);
+  }
+
+  // Note that this instruction sign-extends its 32-bit immediate field to 64
+  // bits and loads the 64-bit value into a 64-bit register.
+  //
+  // Note also that this is similar to the movl_i32r instruction, except that
+  // movl_i32r *zero*-extends its 32-bit immediate, and it has smaller code
+  // size, so it's preferred for values which could use either.
+  void movq_i32r(int32_t imm, RegisterID dst) {
+    spew("movq       $%d, %s", imm, GPRegName(dst));
+    m_formatter.oneByteOp64(OP_GROUP11_EvIz, dst, GROUP11_MOV);
+    m_formatter.immediate32(imm);
+  }
+
+  void movq_i64r(int64_t imm, RegisterID dst) {
+    spew("movabsq    $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOV_EAXIv, dst);
+    m_formatter.immediate64(imm);
+  }
+
+  void movsbq_rr(RegisterID src, RegisterID dst) {
+    spew("movsbq     %s, %s", GPReg32Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEb, src, dst);
+  }
+  void movsbq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("movsbq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, dst);
+  }
+  void movsbq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+                 RegisterID dst) {
+    spew("movsbq     " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, index, scale, dst);
+  }
+
+  void movswq_rr(RegisterID src, RegisterID dst) {
+    spew("movswq     %s, %s", GPReg32Name(src), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEw, src, dst);
+  }
+  void movswq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("movswq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, dst);
+  }
+  void movswq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+                 RegisterID dst) {
+    spew("movswq     " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, index, scale, dst);
+  }
+
+  void movslq_rr(RegisterID src, RegisterID dst) {
+    spew("movslq     %s, %s", GPReg32Name(src), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOVSXD_GvEv, src, dst);
+  }
+  void movslq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("movslq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, dst);
+  }
+  void movslq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+                 RegisterID dst) {
+    spew("movslq     " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+         GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, index, scale, dst);
+  }
+
+  [[nodiscard]] JmpSrc movl_ripr(RegisterID dst) {
+    m_formatter.oneByteRipOp(OP_MOV_GvEv, 0, (RegisterID)dst);
+    JmpSrc label(m_formatter.size());
+    spew("movl       " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+         GPReg32Name(dst));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc movl_rrip(RegisterID src) {
+    m_formatter.oneByteRipOp(OP_MOV_EvGv, 0, (RegisterID)src);
+    JmpSrc label(m_formatter.size());
+    spew("movl       %s, " MEM_o32r "", GPReg32Name(src),
+         ADDR_o32r(label.offset()));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc movq_ripr(RegisterID dst) {
+    m_formatter.oneByteRipOp64(OP_MOV_GvEv, 0, dst);
+    JmpSrc label(m_formatter.size());
+    spew("movq       " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+         GPRegName(dst));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc movq_rrip(RegisterID src) {
+    m_formatter.oneByteRipOp64(OP_MOV_EvGv, 0, (RegisterID)src);
+    JmpSrc label(m_formatter.size());
+    spew("movq       %s, " MEM_o32r "", GPRegName(src),
+         ADDR_o32r(label.offset()));
+    return label;
+  }
+
+  void leaq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+    spew("leaq       " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+    m_formatter.oneByteOp64(OP_LEA, offset, base, dst);
+  }
+
+  [[nodiscard]] JmpSrc leaq_rip(RegisterID dst) {
+    m_formatter.oneByteRipOp64(OP_LEA, 0, dst);
+    JmpSrc label(m_formatter.size());
+    spew("leaq       " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+         GPRegName(dst));
+    return label;
+  }
+
+  // Flow control:
+
+  void jmp_rip(int ripOffset) {
+    // rip-relative addressing.
+    spew("jmp        *%d(%%rip)", ripOffset);
+    m_formatter.oneByteRipOp(OP_GROUP5_Ev, ripOffset, GROUP5_OP_JMPN);
+  }
+
+  void immediate64(int64_t imm) {
+    spew(".quad      %lld", (long long)imm);
+    m_formatter.immediate64(imm);
+  }
+
+  // SSE operations:
+
+  void vcvtsq2sd_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+    twoByteOpInt64Simd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, src1, src0,
+                       dst);
+  }
+  void vcvtsq2ss_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+    twoByteOpInt64Simd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, src1, src0,
+                       dst);
+  }
+
+  void vcvtsi2sdq_rr(RegisterID src, XMMRegisterID dst) {
+    twoByteOpInt64Simd("vcvtsi2sdq", VEX_SD, OP2_CVTSI2SD_VsdEd, src,
+                       invalid_xmm, dst);
+  }
+
+  void vcvttsd2sq_rr(XMMRegisterID src, RegisterID dst) {
+    twoByteOpSimdInt64("vcvttsd2si", VEX_SD, OP2_CVTTSD2SI_GdWsd, src, dst);
+  }
+
+  void vcvttss2sq_rr(XMMRegisterID src, RegisterID dst) {
+    twoByteOpSimdInt64("vcvttss2si", VEX_SS, OP2_CVTTSD2SI_GdWsd, src, dst);
+  }
+
+  void vmovq_rr(XMMRegisterID src, RegisterID dst) {
+    // While this is called "vmovq", it actually uses the vmovd encoding
+    // with a REX prefix modifying it to be 64-bit.
+    twoByteOpSimdInt64("vmovq", VEX_PD, OP2_MOVD_EdVd, (XMMRegisterID)dst,
+                       (RegisterID)src);
+  }
+
+  void vpextrq_irr(unsigned lane, XMMRegisterID src, RegisterID dst) {
+    MOZ_ASSERT(lane < 2);
+    threeByteOpImmSimdInt64("vpextrq", VEX_PD, OP3_PEXTRQ_EvVdqIb, ESCAPE_3A,
+                            lane, src, dst);
+  }
+
+  void vpinsrq_irr(unsigned lane, RegisterID src1, XMMRegisterID src0,
+                   XMMRegisterID dst) {
+    MOZ_ASSERT(lane < 2);
+    threeByteOpImmInt64Simd("vpinsrq", VEX_PD, OP3_PINSRQ_VdqEvIb, ESCAPE_3A,
+                            lane, src1, src0, dst);
+  }
+
+  void vmovq_rr(RegisterID src, XMMRegisterID dst) {
+    // While this is called "vmovq", it actually uses the vmovd encoding
+    // with a REX prefix modifying it to be 64-bit.
+    twoByteOpInt64Simd("vmovq", VEX_PD, OP2_MOVD_VdEd, src, invalid_xmm, dst);
+  }
+
+  [[nodiscard]] JmpSrc vmovsd_ripr(XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, dst);
+  }
+  [[nodiscard]] JmpSrc vmovss_ripr(XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, dst);
+  }
+  [[nodiscard]] JmpSrc vmovaps_ripr(XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, dst);
+  }
+  [[nodiscard]] JmpSrc vmovdqa_ripr(XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, dst);
+  }
+
+  [[nodiscard]] JmpSrc vpaddb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddq", VEX_PD, OP2_PADDQ_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubq", VEX_PD, OP2_PSUBQ_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmullw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmulld_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpaddusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpsubusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpminsb", VEX_PD, OP3_PMINSB_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpminub", VEX_PD, OP2_PMINUB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpminsw", VEX_PD, OP2_PMINSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpminuw", VEX_PD, OP3_PMINUW_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpminsd", VEX_PD, OP3_PMINSD_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpminud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpminud", VEX_PD, OP3_PMINUD_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmaxsb", VEX_PD, OP3_PMAXSB_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpmaxub", VEX_PD, OP2_PMAXUB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpmaxsw", VEX_PD, OP2_PMAXSW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmaxuw", VEX_PD, OP3_PMAXUW_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmaxsd", VEX_PD, OP3_PMAXSD_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaxud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmaxud", VEX_PD, OP3_PMAXUD_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpand_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpxor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vaddps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, src, dst);
+  }
+  [[nodiscard]] JmpSrc vaddpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vaddpd", VEX_PD, OP2_ADDPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vsubps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, src, dst);
+  }
+  [[nodiscard]] JmpSrc vsubpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vsubpd", VEX_PD, OP2_SUBPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vdivps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, src, dst);
+  }
+  [[nodiscard]] JmpSrc vdivpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vdivpd", VEX_PD, OP2_DIVPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vmulps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, src, dst);
+  }
+  [[nodiscard]] JmpSrc vmulpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vmulpd", VEX_PD, OP2_MULPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vandpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vandpd", VEX_PD, OP2_ANDPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vminpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vminpd", VEX_PD, OP2_MINPD_VpdWpd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpacksswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpacksswb", VEX_PD, OP2_PACKSSWB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpackuswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpackuswb", VEX_PD, OP2_PACKUSWB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpackssdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpackssdw", VEX_PD, OP2_PACKSSDW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpackusdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpackusdw", VEX_PD, OP3_PACKUSDW_VdqWdq,
+                              ESCAPE_38, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpunpckldq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, src,
+                            dst);
+  }
+  [[nodiscard]] JmpSrc vunpcklps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, src, dst);
+  }
+  [[nodiscard]] JmpSrc vptest_ripr(XMMRegisterID lhs) {
+    return threeByteRipOpSimd("vptest", VEX_PD, OP3_PTEST_VdVd, ESCAPE_38, lhs);
+  }
+  [[nodiscard]] JmpSrc vpshufb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpshufb", VEX_PD, OP3_PSHUFB_VdqWdq, ESCAPE_38,
+                              src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaddwd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpmaddwd", VEX_PD, OP2_PMADDWD_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpeqb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpgtb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpeqw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpgtw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpeqd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpcmpgtd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpeqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+                               X86Encoding::ConditionCmp_EQ, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpneqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+                               X86Encoding::ConditionCmp_NEQ, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpltps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+                               X86Encoding::ConditionCmp_LT, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpleps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+                               X86Encoding::ConditionCmp_LE, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpgeps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+                               X86Encoding::ConditionCmp_GE, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpeqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+                               X86Encoding::ConditionCmp_EQ, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpneqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+                               X86Encoding::ConditionCmp_NEQ, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmpltpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+                               X86Encoding::ConditionCmp_LT, src, dst);
+  }
+  [[nodiscard]] JmpSrc vcmplepd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+                               X86Encoding::ConditionCmp_LE, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmaddubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return threeByteRipOpSimd("vpmaddubsw", VEX_PD, OP3_PMADDUBSW_VdqWdq,
+                              ESCAPE_38, src, dst);
+  }
+  [[nodiscard]] JmpSrc vpmuludq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+    return twoByteRipOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, src, dst);
+  }
+
+  // BMI instructions:
+
+  void sarxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+    spew("sarxq      %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+         GPReg64Name(dst));
+
+    RegisterID rm = src;
+    XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+    int reg = dst;
+    m_formatter.threeByteOpVex64(VEX_SS /* = F3 */, OP3_SARX_GyEyBy, ESCAPE_38,
+                                 rm, src0, reg);
+  }
+
+  void shlxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+    spew("shlxq      %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+         GPReg64Name(dst));
+
+    RegisterID rm = src;
+    XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+    int reg = dst;
+    m_formatter.threeByteOpVex64(VEX_PD /* = 66 */, OP3_SHLX_GyEyBy, ESCAPE_38,
+                                 rm, src0, reg);
+  }
+
+  void shrxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+    spew("shrxq      %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+         GPReg64Name(dst));
+
+    RegisterID rm = src;
+    XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+    int reg = dst;
+    m_formatter.threeByteOpVex64(VEX_SD /* = F2 */, OP3_SHRX_GyEyBy, ESCAPE_38,
+                                 rm, src0, reg);
+  }
+
+ private:
+  [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+                                        TwoByteOpcodeID opcode,
+                                        XMMRegisterID reg) {
+    MOZ_ASSERT(!IsXMMReversedOperands(opcode));
+    m_formatter.legacySSEPrefix(ty);
+    m_formatter.twoByteRipOp(opcode, 0, reg);
+    JmpSrc label(m_formatter.size());
+    spew("%-11s " MEM_o32r ", %s", legacySSEOpName(name),
+         ADDR_o32r(label.offset()), XMMRegName(reg));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+                                        TwoByteOpcodeID opcode,
+                                        XMMRegisterID src0, XMMRegisterID dst) {
+    MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+    if (useLegacySSEEncoding(src0, dst)) {
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.twoByteRipOp(opcode, 0, dst);
+      JmpSrc label(m_formatter.size());
+      spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+           ADDR_o32r(label.offset()), XMMRegName(dst));
+      return label;
+    }
+
+    m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+    JmpSrc label(m_formatter.size());
+    spew("%-11s, " MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+         XMMRegName(src0), XMMRegName(dst));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc twoByteRipOpImmSimd(const char* name, VexOperandType ty,
+                                           TwoByteOpcodeID opcode, uint32_t imm,
+                                           XMMRegisterID src0,
+                                           XMMRegisterID dst) {
+    MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+    if (useLegacySSEEncoding(src0, dst)) {
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.twoByteRipOp(opcode, 0, dst);
+      m_formatter.immediate8u(imm);
+      JmpSrc label(m_formatter.size(),
+                   /* bytes trailing the patch field = */ 1);
+      spew("%-11s$0x%x, " MEM_o32r ", %s", legacySSEOpName(name), imm,
+           ADDR_o32r(label.offset()), XMMRegName(dst));
+      return label;
+    }
+
+    m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+    m_formatter.immediate8u(imm);
+    JmpSrc label(m_formatter.size(),
+                 /* bytes trailing the patch field = */ 1);
+    spew("%-11s$0x%x, " MEM_o32r ", %s, %s", name, imm,
+         ADDR_o32r(label.offset()), XMMRegName(src0), XMMRegName(dst));
+    return label;
+  }
+
+  void twoByteOpInt64Simd(const char* name, VexOperandType ty,
+                          TwoByteOpcodeID opcode, RegisterID rm,
+                          XMMRegisterID src0, XMMRegisterID dst) {
+    if (useLegacySSEEncoding(src0, dst)) {
+      if (IsXMMReversedOperands(opcode)) {
+        spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(dst),
+             GPRegName(rm));
+      } else {
+        spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(rm),
+             XMMRegName(dst));
+      }
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.twoByteOp64(opcode, rm, dst);
+      return;
+    }
+
+    if (src0 == invalid_xmm) {
+      if (IsXMMReversedOperands(opcode)) {
+        spew("%-11s%s, %s", name, XMMRegName(dst), GPRegName(rm));
+      } else {
+        spew("%-11s%s, %s", name, GPRegName(rm), XMMRegName(dst));
+      }
+    } else {
+      spew("%-11s%s, %s, %s", name, GPRegName(rm), XMMRegName(src0),
+           XMMRegName(dst));
+    }
+    m_formatter.twoByteOpVex64(ty, opcode, rm, src0, dst);
+  }
+
+  void twoByteOpSimdInt64(const char* name, VexOperandType ty,
+                          TwoByteOpcodeID opcode, XMMRegisterID rm,
+                          RegisterID dst) {
+    if (useLegacySSEEncodingAlways()) {
+      if (IsXMMReversedOperands(opcode)) {
+        spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(dst),
+             XMMRegName(rm));
+      } else if (opcode == OP2_MOVD_EdVd) {
+        spew("%-11s%s, %s", legacySSEOpName(name),
+             XMMRegName((XMMRegisterID)dst), GPRegName((RegisterID)rm));
+      } else {
+        spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+             GPRegName(dst));
+      }
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.twoByteOp64(opcode, (RegisterID)rm, dst);
+      return;
+    }
+
+    if (IsXMMReversedOperands(opcode)) {
+      spew("%-11s%s, %s", name, GPRegName(dst), XMMRegName(rm));
+    } else if (opcode == OP2_MOVD_EdVd) {
+      spew("%-11s%s, %s", name, XMMRegName((XMMRegisterID)dst),
+           GPRegName((RegisterID)rm));
+    } else {
+      spew("%-11s%s, %s", name, XMMRegName(rm), GPRegName(dst));
+    }
+    m_formatter.twoByteOpVex64(ty, opcode, (RegisterID)rm, invalid_xmm,
+                               (XMMRegisterID)dst);
+  }
+
+  [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+                                          ThreeByteOpcodeID opcode,
+                                          ThreeByteEscape escape,
+                                          XMMRegisterID dst) {
+    m_formatter.legacySSEPrefix(ty);
+    m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+    JmpSrc label(m_formatter.size());
+    spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+         ADDR_o32r(label.offset()), XMMRegName(dst));
+    return label;
+  }
+
+  [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+                                          ThreeByteOpcodeID opcode,
+                                          ThreeByteEscape escape,
+                                          XMMRegisterID src0,
+                                          XMMRegisterID dst) {
+    MOZ_ASSERT(src0 != invalid_xmm);
+    if (useLegacySSEEncoding(src0, dst)) {
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+      JmpSrc label(m_formatter.size());
+      spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+           ADDR_o32r(label.offset()), XMMRegName(dst));
+      return label;
+    }
+
+    m_formatter.threeByteRipOpVex(ty, opcode, escape, 0, src0, dst);
+    JmpSrc label(m_formatter.size());
+    spew("%-11s" MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+         XMMRegName(src0), XMMRegName(dst));
+    return label;
+  }
+
+  void threeByteOpImmSimdInt64(const char* name, VexOperandType ty,
+                               ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+                               uint32_t imm, XMMRegisterID src,
+                               RegisterID dst) {
+    spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(dst),
+         XMMRegName(src));
+    m_formatter.legacySSEPrefix(ty);
+    m_formatter.threeByteOp64(opcode, escape, dst, (RegisterID)src);
+    m_formatter.immediate8u(imm);
+  }
+
+  void threeByteOpImmInt64Simd(const char* name, VexOperandType ty,
+                               ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+                               uint32_t imm, RegisterID src1,
+                               XMMRegisterID src0, XMMRegisterID dst) {
+    if (useLegacySSEEncoding(src0, dst)) {
+      spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(src1),
+           XMMRegName(dst));
+      m_formatter.legacySSEPrefix(ty);
+      m_formatter.threeByteOp64(opcode, escape, src1, (RegisterID)dst);
+      m_formatter.immediate8u(imm);
+      return;
+    }
+
+    MOZ_ASSERT(src0 != invalid_xmm);
+    spew("%-11s$0x%x, %s, %s, %s", name, imm, GPReg64Name(src1),
+         XMMRegName(src0), XMMRegName(dst));
+    m_formatter.threeByteOpVex64(ty, opcode, escape, src1, src0,
+                                 (RegisterID)dst);
+    m_formatter.immediate8u(imm);
+  }
+};
+
+using BaseAssemblerSpecific = BaseAssemblerX64;
+
+}  // namespace X86Encoding
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_BaseAssembler_x64_h */
diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp
new file mode 100644
index 0000000000..9e5319842b
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.cpp
@@ -0,0 +1,990 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/CodeGenerator-x64.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "js/ScalarType.h"  // js::Scalar::Type
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+
+CodeGeneratorX64::CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph,
+                                   MacroAssembler* masm)
+    : CodeGeneratorX86Shared(gen, graph, masm) {}
+
+ValueOperand CodeGeneratorX64::ToValue(LInstruction* ins, size_t pos) {
+  return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorX64::ToTempValue(LInstruction* ins, size_t pos) {
+  return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+Operand CodeGeneratorX64::ToOperand64(const LInt64Allocation& a64) {
+  const LAllocation& a = a64.value();
+  MOZ_ASSERT(!a.isFloatReg());
+  if (a.isGeneralReg()) {
+    return Operand(a.toGeneralReg()->reg());
+  }
+  return Operand(ToAddress(a));
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+  ValueOperand result = ToOutValue(value);
+  masm.moveValue(value->value(), result);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+  const LAllocation* in = box->getOperand(0);
+  ValueOperand result = ToOutValue(box);
+
+  masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+
+  if (JitOptions.spectreValueMasking && IsFloatingPointType(box->type())) {
+    ScratchRegisterScope scratch(masm);
+    masm.movePtr(ImmWord(JSVAL_SHIFTED_TAG_MAX_DOUBLE), scratch);
+    masm.cmpPtrMovePtr(Assembler::Below, scratch, result.valueReg(), scratch,
+                       result.valueReg());
+  }
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+  MUnbox* mir = unbox->mir();
+
+  Register result = ToRegister(unbox->output());
+
+  if (mir->fallible()) {
+    const ValueOperand value = ToValue(unbox, LUnbox::Input);
+    Label bail;
+    switch (mir->type()) {
+      case MIRType::Int32:
+        masm.fallibleUnboxInt32(value, result, &bail);
+        break;
+      case MIRType::Boolean:
+        masm.fallibleUnboxBoolean(value, result, &bail);
+        break;
+      case MIRType::Object:
+        masm.fallibleUnboxObject(value, result, &bail);
+        break;
+      case MIRType::String:
+        masm.fallibleUnboxString(value, result, &bail);
+        break;
+      case MIRType::Symbol:
+        masm.fallibleUnboxSymbol(value, result, &bail);
+        break;
+      case MIRType::BigInt:
+        masm.fallibleUnboxBigInt(value, result, &bail);
+        break;
+      default:
+        MOZ_CRASH("Given MIRType cannot be unboxed.");
+    }
+    bailoutFrom(&bail, unbox->snapshot());
+    return;
+  }
+
+  // Infallible unbox.
+
+  Operand input = ToOperand(unbox->getOperand(LUnbox::Input));
+
+#ifdef DEBUG
+  // Assert the types match.
+  JSValueTag tag = MIRTypeToTag(mir->type());
+  Label ok;
+  masm.splitTag(input, ScratchReg);
+  masm.branch32(Assembler::Equal, ScratchReg, Imm32(tag), &ok);
+  masm.assumeUnreachable("Infallible unbox type mismatch");
+  masm.bind(&ok);
+#endif
+
+  switch (mir->type()) {
+    case MIRType::Int32:
+      masm.unboxInt32(input, result);
+      break;
+    case MIRType::Boolean:
+      masm.unboxBoolean(input, result);
+      break;
+    case MIRType::Object:
+      masm.unboxObject(input, result);
+      break;
+    case MIRType::String:
+      masm.unboxString(input, result);
+      break;
+    case MIRType::Symbol:
+      masm.unboxSymbol(input, result);
+      break;
+    case MIRType::BigInt:
+      masm.unboxBigInt(input, result);
+      break;
+    default:
+      MOZ_CRASH("Given MIRType cannot be unboxed.");
+  }
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+  MCompare* mir = lir->mir();
+  MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+             mir->compareType() == MCompare::Compare_UInt64);
+
+  const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+  const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+  Register lhsReg = ToRegister64(lhs).reg;
+  Register output = ToRegister(lir->output());
+
+  if (IsConstant(rhs)) {
+    masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+  } else {
+    masm.cmpPtr(lhsReg, ToOperand64(rhs));
+  }
+
+  bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+  masm.emitSet(JSOpToCondition(lir->jsop(), isSigned), output);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+  MCompare* mir = lir->cmpMir();
+  MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+             mir->compareType() == MCompare::Compare_UInt64);
+
+  LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+  LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+  Register lhsReg = ToRegister64(lhs).reg;
+
+  if (IsConstant(rhs)) {
+    masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+  } else {
+    masm.cmpPtr(lhsReg, ToOperand64(rhs));
+  }
+
+  bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+  emitBranch(JSOpToCondition(lir->jsop(), isSigned), lir->ifTrue(),
+             lir->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+  Register regL = ToRegister(baab->left());
+  if (baab->is64()) {
+    if (baab->right()->isConstant()) {
+      masm.test64(regL, Imm64(ToInt64(baab->right())));
+    } else {
+      masm.test64(regL, ToRegister(baab->right()));
+    }
+  } else {
+    if (baab->right()->isConstant()) {
+      masm.test32(regL, Imm32(ToInt32(baab->right())));
+    } else {
+      masm.test32(regL, ToRegister(baab->right()));
+    }
+  }
+  emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+  Register lhs = ToRegister(lir->lhs());
+  Register rhs = ToRegister(lir->rhs());
+  Register output = ToRegister(lir->output());
+
+  MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+  MOZ_ASSERT(rhs != rdx);
+  MOZ_ASSERT_IF(output == rax, ToRegister(lir->remainder()) == rdx);
+  MOZ_ASSERT_IF(output == rdx, ToRegister(lir->remainder()) == rax);
+
+  Label done;
+
+  // Put the lhs in rax.
+  if (lhs != rax) {
+    masm.mov(lhs, rax);
+  }
+
+  // Handle divide by zero.
+  if (lir->canBeDivideByZero()) {
+    Label nonZero;
+    masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+    masm.bind(&nonZero);
+  }
+
+  // Handle an integer overflow exception from INT64_MIN / -1.
+  if (lir->canBeNegativeOverflow()) {
+    Label notOverflow;
+    masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
+    masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
+    if (lir->mir()->isMod()) {
+      masm.xorl(output, output);
+    } else {
+      masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+    }
+    masm.jump(&done);
+    masm.bind(&notOverflow);
+  }
+
+  // Sign extend the lhs into rdx to make rdx:rax.
+  masm.cqo();
+  masm.idivq(rhs);
+
+  masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+  Register lhs = ToRegister(lir->lhs());
+  Register rhs = ToRegister(lir->rhs());
+
+  DebugOnly<Register> output = ToRegister(lir->output());
+  MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+  MOZ_ASSERT(rhs != rdx);
+  MOZ_ASSERT_IF(output.value == rax, ToRegister(lir->remainder()) == rdx);
+  MOZ_ASSERT_IF(output.value == rdx, ToRegister(lir->remainder()) == rax);
+
+  // Put the lhs in rax.
+  if (lhs != rax) {
+    masm.mov(lhs, rax);
+  }
+
+  Label done;
+
+  // Prevent divide by zero.
+  if (lir->canBeDivideByZero()) {
+    Label nonZero;
+    masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+    masm.bind(&nonZero);
+  }
+
+  // Zero extend the lhs into rdx to make (rdx:rax).
+  masm.xorl(rdx, rdx);
+  masm.udivq(rhs);
+
+  masm.bind(&done);
+}
+
+void CodeGeneratorX64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+                                     Register divisor, Register output,
+                                     Label* fail) {
+  // Callers handle division by zero and integer overflow.
+
+  MOZ_ASSERT(dividend == rax);
+  MOZ_ASSERT(output == rdx);
+
+  // Sign extend the lhs into rdx to make rdx:rax.
+  masm.cqo();
+
+  masm.idivq(divisor);
+
+  // Create and return the result.
+  masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+  masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorX64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+                                     Register divisor, Register output,
+                                     Label* fail) {
+  // Callers handle division by zero and integer overflow.
+
+  MOZ_ASSERT(dividend == rax);
+  MOZ_ASSERT(output == rdx);
+
+  // Sign extend the lhs into rdx to make rdx:rax.
+  masm.cqo();
+
+  masm.idivq(divisor);
+
+  // Move the remainder from rdx.
+  masm.movq(output, dividend);
+
+  // Create and return the result.
+  masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+  masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+  Register elements = ToRegister(lir->elements());
+  Register temp = ToRegister(lir->temp());
+  Register64 temp64 = ToRegister64(lir->temp64());
+  Register out = ToRegister(lir->output());
+
+  const MLoadUnboxedScalar* mir = lir->mir();
+
+  Scalar::Type storageType = mir->storageType();
+
+  // NOTE: the generated code must match the assembly code in gen_load in
+  // GenerateAtomicOperations.py
+  auto sync = Synchronization::Load();
+
+  masm.memoryBarrierBefore(sync);
+  if (lir->index()->isConstant()) {
+    Address source =
+        ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+    masm.load64(source, temp64);
+  } else {
+    BaseIndex source(elements, ToRegister(lir->index()),
+                     ScaleFromScalarType(storageType), mir->offsetAdjustment());
+    masm.load64(source, temp64);
+  }
+  masm.memoryBarrierAfter(sync);
+
+  emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+  Register elements = ToRegister(lir->elements());
+  Register value = ToRegister(lir->value());
+  Register64 temp1 = ToRegister64(lir->temp1());
+
+  Scalar::Type writeType = lir->mir()->writeType();
+
+  masm.loadBigInt64(value, temp1);
+
+  // NOTE: the generated code must match the assembly code in gen_store in
+  // GenerateAtomicOperations.py
+  auto sync = Synchronization::Store();
+
+  masm.memoryBarrierBefore(sync);
+  if (lir->index()->isConstant()) {
+    Address dest = ToAddress(elements, lir->index(), writeType);
+    masm.store64(temp1, dest);
+  } else {
+    BaseIndex dest(elements, ToRegister(lir->index()),
+                   ScaleFromScalarType(writeType));
+    masm.store64(temp1, dest);
+  }
+  masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+    LCompareExchangeTypedArrayElement64* lir) {
+  Register elements = ToRegister(lir->elements());
+  Register oldval = ToRegister(lir->oldval());
+  Register newval = ToRegister(lir->newval());
+  Register64 temp1 = ToRegister64(lir->temp1());
+  Register64 temp2 = ToRegister64(lir->temp2());
+  Register out = ToRegister(lir->output());
+
+  MOZ_ASSERT(temp1.reg == rax);
+
+  Scalar::Type arrayType = lir->mir()->arrayType();
+
+  masm.loadBigInt64(oldval, temp1);
+  masm.loadBigInt64(newval, temp2);
+
+  if (lir->index()->isConstant()) {
+    Address dest = ToAddress(elements, lir->index(), arrayType);
+    masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+  } else {
+    BaseIndex dest(elements, ToRegister(lir->index()),
+                   ScaleFromScalarType(arrayType));
+    masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+  }
+
+  emitCreateBigInt(lir, arrayType, temp1, out, temp2.reg);
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+    LAtomicExchangeTypedArrayElement64* lir) {
+  Register elements = ToRegister(lir->elements());
+  Register value = ToRegister(lir->value());
+  Register64 temp1 = ToRegister64(lir->temp1());
+  Register temp2 = ToRegister(lir->temp2());
+  Register out = ToRegister(lir->output());
+
+  Scalar::Type arrayType = lir->mir()->arrayType();
+
+  masm.loadBigInt64(value, temp1);
+
+  if (lir->index()->isConstant()) {
+    Address dest = ToAddress(elements, lir->index(), arrayType);
+    masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+  } else {
+    BaseIndex dest(elements, ToRegister(lir->index()),
+                   ScaleFromScalarType(arrayType));
+    masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+  }
+
+  emitCreateBigInt(lir, arrayType, temp1, out, temp2);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+    LAtomicTypedArrayElementBinop64* lir) {
+  MOZ_ASSERT(!lir->mir()->isForEffect());
+
+  Register elements = ToRegister(lir->elements());
+  Register value = ToRegister(lir->value());
+  Register64 temp1 = ToRegister64(lir->temp1());
+  Register64 temp2 = ToRegister64(lir->temp2());
+  Register out = ToRegister(lir->output());
+
+  Scalar::Type arrayType = lir->mir()->arrayType();
+  AtomicOp atomicOp = lir->mir()->operation();
+
+  masm.loadBigInt64(value, temp1);
+
+  Register64 fetchTemp = Register64(out);
+  Register64 fetchOut = temp2;
+  Register createTemp = temp1.reg;
+
+  // Add and Sub don't need |fetchTemp| and can save a `mov` when the value and
+  // output register are equal to each other.
+  if (atomicOp == AtomicFetchAddOp || atomicOp == AtomicFetchSubOp) {
+    fetchTemp = Register64::Invalid();
+    fetchOut = temp1;
+    createTemp = temp2.reg;
+  } else {
+    MOZ_ASSERT(temp2.reg == rax);
+  }
+
+  if (lir->index()->isConstant()) {
+    Address dest = ToAddress(elements, lir->index(), arrayType);
+    masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+                         fetchTemp, fetchOut);
+  } else {
+    BaseIndex dest(elements, ToRegister(lir->index()),
+                   ScaleFromScalarType(arrayType));
+    masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+                         fetchTemp, fetchOut);
+  }
+
+  emitCreateBigInt(lir, arrayType, fetchOut, out, createTemp);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+    LAtomicTypedArrayElementBinopForEffect64* lir) {
+  MOZ_ASSERT(lir->mir()->isForEffect());
+
+  Register elements = ToRegister(lir->elements());
+  Register value = ToRegister(lir->value());
+  Register64 temp1 = ToRegister64(lir->temp1());
+
+  Scalar::Type arrayType = lir->mir()->arrayType();
+  AtomicOp atomicOp = lir->mir()->operation();
+
+  masm.loadBigInt64(value, temp1);
+
+  if (lir->index()->isConstant()) {
+    Address dest = ToAddress(elements, lir->index(), arrayType);
+    masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+  } else {
+    BaseIndex dest(elements, ToRegister(lir->index()),
+                   ScaleFromScalarType(arrayType));
+    masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+  }
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+  MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+  Register cond = ToRegister(lir->condExpr());
+
+  Operand falseExpr = ToOperandOrRegister64(lir->falseExpr());
+
+  Register64 out = ToOutRegister64(lir);
+  MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+             "true expr is reused for input");
+
+  masm.test32(cond, cond);
+  masm.cmovzq(falseExpr, out.reg);
+}
+
+// We expect to handle only the cases: compare is {U,}Int{32,64}, and select
+// is {U,}Int{32,64}, independently.  Some values may be stack allocated, and
+// the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+  bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+                    ins->compareType() == MCompare::Compare_UInt32;
+  bool cmpIs64bit = ins->compareType() == MCompare::Compare_Int64 ||
+                    ins->compareType() == MCompare::Compare_UInt64;
+  bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+  bool selIs64bit = ins->mir()->type() == MIRType::Int64;
+
+  // Throw out unhandled cases
+  MOZ_RELEASE_ASSERT(
+      cmpIs32bit != cmpIs64bit && selIs32bit != selIs64bit,
+      "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+  using C = Assembler::Condition;
+  using R = Register;
+  using A = const Address&;
+
+  // Identify macroassembler methods to generate instructions, based on the
+  // type of the comparison and the select.  This avoids having to duplicate
+  // the code-generation tree below 4 times.  These assignments to
+  // `cmpMove_CRRRR` et al are unambiguous as a result of the combination of
+  // the template parameters and the 5 argument types ((C, R, R, R, R) etc).
+  void (MacroAssembler::*cmpMove_CRRRR)(C, R, R, R, R) = nullptr;
+  void (MacroAssembler::*cmpMove_CRARR)(C, R, A, R, R) = nullptr;
+  void (MacroAssembler::*cmpLoad_CRRAR)(C, R, R, A, R) = nullptr;
+  void (MacroAssembler::*cmpLoad_CRAAR)(C, R, A, A, R) = nullptr;
+
+  if (cmpIs32bit) {
+    if (selIs32bit) {
+      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 32>;
+      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 32>;
+      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+    } else {
+      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 64>;
+      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 64>;
+      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+    }
+  } else {
+    if (selIs32bit) {
+      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 32>;
+      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 32>;
+      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+    } else {
+      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 64>;
+      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 64>;
+      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+    }
+  }
+
+  Register trueExprAndDest = ToRegister(ins->output());
+  MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+             "true expr input is reused for output");
+
+  Assembler::Condition cond = Assembler::InvertCondition(
+      JSOpToCondition(ins->compareType(), ins->jsop()));
+  const LAllocation* rhs = ins->rightExpr();
+  const LAllocation* falseExpr = ins->ifFalseExpr();
+  Register lhs = ToRegister(ins->leftExpr());
+
+  // We generate one of four cmp+cmov pairings, depending on whether one of
+  // the cmp args and one of the cmov args is in memory or a register.
+  if (rhs->isRegister()) {
+    if (falseExpr->isRegister()) {
+      (masm.*cmpMove_CRRRR)(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+                            trueExprAndDest);
+    } else {
+      (masm.*cmpLoad_CRRAR)(cond, lhs, ToRegister(rhs), ToAddress(falseExpr),
+                            trueExprAndDest);
+    }
+  } else {
+    if (falseExpr->isRegister()) {
+      (masm.*cmpMove_CRARR)(cond, lhs, ToAddress(rhs), ToRegister(falseExpr),
+                            trueExprAndDest);
+    } else {
+      (masm.*cmpLoad_CRAAR)(cond, lhs, ToAddress(rhs), ToAddress(falseExpr),
+                            trueExprAndDest);
+    }
+  }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+  MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+  MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+  masm.vmovq(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+  MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+  MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+  masm.vmovq(ToFloatRegister(lir->input()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+  masm.convertUInt32ToDouble(ToRegister(lir->input()),
+                             ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+  masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+                              ToFloatRegister(lir->output()));
+}
+
+void CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access,
+                                 const LAllocation* value, Operand dstAddr) {
+  if (value->isConstant()) {
+    masm.memoryBarrierBefore(access.sync());
+
+    const MConstant* mir = value->toConstant();
+    Imm32 cst =
+        Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64());
+
+    switch (access.type()) {
+      case Scalar::Int8:
+      case Scalar::Uint8:
+        masm.append(access, wasm::TrapMachineInsn::Store8,
+                    FaultingCodeOffset(masm.currentOffset()));
+        masm.movb(cst, dstAddr);
+        break;
+      case Scalar::Int16:
+      case Scalar::Uint16:
+        masm.append(access, wasm::TrapMachineInsn::Store16,
+                    FaultingCodeOffset(masm.currentOffset()));
+        masm.movw(cst, dstAddr);
+        break;
+      case Scalar::Int32:
+      case Scalar::Uint32:
+        masm.append(access, wasm::TrapMachineInsn::Store32,
+                    FaultingCodeOffset(masm.currentOffset()));
+        masm.movl(cst, dstAddr);
+        break;
+      case Scalar::Int64:
+      case Scalar::Simd128:
+      case Scalar::Float32:
+      case Scalar::Float64:
+      case Scalar::Uint8Clamped:
+      case Scalar::BigInt64:
+      case Scalar::BigUint64:
+      case Scalar::MaxTypedArrayViewType:
+        MOZ_CRASH("unexpected array type");
+    }
+
+    masm.memoryBarrierAfter(access.sync());
+  } else {
+    masm.wasmStore(access, ToAnyRegister(value), dstAddr);
+  }
+}
+
+template <typename T>
+void CodeGeneratorX64::emitWasmLoad(T* ins) {
+  const MWasmLoad* mir = ins->mir();
+
+  mir->access().assertOffsetInGuardPages();
+  uint32_t offset = mir->access().offset();
+
+  // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+  // true 64-bit value.
+  const LAllocation* ptr = ins->ptr();
+  Register memoryBase = ToRegister(ins->memoryBase());
+  Operand srcAddr =
+      ptr->isBogus() ? Operand(memoryBase, offset)
+                     : Operand(memoryBase, ToRegister(ptr), TimesOne, offset);
+
+  if (mir->type() == MIRType::Int64) {
+    masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins));
+  } else {
+    masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output()));
+  }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); }
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); }
+
+template <typename T>
+void CodeGeneratorX64::emitWasmStore(T* ins) {
+  const MWasmStore* mir = ins->mir();
+  const wasm::MemoryAccessDesc& access = mir->access();
+
+  mir->access().assertOffsetInGuardPages();
+  uint32_t offset = access.offset();
+
+  const LAllocation* value = ins->getOperand(ins->ValueIndex);
+  const LAllocation* ptr = ins->ptr();
+  Register memoryBase = ToRegister(ins->memoryBase());
+  Operand dstAddr =
+      ptr->isBogus() ? Operand(memoryBase, offset)
+                     : Operand(memoryBase, ToRegister(ptr), TimesOne, offset);
+
+  wasmStore(access, value, dstAddr);
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); }
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) {
+  MOZ_CRASH("Unused on this platform");
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+    LWasmCompareExchangeHeap* ins) {
+  MWasmCompareExchangeHeap* mir = ins->mir();
+
+  Register ptr = ToRegister(ins->ptr());
+  Register oldval = ToRegister(ins->oldValue());
+  Register newval = ToRegister(ins->newValue());
+  Register memoryBase = ToRegister(ins->memoryBase());
+  MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+  Scalar::Type accessType = mir->access().type();
+  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset());
+
+  if (accessType == Scalar::Int64) {
+    masm.wasmCompareExchange64(mir->access(), srcAddr, Register64(oldval),
+                               Register64(newval), ToOutRegister64(ins));
+  } else {
+    masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval,
+                             ToRegister(ins->output()));
+  }
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+  MWasmAtomicExchangeHeap* mir = ins->mir();
+
+  Register ptr = ToRegister(ins->ptr());
+  Register value = ToRegister(ins->value());
+  Register memoryBase = ToRegister(ins->memoryBase());
+  MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+  Scalar::Type accessType = mir->access().type();
+
+  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset());
+
+  if (accessType == Scalar::Int64) {
+    masm.wasmAtomicExchange64(mir->access(), srcAddr, Register64(value),
+                              ToOutRegister64(ins));
+  } else {
+    masm.wasmAtomicExchange(mir->access(), srcAddr, value,
+                            ToRegister(ins->output()));
+  }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+  MWasmAtomicBinopHeap* mir = ins->mir();
+  MOZ_ASSERT(mir->hasUses());
+
+  Register ptr = ToRegister(ins->ptr());
+  Register memoryBase = ToRegister(ins->memoryBase());
+  const LAllocation* value = ins->value();
+  Register temp =
+      ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
+  Register output = ToRegister(ins->output());
+  MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+  Scalar::Type accessType = mir->access().type();
+  if (accessType == Scalar::Uint32) {
+    accessType = Scalar::Int32;
+  }
+
+  AtomicOp op = mir->operation();
+  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset());
+
+  if (accessType == Scalar::Int64) {
+    Register64 val = Register64(ToRegister(value));
+    Register64 out = Register64(output);
+    Register64 tmp = Register64(temp);
+    masm.wasmAtomicFetchOp64(mir->access(), op, val, srcAddr, tmp, out);
+  } else if (value->isConstant()) {
+    masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), srcAddr,
+                           temp, output);
+  } else {
+    masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr, temp,
+                           output);
+  }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+    LWasmAtomicBinopHeapForEffect* ins) {
+  MWasmAtomicBinopHeap* mir = ins->mir();
+  MOZ_ASSERT(!mir->hasUses());
+
+  Register ptr = ToRegister(ins->ptr());
+  Register memoryBase = ToRegister(ins->memoryBase());
+  const LAllocation* value = ins->value();
+  MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+  Scalar::Type accessType = mir->access().type();
+  AtomicOp op = mir->operation();
+
+  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset());
+
+  if (accessType == Scalar::Int64) {
+    Register64 val = Register64(ToRegister(value));
+    masm.wasmAtomicEffectOp64(mir->access(), op, val, srcAddr);
+  } else if (value->isConstant()) {
+    Imm32 c(0);
+    if (value->toConstant()->type() == MIRType::Int64) {
+      c = Imm32(ToInt64(value));
+    } else {
+      c = Imm32(ToInt32(value));
+    }
+    masm.wasmAtomicEffectOp(mir->access(), op, c, srcAddr, InvalidReg);
+  } else {
+    masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
+                            InvalidReg);
+  }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+  FloatRegister input = ToFloatRegister(ins->input());
+  Register output = ToRegister(ins->output());
+
+  // On x64, branchTruncateDouble uses vcvttsd2sq. Unlike the x86
+  // implementation, this should handle most doubles and we can just
+  // call a stub if it fails.
+  emitTruncateDouble(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+    LWasmBuiltinTruncateDToInt32* lir) {
+  FloatRegister input = ToFloatRegister(lir->getOperand(0));
+  Register output = ToRegister(lir->getDef(0));
+
+  emitTruncateDouble(input, output, lir->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+    LWasmBuiltinTruncateFToInt32* lir) {
+  FloatRegister input = ToFloatRegister(lir->getOperand(0));
+  Register output = ToRegister(lir->getDef(0));
+
+  emitTruncateFloat32(input, output, lir->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+  FloatRegister input = ToFloatRegister(ins->input());
+  Register output = ToRegister(ins->output());
+
+  // On x64, branchTruncateFloat32 uses vcvttss2sq. Unlike the x86
+  // implementation, this should handle most floats and we can just
+  // call a stub if it fails.
+  emitTruncateFloat32(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+  const LAllocation* input = lir->getOperand(0);
+  Register output = ToRegister(lir->output());
+
+  if (lir->mir()->bottomHalf()) {
+    masm.movl(ToOperand(input), output);
+  } else {
+    MOZ_CRASH("Not implemented.");
+  }
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+  const LAllocation* input = lir->getOperand(0);
+  Register output = ToRegister(lir->output());
+
+  if (lir->mir()->isUnsigned()) {
+    masm.movl(ToOperand(input), output);
+  } else {
+    masm.movslq(ToOperand(input), output);
+  }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+  // Generates no code on this platform because the input is assumed to have
+  // canonical form.
+  Register output = ToRegister(lir->output());
+  MOZ_ASSERT(ToRegister(lir->input()) == output);
+  masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+  // Generates no code on this platform because the input is assumed to have
+  // canonical form.
+  Register output = ToRegister(lir->output());
+  MOZ_ASSERT(ToRegister(lir->input()) == output);
+  masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) {
+  Register64 input = ToRegister64(ins->getInt64Operand(0));
+  Register64 output = ToOutRegister64(ins);
+  switch (ins->mode()) {
+    case MSignExtendInt64::Byte:
+      masm.movsbq(Operand(input.reg), output.reg);
+      break;
+    case MSignExtendInt64::Half:
+      masm.movswq(Operand(input.reg), output.reg);
+      break;
+    case MSignExtendInt64::Word:
+      masm.movslq(Operand(input.reg), output.reg);
+      break;
+  }
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+  FloatRegister input = ToFloatRegister(lir->input());
+  Register64 output = ToOutRegister64(lir);
+
+  MWasmTruncateToInt64* mir = lir->mir();
+  MIRType inputType = mir->input()->type();
+
+  MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
+
+  auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+  addOutOfLineCode(ool, mir);
+
+  FloatRegister temp =
+      mir->isUnsigned() ? ToFloatRegister(lir->temp()) : InvalidFloatReg;
+
+  Label* oolEntry = ool->entry();
+  Label* oolRejoin = ool->rejoin();
+  bool isSaturating = mir->isSaturating();
+  if (inputType == MIRType::Double) {
+    if (mir->isUnsigned()) {
+      masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+                                      oolRejoin, temp);
+    } else {
+      masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+                                     oolRejoin, temp);
+    }
+  } else {
+    if (mir->isUnsigned()) {
+      masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+                                       oolRejoin, temp);
+    } else {
+      masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+                                      oolRejoin, temp);
+    }
+  }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+  Register64 input = ToRegister64(lir->getInt64Operand(0));
+  FloatRegister output = ToFloatRegister(lir->output());
+
+  MInt64ToFloatingPoint* mir = lir->mir();
+  bool isUnsigned = mir->isUnsigned();
+
+  MIRType outputType = mir->type();
+  MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+  MOZ_ASSERT(isUnsigned == !lir->getTemp(0)->isBogusTemp());
+
+  if (outputType == MIRType::Double) {
+    if (isUnsigned) {
+      masm.convertUInt64ToDouble(input, output, ToRegister(lir->getTemp(0)));
+    } else {
+      masm.convertInt64ToDouble(input, output);
+    }
+  } else {
+    if (isUnsigned) {
+      masm.convertUInt64ToFloat32(input, output, ToRegister(lir->getTemp(0)));
+    } else {
+      masm.convertInt64ToFloat32(input, output);
+    }
+  }
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+  masm.cmpq(Imm32(0), ToRegister(lir->input()));
+  masm.emitSet(Assembler::Equal, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+  Register64 input = ToRegister64(lir->getInt64Operand(0));
+  Register64 output = ToOutRegister64(lir);
+  masm.clz64(input, output.reg);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+  Register64 input = ToRegister64(lir->getInt64Operand(0));
+  Register64 output = ToOutRegister64(lir);
+  masm.ctz64(input, output.reg);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+  const LAllocation* input = ins->getOperand(0);
+  MOZ_ASSERT(!input->isConstant());
+  Register inputR = ToRegister(input);
+  MOZ_ASSERT(inputR == ToRegister(ins->output()));
+  masm.notq(inputR);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+  Register input = ToRegister(lir->input());
+  masm.testq(input, input);
+  emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse());
+}
diff --git a/js/src/jit/x64/CodeGenerator-x64.h b/js/src/jit/x64/CodeGenerator-x64.h
new file mode 100644
index 0000000000..c3359d0190
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_CodeGenerator_x64_h
+#define jit_x64_CodeGenerator_x64_h
+
+#include "jit/x86-shared/CodeGenerator-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorX64 : public CodeGeneratorX86Shared {
+ protected:
+  CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+  Operand ToOperand64(const LInt64Allocation& a);
+  ValueOperand ToValue(LInstruction* ins, size_t pos);
+  ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+  void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+                     Register output, Label* fail);
+  void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+                     Register output, Label* fail);
+
+  void wasmStore(const wasm::MemoryAccessDesc& access, const LAllocation* value,
+                 Operand dstAddr);
+  template <typename T>
+  void emitWasmLoad(T* ins);
+  template <typename T>
+  void emitWasmStore(T* ins);
+};
+
+using CodeGeneratorSpecific = CodeGeneratorX64;
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_CodeGenerator_x64_h */
diff --git a/js/src/jit/x64/LIR-x64.h b/js/src/jit/x64/LIR-x64.h
new file mode 100644
index 0000000000..efaedc4499
--- /dev/null
+++ b/js/src/jit/x64/LIR-x64.h
@@ -0,0 +1,170 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_LIR_x64_h
+#define jit_x64_LIR_x64_h
+
+namespace js {
+namespace jit {
+
+// Given an untyped input, guards on whether it's a specific type and returns
+// the unboxed payload.
+class LUnboxBase : public LInstructionHelper<1, 1, 0> {
+ public:
+  LUnboxBase(LNode::Opcode op, const LAllocation& input)
+      : LInstructionHelper<1, 1, 0>(op) {
+    setOperand(0, input);
+  }
+
+  static const size_t Input = 0;
+
+  MUnbox* mir() const { return mir_->toUnbox(); }
+};
+
+class LUnbox : public LUnboxBase {
+ public:
+  LIR_HEADER(Unbox)
+
+  explicit LUnbox(const LAllocation& input) : LUnboxBase(classOpcode, input) {}
+
+  const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnboxBase {
+  MIRType type_;
+
+ public:
+  LIR_HEADER(UnboxFloatingPoint)
+
+  LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+      : LUnboxBase(classOpcode, input), type_(type) {}
+
+  MIRType type() const { return type_; }
+  const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+  LIR_HEADER(WasmUint32ToDouble)
+
+  explicit LWasmUint32ToDouble(const LAllocation& input)
+      : LInstructionHelper(classOpcode) {
+    setOperand(0, input);
+  }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+  LIR_HEADER(WasmUint32ToFloat32)
+
+  explicit LWasmUint32ToFloat32(const LAllocation& input)
+      : LInstructionHelper(classOpcode) {
+    setOperand(0, input);
+  }
+};
+
+class LDivOrModI64 : public LBinaryMath<1> {
+ public:
+  LIR_HEADER(DivOrModI64)
+
+  LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+               const LDefinition& temp)
+      : LBinaryMath(classOpcode) {
+    setOperand(0, lhs);
+    setOperand(1, rhs);
+    setTemp(0, temp);
+  }
+
+  const LDefinition* remainder() { return getTemp(0); }
+
+  MBinaryArithInstruction* mir() const {
+    MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+    return static_cast<MBinaryArithInstruction*>(mir_);
+  }
+  bool canBeDivideByZero() const {
+    if (mir_->isMod()) {
+      return mir_->toMod()->canBeDivideByZero();
+    }
+    return mir_->toDiv()->canBeDivideByZero();
+  }
+  bool canBeNegativeOverflow() const {
+    if (mir_->isMod()) {
+      return mir_->toMod()->canBeNegativeDividend();
+    }
+    return mir_->toDiv()->canBeNegativeOverflow();
+  }
+  wasm::BytecodeOffset bytecodeOffset() const {
+    MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+    if (mir_->isMod()) {
+      return mir_->toMod()->bytecodeOffset();
+    }
+    return mir_->toDiv()->bytecodeOffset();
+  }
+};
+
+// This class performs a simple x86 'div', yielding either a quotient or
+// remainder depending on whether this instruction is defined to output
+// rax (quotient) or rdx (remainder).
+class LUDivOrModI64 : public LBinaryMath<1> {
+ public:
+  LIR_HEADER(UDivOrModI64);
+
+  LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+                const LDefinition& temp)
+      : LBinaryMath(classOpcode) {
+    setOperand(0, lhs);
+    setOperand(1, rhs);
+    setTemp(0, temp);
+  }
+
+  const LDefinition* remainder() { return getTemp(0); }
+
+  const char* extraName() const {
+    return mir()->isTruncated() ? "Truncated" : nullptr;
+  }
+
+  MBinaryArithInstruction* mir() const {
+    MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+    return static_cast<MBinaryArithInstruction*>(mir_);
+  }
+
+  bool canBeDivideByZero() const {
+    if (mir_->isMod()) {
+      return mir_->toMod()->canBeDivideByZero();
+    }
+    return mir_->toDiv()->canBeDivideByZero();
+  }
+
+  wasm::BytecodeOffset bytecodeOffset() const {
+    MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+    if (mir_->isMod()) {
+      return mir_->toMod()->bytecodeOffset();
+    }
+    return mir_->toDiv()->bytecodeOffset();
+  }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 1> {
+ public:
+  LIR_HEADER(WasmTruncateToInt64);
+
+  LWasmTruncateToInt64(const LAllocation& in, const LDefinition& temp)
+      : LInstructionHelper(classOpcode) {
+    setOperand(0, in);
+    setTemp(0, temp);
+  }
+
+  MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+
+  const LDefinition* temp() { return getTemp(0); }
+};
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_LIR_x64_h */
diff --git a/js/src/jit/x64/Lowering-x64.cpp b/js/src/jit/x64/Lowering-x64.cpp
new file mode 100644
index 0000000000..55d83e3f05
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.cpp
@@ -0,0 +1,581 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Lowering-x64.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/x64/Assembler-x64.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+LBoxAllocation LIRGeneratorX64::useBoxFixed(MDefinition* mir, Register reg1,
+                                            Register, bool useAtStart) {
+  MOZ_ASSERT(mir->type() == MIRType::Value);
+
+  ensureDefined(mir);
+  return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegister(MDefinition* mir) {
+  return useRegister(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterAtStart(MDefinition* mir) {
+  return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterOrNonDoubleConstant(
+    MDefinition* mir) {
+  return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorX64::tempByteOpRegister() { return temp(); }
+
+LDefinition LIRGeneratorX64::tempToUnbox() { return temp(); }
+
+void LIRGeneratorX64::lowerForALUInt64(
+    LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+    MDefinition* input) {
+  ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+  defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForALUInt64(
+    LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+    MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+  ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+  ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+                                         ? useInt64OrConstant(rhs)
+                                         : useInt64OrConstantAtStart(rhs));
+  defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForMulInt64(LMulI64* ins, MMul* mir,
+                                       MDefinition* lhs, MDefinition* rhs) {
+  // X64 doesn't need a temp for 64bit multiplication.
+  ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+  ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+                                         ? useInt64OrConstant(rhs)
+                                         : useInt64OrConstantAtStart(rhs));
+  defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+  MDefinition* opd = box->getOperand(0);
+
+  // If the operand is a constant, emit near its uses.
+  if (opd->isConstant() && box->canEmitAtUses()) {
+    emitAtUses(box);
+    return;
+  }
+
+  if (opd->isConstant()) {
+    define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+           LDefinition(LDefinition::BOX));
+  } else {
+    LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+    define(ins, box, LDefinition(LDefinition::BOX));
+  }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+  MDefinition* box = unbox->getOperand(0);
+  MOZ_ASSERT(box->type() == MIRType::Value);
+
+  LUnboxBase* lir;
+  if (IsFloatingPointType(unbox->type())) {
+    lir = new (alloc())
+        LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+  } else if (unbox->fallible()) {
+    // If the unbox is fallible, load the Value in a register first to
+    // avoid multiple loads.
+    lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+  } else {
+    lir = new (alloc()) LUnbox(useAtStart(box));
+  }
+
+  if (unbox->fallible()) {
+    assignSnapshot(lir, unbox->bailoutKind());
+  }
+
+  define(lir, unbox);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+  MOZ_ASSERT(opd->type() == MIRType::Value);
+
+  LReturn* ins = new (alloc()) LReturn(isGenerator);
+  ins->setOperand(0, useFixed(opd, JSReturnReg));
+  add(ins);
+}
+
+void LIRGeneratorX64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+                                           LBlock* block, size_t lirIndex) {
+  lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGeneratorX64::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+  defineTypedPhi(phi, lirIndex);
+}
+
+void LIRGeneratorX64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+                                         LBlock* block, size_t lirIndex) {
+  lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+    MCompareExchangeTypedArrayElement* ins) {
+  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+  if (Scalar::isBigIntType(ins->arrayType())) {
+    LUse elements = useRegister(ins->elements());
+    LAllocation index =
+        useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+    LUse oldval = useRegister(ins->oldval());
+    LUse newval = useRegister(ins->newval());
+    LInt64Definition temp1 = tempInt64Fixed(Register64(rax));
+    LInt64Definition temp2 = tempInt64();
+
+    auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+        elements, index, oldval, newval, temp1, temp2);
+    define(lir, ins);
+    assignSafepoint(lir, ins);
+    return;
+  }
+
+  lowerCompareExchangeTypedArrayElement(ins,
+                                        /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+    MAtomicExchangeTypedArrayElement* ins) {
+  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+  if (Scalar::isBigIntType(ins->arrayType())) {
+    LUse elements = useRegister(ins->elements());
+    LAllocation index =
+        useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+    LAllocation value = useRegister(ins->value());
+    LInt64Definition temp1 = tempInt64();
+    LDefinition temp2 = temp();
+
+    auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+        elements, index, value, temp1, temp2);
+    define(lir, ins);
+    assignSafepoint(lir, ins);
+    return;
+  }
+
+  lowerAtomicExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+    MAtomicTypedArrayElementBinop* ins) {
+  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+  if (Scalar::isBigIntType(ins->arrayType())) {
+    LUse elements = useRegister(ins->elements());
+    LAllocation index =
+        useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+    LAllocation value = useRegister(ins->value());
+
+    // Case 1: the result of the operation is not used.
+    //
+    // We can omit allocating the result BigInt.
+
+    if (ins->isForEffect()) {
+      LInt64Definition temp = tempInt64();
+
+      auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+          elements, index, value, temp);
+      add(lir, ins);
+      return;
+    }
+
+    // Case 2: the result of the operation is used.
+    //
+    // For ADD and SUB we'll use XADD.
+    //
+    // For AND/OR/XOR we need to use a CMPXCHG loop with rax as a temp register.
+
+    bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+                   ins->operation() == AtomicFetchSubOp);
+
+    LInt64Definition temp1 = tempInt64();
+    LInt64Definition temp2;
+    if (bitOp) {
+      temp2 = tempInt64Fixed(Register64(rax));
+    } else {
+      temp2 = tempInt64();
+    }
+
+    auto* lir = new (alloc())
+        LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+    define(lir, ins);
+    assignSafepoint(lir, ins);
+    return;
+  }
+
+  lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGeneratorX64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+  const LUse elements = useRegister(ins->elements());
+  const LAllocation index =
+      useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+  auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+  define(lir, ins);
+  assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+  LUse elements = useRegister(ins->elements());
+  LAllocation index =
+      useRegisterOrIndexConstant(ins->index(), ins->writeType());
+  LAllocation value = useRegister(ins->value());
+
+  add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+  MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+  LWasmUint32ToDouble* lir =
+      new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+  define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+  MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+  LWasmUint32ToFloat32* lir =
+      new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+  define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+  MDefinition* base = ins->base();
+  // 'base' is a GPR but may be of either type.  If it is 32-bit it is
+  // zero-extended and can act as 64-bit.
+  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+  LAllocation memoryBase =
+      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
+                           : LGeneralReg(HeapReg);
+
+  if (ins->type() != MIRType::Int64) {
+    auto* lir =
+        new (alloc()) LWasmLoad(useRegisterOrZeroAtStart(base), memoryBase);
+    define(lir, ins);
+    return;
+  }
+
+  auto* lir =
+      new (alloc()) LWasmLoadI64(useRegisterOrZeroAtStart(base), memoryBase);
+  defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+  MDefinition* base = ins->base();
+  // See comment in visitWasmLoad re the type of 'base'.
+  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+  MDefinition* value = ins->value();
+  LAllocation valueAlloc;
+  switch (ins->access().type()) {
+    case Scalar::Int8:
+    case Scalar::Uint8:
+    case Scalar::Int16:
+    case Scalar::Uint16:
+    case Scalar::Int32:
+    case Scalar::Uint32:
+      valueAlloc = useRegisterOrConstantAtStart(value);
+      break;
+    case Scalar::Int64:
+      // No way to encode an int64-to-memory move on x64.
+      if (value->isConstant() && value->type() != MIRType::Int64) {
+        valueAlloc = useOrConstantAtStart(value);
+      } else {
+        valueAlloc = useRegisterAtStart(value);
+      }
+      break;
+    case Scalar::Float32:
+    case Scalar::Float64:
+      valueAlloc = useRegisterAtStart(value);
+      break;
+    case Scalar::Simd128:
+#ifdef ENABLE_WASM_SIMD
+      valueAlloc = useRegisterAtStart(value);
+      break;
+#else
+      MOZ_CRASH("unexpected array type");
+#endif
+    case Scalar::BigInt64:
+    case Scalar::BigUint64:
+    case Scalar::Uint8Clamped:
+    case Scalar::MaxTypedArrayViewType:
+      MOZ_CRASH("unexpected array type");
+  }
+
+  LAllocation baseAlloc = useRegisterOrZeroAtStart(base);
+  LAllocation memoryBaseAlloc =
+      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
+                           : LGeneralReg(HeapReg);
+  auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc, memoryBaseAlloc);
+  add(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+  MDefinition* base = ins->base();
+  // See comment in visitWasmLoad re the type of 'base'.
+  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+  // The output may not be used but will be clobbered regardless, so
+  // pin the output to eax.
+  //
+  // The input values must both be in registers.
+
+  const LAllocation oldval = useRegister(ins->oldValue());
+  const LAllocation newval = useRegister(ins->newValue());
+  const LAllocation memoryBase =
+      ins->hasMemoryBase() ? LAllocation(useRegister(ins->memoryBase()))
+                           : LGeneralReg(HeapReg);
+
+  LWasmCompareExchangeHeap* lir = new (alloc())
+      LWasmCompareExchangeHeap(useRegister(base), oldval, newval, memoryBase);
+
+  defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+  // See comment in visitWasmLoad re the type of 'base'.
+  MOZ_ASSERT(ins->base()->type() == MIRType::Int32 ||
+             ins->base()->type() == MIRType::Int64);
+
+  const LAllocation base = useRegister(ins->base());
+  const LAllocation value = useRegister(ins->value());
+  const LAllocation memoryBase =
+      ins->hasMemoryBase() ? LAllocation(useRegister(ins->memoryBase()))
+                           : LGeneralReg(HeapReg);
+
+  // The output may not be used but will be clobbered regardless,
+  // so ignore the case where we're not using the value and just
+  // use the output register as a temp.
+
+  LWasmAtomicExchangeHeap* lir =
+      new (alloc()) LWasmAtomicExchangeHeap(base, value, memoryBase);
+  define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+  MDefinition* base = ins->base();
+  // See comment in visitWasmLoad re the type of 'base'.
+  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+  const LAllocation memoryBase =
+      ins->hasMemoryBase() ? LAllocation(useRegister(ins->memoryBase()))
+                           : LGeneralReg(HeapReg);
+
+  // No support for 64-bit operations with constants at the masm level.
+
+  bool canTakeConstant = ins->access().type() != Scalar::Int64;
+
+  // Case 1: the result of the operation is not used.
+  //
+  // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
+  // LOCK OR, or LOCK XOR.
+
+  if (!ins->hasUses()) {
+    LAllocation value = canTakeConstant ? useRegisterOrConstant(ins->value())
+                                        : useRegister(ins->value());
+    LWasmAtomicBinopHeapForEffect* lir =
+        new (alloc()) LWasmAtomicBinopHeapForEffect(
+            useRegister(base), value, LDefinition::BogusTemp(), memoryBase);
+    add(lir, ins);
+    return;
+  }
+
+  // Case 2: the result of the operation is used.
+  //
+  // For ADD and SUB we'll use XADD with word and byte ops as
+  // appropriate.  Any output register can be used and if value is a
+  // register it's best if it's the same as output:
+  //
+  //    movl       value, output  ; if value != output
+  //    lock xaddl output, mem
+  //
+  // For AND/OR/XOR we need to use a CMPXCHG loop, and the output is
+  // always in rax:
+  //
+  //    movl          *mem, rax
+  // L: mov           rax, temp
+  //    andl          value, temp
+  //    lock cmpxchg  temp, mem  ; reads rax also
+  //    jnz           L
+  //    ; result in rax
+  //
+  // Note the placement of L, cmpxchg will update rax with *mem if
+  // *mem does not have the expected value, so reloading it at the
+  // top of the loop would be redundant.
+
+  bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+                 ins->operation() == AtomicFetchSubOp);
+  bool reuseInput = false;
+  LAllocation value;
+
+  if (bitOp || ins->value()->isConstant()) {
+    value = canTakeConstant ? useRegisterOrConstant(ins->value())
+                            : useRegister(ins->value());
+  } else {
+    reuseInput = true;
+    value = useRegisterAtStart(ins->value());
+  }
+
+  auto* lir = new (alloc()) LWasmAtomicBinopHeap(
+      useRegister(base), value, bitOp ? temp() : LDefinition::BogusTemp(),
+      LDefinition::BogusTemp(), memoryBase);
+
+  if (reuseInput) {
+    defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp);
+  } else if (bitOp) {
+    defineFixed(lir, ins, LAllocation(AnyRegister(rax)));
+  } else {
+    define(lir, ins);
+  }
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+  LSubstr* lir = new (alloc())
+      LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+              useRegister(ins->length()), temp(), temp(), tempByteOpRegister());
+  define(lir, ins);
+  assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerDivI64(MDiv* div) {
+  if (div->isUnsigned()) {
+    lowerUDivI64(div);
+    return;
+  }
+
+  LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+      useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+  defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+  MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorX64::lowerModI64(MMod* mod) {
+  if (mod->isUnsigned()) {
+    lowerUModI64(mod);
+    return;
+  }
+
+  LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+      useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+  defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+  MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorX64::lowerUDivI64(MDiv* div) {
+  LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+      useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+  defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerUModI64(MMod* mod) {
+  LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+      useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+  defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerBigIntDiv(MBigIntDiv* ins) {
+  auto* lir = new (alloc()) LBigIntDiv(
+      useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+  defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+  assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerBigIntMod(MBigIntMod* ins) {
+  auto* lir = new (alloc()) LBigIntMod(
+      useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+  defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+  assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+  MDefinition* opd = ins->input();
+  MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+  LDefinition maybeTemp =
+      ins->isUnsigned() ? tempDouble() : LDefinition::BogusTemp();
+  defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), maybeTemp),
+              ins);
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinTruncateToInt64(
+    MWasmBuiltinTruncateToInt64* ins) {
+  MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+  MDefinition* opd = ins->input();
+  MOZ_ASSERT(opd->type() == MIRType::Int64);
+  MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+  LDefinition maybeTemp = ins->isUnsigned() ? temp() : LDefinition::BogusTemp();
+  define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp),
+         ins);
+}
+
+void LIRGeneratorX64::lowerBuiltinInt64ToFloatingPoint(
+    MBuiltinInt64ToFloatingPoint* ins) {
+  MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+  defineInt64(new (alloc()) LExtendInt32ToInt64(useAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+  defineInt64(new (alloc())
+                  LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+              ins);
+}
+
+// On x64 we specialize the cases: compare is {U,}Int{32,64}, and select is
+// {U,}Int{32,64}, independently.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+    MCompare::CompareType compTy, MIRType insTy) {
+  return (insTy == MIRType::Int32 || insTy == MIRType::Int64) &&
+         (compTy == MCompare::Compare_Int32 ||
+          compTy == MCompare::Compare_UInt32 ||
+          compTy == MCompare::Compare_Int64 ||
+          compTy == MCompare::Compare_UInt64);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+                                                   MDefinition* lhs,
+                                                   MDefinition* rhs,
+                                                   MCompare::CompareType compTy,
+                                                   JSOp jsop) {
+  MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+  auto* lir = new (alloc()) LWasmCompareAndSelect(
+      useRegister(lhs), useAny(rhs), compTy, jsop,
+      useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr()));
+  defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
diff --git a/js/src/jit/x64/Lowering-x64.h b/js/src/jit/x64/Lowering-x64.h
new file mode 100644
index 0000000000..1c34ea8693
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Lowering_x64_h
+#define jit_x64_Lowering_x64_h
+
+#include "jit/x86-shared/Lowering-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorX64 : public LIRGeneratorX86Shared {
+ protected:
+  LIRGeneratorX64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+      : LIRGeneratorX86Shared(gen, graph, lirGraph) {}
+
+  void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+                            size_t lirIndex);
+  void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+                          size_t lirIndex);
+  void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+  void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+                        MDefinition* mir, MDefinition* input);
+  void lowerForALUInt64(
+      LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+      MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+  void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+                        MDefinition* rhs);
+
+  // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+  LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register,
+                             bool useAtStart = false);
+
+  // x86 has constraints on what registers can be formatted for 1-byte
+  // stores and loads; on x64 all registers are okay.
+  LAllocation useByteOpRegister(MDefinition* mir);
+  LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+  LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+  LDefinition tempByteOpRegister();
+
+  LDefinition tempToUnbox();
+
+  bool needTempForPostBarrier() { return true; }
+
+  void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+  void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+  void lowerDivI64(MDiv* div);
+  void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+  void lowerModI64(MMod* mod);
+  void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+  void lowerUDivI64(MDiv* div);
+  void lowerUModI64(MMod* mod);
+
+  void lowerBigIntDiv(MBigIntDiv* ins);
+  void lowerBigIntMod(MBigIntMod* ins);
+
+  void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+  void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+};
+
+using LIRGeneratorSpecific = LIRGeneratorX64;
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_Lowering_x64_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h
new file mode 100644
index 0000000000..6869e6c4b6
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64-inl.h
@@ -0,0 +1,1099 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_inl_h
+#define jit_x64_MacroAssembler_x64_inl_h
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+// ===============================================================
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+  // Use mov instead of movq because it has special optimizations for imm == 0.
+  mov(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+  movq(src.reg, dest.reg);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+  vmovq(src, dest.reg);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+  vmovq(src.reg, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+  movl(src.reg, dest);
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+  movl(src, dest.reg);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+  movsbq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+  movswq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+  movslq(src, dest.reg);
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+  movslq(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+  movl(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+  movslq(Operand(src), dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::notPtr(Register reg) { notq(reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { andq(src, dest); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { andq(imm, dest); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+  if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+    andq(Imm32(imm.value), dest.reg);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    movq(ImmWord(uintptr_t(imm.value)), scratch);
+    andq(scratch, dest.reg);
+  }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+  if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+    orq(Imm32(imm.value), dest.reg);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    movq(ImmWord(uintptr_t(imm.value)), scratch);
+    orq(scratch, dest.reg);
+  }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+  if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+    xorq(Imm32(imm.value), dest.reg);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    movq(ImmWord(uintptr_t(imm.value)), scratch);
+    xorq(scratch, dest.reg);
+  }
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { orq(src, dest); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { orq(imm, dest); }
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+  andq(src.reg, dest.reg);
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+  orq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+  xorq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { xorq(src, dest); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorq(imm, dest); }
+
+void MacroAssembler::and64(const Operand& src, Register64 dest) {
+  andq(src, dest.reg);
+}
+
+void MacroAssembler::or64(const Operand& src, Register64 dest) {
+  orq(src, dest.reg);
+}
+
+void MacroAssembler::xor64(const Operand& src, Register64 dest) {
+  xorq(src, dest.reg);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg) { bswapq(reg.reg); }
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) { addq(src, dest); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { addq(imm, dest); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(dest != scratch);
+  if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+    addq(Imm32((int32_t)imm.value), dest);
+  } else {
+    mov(imm, scratch);
+    addq(scratch, dest);
+  }
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+  addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
+  addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+  addq(Operand(src), dest);
+}
+
+void MacroAssembler::add64(const Operand& src, Register64 dest) {
+  addq(src, dest.reg);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+  addq(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) { addq(imm, dest.reg); }
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+  addPtr(ImmWord(imm.value), dest.reg);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+  moveStackPtrTo(dest);
+  addqWithPatch(Imm32(0), dest);
+  return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+  patchAddq(offset, -imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { subq(src, dest); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+  subq(src, Operand(dest));
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) { subq(imm, dest); }
+
+void MacroAssembler::subPtr(ImmWord imm, Register dest) {
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(dest != scratch);
+  if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+    subq(Imm32((int32_t)imm.value), dest);
+  } else {
+    mov(imm, scratch);
+    subq(scratch, dest);
+  }
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+  subq(Operand(addr), dest);
+}
+
+void MacroAssembler::sub64(const Operand& src, Register64 dest) {
+  subq(src, dest.reg);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+  subq(src.reg, dest.reg);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+  subPtr(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+  // To compute the unsigned multiplication using imulq, we have to ensure both
+  // operands don't have any bits set in the high word.
+
+  if (imm.value >= 0) {
+    // Clear the high word of |src|.
+    movl(src, src);
+
+    // |imm| and |src| are both positive, so directly perform imulq.
+    imulq(imm, src, dest);
+  } else {
+    // Store the low word of |src| into |dest|.
+    movl(src, dest);
+
+    // Compute the unsigned value of |imm| before performing imulq.
+    movl(imm, ScratchReg);
+    imulq(ScratchReg, dest);
+  }
+
+  // Move the high word into |dest|.
+  shrq(Imm32(32), dest);
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+  imulq(rhs, srcDest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+                           const Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  mul64(imm, dest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+  if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+    imulq(Imm32((int32_t)imm.value), dest.reg, dest.reg);
+  } else {
+    movq(ImmWord(uintptr_t(imm.value)), ScratchReg);
+    imulq(ScratchReg, dest.reg);
+  }
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+                           const Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  mul64(Operand(src.reg), dest);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest) {
+  imulq(src, dest.reg);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest,
+                           const Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  mul64(src, dest);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+  lea(Operand(src, src, TimesTwo), dest);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+                                  FloatRegister dest) {
+  movq(imm, ScratchReg);
+  vmulsd(Operand(ScratchReg, 0), dest, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+  if (X86Encoding::IsAddressImmediate(dest.addr)) {
+    addPtr(Imm32(1), dest);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    mov(ImmPtr(dest.addr), scratch);
+    addPtr(Imm32(1), Address(scratch, 0));
+  }
+}
+
+void MacroAssembler::neg64(Register64 reg) { negq(reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) { negq(reg); }
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+  shlq(imm, dest);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
+  if (Assembler::HasBMI2()) {
+    shlxq(srcDest, shift, srcDest);
+    return;
+  }
+  MOZ_ASSERT(shift == rcx);
+  shlq_cl(srcDest);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+  lshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+  if (Assembler::HasBMI2()) {
+    shlxq(srcDest.reg, shift, srcDest.reg);
+    return;
+  }
+  MOZ_ASSERT(shift == rcx);
+  shlq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+  shrq(imm, dest);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
+  if (Assembler::HasBMI2()) {
+    shrxq(srcDest, shift, srcDest);
+    return;
+  }
+  MOZ_ASSERT(shift == rcx);
+  shrq_cl(srcDest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+  rshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+  if (Assembler::HasBMI2()) {
+    shrxq(srcDest.reg, shift, srcDest.reg);
+    return;
+  }
+  MOZ_ASSERT(shift == rcx);
+  shrq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+  sarq(imm, dest);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+  rshiftPtrArithmetic(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+  if (Assembler::HasBMI2()) {
+    sarxq(srcDest.reg, shift, srcDest.reg);
+    return;
+  }
+  MOZ_ASSERT(shift == rcx);
+  sarq_cl(srcDest.reg);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+                                  Register64 dest) {
+  MOZ_ASSERT(src == dest, "defineReuseInput");
+  MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+  rolq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+                                  Register64 dest, Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+                                   Register64 dest) {
+  MOZ_ASSERT(src == dest, "defineReuseInput");
+  MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+  rorq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+                                   Register64 dest, Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  rotateRight64(count, src, dest);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src,
+                                  Register64 dest) {
+  MOZ_ASSERT(src == dest, "defineReuseInput");
+  rolq(count, dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+                                  Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src,
+                                   Register64 dest) {
+  MOZ_ASSERT(src == dest, "defineReuseInput");
+  rorq(count, dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+                                   Register temp) {
+  MOZ_ASSERT(temp == InvalidReg);
+  rotateRight64(count, src, dest);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+                              Register dest) {
+  cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+  cmpPtr(lhs, rhs);
+  emitSet(cond, dest);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+  if (AssemblerX86Shared::HasLZCNT()) {
+    lzcntq(src.reg, dest);
+    return;
+  }
+
+  Label nonzero;
+  bsrq(src.reg, dest);
+  j(Assembler::NonZero, &nonzero);
+  movq(ImmWord(0x7F), dest);
+  bind(&nonzero);
+  xorq(Imm32(0x3F), dest);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+  if (AssemblerX86Shared::HasBMI1()) {
+    tzcntq(src.reg, dest);
+    return;
+  }
+
+  Label nonzero;
+  bsfq(src.reg, dest);
+  j(Assembler::NonZero, &nonzero);
+  movq(ImmWord(64), dest);
+  bind(&nonzero);
+}
+
+void MacroAssembler::popcnt64(Register64 src64, Register64 dest64,
+                              Register tmp) {
+  Register src = src64.reg;
+  Register dest = dest64.reg;
+
+  if (AssemblerX86Shared::HasPOPCNT()) {
+    MOZ_ASSERT(tmp == InvalidReg);
+    popcntq(src, dest);
+    return;
+  }
+
+  if (src != dest) {
+    movq(src, dest);
+  }
+
+  MOZ_ASSERT(tmp != dest);
+
+  ScratchRegisterScope scratch(*this);
+
+  // Equivalent to mozilla::CountPopulation32, adapted for 64 bits.
+  // x -= (x >> 1) & m1;
+  movq(src, tmp);
+  movq(ImmWord(0x5555555555555555), scratch);
+  shrq(Imm32(1), tmp);
+  andq(scratch, tmp);
+  subq(tmp, dest);
+
+  // x = (x & m2) + ((x >> 2) & m2);
+  movq(dest, tmp);
+  movq(ImmWord(0x3333333333333333), scratch);
+  andq(scratch, dest);
+  shrq(Imm32(2), tmp);
+  andq(scratch, tmp);
+  addq(tmp, dest);
+
+  // x = (x + (x >> 4)) & m4;
+  movq(dest, tmp);
+  movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch);
+  shrq(Imm32(4), tmp);
+  addq(tmp, dest);
+  andq(scratch, dest);
+
+  // (x * h01) >> 56
+  movq(ImmWord(0x0101010101010101), scratch);
+  imulq(scratch, dest);
+  shrq(Imm32(56), dest);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+                              Register rhs, Label* label) {
+  if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+    branch32(cond, Operand(lhs), rhs, label);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    mov(ImmPtr(lhs.addr), scratch);
+    branch32(cond, Address(scratch, 0), rhs, label);
+  }
+}
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+                              Imm32 rhs, Label* label) {
+  if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+    branch32(cond, Operand(lhs), rhs, label);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    mov(ImmPtr(lhs.addr), scratch);
+    branch32(cond, Address(scratch, 0), rhs, label);
+  }
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+                              Imm32 rhs, Label* label) {
+  ScratchRegisterScope scratch(*this);
+  mov(lhs, scratch);
+  branch32(cond, Address(scratch, 0), rhs, label);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+                              Label* success, Label* fail) {
+  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+                 cond == Assembler::LessThan ||
+                 cond == Assembler::LessThanOrEqual ||
+                 cond == Assembler::GreaterThan ||
+                 cond == Assembler::GreaterThanOrEqual ||
+                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+             "other condition codes not supported");
+
+  branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+  if (fail) {
+    jump(fail);
+  }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+                              Label* success, Label* fail) {
+  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+                 cond == Assembler::LessThan ||
+                 cond == Assembler::LessThanOrEqual ||
+                 cond == Assembler::GreaterThan ||
+                 cond == Assembler::GreaterThanOrEqual ||
+                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+             "other condition codes not supported");
+
+  branchPtr(cond, lhs.reg, rhs.reg, success);
+  if (fail) {
+    jump(fail);
+  }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+                              Label* label) {
+  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+             "other condition codes not supported");
+
+  branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+                              Register64 rhs, Label* label) {
+  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+             "other condition codes not supported");
+
+  branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+                              const Address& rhs, Register scratch,
+                              Label* label) {
+  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+             "other condition codes not supported");
+  MOZ_ASSERT(lhs.base != scratch);
+  MOZ_ASSERT(rhs.base != scratch);
+
+  loadPtr(rhs, scratch);
+  branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+                               Register rhs, Label* label) {
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(rhs != scratch);
+  if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+    branchPtrImpl(cond, Operand(lhs), rhs, label);
+  } else {
+    mov(ImmPtr(lhs.addr), scratch);
+    branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+  }
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+                               ImmWord rhs, Label* label) {
+  if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+    branchPtrImpl(cond, Operand(lhs), rhs, label);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    mov(ImmPtr(lhs.addr), scratch);
+    branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+  }
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+                               Register rhs, Label* label) {
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(rhs != scratch);
+  mov(lhs, scratch);
+  branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+                                      Register rhs, Label* label) {
+  branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
+                                                Register dest, Label* fail) {
+  vcvttss2sq(src, dest);
+
+  // Same trick as for Doubles
+  cmpPtr(dest, Imm32(1));
+  j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+                                                         Register dest,
+                                                         Label* fail) {
+  branchTruncateFloat32ToPtr(src, dest, fail);
+  movl(dest, dest);  // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+                                                  Register dest, Label* fail) {
+  branchTruncateFloat32ToPtr(src, dest, fail);
+
+  // Check that the result is in the int32_t range.
+  ScratchRegisterScope scratch(*this);
+  move32To64SignExtend(dest, Register64(scratch));
+  cmpPtr(dest, scratch);
+  j(Assembler::NotEqual, fail);
+
+  movl(dest, dest);  // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
+                                               Label* fail) {
+  vcvttsd2sq(src, dest);
+
+  // vcvttsd2sq returns 0x8000000000000000 on failure. Test for it by
+  // subtracting 1 and testing overflow (this avoids the need to
+  // materialize that value in a register).
+  cmpPtr(dest, Imm32(1));
+  j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+                                                        Register dest,
+                                                        Label* fail) {
+  branchTruncateDoubleToPtr(src, dest, fail);
+  movl(dest, dest);  // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+                                                 Register dest, Label* fail) {
+  branchTruncateDoubleToPtr(src, dest, fail);
+
+  // Check that the result is in the int32_t range.
+  ScratchRegisterScope scratch(*this);
+  move32To64SignExtend(dest, Register64(scratch));
+  cmpPtr(dest, scratch);
+  j(Assembler::NotEqual, fail);
+
+  movl(dest, dest);  // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+                                  Imm32 rhs, Label* label) {
+  if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+    test32(Operand(lhs), rhs);
+  } else {
+    ScratchRegisterScope scratch(*this);
+    mov(ImmPtr(lhs.addr), scratch);
+    test32(Operand(scratch, 0), rhs);
+  }
+  j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+                                  Register64 rhs, Register temp, L label) {
+  branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+                                             const ValueOperand& value,
+                                             Label* label) {
+  test32(value.valueReg(), value.valueReg());
+  j(truthy ? NonZero : Zero, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+                                     JSWhyMagic why, Label* label) {
+  uint64_t magic = MagicValue(why).asRawBits();
+  cmpPtr(valaddr, ImmWord(magic));
+  j(cond, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+                                     const ValueOperand& rhs, Label* label) {
+  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+  branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& address) {
+  jmp(Operand(address));
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+                                   Register src, Register dest) {
+  cmpPtr(lhs, rhs);
+  cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+                                   const Address& rhs, Register src,
+                                   Register dest) {
+  cmpPtr(lhs, Operand(rhs));
+  cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+                                  Register src, Register dest) {
+  cmp32(lhs, rhs);
+  cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+                                  const Address& src, Register dest) {
+  cmp32(lhs, rhs);
+  cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+                                   Imm32 mask, const Address& src,
+                                   Register dest) {
+  MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+  test32(addr, mask);
+  cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+                                   Imm32 mask, Register src, Register dest) {
+  MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+  test32(addr, mask);
+  cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+                                    Register dest) {
+  cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+                                          Register maybeScratch,
+                                          Label* failure) {
+  MOZ_ASSERT(length != maybeScratch);
+  MOZ_ASSERT(index != maybeScratch);
+
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(index != scratch);
+  MOZ_ASSERT(length != scratch);
+
+  if (JitOptions.spectreIndexMasking) {
+    move32(Imm32(0), scratch);
+  }
+
+  cmp32(index, length);
+  j(Assembler::AboveOrEqual, failure);
+
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCl(Assembler::AboveOrEqual, scratch, index);
+  }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+                                          Register maybeScratch,
+                                          Label* failure) {
+  MOZ_ASSERT(index != length.base);
+  MOZ_ASSERT(length.base != maybeScratch);
+  MOZ_ASSERT(index != maybeScratch);
+
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(index != scratch);
+  MOZ_ASSERT(length.base != scratch);
+
+  if (JitOptions.spectreIndexMasking) {
+    move32(Imm32(0), scratch);
+  }
+
+  cmp32(index, Operand(length));
+  j(Assembler::AboveOrEqual, failure);
+
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCl(Assembler::AboveOrEqual, scratch, index);
+  }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+                                           Register maybeScratch,
+                                           Label* failure) {
+  MOZ_ASSERT(length != maybeScratch);
+  MOZ_ASSERT(index != maybeScratch);
+
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(index != scratch);
+  MOZ_ASSERT(length != scratch);
+
+  if (JitOptions.spectreIndexMasking) {
+    movePtr(ImmWord(0), scratch);
+  }
+
+  cmpPtr(index, length);
+  j(Assembler::AboveOrEqual, failure);
+
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCq(Assembler::AboveOrEqual, scratch, index);
+  }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+                                           const Address& length,
+                                           Register maybeScratch,
+                                           Label* failure) {
+  MOZ_ASSERT(index != length.base);
+  MOZ_ASSERT(length.base != maybeScratch);
+  MOZ_ASSERT(index != maybeScratch);
+
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(index != scratch);
+  MOZ_ASSERT(length.base != scratch);
+
+  if (JitOptions.spectreIndexMasking) {
+    movePtr(ImmWord(0), scratch);
+  }
+
+  cmpPtr(index, Operand(length));
+  j(Assembler::AboveOrEqual, failure);
+
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCq(Assembler::AboveOrEqual, scratch, index);
+  }
+}
+
+// ========================================================================
+// SIMD.
+
+// Extract lane as scalar
+
+void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
+                                        Register64 dest) {
+  if (lane == 0) {
+    vmovq(src, dest.reg);
+  } else {
+    vpextrq(lane, src, dest.reg);
+  }
+}
+
+// Replace lane value
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
+                                        FloatRegister lhsDest) {
+  vpinsrq(lane, rhs.reg, lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
+                                        Register64 rhs, FloatRegister dest) {
+  vpinsrq(lane, rhs.reg, lhs, dest);
+}
+
+// Splat
+
+void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
+  vmovq(src.reg, dest);
+  if (HasAVX2()) {
+    vbroadcastq(Operand(dest), dest);
+  } else {
+    vpunpcklqdq(dest, dest, dest);
+  }
+}
+
+// ========================================================================
+// Truncate floating point.
+
+void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
+                                             Register temp,
+                                             FloatRegister floatTemp) {
+  Label done;
+
+  loadFloat32(src, floatTemp);
+
+  truncateFloat32ToInt64(src, dest, temp);
+
+  // For unsigned conversion the case of [INT64, UINT64] needs to get handled
+  // separately.
+  loadPtr(dest, temp);
+  branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+  // Move the value inside INT64 range.
+  storeFloat32(floatTemp, dest);
+  loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
+  vaddss(Operand(dest), floatTemp, floatTemp);
+  storeFloat32(floatTemp, dest);
+  truncateFloat32ToInt64(dest, dest, temp);
+
+  loadPtr(dest, temp);
+  or64(Imm64(0x8000000000000000), Register64(temp));
+  storePtr(temp, dest);
+
+  bind(&done);
+}
+
+void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
+                                            Register temp,
+                                            FloatRegister floatTemp) {
+  Label done;
+
+  loadDouble(src, floatTemp);
+
+  truncateDoubleToInt64(src, dest, temp);
+
+  // For unsigned conversion the case of [INT64, UINT64] needs to get handle
+  // seperately.
+  loadPtr(dest, temp);
+  branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+  // Move the value inside INT64 range.
+  storeDouble(floatTemp, dest);
+  loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
+  vaddsd(Operand(dest), floatTemp, floatTemp);
+  storeDouble(floatTemp, dest);
+  truncateDoubleToInt64(dest, dest, temp);
+
+  loadPtr(dest, temp);
+  or64(Imm64(0x8000000000000000), Register64(temp));
+  storePtr(temp, dest);
+
+  bind(&done);
+}
+
+void MacroAssemblerX64::fallibleUnboxPtrImpl(const Operand& src, Register dest,
+                                             JSValueType type, Label* fail) {
+  MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+             type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+  // dest := src XOR mask
+  // scratch := dest >> JSVAL_TAG_SHIFT
+  // fail if scratch != 0
+  //
+  // Note: src and dest can be the same register.
+  ScratchRegisterScope scratch(asMasm());
+  mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+  xorq(src, scratch);
+  mov(scratch, dest);
+  shrq(Imm32(JSVAL_TAG_SHIFT), scratch);
+  j(Assembler::NonZero, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+                                      JSValueType type, Label* fail) {
+  fallibleUnboxPtrImpl(Operand(src.valueReg()), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+                                      JSValueType type, Label* fail) {
+  fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+                                      JSValueType type, Label* fail) {
+  fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssemblerX64::incrementInt32Value(const Address& addr) {
+  asMasm().addPtr(Imm32(1), addr);
+}
+
+void MacroAssemblerX64::unboxValue(const ValueOperand& src, AnyRegister dest,
+                                   JSValueType type) {
+  if (dest.isFloat()) {
+    Label notInt32, end;
+    asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+    convertInt32ToDouble(src.valueReg(), dest.fpu());
+    jump(&end);
+    bind(&notInt32);
+    unboxDouble(src, dest.fpu());
+    bind(&end);
+  } else {
+    unboxNonDouble(src, dest.gpr(), type);
+  }
+}
+
+template <typename T>
+void MacroAssemblerX64::loadInt32OrDouble(const T& src, FloatRegister dest) {
+  Label notInt32, end;
+  asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+  convertInt32ToDouble(src, dest);
+  jump(&end);
+  bind(&notInt32);
+  unboxDouble(src, dest);
+  bind(&end);
+}
+
+// If source is a double, load it into dest. If source is int32,
+// convert it to double. Else, branch to failure.
+void MacroAssemblerX64::ensureDouble(const ValueOperand& source,
+                                     FloatRegister dest, Label* failure) {
+  Label isDouble, done;
+  {
+    ScratchTagScope tag(asMasm(), source);
+    splitTagForTest(source, tag);
+    asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+    asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+  }
+
+  {
+    ScratchRegisterScope scratch(asMasm());
+    unboxInt32(source, scratch);
+    convertInt32ToDouble(scratch, dest);
+  }
+  jump(&done);
+
+  bind(&isDouble);
+  unboxDouble(source, dest);
+
+  bind(&done);
+}
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_inl_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp
new file mode 100644
index 0000000000..5106e7e382
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.cpp
@@ -0,0 +1,1820 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h"  // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MacroAssemblerX64::loadConstantDouble(double d, FloatRegister dest) {
+  if (maybeInlineDouble(d, dest)) {
+    return;
+  }
+  Double* dbl = getDouble(d);
+  if (!dbl) {
+    return;
+  }
+  // The constants will be stored in a pool appended to the text (see
+  // finish()), so they will always be a fixed distance from the
+  // instructions which reference them. This allows the instructions to use
+  // PC-relative addressing. Use "jump" label support code, because we need
+  // the same PC-relative address patching that jumps use.
+  JmpSrc j = masm.vmovsd_ripr(dest.encoding());
+  propagateOOM(dbl->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantFloat32(float f, FloatRegister dest) {
+  if (maybeInlineFloat(f, dest)) {
+    return;
+  }
+  Float* flt = getFloat(f);
+  if (!flt) {
+    return;
+  }
+  // See comment in loadConstantDouble
+  JmpSrc j = masm.vmovss_ripr(dest.encoding());
+  propagateOOM(flt->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+    const SimdConstant& v, FloatRegister reg,
+    JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+        X86Encoding::XMMRegisterID id)) {
+  SimdData* val = getSimdData(v);
+  if (!val) {
+    return;
+  }
+  JmpSrc j = (masm.*op)(reg.encoding());
+  propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+    const SimdConstant& v, FloatRegister src, FloatRegister dest,
+    JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+        X86Encoding::XMMRegisterID srcId, X86Encoding::XMMRegisterID destId)) {
+  SimdData* val = getSimdData(v);
+  if (!val) {
+    return;
+  }
+  JmpSrc j = (masm.*op)(src.encoding(), dest.encoding());
+  propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantSimd128Int(const SimdConstant& v,
+                                               FloatRegister dest) {
+  if (maybeInlineSimd128Int(v, dest)) {
+    return;
+  }
+  vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovdqa_ripr);
+}
+
+void MacroAssemblerX64::loadConstantSimd128Float(const SimdConstant& v,
+                                                 FloatRegister dest) {
+  if (maybeInlineSimd128Float(v, dest)) {
+    return;
+  }
+  vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovaps_ripr);
+}
+
+void MacroAssemblerX64::vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddb_ripr);
+}
+
+void MacroAssemblerX64::vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddw_ripr);
+}
+
+void MacroAssemblerX64::vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddd_ripr);
+}
+
+void MacroAssemblerX64::vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddq_ripr);
+}
+
+void MacroAssemblerX64::vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubb_ripr);
+}
+
+void MacroAssemblerX64::vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubw_ripr);
+}
+
+void MacroAssemblerX64::vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubd_ripr);
+}
+
+void MacroAssemblerX64::vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubq_ripr);
+}
+
+void MacroAssemblerX64::vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmullw_ripr);
+}
+
+void MacroAssemblerX64::vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmulld_ripr);
+}
+
+void MacroAssemblerX64::vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsb_ripr);
+}
+
+void MacroAssemblerX64::vpaddusbSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusb_ripr);
+}
+
+void MacroAssemblerX64::vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsw_ripr);
+}
+
+void MacroAssemblerX64::vpadduswSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusw_ripr);
+}
+
+void MacroAssemblerX64::vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsb_ripr);
+}
+
+void MacroAssemblerX64::vpsubusbSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusb_ripr);
+}
+
+void MacroAssemblerX64::vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsw_ripr);
+}
+
+void MacroAssemblerX64::vpsubuswSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusw_ripr);
+}
+
+void MacroAssemblerX64::vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsb_ripr);
+}
+
+void MacroAssemblerX64::vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminub_ripr);
+}
+
+void MacroAssemblerX64::vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsw_ripr);
+}
+
+void MacroAssemblerX64::vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminuw_ripr);
+}
+
+void MacroAssemblerX64::vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsd_ripr);
+}
+
+void MacroAssemblerX64::vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminud_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsb_ripr);
+}
+
+void MacroAssemblerX64::vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxub_ripr);
+}
+
+void MacroAssemblerX64::vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxuw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsd_ripr);
+}
+
+void MacroAssemblerX64::vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxud_ripr);
+}
+
+void MacroAssemblerX64::vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+                                     FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpand_ripr);
+}
+
+void MacroAssemblerX64::vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+                                     FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpxor_ripr);
+}
+
+void MacroAssemblerX64::vporSimd128(const SimdConstant& v, FloatRegister lhs,
+                                    FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpor_ripr);
+}
+
+void MacroAssemblerX64::vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddps_ripr);
+}
+
+void MacroAssemblerX64::vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddpd_ripr);
+}
+
+void MacroAssemblerX64::vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubps_ripr);
+}
+
+void MacroAssemblerX64::vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubpd_ripr);
+}
+
+void MacroAssemblerX64::vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivps_ripr);
+}
+
+void MacroAssemblerX64::vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivpd_ripr);
+}
+
+void MacroAssemblerX64::vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulps_ripr);
+}
+
+void MacroAssemblerX64::vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulpd_ripr);
+}
+
+void MacroAssemblerX64::vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vandpd_ripr);
+}
+
+void MacroAssemblerX64::vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                                      FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vminpd_ripr);
+}
+
+void MacroAssemblerX64::vpacksswbSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpacksswb_ripr);
+}
+
+void MacroAssemblerX64::vpackuswbSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackuswb_ripr);
+}
+
+void MacroAssemblerX64::vpackssdwSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackssdw_ripr);
+}
+
+void MacroAssemblerX64::vpackusdwSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackusdw_ripr);
+}
+
+void MacroAssemblerX64::vpunpckldqSimd128(const SimdConstant& v,
+                                          FloatRegister lhs,
+                                          FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest,
+                  &X86Encoding::BaseAssemblerX64::vpunpckldq_ripr);
+}
+
+void MacroAssemblerX64::vunpcklpsSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vunpcklps_ripr);
+}
+
+void MacroAssemblerX64::vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+                                       FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpshufb_ripr);
+}
+
+void MacroAssemblerX64::vptestSimd128(const SimdConstant& v,
+                                      FloatRegister lhs) {
+  vpRiprOpSimd128(v, lhs, &X86Encoding::BaseAssemblerX64::vptest_ripr);
+}
+
+void MacroAssemblerX64::vpmaddwdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaddwd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqbSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtbSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqwSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtwSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtd_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpsSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpsSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpsSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltps_ripr);
+}
+
+void MacroAssemblerX64::vcmplepsSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpleps_ripr);
+}
+
+void MacroAssemblerX64::vcmpgepsSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpgeps_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpdSimd128(const SimdConstant& v,
+                                         FloatRegister lhs,
+                                         FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltpd_ripr);
+}
+
+void MacroAssemblerX64::vcmplepdSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmplepd_ripr);
+}
+
+void MacroAssemblerX64::vpmaddubswSimd128(const SimdConstant& v,
+                                          FloatRegister lhs,
+                                          FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest,
+                  &X86Encoding::BaseAssemblerX64::vpmaddubsw_ripr);
+}
+
+void MacroAssemblerX64::vpmuludqSimd128(const SimdConstant& v,
+                                        FloatRegister lhs, FloatRegister dest) {
+  vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmuludq_ripr);
+}
+
+void MacroAssemblerX64::bindOffsets(
+    const MacroAssemblerX86Shared::UsesVector& uses) {
+  for (JmpSrc src : uses) {
+    JmpDst dst(currentOffset());
+    // Using linkJump here is safe, as explained in the comment in
+    // loadConstantDouble.
+    masm.linkJump(src, dst);
+  }
+}
+
+void MacroAssemblerX64::finish() {
+  if (!doubles_.empty()) {
+    masm.haltingAlign(sizeof(double));
+  }
+  for (const Double& d : doubles_) {
+    bindOffsets(d.uses);
+    masm.doubleConstant(d.value);
+  }
+
+  if (!floats_.empty()) {
+    masm.haltingAlign(sizeof(float));
+  }
+  for (const Float& f : floats_) {
+    bindOffsets(f.uses);
+    masm.floatConstant(f.value);
+  }
+
+  // SIMD memory values must be suitably aligned.
+  if (!simds_.empty()) {
+    masm.haltingAlign(SimdMemoryAlignment);
+  }
+  for (const SimdData& v : simds_) {
+    bindOffsets(v.uses);
+    masm.simd128Constant(v.value.bytes());
+  }
+
+  MacroAssemblerX86Shared::finish();
+}
+
+void MacroAssemblerX64::boxValue(JSValueType type, Register src,
+                                 Register dest) {
+  MOZ_ASSERT(src != dest);
+
+  JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+#ifdef DEBUG
+  if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+    Label upper32BitsZeroed;
+    movePtr(ImmWord(UINT32_MAX), dest);
+    asMasm().branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed);
+    breakpoint();
+    bind(&upper32BitsZeroed);
+  }
+#endif
+  mov(ImmShiftedTag(tag), dest);
+  orq(src, dest);
+}
+
+void MacroAssemblerX64::handleFailureWithHandlerTail(Label* profilerExitTail,
+                                                     Label* bailoutTail) {
+  // Reserve space for exception information.
+  subq(Imm32(sizeof(ResumeFromException)), rsp);
+  movq(rsp, rax);
+
+  // Call the handler.
+  using Fn = void (*)(ResumeFromException* rfe);
+  asMasm().setupUnalignedABICall(rcx);
+  asMasm().passABIArg(rax);
+  asMasm().callWithABI<Fn, HandleException>(
+      ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+  Label entryFrame;
+  Label catch_;
+  Label finally;
+  Label returnBaseline;
+  Label returnIon;
+  Label bailout;
+  Label wasm;
+  Label wasmCatch;
+
+  load32(Address(rsp, ResumeFromException::offsetOfKind()), rax);
+  asMasm().branch32(Assembler::Equal, rax,
+                    Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+  asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Catch),
+                    &catch_);
+  asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Finally),
+                    &finally);
+  asMasm().branch32(Assembler::Equal, rax,
+                    Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+                    &returnBaseline);
+  asMasm().branch32(Assembler::Equal, rax,
+                    Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+  asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Bailout),
+                    &bailout);
+  asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Wasm),
+                    &wasm);
+  asMasm().branch32(Assembler::Equal, rax,
+                    Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch);
+
+  breakpoint();  // Invalid kind.
+
+  // No exception handler. Load the error value, restore state and return from
+  // the entry frame.
+  bind(&entryFrame);
+  asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  ret();
+
+  // If we found a catch handler, this must be a baseline frame. Restore state
+  // and jump to the catch block.
+  bind(&catch_);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  jmp(Operand(rax));
+
+  // If we found a finally block, this must be a baseline frame. Push three
+  // values expected by the finally block: the exception, the exception stack,
+  // and BooleanValue(true).
+  bind(&finally);
+  ValueOperand exception = ValueOperand(rcx);
+  loadValue(Address(rsp, ResumeFromException::offsetOfException()), exception);
+
+  ValueOperand exceptionStack = ValueOperand(rdx);
+  loadValue(Address(rsp, ResumeFromException::offsetOfExceptionStack()),
+            exceptionStack);
+
+  loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+  pushValue(exception);
+  pushValue(exceptionStack);
+  pushValue(BooleanValue(true));
+  jmp(Operand(rax));
+
+  // Return BaselineFrame->returnValue() to the caller.
+  // Used in debug mode and for GeneratorReturn.
+  Label profilingInstrumentation;
+  bind(&returnBaseline);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  loadValue(Address(rbp, BaselineFrame::reverseOffsetOfReturnValue()),
+            JSReturnOperand);
+  jmp(&profilingInstrumentation);
+
+  // Return the given value to the caller.
+  bind(&returnIon);
+  loadValue(Address(rsp, ResumeFromException::offsetOfException()),
+            JSReturnOperand);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+  // If profiling is enabled, then update the lastProfilingFrame to refer to
+  // caller frame before returning. This code is shared by ForcedReturnIon
+  // and ForcedReturnBaseline.
+  bind(&profilingInstrumentation);
+  {
+    Label skipProfilingInstrumentation;
+    AbsoluteAddress addressOfEnabled(
+        asMasm().runtime()->geckoProfiler().addressOfEnabled());
+    asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+                      &skipProfilingInstrumentation);
+    jump(profilerExitTail);
+    bind(&skipProfilingInstrumentation);
+  }
+
+  movq(rbp, rsp);
+  pop(rbp);
+  ret();
+
+  // If we are bailing out to baseline to handle an exception, jump to the
+  // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+  bind(&bailout);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfBailoutInfo()), r9);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  move32(Imm32(1), ReturnReg);
+  jump(bailoutTail);
+
+  // If we are throwing and the innermost frame was a wasm frame, reset SP and
+  // FP; SP is pointing to the unwound return address to the wasm entry, so
+  // we can just ret().
+  bind(&wasm);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  movePtr(ImmPtr((const void*)wasm::FailInstanceReg), InstanceReg);
+  masm.ret();
+
+  // Found a wasm catch handler, restore state and jump to it.
+  bind(&wasmCatch);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+  loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  jmp(Operand(rax));
+}
+
+void MacroAssemblerX64::profilerEnterFrame(Register framePtr,
+                                           Register scratch) {
+  asMasm().loadJSContext(scratch);
+  loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+  storePtr(framePtr,
+           Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+  storePtr(ImmPtr(nullptr),
+           Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerX64::profilerExitFrame() {
+  jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+Assembler::Condition MacroAssemblerX64::testStringTruthy(
+    bool truthy, const ValueOperand& value) {
+  ScratchRegisterScope scratch(asMasm());
+  unboxString(value, scratch);
+  cmp32(Operand(scratch, JSString::offsetOfLength()), Imm32(0));
+  return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+Assembler::Condition MacroAssemblerX64::testBigIntTruthy(
+    bool truthy, const ValueOperand& value) {
+  ScratchRegisterScope scratch(asMasm());
+  unboxBigInt(value, scratch);
+  cmp32(Operand(scratch, JS::BigInt::offsetOfDigitLength()), Imm32(0));
+  return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+MacroAssembler& MacroAssemblerX64::asMasm() {
+  return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerX64::asMasm() const {
+  return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+  if (imm32.value) {
+    // On windows, we cannot skip very far down the stack without touching the
+    // memory pages in-between.  This is a corner-case code for situations where
+    // the Ion frame data for a piece of code is very large.  To handle this
+    // special case, for frames over 4k in size we allocate memory on the stack
+    // incrementally, touching it as we go.
+    //
+    // When the amount is quite large, which it can be, we emit an actual loop,
+    // in order to keep the function prologue compact.  Compactness is a
+    // requirement for eg Wasm's CodeRange data structure, which can encode only
+    // 8-bit offsets.
+    uint32_t amountLeft = imm32.value;
+    uint32_t fullPages = amountLeft / 4096;
+    if (fullPages <= 8) {
+      while (amountLeft > 4096) {
+        subq(Imm32(4096), StackPointer);
+        store32(Imm32(0), Address(StackPointer, 0));
+        amountLeft -= 4096;
+      }
+      subq(Imm32(amountLeft), StackPointer);
+    } else {
+      ScratchRegisterScope scratch(*this);
+      Label top;
+      move32(Imm32(fullPages), scratch);
+      bind(&top);
+      subq(Imm32(4096), StackPointer);
+      store32(Imm32(0), Address(StackPointer, 0));
+      subl(Imm32(1), scratch);
+      j(Assembler::NonZero, &top);
+      amountLeft -= fullPages * 4096;
+      if (amountLeft) {
+        subq(Imm32(amountLeft), StackPointer);
+      }
+    }
+  }
+}
+
+void MacroAssemblerX64::convertDoubleToPtr(FloatRegister src, Register dest,
+                                           Label* fail,
+                                           bool negativeZeroCheck) {
+  // Check for -0.0
+  if (negativeZeroCheck) {
+    branchNegativeZero(src, dest, fail);
+  }
+
+  ScratchDoubleScope scratch(asMasm());
+  vcvttsd2sq(src, dest);
+  asMasm().convertInt64ToDouble(Register64(dest), scratch);
+  vucomisd(scratch, src);
+  j(Assembler::Parity, fail);
+  j(Assembler::NotEqual, fail);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+  setupNativeABICall();
+  dynamicAlignment_ = true;
+
+  movq(rsp, scratch);
+  andq(Imm32(~(ABIStackAlignment - 1)), rsp);
+  push(scratch);
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+  MOZ_ASSERT(inCall_);
+  uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+  if (dynamicAlignment_) {
+    // sizeof(intptr_t) accounts for the saved stack pointer pushed by
+    // setupUnalignedABICall.
+    stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
+                                         ABIStackAlignment);
+  } else {
+    uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+    stackForCall += ComputeByteAlignment(
+        stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+  }
+
+  *stackAdjust = stackForCall;
+  reserveStack(stackForCall);
+
+  // Position all arguments.
+  {
+    enoughMemory_ &= moveResolver_.resolve();
+    if (!enoughMemory_) {
+      return;
+    }
+
+    MoveEmitter emitter(*this);
+    emitter.emit(moveResolver_);
+    emitter.finish();
+  }
+
+  assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result,
+                                     bool callFromWasm) {
+  freeStack(stackAdjust);
+  if (dynamicAlignment_) {
+    pop(rsp);
+  }
+
+#ifdef DEBUG
+  MOZ_ASSERT(inCall_);
+  inCall_ = false;
+#endif
+}
+
+static bool IsIntArgReg(Register reg) {
+  for (uint32_t i = 0; i < NumIntArgRegs; i++) {
+    if (IntArgRegs[i] == reg) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, ABIType result) {
+  if (IsIntArgReg(fun)) {
+    // Callee register may be clobbered for an argument. Move the callee to
+    // r10, a volatile, non-argument register.
+    propagateOOM(moveResolver_.addMove(MoveOperand(fun), MoveOperand(r10),
+                                       MoveOp::GENERAL));
+    fun = r10;
+  }
+
+  MOZ_ASSERT(!IsIntArgReg(fun));
+
+  uint32_t stackAdjust;
+  callWithABIPre(&stackAdjust);
+  call(fun);
+  callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) {
+  Address safeFun = fun;
+  if (IsIntArgReg(safeFun.base)) {
+    // Callee register may be clobbered for an argument. Move the callee to
+    // r10, a volatile, non-argument register.
+    propagateOOM(moveResolver_.addMove(MoveOperand(fun.base), MoveOperand(r10),
+                                       MoveOp::GENERAL));
+    safeFun.base = r10;
+  }
+
+  MOZ_ASSERT(!IsIntArgReg(safeFun.base));
+
+  uint32_t stackAdjust;
+  callWithABIPre(&stackAdjust);
+  call(safeFun);
+  callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+                               const ValueOperand& dest) {
+  if (src.hasValue()) {
+    moveValue(src.valueReg(), dest);
+    return;
+  }
+
+  MIRType type = src.type();
+  AnyRegister reg = src.typedReg();
+
+  if (!IsFloatingPointType(type)) {
+    boxValue(ValueTypeFromMIRType(type), reg.gpr(), dest.valueReg());
+    return;
+  }
+
+  ScratchDoubleScope scratch(*this);
+  FloatRegister freg = reg.fpu();
+  if (type == MIRType::Float32) {
+    convertFloat32ToDouble(freg, scratch);
+    freg = scratch;
+  }
+  boxDouble(freg, dest, freg);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+                               const ValueOperand& dest) {
+  if (src == dest) {
+    return;
+  }
+  movq(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+  movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+  writeDataRelocation(src);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+  if (ptr != buffer) {
+    movePtr(ptr, buffer);
+  }
+  andPtr(Imm32(int32_t(~gc::ChunkMask)), buffer);
+  loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+                                             Register temp, Label* label) {
+  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(ptr != temp);
+  MOZ_ASSERT(ptr != scratch);
+
+  movePtr(ptr, scratch);
+  andPtr(Imm32(int32_t(~gc::ChunkMask)), scratch);
+  branchPtr(InvertCondition(cond), Address(scratch, gc::ChunkStoreBufferOffset),
+            ImmWord(0), label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+                                                  const T& value, Register temp,
+                                                  Label* label) {
+  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+  MOZ_ASSERT(temp != InvalidReg);
+
+  Label done;
+  branchTestGCThing(Assembler::NotEqual, value,
+                    cond == Assembler::Equal ? &done : label);
+
+  getGCThingValueChunk(value, temp);
+  branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+            ImmWord(0), label);
+
+  bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+                                              const Address& address,
+                                              Register temp, Label* label) {
+  branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+                                              ValueOperand value, Register temp,
+                                              Label* label) {
+  branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+                                     const Value& rhs, Label* label) {
+  MOZ_ASSERT(cond == Equal || cond == NotEqual);
+  ScratchRegisterScope scratch(*this);
+  MOZ_ASSERT(lhs.valueReg() != scratch);
+  moveValue(rhs, ValueOperand(scratch));
+  cmpPtr(lhs.valueReg(), scratch);
+  j(cond, label);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+                                       MIRType valueType, const T& dest) {
+  MOZ_ASSERT(valueType < MIRType::Value);
+
+  if (valueType == MIRType::Double) {
+    boxDouble(value.reg().typedReg().fpu(), dest);
+    return;
+  }
+
+  if (value.constant()) {
+    storeValue(value.value(), dest);
+  } else {
+    storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+               dest);
+  }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+                                                MIRType valueType,
+                                                const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+    const ConstantOrRegister& value, MIRType valueType,
+    const BaseObjectElementIndex& dest);
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+  subq(Imm32(sizeof(double)), StackPointer);
+  boxDouble(reg, Address(StackPointer, 0));
+  adjustFrame(sizeof(double));
+}
+
+// ========================================================================
+// wasm support
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+                              Operand srcAddr, AnyRegister out) {
+  // NOTE: the generated code must match the assembly code in gen_load in
+  // GenerateAtomicOperations.py
+  memoryBarrierBefore(access.sync());
+
+  MOZ_ASSERT_IF(
+      access.isZeroExtendSimd128Load(),
+      access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+  MOZ_ASSERT_IF(
+      access.isSplatSimd128Load(),
+      access.type() == Scalar::Uint8 || access.type() == Scalar::Uint16 ||
+          access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+  MOZ_ASSERT_IF(access.isWidenSimd128Load(), access.type() == Scalar::Float64);
+
+  switch (access.type()) {
+    case Scalar::Int8:
+      append(access, wasm::TrapMachineInsn::Load8,
+             FaultingCodeOffset(currentOffset()));
+      movsbl(srcAddr, out.gpr());
+      break;
+    case Scalar::Uint8:
+      append(access, wasm::TrapMachineInsn::Load8,
+             FaultingCodeOffset(currentOffset()));
+      if (access.isSplatSimd128Load()) {
+        vbroadcastb(srcAddr, out.fpu());
+      } else {
+        movzbl(srcAddr, out.gpr());
+      }
+      break;
+    case Scalar::Int16:
+      append(access, wasm::TrapMachineInsn::Load16,
+             FaultingCodeOffset(currentOffset()));
+      movswl(srcAddr, out.gpr());
+      break;
+    case Scalar::Uint16:
+      append(access, wasm::TrapMachineInsn::Load16,
+             FaultingCodeOffset(currentOffset()));
+      if (access.isSplatSimd128Load()) {
+        vbroadcastw(srcAddr, out.fpu());
+      } else {
+        movzwl(srcAddr, out.gpr());
+      }
+      break;
+    case Scalar::Int32:
+    case Scalar::Uint32:
+      append(access, wasm::TrapMachineInsn::Load32,
+             FaultingCodeOffset(currentOffset()));
+      movl(srcAddr, out.gpr());
+      break;
+    case Scalar::Float32:
+      append(access, wasm::TrapMachineInsn::Load32,
+             FaultingCodeOffset(currentOffset()));
+      if (access.isSplatSimd128Load()) {
+        vbroadcastss(srcAddr, out.fpu());
+      } else {
+        // vmovss does the right thing also for access.isZeroExtendSimd128Load()
+        vmovss(srcAddr, out.fpu());
+      }
+      break;
+    case Scalar::Float64:
+      append(access, wasm::TrapMachineInsn::Load64,
+             FaultingCodeOffset(currentOffset()));
+      if (access.isSplatSimd128Load()) {
+        vmovddup(srcAddr, out.fpu());
+      } else if (access.isWidenSimd128Load()) {
+        switch (access.widenSimdOp()) {
+          case wasm::SimdOp::V128Load8x8S:
+            vpmovsxbw(srcAddr, out.fpu());
+            break;
+          case wasm::SimdOp::V128Load8x8U:
+            vpmovzxbw(srcAddr, out.fpu());
+            break;
+          case wasm::SimdOp::V128Load16x4S:
+            vpmovsxwd(srcAddr, out.fpu());
+            break;
+          case wasm::SimdOp::V128Load16x4U:
+            vpmovzxwd(srcAddr, out.fpu());
+            break;
+          case wasm::SimdOp::V128Load32x2S:
+            vpmovsxdq(srcAddr, out.fpu());
+            break;
+          case wasm::SimdOp::V128Load32x2U:
+            vpmovzxdq(srcAddr, out.fpu());
+            break;
+          default:
+            MOZ_CRASH("Unexpected widening op for wasmLoad");
+        }
+      } else {
+        // vmovsd does the right thing also for access.isZeroExtendSimd128Load()
+        vmovsd(srcAddr, out.fpu());
+      }
+      break;
+    case Scalar::Simd128: {
+      FaultingCodeOffset fco =
+          MacroAssemblerX64::loadUnalignedSimd128(srcAddr, out.fpu());
+      append(access, wasm::TrapMachineInsn::Load128, fco);
+      break;
+    }
+    case Scalar::Int64:
+      MOZ_CRASH("int64 loads must use load64");
+    case Scalar::BigInt64:
+    case Scalar::BigUint64:
+    case Scalar::Uint8Clamped:
+    case Scalar::MaxTypedArrayViewType:
+      MOZ_CRASH("unexpected scalar type for wasmLoad");
+  }
+
+  memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+                                 Operand srcAddr, Register64 out) {
+  // NOTE: the generated code must match the assembly code in gen_load in
+  // GenerateAtomicOperations.py
+  memoryBarrierBefore(access.sync());
+
+  switch (access.type()) {
+    case Scalar::Int8:
+      append(access, wasm::TrapMachineInsn::Load8,
+             FaultingCodeOffset(currentOffset()));
+      movsbq(srcAddr, out.reg);
+      break;
+    case Scalar::Uint8:
+      append(access, wasm::TrapMachineInsn::Load8,
+             FaultingCodeOffset(currentOffset()));
+      movzbq(srcAddr, out.reg);
+      break;
+    case Scalar::Int16:
+      append(access, wasm::TrapMachineInsn::Load16,
+             FaultingCodeOffset(currentOffset()));
+      movswq(srcAddr, out.reg);
+      break;
+    case Scalar::Uint16:
+      append(access, wasm::TrapMachineInsn::Load16,
+             FaultingCodeOffset(currentOffset()));
+      movzwq(srcAddr, out.reg);
+      break;
+    case Scalar::Int32:
+      append(access, wasm::TrapMachineInsn::Load32,
+             FaultingCodeOffset(currentOffset()));
+      movslq(srcAddr, out.reg);
+      break;
+    // Int32 to int64 moves zero-extend by default.
+    case Scalar::Uint32:
+      append(access, wasm::TrapMachineInsn::Load32,
+             FaultingCodeOffset(currentOffset()));
+      movl(srcAddr, out.reg);
+      break;
+    case Scalar::Int64:
+      append(access, wasm::TrapMachineInsn::Load64,
+             FaultingCodeOffset(currentOffset()));
+      movq(srcAddr, out.reg);
+      break;
+    case Scalar::Float32:
+    case Scalar::Float64:
+    case Scalar::Simd128:
+      MOZ_CRASH("float loads must use wasmLoad");
+    case Scalar::Uint8Clamped:
+    case Scalar::BigInt64:
+    case Scalar::BigUint64:
+    case Scalar::MaxTypedArrayViewType:
+      MOZ_CRASH("unexpected scalar type for wasmLoadI64");
+  }
+
+  memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+                               AnyRegister value, Operand dstAddr) {
+  // NOTE: the generated code must match the assembly code in gen_store in
+  // GenerateAtomicOperations.py
+  memoryBarrierBefore(access.sync());
+
+  switch (access.type()) {
+    case Scalar::Int8:
+    case Scalar::Uint8:
+      append(access, wasm::TrapMachineInsn::Store8,
+             FaultingCodeOffset(currentOffset()));
+      movb(value.gpr(), dstAddr);
+      break;
+    case Scalar::Int16:
+    case Scalar::Uint16:
+      append(access, wasm::TrapMachineInsn::Store16,
+             FaultingCodeOffset(currentOffset()));
+      movw(value.gpr(), dstAddr);
+      break;
+    case Scalar::Int32:
+    case Scalar::Uint32:
+      append(access, wasm::TrapMachineInsn::Store32,
+             FaultingCodeOffset(currentOffset()));
+      movl(value.gpr(), dstAddr);
+      break;
+    case Scalar::Int64:
+      append(access, wasm::TrapMachineInsn::Store64,
+             FaultingCodeOffset(currentOffset()));
+      movq(value.gpr(), dstAddr);
+      break;
+    case Scalar::Float32: {
+      FaultingCodeOffset fco =
+          storeUncanonicalizedFloat32(value.fpu(), dstAddr);
+      append(access, wasm::TrapMachineInsn::Store32, fco);
+      break;
+    }
+    case Scalar::Float64: {
+      FaultingCodeOffset fco = storeUncanonicalizedDouble(value.fpu(), dstAddr);
+      append(access, wasm::TrapMachineInsn::Store64, fco);
+      break;
+    }
+    case Scalar::Simd128: {
+      FaultingCodeOffset fco =
+          MacroAssemblerX64::storeUnalignedSimd128(value.fpu(), dstAddr);
+      append(access, wasm::TrapMachineInsn::Store128, fco);
+      break;
+    }
+    case Scalar::Uint8Clamped:
+    case Scalar::BigInt64:
+    case Scalar::BigUint64:
+    case Scalar::MaxTypedArrayViewType:
+      MOZ_CRASH("unexpected array type");
+  }
+
+  memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+                                                Register output,
+                                                bool isSaturating,
+                                                Label* oolEntry) {
+  vcvttsd2sq(input, output);
+
+  // Check that the result is in the uint32_t range.
+  ScratchRegisterScope scratch(*this);
+  move32(Imm32(0xffffffff), scratch);
+  cmpq(scratch, output);
+  j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+                                                 Register output,
+                                                 bool isSaturating,
+                                                 Label* oolEntry) {
+  vcvttss2sq(input, output);
+
+  // Check that the result is in the uint32_t range.
+  ScratchRegisterScope scratch(*this);
+  move32(Imm32(0xffffffff), scratch);
+  cmpq(scratch, output);
+  j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+    Label* oolRejoin, FloatRegister tempReg) {
+  vcvttsd2sq(input, output.reg);
+  cmpq(Imm32(1), output.reg);
+  j(Assembler::Overflow, oolEntry);
+  bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+    Label* oolRejoin, FloatRegister tempReg) {
+  vcvttss2sq(input, output.reg);
+  cmpq(Imm32(1), output.reg);
+  j(Assembler::Overflow, oolEntry);
+  bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+    Label* oolRejoin, FloatRegister tempReg) {
+  // If the input < INT64_MAX, vcvttsd2sq will do the right thing, so
+  // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+  // and then add INT64_MAX to the result.
+
+  Label isLarge;
+
+  ScratchDoubleScope scratch(*this);
+  loadConstantDouble(double(0x8000000000000000), scratch);
+  branchDouble(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+  vcvttsd2sq(input, output.reg);
+  testq(output.reg, output.reg);
+  j(Assembler::Signed, oolEntry);
+  jump(oolRejoin);
+
+  bind(&isLarge);
+
+  moveDouble(input, tempReg);
+  vsubsd(scratch, tempReg, tempReg);
+  vcvttsd2sq(tempReg, output.reg);
+  testq(output.reg, output.reg);
+  j(Assembler::Signed, oolEntry);
+  or64(Imm64(0x8000000000000000), output);
+
+  bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+    Label* oolRejoin, FloatRegister tempReg) {
+  // If the input < INT64_MAX, vcvttss2sq will do the right thing, so
+  // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+  // and then add INT64_MAX to the result.
+
+  Label isLarge;
+
+  ScratchFloat32Scope scratch(*this);
+  loadConstantFloat32(float(0x8000000000000000), scratch);
+  branchFloat(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+  vcvttss2sq(input, output.reg);
+  testq(output.reg, output.reg);
+  j(Assembler::Signed, oolEntry);
+  jump(oolRejoin);
+
+  bind(&isLarge);
+
+  moveFloat32(input, tempReg);
+  vsubss(scratch, tempReg, tempReg);
+  vcvttss2sq(tempReg, output.reg);
+  testq(output.reg, output.reg);
+  j(Assembler::Signed, oolEntry);
+  or64(Imm64(0x8000000000000000), output);
+
+  bind(oolRejoin);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+  move32To64ZeroExtend(r, Register64(r));
+}
+
+// ========================================================================
+// Convert floating point.
+
+void MacroAssembler::convertInt64ToDouble(Register64 input,
+                                          FloatRegister output) {
+  // Zero the output register to break dependencies, see convertInt32ToDouble.
+  zeroDouble(output);
+
+  vcvtsq2sd(input.reg, output, output);
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 input,
+                                           FloatRegister output) {
+  // Zero the output register to break dependencies, see convertInt32ToDouble.
+  zeroFloat32(output);
+
+  vcvtsq2ss(input.reg, output, output);
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return true; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 input,
+                                           FloatRegister output,
+                                           Register temp) {
+  // Zero the output register to break dependencies, see convertInt32ToDouble.
+  zeroDouble(output);
+
+  // If the input's sign bit is not set we use vcvtsq2sd directly.
+  // Else, we divide by 2 and keep the LSB, convert to double, and multiply
+  // the result by 2.
+  Label done;
+  Label isSigned;
+
+  testq(input.reg, input.reg);
+  j(Assembler::Signed, &isSigned);
+  vcvtsq2sd(input.reg, output, output);
+  jump(&done);
+
+  bind(&isSigned);
+
+  ScratchRegisterScope scratch(*this);
+  mov(input.reg, scratch);
+  mov(input.reg, temp);
+  shrq(Imm32(1), scratch);
+  andq(Imm32(1), temp);
+  orq(temp, scratch);
+
+  vcvtsq2sd(scratch, output, output);
+  vaddsd(output, output, output);
+
+  bind(&done);
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 input,
+                                            FloatRegister output,
+                                            Register temp) {
+  // Zero the output register to break dependencies, see convertInt32ToDouble.
+  zeroFloat32(output);
+
+  // See comment in convertUInt64ToDouble.
+  Label done;
+  Label isSigned;
+
+  testq(input.reg, input.reg);
+  j(Assembler::Signed, &isSigned);
+  vcvtsq2ss(input.reg, output, output);
+  jump(&done);
+
+  bind(&isSigned);
+
+  ScratchRegisterScope scratch(*this);
+  mov(input.reg, scratch);
+  mov(input.reg, temp);
+  shrq(Imm32(1), scratch);
+  andq(Imm32(1), temp);
+  orq(temp, scratch);
+
+  vcvtsq2ss(scratch, output, output);
+  vaddss(output, output, output);
+
+  bind(&done);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+  convertInt64ToDouble(Register64(src), dest);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+                                           const Address& mem,
+                                           Register64 expected,
+                                           Register64 replacement,
+                                           Register64 output) {
+  MOZ_ASSERT(output.reg == rax);
+  if (expected != output) {
+    movq(expected.reg, output.reg);
+  }
+  append(access, wasm::TrapMachineInsn::Atomic,
+         FaultingCodeOffset(currentOffset()));
+  lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+                                           const BaseIndex& mem,
+                                           Register64 expected,
+                                           Register64 replacement,
+                                           Register64 output) {
+  MOZ_ASSERT(output.reg == rax);
+  if (expected != output) {
+    movq(expected.reg, output.reg);
+  }
+  append(access, wasm::TrapMachineInsn::Atomic,
+         FaultingCodeOffset(currentOffset()));
+  lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+                                          const Address& mem, Register64 value,
+                                          Register64 output) {
+  if (value != output) {
+    movq(value.reg, output.reg);
+  }
+  append(access, wasm::TrapMachineInsn::Atomic,
+         FaultingCodeOffset(masm.currentOffset()));
+  xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+                                          const BaseIndex& mem,
+                                          Register64 value, Register64 output) {
+  if (value != output) {
+    movq(value.reg, output.reg);
+  }
+  append(access, wasm::TrapMachineInsn::Atomic,
+         FaultingCodeOffset(masm.currentOffset()));
+  xchgq(output.reg, Operand(mem));
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+                            const wasm::MemoryAccessDesc* access, AtomicOp op,
+                            Register value, const T& mem, Register temp,
+                            Register output) {
+  // NOTE: the generated code must match the assembly code in gen_fetchop in
+  // GenerateAtomicOperations.py
+  if (op == AtomicFetchAddOp) {
+    if (value != output) {
+      masm.movq(value, output);
+    }
+    if (access) {
+      masm.append(*access, wasm::TrapMachineInsn::Atomic,
+                  FaultingCodeOffset(masm.currentOffset()));
+    }
+    masm.lock_xaddq(output, Operand(mem));
+  } else if (op == AtomicFetchSubOp) {
+    if (value != output) {
+      masm.movq(value, output);
+    }
+    masm.negq(output);
+    if (access) {
+      masm.append(*access, wasm::TrapMachineInsn::Atomic,
+                  FaultingCodeOffset(masm.currentOffset()));
+    }
+    masm.lock_xaddq(output, Operand(mem));
+  } else {
+    Label again;
+    MOZ_ASSERT(output == rax);
+    MOZ_ASSERT(value != output);
+    MOZ_ASSERT(value != temp);
+    MOZ_ASSERT(temp != output);
+    if (access) {
+      masm.append(*access, wasm::TrapMachineInsn::Load64,
+                  FaultingCodeOffset(masm.currentOffset()));
+    }
+    masm.movq(Operand(mem), rax);
+    masm.bind(&again);
+    masm.movq(rax, temp);
+    switch (op) {
+      case AtomicFetchAndOp:
+        masm.andq(value, temp);
+        break;
+      case AtomicFetchOrOp:
+        masm.orq(value, temp);
+        break;
+      case AtomicFetchXorOp:
+        masm.xorq(value, temp);
+        break;
+      default:
+        MOZ_CRASH();
+    }
+    masm.lock_cmpxchgq(temp, Operand(mem));
+    masm.j(MacroAssembler::NonZero, &again);
+  }
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+                                         AtomicOp op, Register64 value,
+                                         const Address& mem, Register64 temp,
+                                         Register64 output) {
+  AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+                                         AtomicOp op, Register64 value,
+                                         const BaseIndex& mem, Register64 temp,
+                                         Register64 output) {
+  AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+template <typename T>
+static void AtomicEffectOp64(MacroAssembler& masm,
+                             const wasm::MemoryAccessDesc* access, AtomicOp op,
+                             Register value, const T& mem) {
+  if (access) {
+    masm.append(*access, wasm::TrapMachineInsn::Atomic,
+                FaultingCodeOffset(masm.currentOffset()));
+  }
+  switch (op) {
+    case AtomicFetchAddOp:
+      masm.lock_addq(value, Operand(mem));
+      break;
+    case AtomicFetchSubOp:
+      masm.lock_subq(value, Operand(mem));
+      break;
+    case AtomicFetchAndOp:
+      masm.lock_andq(value, Operand(mem));
+      break;
+    case AtomicFetchOrOp:
+      masm.lock_orq(value, Operand(mem));
+      break;
+    case AtomicFetchXorOp:
+      masm.lock_xorq(value, Operand(mem));
+      break;
+    default:
+      MOZ_CRASH();
+  }
+}
+
+void MacroAssembler::wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access,
+                                          AtomicOp op, Register64 value,
+                                          const BaseIndex& mem) {
+  AtomicEffectOp64(*this, &access, op, value.reg, mem);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+                                       const Address& mem, Register64 expected,
+                                       Register64 replacement,
+                                       Register64 output) {
+  // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+  // GenerateAtomicOperations.py
+  MOZ_ASSERT(output.reg == rax);
+  if (expected != output) {
+    movq(expected.reg, output.reg);
+  }
+  lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+                                       const BaseIndex& mem,
+                                       Register64 expected,
+                                       Register64 replacement,
+                                       Register64 output) {
+  MOZ_ASSERT(output.reg == rax);
+  if (expected != output) {
+    movq(expected.reg, output.reg);
+  }
+  lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+                                      const Address& mem, Register64 value,
+                                      Register64 output) {
+  // NOTE: the generated code must match the assembly code in gen_exchange in
+  // GenerateAtomicOperations.py
+  if (value != output) {
+    movq(value.reg, output.reg);
+  }
+  xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+                                      const BaseIndex& mem, Register64 value,
+                                      Register64 output) {
+  if (value != output) {
+    movq(value.reg, output.reg);
+  }
+  xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+                                     Register64 value, const Address& mem,
+                                     Register64 temp, Register64 output) {
+  AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+                                     Register64 value, const BaseIndex& mem,
+                                     Register64 temp, Register64 output) {
+  AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+                                      Register64 value, const Address& mem) {
+  AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+                                      Register64 value, const BaseIndex& mem) {
+  AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+  return leaRipRelative(dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+                                          CodeLocationLabel target) {
+  ptrdiff_t off = target - loc;
+  MOZ_ASSERT(off > ptrdiff_t(INT32_MIN));
+  MOZ_ASSERT(off < ptrdiff_t(INT32_MAX));
+  PatchWrite_Imm32(loc, Imm32(off));
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+                                       Register64 boundsCheckLimit, Label* ok) {
+  cmpPtr(index.reg, boundsCheckLimit.reg);
+  j(cond, ok);
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCq(cond, Operand(boundsCheckLimit.reg), index.reg);
+  }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+                                       Address boundsCheckLimit, Label* ok) {
+  cmpPtr(index.reg, Operand(boundsCheckLimit));
+  j(cond, ok);
+  if (JitOptions.spectreIndexMasking) {
+    cmovCCq(cond, Operand(boundsCheckLimit), index.reg);
+  }
+}
+
+#ifdef ENABLE_WASM_TAIL_CALLS
+void MacroAssembler::wasmMarkSlowCall() {
+  static_assert(InstanceReg == r14);
+  orPtr(Imm32(0), r14);
+}
+
+const int32_t SlowCallMarker = 0x00ce8349;  // OR r14, 0
+
+void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
+                                           Register temp1, Register temp2) {
+  // Check if RA has slow marker.
+  cmp32(Address(ra, 0), Imm32(SlowCallMarker));
+  j(Assembler::NotEqual, notSlow);
+}
+#endif  // ENABLE_WASM_TAIL_CALLS
+
+// ========================================================================
+// Integer compare-then-conditionally-load/move operations.
+
+// cmpMove, Cond-Reg-Reg-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs, Register rhs,
+                                Register falseVal, Register trueValAndDest) {
+  if constexpr (CmpSize == 32) {
+    cmp32(lhs, rhs);
+  } else {
+    static_assert(CmpSize == 64);
+    cmpPtr(lhs, rhs);
+  }
+  if constexpr (MoveSize == 32) {
+    cmovCCl(cond, Operand(falseVal), trueValAndDest);
+  } else {
+    static_assert(MoveSize == 64);
+    cmovCCq(cond, Operand(falseVal), trueValAndDest);
+  }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+
+// cmpMove, Cond-Reg-Addr-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs,
+                                const Address& rhs, Register falseVal,
+                                Register trueValAndDest) {
+  if constexpr (CmpSize == 32) {
+    cmp32(lhs, Operand(rhs));
+  } else {
+    static_assert(CmpSize == 64);
+    cmpPtr(lhs, Operand(rhs));
+  }
+  if constexpr (MoveSize == 32) {
+    cmovCCl(cond, Operand(falseVal), trueValAndDest);
+  } else {
+    static_assert(MoveSize == 64);
+    cmovCCq(cond, Operand(falseVal), trueValAndDest);
+  }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 Register falseVal,
+                                                 Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Reg-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs, Register rhs,
+                                const Address& falseVal,
+                                Register trueValAndDest) {
+  if constexpr (CmpSize == 32) {
+    cmp32(lhs, rhs);
+  } else {
+    static_assert(CmpSize == 64);
+    cmpPtr(lhs, rhs);
+  }
+  if constexpr (LoadSize == 32) {
+    cmovCCl(cond, Operand(falseVal), trueValAndDest);
+  } else {
+    static_assert(LoadSize == 64);
+    cmovCCq(cond, Operand(falseVal), trueValAndDest);
+  }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+                                                 Register rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Addr-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs,
+                                const Address& rhs, const Address& falseVal,
+                                Register trueValAndDest) {
+  if constexpr (CmpSize == 32) {
+    cmp32(lhs, Operand(rhs));
+  } else {
+    static_assert(CmpSize == 64);
+    cmpPtr(lhs, Operand(rhs));
+  }
+  if constexpr (LoadSize == 32) {
+    cmovCCl(cond, Operand(falseVal), trueValAndDest);
+  } else {
+    static_assert(LoadSize == 64);
+    cmovCCq(cond, Operand(falseVal), trueValAndDest);
+  }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+                                                 const Address& rhs,
+                                                 const Address& falseVal,
+                                                 Register trueValAndDest);
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h
new file mode 100644
index 0000000000..12c8632dfa
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.h
@@ -0,0 +1,1245 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_h
+#define jit_x64_MacroAssembler_x64_h
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+#include "js/HeapAPI.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+struct ImmShiftedTag : public ImmWord {
+  explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+  explicit ImmShiftedTag(JSValueType type)
+      : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {}
+};
+
+struct ImmTag : public Imm32 {
+  explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
+};
+
+// ScratchTagScope and ScratchTagScopeRelease are used to manage the tag
+// register for splitTagForTest(), which has different register management on
+// different platforms.  On 64-bit platforms it requires a scratch register that
+// does not interfere with other operations; on 32-bit platforms it uses a
+// register that is already part of the Value.
+//
+// The ScratchTagScope RAII type acquires the appropriate register; a reference
+// to a variable of this type is then passed to splitTagForTest().
+//
+// On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register
+// available in a dynamic scope during compilation.  However it is important to
+// remember that that does not preserve the register value in any way, so this
+// RAII type should only be used along paths that eventually branch past further
+// uses of the extracted tag value.
+//
+// On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not
+// manage a register, it only aliases a register in the ValueOperand.
+
+class ScratchTagScope : public ScratchRegisterScope {
+ public:
+  ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+      : ScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+  ScratchTagScope* ts_;
+
+ public:
+  explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+    ts_->release();
+  }
+  ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerX64 : public MacroAssemblerX86Shared {
+ private:
+  // Perform a downcast. Should be removed by Bug 996602.
+  MacroAssembler& asMasm();
+  const MacroAssembler& asMasm() const;
+
+  void bindOffsets(const MacroAssemblerX86Shared::UsesVector&);
+
+  void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg,
+                       JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+                           X86Encoding::XMMRegisterID id));
+
+  void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest,
+                       JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+                           X86Encoding::XMMRegisterID srcId,
+                           X86Encoding::XMMRegisterID destId));
+
+ public:
+  using MacroAssemblerX86Shared::load32;
+  using MacroAssemblerX86Shared::store16;
+  using MacroAssemblerX86Shared::store32;
+
+  MacroAssemblerX64() = default;
+
+  // The buffer is about to be linked, make sure any constant pools or excess
+  // bookkeeping has been flushed to the instruction stream.
+  void finish();
+
+  /////////////////////////////////////////////////////////////////
+  // X64 helpers.
+  /////////////////////////////////////////////////////////////////
+  void writeDataRelocation(const Value& val) {
+    // Raw GC pointer relocations and Value relocations both end up in
+    // Assembler::TraceDataRelocations.
+    if (val.isGCThing()) {
+      gc::Cell* cell = val.toGCThing();
+      if (cell && gc::IsInsideNursery(cell)) {
+        embedsNurseryPointers_ = true;
+      }
+      dataRelocations_.writeUnsigned(masm.currentOffset());
+    }
+  }
+
+  // Refers to the upper 32 bits of a 64-bit Value operand.
+  // On x86_64, the upper 32 bits do not necessarily only contain the type.
+  Operand ToUpper32(Operand base) {
+    switch (base.kind()) {
+      case Operand::MEM_REG_DISP:
+        return Operand(Register::FromCode(base.base()), base.disp() + 4);
+
+      case Operand::MEM_SCALE:
+        return Operand(Register::FromCode(base.base()),
+                       Register::FromCode(base.index()), base.scale(),
+                       base.disp() + 4);
+
+      default:
+        MOZ_CRASH("unexpected operand kind");
+    }
+  }
+  static inline Operand ToUpper32(const Address& address) {
+    return Operand(address.base, address.offset + 4);
+  }
+  static inline Operand ToUpper32(const BaseIndex& address) {
+    return Operand(address.base, address.index, address.scale,
+                   address.offset + 4);
+  }
+
+  uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); }
+
+  JSValueShiftedTag GetShiftedTag(JSValueType type) {
+    return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+  }
+
+  /////////////////////////////////////////////////////////////////
+  // X86/X64-common interface.
+  /////////////////////////////////////////////////////////////////
+
+  void storeValue(ValueOperand val, Operand dest) {
+    movq(val.valueReg(), dest);
+  }
+  void storeValue(ValueOperand val, const Address& dest) {
+    storeValue(val, Operand(dest));
+  }
+  template <typename T>
+  void storeValue(JSValueType type, Register reg, const T& dest) {
+    // Value types with 32-bit payloads can be emitted as two 32-bit moves.
+    if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+      movl(reg, Operand(dest));
+      movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest)));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      boxValue(type, reg, scratch);
+      movq(scratch, Operand(dest));
+    }
+  }
+  template <typename T>
+  void storeValue(const Value& val, const T& dest) {
+    ScratchRegisterScope scratch(asMasm());
+    if (val.isGCThing()) {
+      movWithPatch(ImmWord(val.asRawBits()), scratch);
+      writeDataRelocation(val);
+    } else {
+      mov(ImmWord(val.asRawBits()), scratch);
+    }
+    movq(scratch, Operand(dest));
+  }
+  void storeValue(ValueOperand val, BaseIndex dest) {
+    storeValue(val, Operand(dest));
+  }
+  void storeValue(const Address& src, const Address& dest, Register temp) {
+    loadPtr(src, temp);
+    storePtr(temp, dest);
+  }
+  void storePrivateValue(Register src, const Address& dest) {
+    storePtr(src, dest);
+  }
+  void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+    storePtr(imm, dest);
+  }
+  void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); }
+  void loadValue(Address src, ValueOperand val) {
+    loadValue(Operand(src), val);
+  }
+  void loadValue(const BaseIndex& src, ValueOperand val) {
+    loadValue(Operand(src), val);
+  }
+  void loadUnalignedValue(const Address& src, ValueOperand dest) {
+    loadValue(src, dest);
+  }
+  void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+    ScratchRegisterScope scratch(asMasm());
+    MOZ_ASSERT(dest.valueReg() != scratch);
+    if (payload != dest.valueReg()) {
+      movq(payload, dest.valueReg());
+    }
+    mov(ImmShiftedTag(type), scratch);
+    orq(scratch, dest.valueReg());
+  }
+  void pushValue(ValueOperand val) { push(val.valueReg()); }
+  void popValue(ValueOperand val) { pop(val.valueReg()); }
+  void pushValue(const Value& val) {
+    if (val.isGCThing()) {
+      ScratchRegisterScope scratch(asMasm());
+      movWithPatch(ImmWord(val.asRawBits()), scratch);
+      writeDataRelocation(val);
+      push(scratch);
+    } else {
+      push(ImmWord(val.asRawBits()));
+    }
+  }
+  void pushValue(JSValueType type, Register reg) {
+    ScratchRegisterScope scratch(asMasm());
+    boxValue(type, reg, scratch);
+    push(scratch);
+  }
+  void pushValue(const Address& addr) { push(Operand(addr)); }
+
+  void pushValue(const BaseIndex& addr, Register scratch) {
+    push(Operand(addr));
+  }
+
+  void boxValue(JSValueType type, Register src, Register dest);
+
+  Condition testUndefined(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+    return cond;
+  }
+  Condition testInt32(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_INT32));
+    return cond;
+  }
+  Condition testBoolean(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+    return cond;
+  }
+  Condition testNull(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_NULL));
+    return cond;
+  }
+  Condition testString(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_STRING));
+    return cond;
+  }
+  Condition testSymbol(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
+    return cond;
+  }
+  Condition testBigInt(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
+    return cond;
+  }
+  Condition testObject(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
+    return cond;
+  }
+  Condition testDouble(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
+    return cond == Equal ? BelowOrEqual : Above;
+  }
+  Condition testNumber(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag));
+    return cond == Equal ? BelowOrEqual : Above;
+  }
+  Condition testGCThing(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag));
+    return cond == Equal ? AboveOrEqual : Below;
+  }
+
+  Condition testMagic(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
+    return cond;
+  }
+  Condition testError(Condition cond, Register tag) {
+    return testMagic(cond, tag);
+  }
+  Condition testPrimitive(Condition cond, Register tag) {
+    MOZ_ASSERT(cond == Equal || cond == NotEqual);
+    cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+    return cond == Equal ? Below : AboveOrEqual;
+  }
+
+  Condition testUndefined(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testUndefined(cond, scratch);
+  }
+  Condition testInt32(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testInt32(cond, scratch);
+  }
+  Condition testBoolean(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testBoolean(cond, scratch);
+  }
+  Condition testDouble(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testDouble(cond, scratch);
+  }
+  Condition testNumber(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testNumber(cond, scratch);
+  }
+  Condition testNull(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testNull(cond, scratch);
+  }
+  Condition testString(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testString(cond, scratch);
+  }
+  Condition testSymbol(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testSymbol(cond, scratch);
+  }
+  Condition testBigInt(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testBigInt(cond, scratch);
+  }
+  Condition testObject(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testObject(cond, scratch);
+  }
+  Condition testGCThing(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testGCThing(cond, scratch);
+  }
+  Condition testPrimitive(Condition cond, const ValueOperand& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testPrimitive(cond, scratch);
+  }
+
+  Condition testUndefined(Condition cond, const Address& src) {
+    cmp32(ToUpper32(src),
+          Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
+    return cond;
+  }
+  Condition testInt32(Condition cond, const Address& src) {
+    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
+    return cond;
+  }
+  Condition testBoolean(Condition cond, const Address& src) {
+    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
+    return cond;
+  }
+  Condition testDouble(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testDouble(cond, scratch);
+  }
+  Condition testNumber(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testNumber(cond, scratch);
+  }
+  Condition testNull(Condition cond, const Address& src) {
+    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
+    return cond;
+  }
+  Condition testString(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testString(cond, scratch);
+  }
+  Condition testSymbol(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testSymbol(cond, scratch);
+  }
+  Condition testBigInt(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testBigInt(cond, scratch);
+  }
+  Condition testObject(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testObject(cond, scratch);
+  }
+  Condition testPrimitive(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testPrimitive(cond, scratch);
+  }
+  Condition testGCThing(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testGCThing(cond, scratch);
+  }
+  Condition testMagic(Condition cond, const Address& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testMagic(cond, scratch);
+  }
+
+  Condition testUndefined(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testUndefined(cond, scratch);
+  }
+  Condition testNull(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testNull(cond, scratch);
+  }
+  Condition testBoolean(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testBoolean(cond, scratch);
+  }
+  Condition testString(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testString(cond, scratch);
+  }
+  Condition testSymbol(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testSymbol(cond, scratch);
+  }
+  Condition testBigInt(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testBigInt(cond, scratch);
+  }
+  Condition testInt32(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testInt32(cond, scratch);
+  }
+  Condition testObject(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testObject(cond, scratch);
+  }
+  Condition testDouble(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testDouble(cond, scratch);
+  }
+  Condition testMagic(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testMagic(cond, scratch);
+  }
+  Condition testGCThing(Condition cond, const BaseIndex& src) {
+    ScratchRegisterScope scratch(asMasm());
+    splitTag(src, scratch);
+    return testGCThing(cond, scratch);
+  }
+
+  Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) {
+    uint64_t magic = MagicValue(why).asRawBits();
+    cmpPtr(src.valueReg(), ImmWord(magic));
+    return cond;
+  }
+
+  void cmpPtr(Register lhs, const ImmWord rhs) {
+    ScratchRegisterScope scratch(asMasm());
+    MOZ_ASSERT(lhs != scratch);
+    if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) {
+      cmpPtr(lhs, Imm32(int32_t(rhs.value)));
+    } else {
+      movePtr(rhs, scratch);
+      cmpPtr(lhs, scratch);
+    }
+  }
+  void cmpPtr(Register lhs, const ImmPtr rhs) {
+    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+  }
+  void cmpPtr(Register lhs, const ImmGCPtr rhs) {
+    ScratchRegisterScope scratch(asMasm());
+    MOZ_ASSERT(lhs != scratch);
+    movePtr(rhs, scratch);
+    cmpPtr(lhs, scratch);
+  }
+  void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+  void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) {
+    ScratchRegisterScope scratch(asMasm());
+    MOZ_ASSERT(!lhs.containsReg(scratch));
+    movePtr(rhs, scratch);
+    cmpPtr(lhs, scratch);
+  }
+  void cmpPtr(const Operand& lhs, const ImmWord rhs) {
+    if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+      cmpPtr(lhs, Imm32((int32_t)rhs.value));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      movePtr(rhs, scratch);
+      cmpPtr(lhs, scratch);
+    }
+  }
+  void cmpPtr(const Operand& lhs, const ImmPtr rhs) {
+    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+  }
+  void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
+    cmpPtr(Operand(lhs), rhs);
+  }
+  void cmpPtr(const Address& lhs, const ImmWord rhs) {
+    cmpPtr(Operand(lhs), rhs);
+  }
+  void cmpPtr(const Address& lhs, const ImmPtr rhs) {
+    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+  }
+  void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); }
+  void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); }
+  void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+  void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
+  void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); }
+  void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); }
+  void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); }
+  void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); }
+  void test64(Register lhs, Register rhs) { testq(rhs, lhs); }
+  void test64(Register lhs, const Imm64 rhs) {
+    if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+      testq(Imm32((int32_t)rhs.value), lhs);
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      movq(ImmWord(rhs.value), scratch);
+      testq(scratch, lhs);
+    }
+  }
+
+  // Compare-then-conditionally-move/load, for integer types
+  template <size_t CmpSize, size_t MoveSize>
+  void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal,
+               Register trueValAndDest);
+
+  template <size_t CmpSize, size_t MoveSize>
+  void cmpMove(Condition cond, Register lhs, const Address& rhs,
+               Register falseVal, Register trueValAndDest);
+
+  template <size_t CmpSize, size_t LoadSize>
+  void cmpLoad(Condition cond, Register lhs, Register rhs,
+               const Address& falseVal, Register trueValAndDest);
+
+  template <size_t CmpSize, size_t LoadSize>
+  void cmpLoad(Condition cond, Register lhs, const Address& rhs,
+               const Address& falseVal, Register trueValAndDest);
+
+  /////////////////////////////////////////////////////////////////
+  // Common interface.
+  /////////////////////////////////////////////////////////////////
+
+  void movePtr(Register src, Register dest) { movq(src, dest); }
+  void movePtr(Register src, const Operand& dest) { movq(src, dest); }
+  void movePtr(ImmWord imm, Register dest) { mov(imm, dest); }
+  void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); }
+  void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
+  void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); }
+  void loadPtr(AbsoluteAddress address, Register dest) {
+    if (X86Encoding::IsAddressImmediate(address.addr)) {
+      movq(Operand(address), dest);
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmPtr(address.addr), scratch);
+      loadPtr(Address(scratch, 0x0), dest);
+    }
+  }
+  FaultingCodeOffset loadPtr(const Address& address, Register dest) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(Operand(address), dest);
+    return fco;
+  }
+  void load64(const Address& address, Register dest) {
+    movq(Operand(address), dest);
+  }
+  void loadPtr(const Operand& src, Register dest) { movq(src, dest); }
+  FaultingCodeOffset loadPtr(const BaseIndex& src, Register dest) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(Operand(src), dest);
+    return fco;
+  }
+  void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); }
+  void load32(AbsoluteAddress address, Register dest) {
+    if (X86Encoding::IsAddressImmediate(address.addr)) {
+      movl(Operand(address), dest);
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmPtr(address.addr), scratch);
+      load32(Address(scratch, 0x0), dest);
+    }
+  }
+  void load64(const Operand& address, Register64 dest) {
+    movq(address, dest.reg);
+  }
+  FaultingCodeOffset load64(const Address& address, Register64 dest) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(Operand(address), dest.reg);
+    return fco;
+  }
+  FaultingCodeOffset load64(const BaseIndex& address, Register64 dest) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(Operand(address), dest.reg);
+    return fco;
+  }
+  template <typename S>
+  void load64Unaligned(const S& src, Register64 dest) {
+    load64(src, dest);
+  }
+  template <typename T>
+  void storePtr(ImmWord imm, T address) {
+    if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+      movq(Imm32((int32_t)imm.value), Operand(address));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(imm, scratch);
+      movq(scratch, Operand(address));
+    }
+  }
+  template <typename T>
+  void storePtr(ImmPtr imm, T address) {
+    storePtr(ImmWord(uintptr_t(imm.value)), address);
+  }
+  template <typename T>
+  void storePtr(ImmGCPtr imm, T address) {
+    ScratchRegisterScope scratch(asMasm());
+    movq(imm, scratch);
+    movq(scratch, Operand(address));
+  }
+  FaultingCodeOffset storePtr(Register src, const Address& address) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(src, Operand(address));
+    return fco;
+  }
+  void store64(Register src, const Address& address) {
+    movq(src, Operand(address));
+  }
+  void store64(Register64 src, const Operand& address) {
+    movq(src.reg, address);
+  }
+  FaultingCodeOffset storePtr(Register src, const BaseIndex& address) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    movq(src, Operand(address));
+    return fco;
+  }
+  void storePtr(Register src, const Operand& dest) { movq(src, dest); }
+  void storePtr(Register src, AbsoluteAddress address) {
+    if (X86Encoding::IsAddressImmediate(address.addr)) {
+      movq(src, Operand(address));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmPtr(address.addr), scratch);
+      storePtr(src, Address(scratch, 0x0));
+    }
+  }
+  void store32(Register src, AbsoluteAddress address) {
+    if (X86Encoding::IsAddressImmediate(address.addr)) {
+      movl(src, Operand(address));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmPtr(address.addr), scratch);
+      store32(src, Address(scratch, 0x0));
+    }
+  }
+  void store16(Register src, AbsoluteAddress address) {
+    if (X86Encoding::IsAddressImmediate(address.addr)) {
+      movw(src, Operand(address));
+    } else {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmPtr(address.addr), scratch);
+      store16(src, Address(scratch, 0x0));
+    }
+  }
+  FaultingCodeOffset store64(Register64 src, Address address) {
+    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
+    storePtr(src.reg, address);
+    return fco;
+  }
+  FaultingCodeOffset store64(Register64 src, const BaseIndex& address) {
+    return storePtr(src.reg, address);
+  }
+  void store64(Imm64 imm, Address address) {
+    storePtr(ImmWord(imm.value), address);
+  }
+  void store64(Imm64 imm, const BaseIndex& address) {
+    storePtr(ImmWord(imm.value), address);
+  }
+  template <typename S, typename T>
+  void store64Unaligned(const S& src, const T& dest) {
+    store64(src, dest);
+  }
+
+  void splitTag(Register src, Register dest) {
+    if (src != dest) {
+      movq(src, dest);
+    }
+    shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+  }
+  void splitTag(const ValueOperand& operand, Register dest) {
+    splitTag(operand.valueReg(), dest);
+  }
+  void splitTag(const Operand& operand, Register dest) {
+    movq(operand, dest);
+    shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+  }
+  void splitTag(const Address& operand, Register dest) {
+    splitTag(Operand(operand), dest);
+  }
+  void splitTag(const BaseIndex& operand, Register dest) {
+    splitTag(Operand(operand), dest);
+  }
+
+  // Extracts the tag of a value and places it in tag.
+  void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+    splitTag(value, tag);
+  }
+  void cmpTag(const ValueOperand& operand, ImmTag tag) {
+    ScratchTagScope reg(asMasm(), operand);
+    splitTagForTest(operand, reg);
+    cmp32(reg, tag);
+  }
+
+  Condition testMagic(Condition cond, const ValueOperand& src) {
+    ScratchTagScope scratch(asMasm(), src);
+    splitTagForTest(src, scratch);
+    return testMagic(cond, scratch);
+  }
+  Condition testError(Condition cond, const ValueOperand& src) {
+    return testMagic(cond, src);
+  }
+
+  void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+    cond = testNull(cond, value);
+    emitSet(cond, dest);
+  }
+
+  void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+    cond = testObject(cond, value);
+    emitSet(cond, dest);
+  }
+
+  void testUndefinedSet(Condition cond, const ValueOperand& value,
+                        Register dest) {
+    cond = testUndefined(cond, value);
+    emitSet(cond, dest);
+  }
+
+  void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
+    vmovq(src, dest.valueReg());
+  }
+  void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+    MOZ_ASSERT(src != dest.valueReg());
+    boxValue(type, src, dest.valueReg());
+  }
+
+  // Note that the |dest| register here may be ScratchReg, so we shouldn't
+  // use it.
+  void unboxInt32(const ValueOperand& src, Register dest) {
+    movl(src.valueReg(), dest);
+  }
+  void unboxInt32(const Operand& src, Register dest) { movl(src, dest); }
+  void unboxInt32(const Address& src, Register dest) {
+    unboxInt32(Operand(src), dest);
+  }
+  void unboxInt32(const BaseIndex& src, Register dest) {
+    unboxInt32(Operand(src), dest);
+  }
+  template <typename T>
+  void unboxDouble(const T& src, FloatRegister dest) {
+    loadDouble(Operand(src), dest);
+  }
+
+  void unboxArgObjMagic(const ValueOperand& src, Register dest) {
+    unboxArgObjMagic(Operand(src.valueReg()), dest);
+  }
+  void unboxArgObjMagic(const Operand& src, Register dest) {
+    mov(ImmWord(0), dest);
+  }
+  void unboxArgObjMagic(const Address& src, Register dest) {
+    unboxArgObjMagic(Operand(src), dest);
+  }
+
+  void unboxBoolean(const ValueOperand& src, Register dest) {
+    movl(src.valueReg(), dest);
+  }
+  void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); }
+  void unboxBoolean(const Address& src, Register dest) {
+    unboxBoolean(Operand(src), dest);
+  }
+  void unboxBoolean(const BaseIndex& src, Register dest) {
+    unboxBoolean(Operand(src), dest);
+  }
+
+  void unboxMagic(const ValueOperand& src, Register dest) {
+    movl(src.valueReg(), dest);
+  }
+
+  void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+    vmovq(src.valueReg(), dest);
+  }
+
+  void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); }
+
+  void unboxNonDouble(const ValueOperand& src, Register dest,
+                      JSValueType type) {
+    MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+    if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+      movl(src.valueReg(), dest);
+      return;
+    }
+    if (src.valueReg() == dest) {
+      ScratchRegisterScope scratch(asMasm());
+      mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+      xorq(scratch, dest);
+    } else {
+      mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+      xorq(src.valueReg(), dest);
+    }
+  }
+  void unboxNonDouble(const Operand& src, Register dest, JSValueType type) {
+    MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+    if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+      movl(src, dest);
+      return;
+    }
+    // Explicitly permits |dest| to be used in |src|.
+    ScratchRegisterScope scratch(asMasm());
+    MOZ_ASSERT(dest != scratch);
+    if (src.containsReg(dest)) {
+      mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+      // If src is already a register, then src and dest are the same
+      // thing and we don't need to move anything into dest.
+      if (src.kind() != Operand::REG) {
+        movq(src, dest);
+      }
+      xorq(scratch, dest);
+    } else {
+      mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+      xorq(src, dest);
+    }
+  }
+  void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
+    unboxNonDouble(Operand(src), dest, type);
+  }
+  void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
+    unboxNonDouble(Operand(src), dest, type);
+  }
+
+  void unboxString(const ValueOperand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+  }
+  void unboxString(const Operand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+  }
+  void unboxString(const Address& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+  }
+
+  void unboxSymbol(const ValueOperand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+  }
+  void unboxSymbol(const Operand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+  }
+
+  void unboxBigInt(const ValueOperand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+  }
+  void unboxBigInt(const Operand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+  }
+  void unboxBigInt(const Address& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+  }
+
+  void unboxObject(const ValueOperand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+  }
+  void unboxObject(const Operand& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+  }
+  void unboxObject(const Address& src, Register dest) {
+    unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+  }
+  void unboxObject(const BaseIndex& src, Register dest) {
+    unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+  }
+
+  template <typename T>
+  void unboxObjectOrNull(const T& src, Register dest) {
+    unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+    ScratchRegisterScope scratch(asMasm());
+    mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+    andq(scratch, dest);
+  }
+
+  // This should only be used for GC barrier code, to unbox a GC thing Value.
+  // It's fine there because we don't depend on the actual Value type (all Cells
+  // are treated the same way). In almost all other cases this would be
+  // Spectre-unsafe - use unboxNonDouble and friends instead.
+  void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+    movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+    andq(Operand(src), dest);
+  }
+  void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+    MOZ_ASSERT(src.valueReg() != dest);
+    movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+    andq(src.valueReg(), dest);
+  }
+
+  void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
+    movq(ImmWord(wasm::AnyRef::GCThingMask), dest);
+    andq(Operand(src), dest);
+  }
+
+  // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+  void getGCThingValueChunk(const Address& src, Register dest) {
+    movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+    andq(Operand(src), dest);
+  }
+  void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+    MOZ_ASSERT(src.valueReg() != dest);
+    movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+    andq(src.valueReg(), dest);
+  }
+
+  void getWasmAnyRefGCThingChunk(Register src, Register dest) {
+    MOZ_ASSERT(src != dest);
+    movq(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
+    andq(src, dest);
+  }
+
+  inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
+                                   JSValueType type, Label* fail);
+
+  // Extended unboxing API. If the payload is already in a register, returns
+  // that register. Otherwise, provides a move to the given scratch register,
+  // and returns that.
+  [[nodiscard]] Register extractObject(const Address& address,
+                                       Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    unboxObject(address, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractObject(const ValueOperand& value,
+                                       Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    unboxObject(value, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+                                       Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    unboxSymbol(value, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractInt32(const ValueOperand& value,
+                                      Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    unboxInt32(value, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+                                        Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    unboxBoolean(value, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    loadPtr(address, scratch);
+    splitTag(scratch, scratch);
+    return scratch;
+  }
+  [[nodiscard]] Register extractTag(const ValueOperand& value,
+                                    Register scratch) {
+    MOZ_ASSERT(scratch != ScratchReg);
+    splitTag(value, scratch);
+    return scratch;
+  }
+
+  inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+                         JSValueType type);
+
+  // These two functions use the low 32-bits of the full value register.
+  void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+    convertInt32ToDouble(operand.valueReg(), dest);
+  }
+  void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+    convertInt32ToDouble(operand.valueReg(), dest);
+  }
+
+  void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+    convertInt32ToFloat32(operand.valueReg(), dest);
+  }
+  void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+    convertInt32ToFloat32(operand.valueReg(), dest);
+  }
+
+  void loadConstantDouble(double d, FloatRegister dest);
+  void loadConstantFloat32(float f, FloatRegister dest);
+
+  void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
+  void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
+  void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+                    FloatRegister dest);
+  void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+                    FloatRegister dest);
+  void vporSimd128(const SimdConstant& v, FloatRegister lhs,
+                   FloatRegister dest);
+  void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                     FloatRegister dest);
+  void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
+                         FloatRegister dest);
+  void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+                      FloatRegister dest);
+  void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
+  void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                        FloatRegister dest);
+  void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+  void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
+                         FloatRegister dest);
+  void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
+                       FloatRegister dest);
+
+ public:
+  Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+    test32(operand.valueReg(), operand.valueReg());
+    return truthy ? NonZero : Zero;
+  }
+  Condition testStringTruthy(bool truthy, const ValueOperand& value);
+  Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+  template <typename T>
+  inline void loadInt32OrDouble(const T& src, FloatRegister dest);
+
+  template <typename T>
+  void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) {
+    if (dest.isFloat()) {
+      loadInt32OrDouble(src, dest.fpu());
+    } else {
+      unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type));
+    }
+  }
+
+  template <typename T>
+  void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+                           JSValueType type) {
+    switch (nbytes) {
+      case 8: {
+        ScratchRegisterScope scratch(asMasm());
+        unboxNonDouble(value, scratch, type);
+        storePtr(scratch, address);
+        if (type == JSVAL_TYPE_OBJECT) {
+          // Ideally we would call unboxObjectOrNull, but we need an extra
+          // scratch register for that. So unbox as object, then clear the
+          // object-or-null bit.
+          mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+          andq(scratch, Operand(address));
+        }
+        return;
+      }
+      case 4:
+        store32(value.valueReg(), address);
+        return;
+      case 1:
+        store8(value.valueReg(), address);
+        return;
+      default:
+        MOZ_CRASH("Bad payload width");
+    }
+  }
+
+  // Checks whether a double is representable as a 64-bit integer. If so, the
+  // integer is written to the output register. Otherwise, a bailout is taken to
+  // the given snapshot. This function overwrites the scratch float register.
+  void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+                          bool negativeZeroCheck = true);
+
+  void convertUInt32ToDouble(Register src, FloatRegister dest) {
+    // Zero the output register to break dependencies, see convertInt32ToDouble.
+    zeroDouble(dest);
+
+    vcvtsq2sd(src, dest, dest);
+  }
+
+  void convertUInt32ToFloat32(Register src, FloatRegister dest) {
+    // Zero the output register to break dependencies, see convertInt32ToDouble.
+    zeroDouble(dest);
+
+    vcvtsq2ss(src, dest, dest);
+  }
+
+  inline void incrementInt32Value(const Address& addr);
+
+  inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+                           Label* failure);
+
+ public:
+  void handleFailureWithHandlerTail(Label* profilerExitTail,
+                                    Label* bailoutTail);
+
+  // Instrumentation for entering and leaving the profiler.
+  void profilerEnterFrame(Register framePtr, Register scratch);
+  void profilerExitFrame();
+};
+
+using MacroAssemblerSpecific = MacroAssemblerX64;
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64-inl.h b/js/src/jit/x64/SharedICHelpers-x64-inl.h
new file mode 100644
index 0000000000..c40eb45c74
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64-inl.h
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_inl_h
+#define jit_x64_SharedICHelpers_x64_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+                                   uint32_t argSize) {
+#ifdef DEBUG
+  ScratchRegisterScope scratch(masm);
+
+  // We can assume during this that R0 and R1 have been pushed.
+  // Store frame size without VMFunction arguments for debug assertions.
+  masm.movq(FramePointer, scratch);
+  masm.subq(StackPointer, scratch);
+  masm.subq(Imm32(argSize), scratch);
+  Address frameSizeAddr(FramePointer,
+                        BaselineFrame::reverseOffsetOfDebugFrameSize());
+  masm.store32(scratch, frameSizeAddr);
+#endif
+
+  // Push frame descriptor and perform the tail call.
+  masm.pushFrameDescriptor(FrameType::BaselineJS);
+  masm.push(ICTailCallReg);
+  masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+  masm.pushFrameDescriptor(FrameType::BaselineStub);
+  masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register) {
+#ifdef DEBUG
+  // Compute frame size. Because the return address is still on the stack,
+  // this is:
+  //
+  //   FramePointer
+  //   - StackPointer
+  //   - sizeof(return address)
+
+  ScratchRegisterScope scratch(masm);
+  masm.movq(FramePointer, scratch);
+  masm.subq(StackPointer, scratch);
+  masm.subq(Imm32(sizeof(void*)), scratch);  // Return address.
+
+  Address frameSizeAddr(FramePointer,
+                        BaselineFrame::reverseOffsetOfDebugFrameSize());
+  masm.store32(scratch, frameSizeAddr);
+#endif
+
+  // Push the return address that's currently on top of the stack.
+  masm.Push(Operand(StackPointer, 0));
+
+  // Replace the original return address with the frame descriptor.
+  masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+                Address(StackPointer, sizeof(uintptr_t)));
+
+  // Save old frame pointer, stack pointer and stub reg.
+  masm.Push(FramePointer);
+  masm.mov(StackPointer, FramePointer);
+
+  masm.Push(ICStubReg);
+}
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_inl_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64.h b/js/src/jit/x64/SharedICHelpers-x64.h
new file mode 100644
index 0000000000..8233db5735
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_h
+#define jit_x64_SharedICHelpers_x64_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from Stack top to the top Value inside an IC stub (this is the
+// return address).
+static const size_t ICStackValueOffset = sizeof(void*);
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+  masm.Pop(ICTailCallReg);
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+  masm.Push(ICTailCallReg);
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+  // The stub pointer must already be in ICStubReg.
+  // Call the stubcode.
+  masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
+  *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ret(); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+  Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+  masm.loadPtr(stubAddr, ICStubReg);
+
+  masm.mov(FramePointer, StackPointer);
+  masm.Pop(FramePointer);
+
+  // The return address is on top of the stack, followed by the frame
+  // descriptor. Use a pop instruction to overwrite the frame descriptor
+  // with the return address. Note that pop increments the stack pointer
+  // before computing the address.
+  masm.Pop(Operand(StackPointer, 0));
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+                           MIRType type) {
+  masm.guardedCallPreBarrier(addr, type);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+  // Load next stub into ICStubReg
+  masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+  // Return address is already loaded, just jump to the next stubcode.
+  masm.jmp(Operand(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_h */
diff --git a/js/src/jit/x64/SharedICRegisters-x64.h b/js/src/jit/x64/SharedICRegisters-x64.h
new file mode 100644
index 0000000000..8e52e5f3c9
--- /dev/null
+++ b/js/src/jit/x64/SharedICRegisters-x64.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICRegisters_x64_h
+#define jit_x64_SharedICRegisters_x64_h
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/x64/Assembler-x64.h"
+
+namespace js {
+namespace jit {
+
+static constexpr ValueOperand R0(rcx);
+static constexpr ValueOperand R1(rbx);
+static constexpr ValueOperand R2(rax);
+
+static constexpr Register ICTailCallReg = rsi;
+static constexpr Register ICStubReg = rdi;
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = xmm0;
+static constexpr FloatRegister FloatReg1 = xmm1;
+static constexpr FloatRegister FloatReg2 = xmm2;
+static constexpr FloatRegister FloatReg3 = xmm3;
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_x64_SharedICRegisters_x64_h */
diff --git a/js/src/jit/x64/Trampoline-x64.cpp b/js/src/jit/x64/Trampoline-x64.cpp
new file mode 100644
index 0000000000..fdc52ab49d
--- /dev/null
+++ b/js/src/jit/x64/Trampoline-x64.cpp
@@ -0,0 +1,819 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "jit/x64/SharedICRegisters-x64.h"
+#include "vm/JitActivation.h"  // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::IsPowerOfTwo;
+
+// This struct reflects the contents of the stack entry.
+// Given a `CommonFrameLayout* frame`:
+// - `frame->prevType()` should be `FrameType::CppToJSJit`.
+// - Then EnterJITStackEntry starts at:
+//     frame->callerFramePtr() + EnterJITStackEntry::offsetFromFP()
+//     (the offset is negative, so this subtracts from the frame pointer)
+struct EnterJITStackEntry {
+  // Offset from frame pointer to EnterJITStackEntry*.
+  static constexpr int32_t offsetFromFP() {
+    return -int32_t(offsetof(EnterJITStackEntry, rbp));
+  }
+
+  void* result;
+
+#if defined(_WIN64)
+  struct XMM {
+    using XMM128 = char[16];
+    XMM128 xmm6;
+    XMM128 xmm7;
+    XMM128 xmm8;
+    XMM128 xmm9;
+    XMM128 xmm10;
+    XMM128 xmm11;
+    XMM128 xmm12;
+    XMM128 xmm13;
+    XMM128 xmm14;
+    XMM128 xmm15;
+  } xmm;
+
+  // 16-byte aligment for xmm registers above.
+  uint64_t xmmPadding;
+
+  void* rsi;
+  void* rdi;
+#endif
+
+  void* r15;
+  void* r14;
+  void* r13;
+  void* r12;
+  void* rbx;
+  void* rbp;
+
+  // Pushed by CALL.
+  void* rip;
+};
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+    LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+                    FloatRegisterSet(FloatRegisters::AllMask));
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard x64 fastcall
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+  enterJITOffset_ = startTrampolineCode(masm);
+
+  masm.assertStackAlignment(ABIStackAlignment,
+                            -int32_t(sizeof(uintptr_t)) /* return address */);
+
+  const Register reg_code = IntArgReg0;
+  const Register reg_argc = IntArgReg1;
+  const Register reg_argv = IntArgReg2;
+  static_assert(OsrFrameReg == IntArgReg3);
+
+#if defined(_WIN64)
+  const Address token = Address(rbp, 16 + ShadowStackSpace);
+  const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace);
+  const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace);
+  const Operand result = Operand(rbp, 40 + ShadowStackSpace);
+#else
+  const Register token = IntArgReg4;
+  const Register scopeChain = IntArgReg5;
+  const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace);
+  const Operand result = Operand(rbp, 24 + ShadowStackSpace);
+#endif
+
+  // Note: the stack pushes below must match the fields in EnterJITStackEntry.
+
+  // Save old stack frame pointer, set new stack frame pointer.
+  masm.push(rbp);
+  masm.mov(rsp, rbp);
+
+  // Save non-volatile registers. These must be saved by the trampoline, rather
+  // than by the JIT'd code, because they are scanned by the conservative
+  // scanner.
+  masm.push(rbx);
+  masm.push(r12);
+  masm.push(r13);
+  masm.push(r14);
+  masm.push(r15);
+#if defined(_WIN64)
+  masm.push(rdi);
+  masm.push(rsi);
+
+  // 16-byte aligment for vmovdqa
+  masm.subq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+  masm.vmovdqa(xmm6, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)));
+  masm.vmovdqa(xmm7, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)));
+  masm.vmovdqa(xmm8, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)));
+  masm.vmovdqa(xmm9, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)));
+  masm.vmovdqa(xmm10, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)));
+  masm.vmovdqa(xmm11, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)));
+  masm.vmovdqa(xmm12, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)));
+  masm.vmovdqa(xmm13, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)));
+  masm.vmovdqa(xmm14, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)));
+  masm.vmovdqa(xmm15, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)));
+#endif
+
+  // Save arguments passed in registers needed after function call.
+  masm.push(result);
+
+  // End of pushes reflected in EnterJITStackEntry, i.e. EnterJITStackEntry
+  // starts at this rsp.
+
+  // Remember number of bytes occupied by argument vector
+  masm.mov(reg_argc, r13);
+
+  // if we are constructing, that also needs to include newTarget
+  {
+    Label noNewTarget;
+    masm.branchTest32(Assembler::Zero, token,
+                      Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+    masm.addq(Imm32(1), r13);
+
+    masm.bind(&noNewTarget);
+  }
+
+  masm.shll(Imm32(3), r13);  // r13 = argc * sizeof(Value)
+  static_assert(sizeof(Value) == 1 << 3, "Constant is baked in assembly code");
+
+  // Guarantee stack alignment of Jit frames.
+  //
+  // This code compensates for the offset created by the copy of the vector of
+  // arguments, such that the jit frame will be aligned once the return
+  // address is pushed on the stack.
+  //
+  // In the computation of the offset, we omit the size of the JitFrameLayout
+  // which is pushed on the stack, as the JitFrameLayout size is a multiple of
+  // the JitStackAlignment.
+  masm.mov(rsp, r12);
+  masm.subq(r13, r12);
+  static_assert(
+      sizeof(JitFrameLayout) % JitStackAlignment == 0,
+      "No need to consider the JitFrameLayout for aligning the stack");
+  masm.andl(Imm32(JitStackAlignment - 1), r12);
+  masm.subq(r12, rsp);
+
+  /***************************************************************
+  Loop over argv vector, push arguments onto stack in reverse order
+  ***************************************************************/
+
+  // r13 still stores the number of bytes in the argument vector.
+  masm.addq(reg_argv, r13);  // r13 points above last argument or newTarget
+
+  // while r13 > rdx, push arguments.
+  {
+    Label header, footer;
+    masm.bind(&header);
+
+    masm.cmpPtr(r13, reg_argv);
+    masm.j(AssemblerX86Shared::BelowOrEqual, &footer);
+
+    masm.subq(Imm32(8), r13);
+    masm.push(Operand(r13, 0));
+    masm.jmp(&header);
+
+    masm.bind(&footer);
+  }
+
+  // Load the number of actual arguments.  |result| is used to store the
+  // actual number of arguments without adding an extra argument to the enter
+  // JIT.
+  masm.movq(result, reg_argc);
+  masm.unboxInt32(Operand(reg_argc, 0), reg_argc);
+
+  // Push the callee token.
+  masm.push(token);
+
+  // Push the descriptor.
+  masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, reg_argc, reg_argc);
+
+  CodeLabel returnLabel;
+  Label oomReturnLabel;
+  {
+    // Handle Interpreter -> Baseline OSR.
+    AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+    MOZ_ASSERT(!regs.has(rbp));
+    regs.take(OsrFrameReg);
+    regs.take(reg_code);
+
+    Register scratch = regs.takeAny();
+
+    Label notOsr;
+    masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+    Register numStackValues = regs.takeAny();
+    masm.movq(numStackValuesAddr, numStackValues);
+
+    // Push return address
+    masm.mov(&returnLabel, scratch);
+    masm.push(scratch);
+
+    // Frame prologue.
+    masm.push(rbp);
+    masm.mov(rsp, rbp);
+
+    // Reserve frame.
+    masm.subPtr(Imm32(BaselineFrame::Size()), rsp);
+
+    Register framePtrScratch = regs.takeAny();
+    masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+    masm.mov(rsp, framePtrScratch);
+
+    // Reserve space for locals and stack values.
+    Register valuesSize = regs.takeAny();
+    masm.mov(numStackValues, valuesSize);
+    masm.shll(Imm32(3), valuesSize);
+    masm.subPtr(valuesSize, rsp);
+
+    // Enter exit frame.
+    masm.pushFrameDescriptor(FrameType::BaselineJS);
+    masm.push(Imm32(0));  // Fake return address.
+    masm.push(FramePointer);
+    // No GC things to mark, push a bare token.
+    masm.loadJSContext(scratch);
+    masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+    regs.add(valuesSize);
+
+    masm.push(reg_code);
+
+    using Fn = bool (*)(BaselineFrame* frame, InterpreterFrame* interpFrame,
+                        uint32_t numStackValues);
+    masm.setupUnalignedABICall(scratch);
+    masm.passABIArg(framePtrScratch);  // BaselineFrame
+    masm.passABIArg(OsrFrameReg);      // InterpreterFrame
+    masm.passABIArg(numStackValues);
+    masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+        ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+    masm.pop(reg_code);
+
+    MOZ_ASSERT(reg_code != ReturnReg);
+
+    Label error;
+    masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp);
+    masm.branchIfFalseBool(ReturnReg, &error);
+
+    // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+    // if profiler instrumentation is enabled.
+    {
+      Label skipProfilingInstrumentation;
+      AbsoluteAddress addressOfEnabled(
+          cx->runtime()->geckoProfiler().addressOfEnabled());
+      masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+                    &skipProfilingInstrumentation);
+      masm.profilerEnterFrame(rbp, scratch);
+      masm.bind(&skipProfilingInstrumentation);
+    }
+
+    masm.jump(reg_code);
+
+    // OOM: frame epilogue, load error value, discard return address and return.
+    masm.bind(&error);
+    masm.mov(rbp, rsp);
+    masm.pop(rbp);
+    masm.addPtr(Imm32(sizeof(uintptr_t)), rsp);  // Return address.
+    masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+    masm.jump(&oomReturnLabel);
+
+    masm.bind(&notOsr);
+    masm.movq(scopeChain, R1.scratchReg());
+  }
+
+  // The call will push the return address and frame pointer on the stack, thus
+  // we check that the stack would be aligned once the call is complete.
+  masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+  // Call function.
+  masm.callJitNoProfiler(reg_code);
+
+  {
+    // Interpreter -> Baseline OSR will return here.
+    masm.bind(&returnLabel);
+    masm.addCodeLabel(returnLabel);
+    masm.bind(&oomReturnLabel);
+  }
+
+  // Discard arguments and padding. Set rsp to the address of the
+  // EnterJITStackEntry on the stack.
+  masm.lea(Operand(rbp, EnterJITStackEntry::offsetFromFP()), rsp);
+
+  /*****************************************************************
+  Place return value where it belongs, pop all saved registers
+  *****************************************************************/
+  masm.pop(r12);  // vp
+  masm.storeValue(JSReturnOperand, Operand(r12, 0));
+
+  // Restore non-volatile registers.
+#if defined(_WIN64)
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)), xmm6);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)), xmm7);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)), xmm8);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)), xmm9);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)), xmm10);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)), xmm11);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)), xmm12);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)), xmm13);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)), xmm14);
+  masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)), xmm15);
+
+  masm.addq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+  masm.pop(rsi);
+  masm.pop(rdi);
+#endif
+  masm.pop(r15);
+  masm.pop(r14);
+  masm.pop(r13);
+  masm.pop(r12);
+  masm.pop(rbx);
+
+  // Restore frame pointer and return.
+  masm.pop(rbp);
+  masm.ret();
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+  if (frameStackAddress->prevType() != FrameType::CppToJSJit) {
+    // This is not a CppToJSJit frame, there are no C++ registers here.
+    return mozilla::Nothing{};
+  }
+
+  // Compute pointer to start of EnterJITStackEntry on the stack.
+  uint8_t* fp = frameStackAddress->callerFramePtr();
+  auto* enterJITStackEntry = reinterpret_cast<EnterJITStackEntry*>(
+      fp + EnterJITStackEntry::offsetFromFP());
+
+  // Extract native function call registers.
+  ::JS::ProfilingFrameIterator::RegisterState registerState;
+  registerState.fp = enterJITStackEntry->rbp;
+  registerState.pc = enterJITStackEntry->rip;
+  // sp should be inside the caller's frame, so set sp to the value of the stack
+  // pointer before the call to the EnterJit trampoline.
+  registerState.sp = &enterJITStackEntry->rip + 1;
+  // No lr in this world.
+  registerState.lr = nullptr;
+  return mozilla::Some(registerState);
+}
+
+// Push AllRegs in a way that is compatible with RegisterDump, regardless of
+// what PushRegsInMask might do to reduce the set size.
+static void DumpAllRegs(MacroAssembler& masm) {
+#ifdef ENABLE_WASM_SIMD
+  masm.PushRegsInMask(AllRegs);
+#else
+  // When SIMD isn't supported, PushRegsInMask reduces the set of float
+  // registers to be double-sized, while the RegisterDump expects each of
+  // the float registers to have the maximal possible size
+  // (Simd128DataSize). To work around this, we just spill the double
+  // registers by hand here, using the register dump offset directly.
+  for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more();
+       ++iter) {
+    masm.Push(*iter);
+  }
+
+  masm.reserveStack(sizeof(RegisterDump::FPUArray));
+  for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more();
+       ++iter) {
+    FloatRegister reg = *iter;
+    Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes());
+    masm.storeDouble(reg, spillAddress);
+  }
+#endif
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+  // See explanatory comment in x86's JitRuntime::generateInvalidator.
+
+  invalidatorOffset_ = startTrampolineCode(masm);
+
+  // Push registers such that we can access them from [base + code].
+  DumpAllRegs(masm);
+
+  masm.movq(rsp, rax);  // Argument to jit::InvalidationBailout.
+
+  // Make space for InvalidationBailout's bailoutInfo outparam.
+  masm.reserveStack(sizeof(void*));
+  masm.movq(rsp, rbx);
+
+  using Fn = bool (*)(InvalidationBailoutStack* sp, BaselineBailoutInfo** info);
+  masm.setupUnalignedABICall(rdx);
+  masm.passABIArg(rax);
+  masm.passABIArg(rbx);
+  masm.callWithABI<Fn, InvalidationBailout>(
+      ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
+
+  masm.pop(r9);  // Get the bailoutInfo outparam.
+
+  // Pop the machine state and the dead frame.
+  masm.moveToStackPtr(FramePointer);
+
+  // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+  masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+                                            ArgumentsRectifierKind kind) {
+  // Do not erase the frame pointer in this function.
+
+  AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+  switch (kind) {
+    case ArgumentsRectifierKind::Normal:
+      argumentsRectifierOffset_ = startTrampolineCode(masm);
+      break;
+    case ArgumentsRectifierKind::TrialInlining:
+      trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+      break;
+  }
+
+  // Caller:
+  // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- rsp
+
+  // Frame prologue.
+  //
+  // NOTE: if this changes, fix the Baseline bailout code too!
+  // See BaselineStackBuilder::calculatePrevFramePtr and
+  // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+  masm.push(FramePointer);
+  masm.movq(rsp, FramePointer);
+
+  // Load argc.
+  masm.loadNumActualArgs(FramePointer, r8);
+
+  // Load |nformals| into %rcx.
+  masm.loadPtr(Address(rbp, RectifierFrameLayout::offsetOfCalleeToken()), rax);
+  masm.mov(rax, rcx);
+  masm.andq(Imm32(uint32_t(CalleeTokenMask)), rcx);
+  masm.loadFunctionArgCount(rcx, rcx);
+
+  // Stash another copy in r11, since we are going to do destructive operations
+  // on rcx
+  masm.mov(rcx, r11);
+
+  static_assert(
+      CalleeToken_FunctionConstructing == 1,
+      "Ensure that we can use the constructing bit to count the value");
+  masm.mov(rax, rdx);
+  masm.andq(Imm32(uint32_t(CalleeToken_FunctionConstructing)), rdx);
+
+  // Including |this|, and |new.target|, there are (|nformals| + 1 +
+  // isConstructing) arguments to push to the stack.  Then we push a
+  // JitFrameLayout.  We compute the padding expressed in the number of extra
+  // |undefined| values to push on the stack.
+  static_assert(
+      sizeof(JitFrameLayout) % JitStackAlignment == 0,
+      "No need to consider the JitFrameLayout for aligning the stack");
+  static_assert(
+      JitStackAlignment % sizeof(Value) == 0,
+      "Ensure that we can pad the stack by pushing extra UndefinedValue");
+  static_assert(IsPowerOfTwo(JitStackValueAlignment),
+                "must have power of two for masm.andl to do its job");
+
+  masm.addl(
+      Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+      rcx);
+  masm.addl(rdx, rcx);
+  masm.andl(Imm32(~(JitStackValueAlignment - 1)), rcx);
+
+  // Load the number of |undefined|s to push into %rcx. Subtract 1 for |this|.
+  masm.subl(r8, rcx);
+  masm.subl(Imm32(1), rcx);
+
+  // Caller:
+  // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- rsp
+  // '--- #r8 ---'
+  //
+  // Rectifier frame:
+  // [rbp'] [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+  //                                                          [descr] [raddr] ]
+  //        '------- #rcx --------' '--- #r8 ---'
+
+  // Copy the number of actual arguments into rdx.
+  masm.mov(r8, rdx);
+
+  masm.moveValue(UndefinedValue(), ValueOperand(r10));
+
+  // Push undefined. (including the padding)
+  {
+    Label undefLoopTop;
+    masm.bind(&undefLoopTop);
+
+    masm.push(r10);
+    masm.subl(Imm32(1), rcx);
+    masm.j(Assembler::NonZero, &undefLoopTop);
+  }
+
+  // Get the topmost argument.
+  static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+
+  // Get the topmost argument.
+  BaseIndex b(FramePointer, r8, TimesEight, sizeof(RectifierFrameLayout));
+  masm.lea(Operand(b), rcx);
+
+  // Push arguments, |nargs| + 1 times (to include |this|).
+  masm.addl(Imm32(1), r8);
+  {
+    Label copyLoopTop;
+
+    masm.bind(&copyLoopTop);
+    masm.push(Operand(rcx, 0x0));
+    masm.subq(Imm32(sizeof(Value)), rcx);
+    masm.subl(Imm32(1), r8);
+    masm.j(Assembler::NonZero, &copyLoopTop);
+  }
+
+  // if constructing, copy newTarget
+  {
+    Label notConstructing;
+
+    masm.branchTest32(Assembler::Zero, rax,
+                      Imm32(CalleeToken_FunctionConstructing),
+                      &notConstructing);
+
+    // thisFrame[numFormals] = prevFrame[argc]
+    ValueOperand newTarget(r10);
+
+    // Load vp[argc]. Add sizeof(Value) for |this|.
+    BaseIndex newTargetSrc(FramePointer, rdx, TimesEight,
+                           sizeof(RectifierFrameLayout) + sizeof(Value));
+    masm.loadValue(newTargetSrc, newTarget);
+
+    // Again, 1 for |this|
+    BaseIndex newTargetDest(rsp, r11, TimesEight, sizeof(Value));
+    masm.storeValue(newTarget, newTargetDest);
+
+    masm.bind(&notConstructing);
+  }
+
+  // Caller:
+  // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+  //
+  //
+  // Rectifier frame:
+  // [rbp'] <- rbp [undef] [undef] [undef] [arg2] [arg1] [this] <- rsp [ [argc]
+  //                                                 [callee] [descr] [raddr] ]
+  //
+
+  // Construct JitFrameLayout.
+  masm.push(rax);  // callee token
+  masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, rdx, rdx);
+
+  // Call the target function.
+  masm.andq(Imm32(uint32_t(CalleeTokenMask)), rax);
+  switch (kind) {
+    case ArgumentsRectifierKind::Normal:
+      masm.loadJitCodeRaw(rax, rax);
+      argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(rax);
+      break;
+    case ArgumentsRectifierKind::TrialInlining:
+      Label noBaselineScript, done;
+      masm.loadBaselineJitCodeRaw(rax, rbx, &noBaselineScript);
+      masm.callJitNoProfiler(rbx);
+      masm.jump(&done);
+
+      // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+      masm.bind(&noBaselineScript);
+      masm.loadJitCodeRaw(rax, rax);
+      masm.callJitNoProfiler(rax);
+      masm.bind(&done);
+      break;
+  }
+
+  masm.mov(FramePointer, StackPointer);
+  masm.pop(FramePointer);
+  masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+  // Push registers such that we can access them from [base + code].
+  DumpAllRegs(masm);
+
+  // Get the stack pointer into a register, pre-alignment.
+  masm.movq(rsp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+  PushBailoutFrame(masm, r8);
+
+  // Make space for Bailout's bailoutInfo outparam.
+  masm.reserveStack(sizeof(void*));
+  masm.movq(rsp, r9);
+
+  // Call the bailout function.
+  using Fn = bool (*)(BailoutStack* sp, BaselineBailoutInfo** info);
+  masm.setupUnalignedABICall(rax);
+  masm.passABIArg(r8);
+  masm.passABIArg(r9);
+  masm.callWithABI<Fn, Bailout>(ABIType::General,
+                                CheckUnsafeCallWithABI::DontCheckOther);
+
+  masm.pop(r9);  // Get the bailoutInfo outparam.
+
+  // Remove both the bailout frame and the topmost Ion frame's stack.
+  masm.moveToStackPtr(FramePointer);
+
+  // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+  masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+                                        Label* bailoutTail) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+  bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+  GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+                                   VMFunctionId id, const VMFunctionData& f,
+                                   DynFn nativeFun, uint32_t* wrapperOffset) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+  *wrapperOffset = startTrampolineCode(masm);
+
+  // Avoid conflicts with argument registers while discarding the result after
+  // the function call.
+  AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+  static_assert(
+      (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+      "Wrapper register set must be a superset of Volatile register set");
+
+  // The context is the first argument.
+  Register cxreg = IntArgReg0;
+  regs.take(cxreg);
+
+  // Stack is:
+  //    ... frame ...
+  //  +12 [args]
+  //  +8  descriptor
+  //  +0  returnAddress
+  //
+  // Push the frame pointer to finish the exit frame, then link it up.
+  masm.Push(FramePointer);
+  masm.moveStackPtrTo(FramePointer);
+  masm.loadJSContext(cxreg);
+  masm.enterExitFrame(cxreg, regs.getAny(), id);
+
+  // Reserve space for the outparameter.
+  masm.reserveVMFunctionOutParamSpace(f);
+
+  masm.setupUnalignedABICallDontSaveRestoreSP();
+  masm.passABIArg(cxreg);
+
+  size_t argDisp = ExitFrameLayout::Size();
+
+  // Copy arguments.
+  for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+    switch (f.argProperties(explicitArg)) {
+      case VMFunctionData::WordByValue:
+        if (f.argPassedInFloatReg(explicitArg)) {
+          masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::Float64);
+        } else {
+          masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::General);
+        }
+        argDisp += sizeof(void*);
+        break;
+      case VMFunctionData::WordByRef:
+        masm.passABIArg(MoveOperand(FramePointer, argDisp,
+                                    MoveOperand::Kind::EffectiveAddress),
+                        ABIType::General);
+        argDisp += sizeof(void*);
+        break;
+      case VMFunctionData::DoubleByValue:
+      case VMFunctionData::DoubleByRef:
+        MOZ_CRASH("NYI: x64 callVM should not be used with 128bits values.");
+    }
+  }
+
+  // Copy the implicit outparam, if any.
+  const int32_t outParamOffset =
+      -int32_t(ExitFooterFrame::Size()) - f.sizeOfOutParamStackSlot();
+  if (f.outParam != Type_Void) {
+    masm.passABIArg(MoveOperand(FramePointer, outParamOffset,
+                                MoveOperand::Kind::EffectiveAddress),
+                    ABIType::General);
+  }
+
+  masm.callWithABI(nativeFun, ABIType::General,
+                   CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+  // Test for failure.
+  switch (f.failType()) {
+    case Type_Cell:
+      masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel());
+      break;
+    case Type_Bool:
+      masm.testb(rax, rax);
+      masm.j(Assembler::Zero, masm.failureLabel());
+      break;
+    case Type_Void:
+      break;
+    default:
+      MOZ_CRASH("unknown failure kind");
+  }
+
+  // Load the outparam.
+  masm.loadVMFunctionOutParam(f, Address(FramePointer, outParamOffset));
+
+  // Until C++ code is instrumented against Spectre, prevent speculative
+  // execution from returning any private data.
+  if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+    masm.speculationBarrier();
+  }
+
+  // Pop frame and restore frame pointer.
+  masm.moveToStackPtr(FramePointer);
+  masm.pop(FramePointer);
+
+  // Return. Subtract sizeof(void*) for the frame pointer.
+  masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+                  f.explicitStackSlots() * sizeof(void*) +
+                  f.extraValuesToPop * sizeof(Value)));
+
+  return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+                                        MIRType type) {
+  AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+  uint32_t offset = startTrampolineCode(masm);
+
+  static_assert(PreBarrierReg == rdx);
+  Register temp1 = rax;
+  Register temp2 = rbx;
+  Register temp3 = rcx;
+  masm.push(temp1);
+  masm.push(temp2);
+  masm.push(temp3);
+
+  Label noBarrier;
+  masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+                              &noBarrier);
+
+  // Call into C++ to mark this GC thing.
+  masm.pop(temp3);
+  masm.pop(temp2);
+  masm.pop(temp1);
+
+  LiveRegisterSet regs =
+      LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+                      FloatRegisterSet(FloatRegisters::VolatileMask));
+  masm.PushRegsInMask(regs);
+
+  masm.mov(ImmPtr(cx->runtime()), rcx);
+
+  masm.setupUnalignedABICall(rax);
+  masm.passABIArg(rcx);
+  masm.passABIArg(rdx);
+  masm.callWithABI(JitPreWriteBarrier(type));
+
+  masm.PopRegsInMask(regs);
+  masm.ret();
+
+  masm.bind(&noBarrier);
+  masm.pop(temp3);
+  masm.pop(temp2);
+  masm.pop(temp1);
+  masm.ret();
+
+  return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+                                         Label* bailoutTail) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+  masm.bind(bailoutTail);
+  masm.generateBailoutTail(rdx, r9);
+}