Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1100 insertions, 0 deletions

diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h
new file mode 100644
index 0000000000..beba576a22
--- /dev/null
+++ b/js/src/jit/MacroAssembler-inl.h
@@ -0,0 +1,1100 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MacroAssembler_inl_h
+#define jit_MacroAssembler_inl_h
+
+#include "jit/MacroAssembler.h"
+
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "gc/Zone.h"
+#include "jit/CalleeToken.h"
+#include "jit/CompileWrappers.h"
+#include "jit/JitFrames.h"
+#include "jit/JSJitFrameIter.h"
+#include "js/Prefs.h"
+#include "util/DifferentialTesting.h"
+#include "vm/BigIntType.h"
+#include "vm/JSObject.h"
+#include "vm/ProxyObject.h"
+#include "vm/Runtime.h"
+#include "vm/StringType.h"
+
+#include "jit/ABIFunctionList-inl.h"
+
+#if defined(JS_CODEGEN_X86)
+#  include "jit/x86/MacroAssembler-x86-inl.h"
+#elif defined(JS_CODEGEN_X64)
+#  include "jit/x64/MacroAssembler-x64-inl.h"
+#elif defined(JS_CODEGEN_ARM)
+#  include "jit/arm/MacroAssembler-arm-inl.h"
+#elif defined(JS_CODEGEN_ARM64)
+#  include "jit/arm64/MacroAssembler-arm64-inl.h"
+#elif defined(JS_CODEGEN_MIPS32)
+#  include "jit/mips32/MacroAssembler-mips32-inl.h"
+#elif defined(JS_CODEGEN_MIPS64)
+#  include "jit/mips64/MacroAssembler-mips64-inl.h"
+#elif defined(JS_CODEGEN_LOONG64)
+#  include "jit/loong64/MacroAssembler-loong64-inl.h"
+#elif defined(JS_CODEGEN_RISCV64)
+#  include "jit/riscv64/MacroAssembler-riscv64-inl.h"
+#elif defined(JS_CODEGEN_WASM32)
+#  include "jit/wasm32/MacroAssembler-wasm32-inl.h"
+#elif !defined(JS_CODEGEN_NONE)
+#  error "Unknown architecture!"
+#endif
+
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+template <typename Sig>
+DynFn DynamicFunction(Sig fun) {
+  ABIFunctionSignature<Sig> sig;
+  return DynFn{sig.address(fun)};
+}
+
+// Helper for generatePreBarrier.
+inline DynFn JitPreWriteBarrier(MIRType type) {
+  switch (type) {
+    case MIRType::Value: {
+      using Fn = void (*)(JSRuntime* rt, Value* vp);
+      return DynamicFunction<Fn>(JitValuePreWriteBarrier);
+    }
+    case MIRType::String: {
+      using Fn = void (*)(JSRuntime* rt, JSString** stringp);
+      return DynamicFunction<Fn>(JitStringPreWriteBarrier);
+    }
+    case MIRType::Object: {
+      using Fn = void (*)(JSRuntime* rt, JSObject** objp);
+      return DynamicFunction<Fn>(JitObjectPreWriteBarrier);
+    }
+    case MIRType::Shape: {
+      using Fn = void (*)(JSRuntime* rt, Shape** shapep);
+      return DynamicFunction<Fn>(JitShapePreWriteBarrier);
+    }
+    case MIRType::WasmAnyRef: {
+      using Fn = void (*)(JSRuntime* rt, wasm::AnyRef* refp);
+      return DynamicFunction<Fn>(JitWasmAnyRefPreWriteBarrier);
+    }
+    default:
+      MOZ_CRASH();
+  }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+CodeOffset MacroAssembler::PushWithPatch(ImmWord word) {
+  framePushed_ += sizeof(word.value);
+  return pushWithPatch(word);
+}
+
+CodeOffset MacroAssembler::PushWithPatch(ImmPtr imm) {
+  return PushWithPatch(ImmWord(uintptr_t(imm.value)));
+}
+
+// ===============================================================
+// Simple call functions.
+
+void MacroAssembler::call(TrampolinePtr code) { call(ImmPtr(code.value)); }
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+                                const Register reg) {
+  CodeOffset l = call(reg);
+  append(desc, l);
+  return l;
+}
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+                                uint32_t funcIndex) {
+  CodeOffset l = callWithPatch();
+  append(desc, l, funcIndex);
+  return l;
+}
+
+void MacroAssembler::call(const wasm::CallSiteDesc& desc, wasm::Trap trap) {
+  CodeOffset l = callWithPatch();
+  append(desc, l, trap);
+}
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+                                wasm::SymbolicAddress imm) {
+  MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm),
+             "only for functions which may appear in profiler");
+  CodeOffset raOffset = call(imm);
+  append(desc, raOffset);
+  return raOffset;
+}
+
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::passABIArg(Register reg) {
+  passABIArg(MoveOperand(reg), ABIType::General);
+}
+
+void MacroAssembler::passABIArg(FloatRegister reg, ABIType type) {
+  passABIArg(MoveOperand(reg), type);
+}
+
+void MacroAssembler::callWithABI(DynFn fun, ABIType result,
+                                 CheckUnsafeCallWithABI check) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  callWithABINoProfiler(fun.address, result, check);
+}
+
+template <typename Sig, Sig fun>
+void MacroAssembler::callWithABI(ABIType result, CheckUnsafeCallWithABI check) {
+  ABIFunction<Sig, fun> abiFun;
+  AutoProfilerCallInstrumentation profiler(*this);
+  callWithABINoProfiler(abiFun.address(), result, check);
+}
+
+void MacroAssembler::callWithABI(Register fun, ABIType result) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  callWithABINoProfiler(fun, result);
+}
+
+void MacroAssembler::callWithABI(const Address& fun, ABIType result) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  callWithABINoProfiler(fun, result);
+}
+
+void MacroAssembler::appendSignatureType(ABIType type) {
+#ifdef JS_SIMULATOR
+  signature_ <<= ABITypeArgShift;
+  signature_ |= uint32_t(type);
+#endif
+}
+
+ABIFunctionType MacroAssembler::signature() const {
+#ifdef JS_SIMULATOR
+#  ifdef DEBUG
+  switch (signature_) {
+    case Args_General0:
+    case Args_General1:
+    case Args_General2:
+    case Args_General3:
+    case Args_General4:
+    case Args_General5:
+    case Args_General6:
+    case Args_General7:
+    case Args_General8:
+    case Args_Double_None:
+    case Args_Int_Double:
+    case Args_Float32_Float32:
+    case Args_Int_Float32:
+    case Args_Double_Double:
+    case Args_Double_Int:
+    case Args_Double_DoubleInt:
+    case Args_Double_DoubleDouble:
+    case Args_Double_IntDouble:
+    case Args_Int_IntDouble:
+    case Args_Int_DoubleInt:
+    case Args_Int_DoubleIntInt:
+    case Args_Int_IntDoubleIntInt:
+    case Args_Double_DoubleDoubleDouble:
+    case Args_Double_DoubleDoubleDoubleDouble:
+    case Args_Int64_GeneralGeneral:
+      break;
+    default:
+      MOZ_CRASH("Unexpected type");
+  }
+#  endif  // DEBUG
+
+  return ABIFunctionType(signature_);
+#else
+  // No simulator enabled.
+  MOZ_CRASH("Only available for making calls within a simulator.");
+#endif
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::callJitNoProfiler(Register callee) {
+#ifdef JS_USE_LINK_REGISTER
+  // The return address is pushed by the callee.
+  call(callee);
+#else
+  callAndPushReturnAddress(callee);
+#endif
+  return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(Register callee) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  uint32_t ret = callJitNoProfiler(callee);
+  return ret;
+}
+
+uint32_t MacroAssembler::callJit(JitCode* callee) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  call(callee);
+  return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(TrampolinePtr code) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  call(code);
+  return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(ImmPtr callee) {
+  AutoProfilerCallInstrumentation profiler(*this);
+  call(callee);
+  return currentOffset();
+}
+
+void MacroAssembler::pushFrameDescriptor(FrameType type) {
+  uint32_t descriptor = MakeFrameDescriptor(type);
+  push(Imm32(descriptor));
+}
+
+void MacroAssembler::PushFrameDescriptor(FrameType type) {
+  uint32_t descriptor = MakeFrameDescriptor(type);
+  Push(Imm32(descriptor));
+}
+
+void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
+                                                   uint32_t argc) {
+  uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
+  push(Imm32(descriptor));
+}
+
+void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
+                                                   uint32_t argc) {
+  uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
+  Push(Imm32(descriptor));
+}
+
+void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
+                                                   Register argc,
+                                                   Register scratch) {
+  if (argc != scratch) {
+    mov(argc, scratch);
+  }
+  lshift32(Imm32(NUMACTUALARGS_SHIFT), scratch);
+  or32(Imm32(int32_t(type)), scratch);
+  push(scratch);
+}
+
+void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
+                                                   Register argc,
+                                                   Register scratch) {
+  pushFrameDescriptorForJitCall(type, argc, scratch);
+  framePushed_ += sizeof(uintptr_t);
+}
+
+void MacroAssembler::loadNumActualArgs(Register framePtr, Register dest) {
+  loadPtr(Address(framePtr, JitFrameLayout::offsetOfDescriptor()), dest);
+  rshift32(Imm32(NUMACTUALARGS_SHIFT), dest);
+}
+
+void MacroAssembler::PushCalleeToken(Register callee, bool constructing) {
+  if (constructing) {
+    orPtr(Imm32(CalleeToken_FunctionConstructing), callee);
+    Push(callee);
+    andPtr(Imm32(uint32_t(CalleeTokenMask)), callee);
+  } else {
+    static_assert(CalleeToken_Function == 0,
+                  "Non-constructing call requires no tagging");
+    Push(callee);
+  }
+}
+
+void MacroAssembler::loadFunctionFromCalleeToken(Address token, Register dest) {
+#ifdef DEBUG
+  Label ok;
+  loadPtr(token, dest);
+  andPtr(Imm32(uint32_t(~CalleeTokenMask)), dest);
+  branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_Function), &ok);
+  branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_FunctionConstructing),
+            &ok);
+  assumeUnreachable("Unexpected CalleeToken tag");
+  bind(&ok);
+#endif
+  loadPtr(token, dest);
+  andPtr(Imm32(uint32_t(CalleeTokenMask)), dest);
+}
+
+uint32_t MacroAssembler::buildFakeExitFrame(Register scratch) {
+  mozilla::DebugOnly<uint32_t> initialDepth = framePushed();
+
+  PushFrameDescriptor(FrameType::IonJS);
+  uint32_t retAddr = pushFakeReturnAddress(scratch);
+  Push(FramePointer);
+
+  MOZ_ASSERT(framePushed() == initialDepth + ExitFrameLayout::Size());
+  return retAddr;
+}
+
+// ===============================================================
+// Exit frame footer.
+
+void MacroAssembler::enterExitFrame(Register cxreg, Register scratch,
+                                    VMFunctionId f) {
+  linkExitFrame(cxreg, scratch);
+  // Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments.
+  // See ExitFooterFrame::data_.
+  uintptr_t type = uintptr_t(ExitFrameType::VMFunction) + uintptr_t(f);
+  MOZ_ASSERT(type <= INT32_MAX);
+  Push(Imm32(type));
+}
+
+void MacroAssembler::enterFakeExitFrame(Register cxreg, Register scratch,
+                                        ExitFrameType type) {
+  linkExitFrame(cxreg, scratch);
+  Push(Imm32(int32_t(type)));
+}
+
+void MacroAssembler::enterFakeExitFrameForNative(Register cxreg,
+                                                 Register scratch,
+                                                 bool isConstructing) {
+  enterFakeExitFrame(cxreg, scratch,
+                     isConstructing ? ExitFrameType::ConstructNative
+                                    : ExitFrameType::CallNative);
+}
+
+void MacroAssembler::leaveExitFrame(size_t extraFrame) {
+  freeStack(ExitFooterFrame::Size() + extraFrame);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const ConstantOrRegister& src,
+                               const ValueOperand& dest) {
+  if (src.constant()) {
+    moveValue(src.value(), dest);
+    return;
+  }
+
+  moveValue(src.reg(), dest);
+}
+
+// ===============================================================
+// Copy instructions
+
+void MacroAssembler::copy64(const Address& src, const Address& dest,
+                            Register scratch) {
+#if JS_BITS_PER_WORD == 32
+  MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
+  load32(LowWord(src), scratch);
+  store32(scratch, LowWord(dest));
+  load32(HighWord(src), scratch);
+  store32(scratch, HighWord(dest));
+#else
+  Register64 scratch64(scratch);
+  load64(src, scratch64);
+  store64(scratch64, dest);
+#endif
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(ImmPtr imm, Register dest) {
+  addPtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+
+// ===============================================================
+// Branch functions
+
+template <class L>
+void MacroAssembler::branchIfFalseBool(Register reg, L label) {
+  // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
+  branchTest32(Assembler::Zero, reg, Imm32(0xFF), label);
+}
+
+void MacroAssembler::branchIfTrueBool(Register reg, Label* label) {
+  // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
+  branchTest32(Assembler::NonZero, reg, Imm32(0xFF), label);
+}
+
+void MacroAssembler::branchIfRope(Register str, Label* label) {
+  Address flags(str, JSString::offsetOfFlags());
+  branchTest32(Assembler::Zero, flags, Imm32(JSString::LINEAR_BIT), label);
+}
+
+void MacroAssembler::branchIfNotRope(Register str, Label* label) {
+  Address flags(str, JSString::offsetOfFlags());
+  branchTest32(Assembler::NonZero, flags, Imm32(JSString::LINEAR_BIT), label);
+}
+
+void MacroAssembler::branchLatin1String(Register string, Label* label) {
+  branchTest32(Assembler::NonZero, Address(string, JSString::offsetOfFlags()),
+               Imm32(JSString::LATIN1_CHARS_BIT), label);
+}
+
+void MacroAssembler::branchTwoByteString(Register string, Label* label) {
+  branchTest32(Assembler::Zero, Address(string, JSString::offsetOfFlags()),
+               Imm32(JSString::LATIN1_CHARS_BIT), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNegative(Register bigInt, Label* label) {
+  branchTest32(Assembler::NonZero, Address(bigInt, BigInt::offsetOfFlags()),
+               Imm32(BigInt::signBitMask()), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt,
+                                                 Label* label) {
+  branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()),
+               Imm32(BigInt::signBitMask()), label);
+}
+
+void MacroAssembler::branchIfBigIntIsZero(Register bigInt, Label* label) {
+  branch32(Assembler::Equal, Address(bigInt, BigInt::offsetOfLength()),
+           Imm32(0), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt, Label* label) {
+  branch32(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()),
+           Imm32(0), label);
+}
+
+void MacroAssembler::branchTestFunctionFlags(Register fun, uint32_t flags,
+                                             Condition cond, Label* label) {
+  Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
+  branchTest32(cond, address, Imm32(flags), label);
+}
+
+void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun,
+                                                         Register scratch,
+                                                         Label* label) {
+  // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
+  // have the SELF_HOSTED flag.
+  // This is equivalent to JSFunction::isNonBuiltinConstructor.
+  constexpr int32_t mask = FunctionFlags::BASESCRIPT |
+                           FunctionFlags::SELF_HOSTED |
+                           FunctionFlags::CONSTRUCTOR;
+  constexpr int32_t expected =
+      FunctionFlags::BASESCRIPT | FunctionFlags::CONSTRUCTOR;
+
+  load32(Address(fun, JSFunction::offsetOfFlagsAndArgCount()), scratch);
+  and32(Imm32(mask), scratch);
+  branch32(Assembler::NotEqual, scratch, Imm32(expected), label);
+}
+
+void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun,
+                                                   bool isConstructing,
+                                                   Label* label) {
+  uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
+  branchTestFunctionFlags(fun, flags, Assembler::Zero, label);
+}
+
+void MacroAssembler::branchIfFunctionHasJitEntry(Register fun,
+                                                 bool isConstructing,
+                                                 Label* label) {
+  uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
+  branchTestFunctionFlags(fun, flags, Assembler::NonZero, label);
+}
+
+void MacroAssembler::branchIfScriptHasJitScript(Register script, Label* label) {
+  static_assert(ScriptWarmUpData::JitScriptTag == 0,
+                "Code below depends on tag value");
+  branchTestPtr(Assembler::Zero,
+                Address(script, JSScript::offsetOfWarmUpData()),
+                Imm32(ScriptWarmUpData::TagMask), label);
+}
+
+void MacroAssembler::branchIfScriptHasNoJitScript(Register script,
+                                                  Label* label) {
+  static_assert(ScriptWarmUpData::JitScriptTag == 0,
+                "Code below depends on tag value");
+  static_assert(BaseScript::offsetOfWarmUpData() ==
+                    SelfHostedLazyScript::offsetOfWarmUpData(),
+                "SelfHostedLazyScript and BaseScript must use same layout for "
+                "warmUpData_");
+  branchTestPtr(Assembler::NonZero,
+                Address(script, JSScript::offsetOfWarmUpData()),
+                Imm32(ScriptWarmUpData::TagMask), label);
+}
+
+void MacroAssembler::loadJitScript(Register script, Register dest) {
+#ifdef DEBUG
+  Label ok;
+  branchIfScriptHasJitScript(script, &ok);
+  assumeUnreachable("Script has no JitScript!");
+  bind(&ok);
+#endif
+
+  static_assert(ScriptWarmUpData::JitScriptTag == 0,
+                "Code below depends on tag value");
+  loadPtr(Address(script, JSScript::offsetOfWarmUpData()), dest);
+}
+
+void MacroAssembler::loadFunctionArgCount(Register func, Register output) {
+  load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output);
+  rshift32(Imm32(JSFunction::ArgCountShift), output);
+}
+
+void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg,
+                                                     Register scratch,
+                                                     Label* slowCheck,
+                                                     Label* label) {
+  MOZ_ASSERT(objReg != scratch);
+
+  Label done;
+  if (JS::Prefs::use_emulates_undefined_fuse()) {
+    loadPtr(AbsoluteAddress(
+                runtime()->addressOfHasSeenObjectEmulateUndefinedFuse()),
+            scratch);
+    branchPtr(Assembler::Equal, scratch, ImmPtr(nullptr), &done);
+  }
+
+  // The branches to out-of-line code here implement a conservative version
+  // of the JSObject::isWrapper test performed in EmulatesUndefined.
+  loadObjClassUnsafe(objReg, scratch);
+
+  branchTestClassIsProxy(true, scratch, slowCheck);
+
+  Address flags(scratch, JSClass::offsetOfFlags());
+  branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED),
+               label);
+  bind(&done);
+}
+
+void MacroAssembler::branchFunctionKind(Condition cond,
+                                        FunctionFlags::FunctionKind kind,
+                                        Register fun, Register scratch,
+                                        Label* label) {
+  Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
+  load32(address, scratch);
+  and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK), scratch);
+  branch32(cond, scratch, Imm32(kind), label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+                                        const JSClass* clasp, Register scratch,
+                                        Register spectreRegToZero,
+                                        Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  MOZ_ASSERT(scratch != spectreRegToZero);
+
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
+            label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreZeroRegister(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestObjClassNoSpectreMitigations(
+    Condition cond, Register obj, const JSClass* clasp, Register scratch,
+    Label* label) {
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
+            label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+                                        const Address& clasp, Register scratch,
+                                        Register spectreRegToZero,
+                                        Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  MOZ_ASSERT(scratch != spectreRegToZero);
+
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+  branchPtr(cond, clasp, scratch, label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreZeroRegister(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestObjClassNoSpectreMitigations(
+    Condition cond, Register obj, const Address& clasp, Register scratch,
+    Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+  branchPtr(cond, clasp, scratch, label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+                                        Register clasp, Register scratch,
+                                        Register spectreRegToZero,
+                                        Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  MOZ_ASSERT(scratch != spectreRegToZero);
+
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+  branchPtr(cond, clasp, scratch, label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreZeroRegister(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestClassIsFunction(Condition cond, Register clasp,
+                                               Label* label) {
+  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+  if (cond == Assembler::Equal) {
+    branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), label);
+    branchPtr(Assembler::Equal, clasp, ImmPtr(&ExtendedFunctionClass), label);
+    return;
+  }
+
+  Label isFunction;
+  branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), &isFunction);
+  branchPtr(Assembler::NotEqual, clasp, ImmPtr(&ExtendedFunctionClass), label);
+  bind(&isFunction);
+}
+
+void MacroAssembler::branchTestObjIsFunction(Condition cond, Register obj,
+                                             Register scratch,
+                                             Register spectreRegToZero,
+                                             Label* label) {
+  MOZ_ASSERT(scratch != spectreRegToZero);
+
+  branchTestObjIsFunctionNoSpectreMitigations(cond, obj, scratch, label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreZeroRegister(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
+    Condition cond, Register obj, Register scratch, Label* label) {
+  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+  MOZ_ASSERT(obj != scratch);
+
+  loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+  loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+  loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+  branchTestClassIsFunction(cond, scratch, label);
+}
+
+void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
+                                        const Shape* shape, Register scratch,
+                                        Register spectreRegToZero,
+                                        Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  MOZ_ASSERT(spectreRegToZero != scratch);
+
+  if (JitOptions.spectreObjectMitigations) {
+    move32(Imm32(0), scratch);
+  }
+
+  branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
+            label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreMovePtr(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
+                                                            Register obj,
+                                                            const Shape* shape,
+                                                            Label* label) {
+  branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
+            label);
+}
+
+void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
+                                        Register shape, Register scratch,
+                                        Register spectreRegToZero,
+                                        Label* label) {
+  MOZ_ASSERT(obj != scratch);
+  MOZ_ASSERT(obj != shape);
+  MOZ_ASSERT(spectreRegToZero != scratch);
+
+  if (JitOptions.spectreObjectMitigations) {
+    move32(Imm32(0), scratch);
+  }
+
+  branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
+
+  if (JitOptions.spectreObjectMitigations) {
+    spectreMovePtr(cond, scratch, spectreRegToZero);
+  }
+}
+
+void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
+                                                            Register obj,
+                                                            Register shape,
+                                                            Label* label) {
+  branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
+}
+
+void MacroAssembler::branchTestObjShapeUnsafe(Condition cond, Register obj,
+                                              Register shape, Label* label) {
+  branchTestObjShapeNoSpectreMitigations(cond, obj, shape, label);
+}
+
+void MacroAssembler::branchTestClassIsProxy(bool proxy, Register clasp,
+                                            Label* label) {
+  branchTest32(proxy ? Assembler::NonZero : Assembler::Zero,
+               Address(clasp, JSClass::offsetOfFlags()),
+               Imm32(JSCLASS_IS_PROXY), label);
+}
+
+void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object,
+                                             Register scratch, Label* label) {
+  constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
+  static_assert(uint32_t(Shape::Kind::Proxy) == 0,
+                "branchTest32 below depends on proxy kind being 0");
+  loadPtr(Address(object, JSObject::offsetOfShape()), scratch);
+  branchTest32(proxy ? Assembler::Zero : Assembler::NonZero,
+               Address(scratch, Shape::offsetOfImmutableFlags()),
+               Imm32(ShiftedMask), label);
+}
+
+void MacroAssembler::branchTestProxyHandlerFamily(Condition cond,
+                                                  Register proxy,
+                                                  Register scratch,
+                                                  const void* handlerp,
+                                                  Label* label) {
+#ifdef DEBUG
+  Label ok;
+  branchTestObjectIsProxy(true, proxy, scratch, &ok);
+  assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
+  bind(&ok);
+#endif
+
+  Address handlerAddr(proxy, ProxyObject::offsetOfHandler());
+  loadPtr(handlerAddr, scratch);
+  Address familyAddr(scratch, BaseProxyHandler::offsetOfFamily());
+  branchPtr(cond, familyAddr, ImmPtr(handlerp), label);
+}
+
+void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond,
+                                                       Label* label) {
+  MOZ_ASSERT(cond == Zero || cond == NonZero);
+  CompileZone* zone = realm()->zone();
+  const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier();
+  branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label);
+}
+
+void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
+    Condition cond, Label* label, Register scratch) {
+  MOZ_ASSERT(cond == Zero || cond == NonZero);
+  if (maybeRealm_) {
+    branchTestNeedsIncrementalBarrier(cond, label);
+  } else {
+    // We are compiling the interpreter or another runtime-wide trampoline, so
+    // we have to load cx->zone.
+    loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch);
+    Address needsBarrierAddr(scratch, Zone::offsetOfNeedsIncrementalBarrier());
+    branchTest32(cond, needsBarrierAddr, Imm32(0x1), label);
+  }
+}
+
+void MacroAssembler::branchTestMagicValue(Condition cond,
+                                          const ValueOperand& val,
+                                          JSWhyMagic why, Label* label) {
+  MOZ_ASSERT(cond == Equal || cond == NotEqual);
+  branchTestValue(cond, val, MagicValue(why), label);
+}
+
+void MacroAssembler::branchDoubleNotInInt64Range(Address src, Register temp,
+                                                 Label* fail) {
+  using mozilla::FloatingPoint;
+
+  // Tests if double is in [INT64_MIN; INT64_MAX] range
+  uint32_t EXPONENT_MASK = 0x7ff00000;
+  uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
+  uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 63)
+                              << EXPONENT_SHIFT;
+
+  load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
+  and32(Imm32(EXPONENT_MASK), temp);
+  branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchDoubleNotInUInt64Range(Address src, Register temp,
+                                                  Label* fail) {
+  using mozilla::FloatingPoint;
+
+  // Note: returns failure on -0.0
+  // Tests if double is in [0; UINT64_MAX] range
+  // Take the sign also in the equation. That way we can compare in one test?
+  uint32_t EXPONENT_MASK = 0xfff00000;
+  uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
+  uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 64)
+                              << EXPONENT_SHIFT;
+
+  load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
+  and32(Imm32(EXPONENT_MASK), temp);
+  branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchFloat32NotInInt64Range(Address src, Register temp,
+                                                  Label* fail) {
+  using mozilla::FloatingPoint;
+
+  // Tests if float is in [INT64_MIN; INT64_MAX] range
+  uint32_t EXPONENT_MASK = 0x7f800000;
+  uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
+  uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 63)
+                              << EXPONENT_SHIFT;
+
+  load32(src, temp);
+  and32(Imm32(EXPONENT_MASK), temp);
+  branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchFloat32NotInUInt64Range(Address src, Register temp,
+                                                   Label* fail) {
+  using mozilla::FloatingPoint;
+
+  // Note: returns failure on -0.0
+  // Tests if float is in [0; UINT64_MAX] range
+  // Take the sign also in the equation. That way we can compare in one test?
+  uint32_t EXPONENT_MASK = 0xff800000;
+  uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
+  uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 64)
+                              << EXPONENT_SHIFT;
+
+  load32(src, temp);
+  and32(Imm32(EXPONENT_MASK), temp);
+  branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+// ========================================================================
+// Canonicalization primitives.
+void MacroAssembler::canonicalizeFloat(FloatRegister reg) {
+  Label notNaN;
+  branchFloat(DoubleOrdered, reg, reg, &notNaN);
+  loadConstantFloat32(float(JS::GenericNaN()), reg);
+  bind(&notNaN);
+}
+
+void MacroAssembler::canonicalizeFloatIfDeterministic(FloatRegister reg) {
+  // See the comment in TypedArrayObjectTemplate::getElement.
+  if (js::SupportDifferentialTesting()) {
+    canonicalizeFloat(reg);
+  }
+}
+
+void MacroAssembler::canonicalizeDouble(FloatRegister reg) {
+  Label notNaN;
+  branchDouble(DoubleOrdered, reg, reg, &notNaN);
+  loadConstantDouble(JS::GenericNaN(), reg);
+  bind(&notNaN);
+}
+
+void MacroAssembler::canonicalizeDoubleIfDeterministic(FloatRegister reg) {
+  // See the comment in TypedArrayObjectTemplate::getElement.
+  if (js::SupportDifferentialTesting()) {
+    canonicalizeDouble(reg);
+  }
+}
+
+// ========================================================================
+// Memory access primitives.
+template <class T>
+FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
+                                               const T& dest) {
+  canonicalizeDoubleIfDeterministic(src);
+  return storeUncanonicalizedDouble(src, dest);
+}
+
+template FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
+                                                        const Address& dest);
+template FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
+                                                        const BaseIndex& dest);
+
+template <class T>
+void MacroAssembler::boxDouble(FloatRegister src, const T& dest) {
+  storeDouble(src, dest);
+}
+
+template <class T>
+FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
+                                                const T& dest) {
+  canonicalizeFloatIfDeterministic(src);
+  return storeUncanonicalizedFloat32(src, dest);
+}
+
+template FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
+                                                         const Address& dest);
+template FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
+                                                         const BaseIndex& dest);
+
+template <typename T>
+void MacroAssembler::fallibleUnboxInt32(const T& src, Register dest,
+                                        Label* fail) {
+  // Int32Value can be unboxed efficiently with unboxInt32, so use that.
+  branchTestInt32(Assembler::NotEqual, src, fail);
+  unboxInt32(src, dest);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxBoolean(const T& src, Register dest,
+                                          Label* fail) {
+  // BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
+  branchTestBoolean(Assembler::NotEqual, src, fail);
+  unboxBoolean(src, dest);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxObject(const T& src, Register dest,
+                                         Label* fail) {
+  fallibleUnboxPtr(src, dest, JSVAL_TYPE_OBJECT, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxString(const T& src, Register dest,
+                                         Label* fail) {
+  fallibleUnboxPtr(src, dest, JSVAL_TYPE_STRING, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxSymbol(const T& src, Register dest,
+                                         Label* fail) {
+  fallibleUnboxPtr(src, dest, JSVAL_TYPE_SYMBOL, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxBigInt(const T& src, Register dest,
+                                         Label* fail) {
+  fallibleUnboxPtr(src, dest, JSVAL_TYPE_BIGINT, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+#ifndef JS_CODEGEN_ARM64
+
+template <typename T>
+void MacroAssembler::branchTestStackPtr(Condition cond, T t, Label* label) {
+  branchTestPtr(cond, getStackPointer(), t, label);
+}
+
+template <typename T>
+void MacroAssembler::branchStackPtr(Condition cond, T rhs, Label* label) {
+  branchPtr(cond, getStackPointer(), rhs, label);
+}
+
+template <typename T>
+void MacroAssembler::branchStackPtrRhs(Condition cond, T lhs, Label* label) {
+  branchPtr(cond, lhs, getStackPointer(), label);
+}
+
+template <typename T>
+void MacroAssembler::addToStackPtr(T t) {
+  addPtr(t, getStackPointer());
+}
+
+template <typename T>
+void MacroAssembler::addStackPtrTo(T t) {
+  addPtr(getStackPointer(), t);
+}
+
+void MacroAssembler::reserveStack(uint32_t amount) {
+  subFromStackPtr(Imm32(amount));
+  adjustFrame(amount);
+}
+#endif  // !JS_CODEGEN_ARM64
+
+void MacroAssembler::loadObjClassUnsafe(Register obj, Register dest) {
+  loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
+  loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
+  loadPtr(Address(dest, BaseShape::offsetOfClasp()), dest);
+}
+
+template <typename EmitPreBarrier>
+void MacroAssembler::storeObjShape(Register shape, Register obj,
+                                   EmitPreBarrier emitPreBarrier) {
+  MOZ_ASSERT(shape != obj);
+  Address shapeAddr(obj, JSObject::offsetOfShape());
+  emitPreBarrier(*this, shapeAddr);
+  storePtr(shape, shapeAddr);
+}
+
+template <typename EmitPreBarrier>
+void MacroAssembler::storeObjShape(Shape* shape, Register obj,
+                                   EmitPreBarrier emitPreBarrier) {
+  Address shapeAddr(obj, JSObject::offsetOfShape());
+  emitPreBarrier(*this, shapeAddr);
+  storePtr(ImmGCPtr(shape), shapeAddr);
+}
+
+void MacroAssembler::loadObjProto(Register obj, Register dest) {
+  loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
+  loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
+  loadPtr(Address(dest, BaseShape::offsetOfProto()), dest);
+}
+
+void MacroAssembler::loadStringLength(Register str, Register dest) {
+  load32(Address(str, JSString::offsetOfLength()), dest);
+}
+
+void MacroAssembler::assertStackAlignment(uint32_t alignment,
+                                          int32_t offset /* = 0 */) {
+#ifdef DEBUG
+  Label ok, bad;
+  MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
+
+  // Wrap around the offset to be a non-negative number.
+  offset %= alignment;
+  if (offset < 0) {
+    offset += alignment;
+  }
+
+  // Test if each bit from offset is set.
+  uint32_t off = offset;
+  while (off) {
+    uint32_t lowestBit = 1 << mozilla::CountTrailingZeroes32(off);
+    branchTestStackPtr(Assembler::Zero, Imm32(lowestBit), &bad);
+    off ^= lowestBit;
+  }
+
+  // Check that all remaining bits are zero.
+  branchTestStackPtr(Assembler::Zero, Imm32((alignment - 1) ^ offset), &ok);
+
+  bind(&bad);
+  breakpoint();
+  bind(&ok);
+#endif
+}
+
+void MacroAssembler::storeCallBoolResult(Register reg) {
+  convertBoolToInt32(ReturnReg, reg);
+}
+
+void MacroAssembler::storeCallInt32Result(Register reg) {
+#if JS_BITS_PER_WORD == 32
+  storeCallPointerResult(reg);
+#else
+  // Ensure the upper 32 bits are cleared.
+  move32(ReturnReg, reg);
+#endif
+}
+
+void MacroAssembler::storeCallResultValue(AnyRegister dest, JSValueType type) {
+  unboxValue(JSReturnOperand, dest, type);
+}
+
+void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest) {
+  if (dest.hasValue()) {
+    storeCallResultValue(dest.valueReg());
+  } else {
+    storeCallResultValue(dest.typedReg(), ValueTypeFromMIRType(dest.type()));
+  }
+}
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_MacroAssembler_inl_h */