summaryrefslogtreecommitdiffstats
path: root/js/src/wasm/WasmBaselineCompile.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/wasm/WasmBaselineCompile.cpp')
-rw-r--r--js/src/wasm/WasmBaselineCompile.cpp11009
1 files changed, 11009 insertions, 0 deletions
diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp
new file mode 100644
index 0000000000..b277ca22c6
--- /dev/null
+++ b/js/src/wasm/WasmBaselineCompile.cpp
@@ -0,0 +1,11009 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright 2016 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * [SMDOC] WebAssembly baseline compiler (RabaldrMonkey)
+ *
+ * For now, see WasmBCClass.h for general comments about the compiler's
+ * structure.
+ *
+ * ----------------
+ *
+ * General assumptions for 32-bit vs 64-bit code:
+ *
+ * - A 32-bit register can be extended in-place to a 64-bit register on 64-bit
+ * systems.
+ *
+ * - Code that knows that Register64 has a '.reg' member on 64-bit systems and
+ * '.high' and '.low' members on 32-bit systems, or knows the implications
+ * thereof, is #ifdef JS_PUNBOX64. All other code is #if(n)?def JS_64BIT.
+ *
+ * Coding standards are a little fluid:
+ *
+ * - In "small" code generating functions (eg emitMultiplyF64, emitQuotientI32,
+ * and surrounding functions; most functions fall into this class) where the
+ * meaning is obvious:
+ *
+ * Old school:
+ * - if there is a single source + destination register, it is called 'r'
+ * - if there is one source and a different destination, they are called 'rs'
+ * and 'rd'
+ * - if there is one source + destination register and another source register
+ * they are called 'r' and 'rs'
+ * - if there are two source registers and a destination register they are
+ * called 'rs0', 'rs1', and 'rd'.
+ *
+ * The new thing:
+ * - what is called 'r' in the old-school naming scheme is increasingly called
+ * 'rsd' in source+dest cases.
+ *
+ * - Generic temp registers are named /temp[0-9]?/ not /tmp[0-9]?/.
+ *
+ * - Registers can be named non-generically for their function ('rp' for the
+ * 'pointer' register and 'rv' for the 'value' register are typical) and those
+ * names may or may not have an 'r' prefix.
+ *
+ * - "Larger" code generating functions make their own rules.
+ */
+
+#include "wasm/WasmBaselineCompile.h"
+
+#include "wasm/WasmBCClass.h"
+#include "wasm/WasmBCDefs.h"
+#include "wasm/WasmBCFrame.h"
+#include "wasm/WasmBCRegDefs.h"
+#include "wasm/WasmBCStk.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "wasm/WasmBCClass-inl.h"
+#include "wasm/WasmBCCodegen-inl.h"
+#include "wasm/WasmBCRegDefs-inl.h"
+#include "wasm/WasmBCRegMgmt-inl.h"
+#include "wasm/WasmBCStkMgmt-inl.h"
+
+namespace js {
+namespace wasm {
+
+using namespace js::jit;
+
+////////////////////////////////////////////////////////////
+//
+// Out of line code management.
+
+// The baseline compiler will use OOL code more sparingly than Ion since our
+// code is not high performance and frills like code density and branch
+// prediction friendliness will be less important.
+class OutOfLineCode : public TempObject {
+ private:
+ NonAssertingLabel entry_;
+ NonAssertingLabel rejoin_;
+ StackHeight stackHeight_;
+
+ public:
+ OutOfLineCode() : stackHeight_(StackHeight::Invalid()) {}
+
+ Label* entry() { return &entry_; }
+ Label* rejoin() { return &rejoin_; }
+
+ void setStackHeight(StackHeight stackHeight) {
+ MOZ_ASSERT(!stackHeight_.isValid());
+ stackHeight_ = stackHeight;
+ }
+
+ void bind(BaseStackFrame* fr, MacroAssembler* masm) {
+ MOZ_ASSERT(stackHeight_.isValid());
+ masm->bind(&entry_);
+ fr->setStackHeight(stackHeight_);
+ }
+
+ // The generate() method must be careful about register use because it will be
+ // invoked when there is a register assignment in the BaseCompiler that does
+ // not correspond to the available registers when the generated OOL code is
+ // executed. The register allocator *must not* be called.
+ //
+ // The best strategy is for the creator of the OOL object to allocate all
+ // temps that the OOL code will need.
+ //
+ // Input, output, and temp registers are embedded in the OOL object and are
+ // known to the code generator.
+ //
+ // Scratch registers are available to use in OOL code.
+ //
+ // All other registers must be explicitly saved and restored by the OOL code
+ // before being used.
+
+ virtual void generate(MacroAssembler* masm) = 0;
+};
+
+OutOfLineCode* BaseCompiler::addOutOfLineCode(OutOfLineCode* ool) {
+ if (!ool || !outOfLine_.append(ool)) {
+ return nullptr;
+ }
+ ool->setStackHeight(fr.stackHeight());
+ return ool;
+}
+
+bool BaseCompiler::generateOutOfLineCode() {
+ for (auto* ool : outOfLine_) {
+ if (!ool->entry()->used()) {
+ continue;
+ }
+ ool->bind(&fr, &masm);
+ ool->generate(&masm);
+ }
+
+ return !masm.oom();
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Sundry code generation.
+
+bool BaseCompiler::addInterruptCheck() {
+#ifdef RABALDR_PIN_INSTANCE
+ Register tmp(InstanceReg);
+#else
+ ScratchI32 tmp(*this);
+ fr.loadInstancePtr(tmp);
+#endif
+ Label ok;
+ masm.branch32(Assembler::Equal,
+ Address(tmp, wasm::Instance::offsetOfInterrupt()), Imm32(0),
+ &ok);
+ masm.wasmTrap(wasm::Trap::CheckInterrupt, bytecodeOffset());
+ masm.bind(&ok);
+ return createStackMap("addInterruptCheck");
+}
+
+void BaseCompiler::checkDivideByZero(RegI32 rhs) {
+ Label nonZero;
+ masm.branchTest32(Assembler::NonZero, rhs, rhs, &nonZero);
+ trap(Trap::IntegerDivideByZero);
+ masm.bind(&nonZero);
+}
+
+void BaseCompiler::checkDivideByZero(RegI64 r) {
+ Label nonZero;
+ ScratchI32 scratch(*this);
+ masm.branchTest64(Assembler::NonZero, r, r, scratch, &nonZero);
+ trap(Trap::IntegerDivideByZero);
+ masm.bind(&nonZero);
+}
+
+void BaseCompiler::checkDivideSignedOverflow(RegI32 rhs, RegI32 srcDest,
+ Label* done, bool zeroOnOverflow) {
+ Label notMin;
+ masm.branch32(Assembler::NotEqual, srcDest, Imm32(INT32_MIN), &notMin);
+ if (zeroOnOverflow) {
+ masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), &notMin);
+ moveImm32(0, srcDest);
+ masm.jump(done);
+ } else {
+ masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), &notMin);
+ trap(Trap::IntegerOverflow);
+ }
+ masm.bind(&notMin);
+}
+
+void BaseCompiler::checkDivideSignedOverflow(RegI64 rhs, RegI64 srcDest,
+ Label* done, bool zeroOnOverflow) {
+ Label notmin;
+ masm.branch64(Assembler::NotEqual, srcDest, Imm64(INT64_MIN), &notmin);
+ masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), &notmin);
+ if (zeroOnOverflow) {
+ masm.xor64(srcDest, srcDest);
+ masm.jump(done);
+ } else {
+ trap(Trap::IntegerOverflow);
+ }
+ masm.bind(&notmin);
+}
+
+void BaseCompiler::jumpTable(const LabelVector& labels, Label* theTable) {
+ // Flush constant pools to ensure that the table is never interrupted by
+ // constant pool entries.
+ masm.flush();
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+ // Prevent nop sequences to appear in the jump table.
+ AutoForbidNops afn(&masm);
+#endif
+ masm.bind(theTable);
+
+ for (const auto& label : labels) {
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(label.offset());
+ masm.addCodeLabel(cl);
+ }
+}
+
+void BaseCompiler::tableSwitch(Label* theTable, RegI32 switchValue,
+ Label* dispatchCode) {
+ masm.bind(dispatchCode);
+
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
+ ScratchI32 scratch(*this);
+ CodeLabel tableCl;
+
+ masm.mov(&tableCl, scratch);
+
+ tableCl.target()->bind(theTable->offset());
+ masm.addCodeLabel(tableCl);
+
+ masm.jmp(Operand(scratch, switchValue, ScalePointer));
+#elif defined(JS_CODEGEN_ARM)
+ // Flush constant pools: offset must reflect the distance from the MOV
+ // to the start of the table; as the address of the MOV is given by the
+ // label, nothing must come between the bind() and the ma_mov().
+ AutoForbidPoolsAndNops afp(&masm,
+ /* number of instructions in scope = */ 5);
+
+ ScratchI32 scratch(*this);
+
+ // Compute the offset from the ma_mov instruction to the jump table.
+ Label here;
+ masm.bind(&here);
+ uint32_t offset = here.offset() - theTable->offset();
+
+ // Read PC+8
+ masm.ma_mov(pc, scratch);
+
+ // ARM scratch register is required by ma_sub.
+ ScratchRegisterScope arm_scratch(*this);
+
+ // Compute the absolute table base pointer into `scratch`, offset by 8
+ // to account for the fact that ma_mov read PC+8.
+ masm.ma_sub(Imm32(offset + 8), scratch, arm_scratch);
+
+ // Jump indirect via table element.
+ masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue, LSL, 2)), pc, Offset,
+ Assembler::Always);
+#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \
+ defined(JS_CODEGEN_RISCV64)
+ ScratchI32 scratch(*this);
+ CodeLabel tableCl;
+
+ masm.ma_li(scratch, &tableCl);
+
+ tableCl.target()->bind(theTable->offset());
+ masm.addCodeLabel(tableCl);
+
+ masm.branchToComputedAddress(BaseIndex(scratch, switchValue, ScalePointer));
+#elif defined(JS_CODEGEN_ARM64)
+ AutoForbidPoolsAndNops afp(&masm,
+ /* number of instructions in scope = */ 4);
+
+ ScratchI32 scratch(*this);
+
+ ARMRegister s(scratch, 64);
+ ARMRegister v(switchValue, 64);
+ masm.Adr(s, theTable);
+ masm.Add(s, s, Operand(v, vixl::LSL, 3));
+ masm.Ldr(s, MemOperand(s, 0));
+ masm.Br(s);
+#else
+ MOZ_CRASH("BaseCompiler platform hook: tableSwitch");
+#endif
+}
+
+#ifdef JS_CODEGEN_X86
+void BaseCompiler::stashI64(RegPtr regForInstance, RegI64 r) {
+ MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8);
+ MOZ_ASSERT(regForInstance != r.low && regForInstance != r.high);
+# ifdef RABALDR_PIN_INSTANCE
+# error "Pinned instance not expected"
+# endif
+ fr.loadInstancePtr(regForInstance);
+ masm.store32(r.low,
+ Address(regForInstance, Instance::offsetOfBaselineScratch()));
+ masm.store32(
+ r.high, Address(regForInstance, Instance::offsetOfBaselineScratch() + 4));
+}
+
+void BaseCompiler::unstashI64(RegPtr regForInstance, RegI64 r) {
+ MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8);
+# ifdef RABALDR_PIN_INSTANCE
+# error "Pinned instance not expected"
+# endif
+ fr.loadInstancePtr(regForInstance);
+ if (regForInstance == r.low) {
+ masm.load32(
+ Address(regForInstance, Instance::offsetOfBaselineScratch() + 4),
+ r.high);
+ masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()),
+ r.low);
+ } else {
+ masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()),
+ r.low);
+ masm.load32(
+ Address(regForInstance, Instance::offsetOfBaselineScratch() + 4),
+ r.high);
+ }
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Function entry and exit
+
+bool BaseCompiler::beginFunction() {
+ AutoCreatedBy acb(masm, "(wasm)BaseCompiler::beginFunction");
+
+ JitSpew(JitSpew_Codegen, "# ========================================");
+ JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code");
+ JitSpew(JitSpew_Codegen,
+ "# beginFunction: start of function prologue for index %d",
+ (int)func_.index);
+
+ // Make a start on the stackmap for this function. Inspect the args so
+ // as to determine which of them are both in-memory and pointer-typed, and
+ // add entries to machineStackTracker as appropriate.
+
+ ArgTypeVector args(funcType());
+ size_t inboundStackArgBytes = StackArgAreaSizeUnaligned(args);
+ MOZ_ASSERT(inboundStackArgBytes % sizeof(void*) == 0);
+ stackMapGenerator_.numStackArgWords = inboundStackArgBytes / sizeof(void*);
+
+ MOZ_ASSERT(stackMapGenerator_.machineStackTracker.length() == 0);
+ if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers(
+ stackMapGenerator_.numStackArgWords)) {
+ return false;
+ }
+
+ // Identify GC-managed pointers passed on the stack.
+ for (WasmABIArgIter i(args); !i.done(); i++) {
+ ABIArg argLoc = *i;
+ if (argLoc.kind() == ABIArg::Stack &&
+ args[i.index()] == MIRType::RefOrNull) {
+ uint32_t offset = argLoc.offsetFromArgBase();
+ MOZ_ASSERT(offset < inboundStackArgBytes);
+ MOZ_ASSERT(offset % sizeof(void*) == 0);
+ stackMapGenerator_.machineStackTracker.setGCPointer(offset /
+ sizeof(void*));
+ }
+ }
+
+ GenerateFunctionPrologue(
+ masm, CallIndirectId::forFunc(moduleEnv_, func_.index),
+ compilerEnv_.mode() == CompileMode::Tier1 ? Some(func_.index) : Nothing(),
+ &offsets_);
+
+ // GenerateFunctionPrologue pushes exactly one wasm::Frame's worth of
+ // stuff, and none of the values are GC pointers. Hence:
+ if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers(
+ sizeof(Frame) / sizeof(void*))) {
+ return false;
+ }
+
+ // Initialize DebugFrame fields before the stack overflow trap so that
+ // we have the invariant that all observable Frames in a debugEnabled
+ // Module have valid DebugFrames.
+ if (compilerEnv_.debugEnabled()) {
+#ifdef JS_CODEGEN_ARM64
+ static_assert(DebugFrame::offsetOfFrame() % WasmStackAlignment == 0,
+ "aligned");
+#endif
+ masm.reserveStack(DebugFrame::offsetOfFrame());
+ if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers(
+ DebugFrame::offsetOfFrame() / sizeof(void*))) {
+ return false;
+ }
+
+ masm.store32(Imm32(func_.index), Address(masm.getStackPointer(),
+ DebugFrame::offsetOfFuncIndex()));
+ masm.store32(Imm32(0),
+ Address(masm.getStackPointer(), DebugFrame::offsetOfFlags()));
+
+ // No need to initialize cachedReturnJSValue_ or any ref-typed spilled
+ // register results, as they are traced if and only if a corresponding
+ // flag (hasCachedReturnJSValue or hasSpilledRefRegisterResult) is set.
+ }
+
+ // Generate a stack-overflow check and its associated stackmap.
+
+ fr.checkStack(ABINonArgReg0, BytecodeOffset(func_.lineOrBytecode));
+
+ ExitStubMapVector extras;
+ if (!stackMapGenerator_.generateStackmapEntriesForTrapExit(args, &extras)) {
+ return false;
+ }
+ if (!createStackMap("stack check", extras, masm.currentOffset(),
+ HasDebugFrameWithLiveRefs::No)) {
+ return false;
+ }
+
+ size_t reservedBytes = fr.fixedAllocSize() - masm.framePushed();
+ MOZ_ASSERT(0 == (reservedBytes % sizeof(void*)));
+
+ masm.reserveStack(reservedBytes);
+ fr.onFixedStackAllocated();
+ if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers(
+ reservedBytes / sizeof(void*))) {
+ return false;
+ }
+
+ // Locals are stack allocated. Mark ref-typed ones in the stackmap
+ // accordingly.
+ for (const Local& l : localInfo_) {
+ // Locals that are stack arguments were already added to the stackmap
+ // before pushing the frame.
+ if (l.type == MIRType::RefOrNull && !l.isStackArgument()) {
+ uint32_t offs = fr.localOffsetFromSp(l);
+ MOZ_ASSERT(0 == (offs % sizeof(void*)));
+ stackMapGenerator_.machineStackTracker.setGCPointer(offs / sizeof(void*));
+ }
+ }
+
+ // Copy arguments from registers to stack.
+ for (WasmABIArgIter i(args); !i.done(); i++) {
+ if (args.isSyntheticStackResultPointerArg(i.index())) {
+ // If there are stack results and the pointer to stack results
+ // was passed in a register, store it to the stack.
+ if (i->argInRegister()) {
+ fr.storeIncomingStackResultAreaPtr(RegPtr(i->gpr()));
+ }
+ // If we're in a debug frame, copy the stack result pointer arg
+ // to a well-known place.
+ if (compilerEnv_.debugEnabled()) {
+ Register target = ABINonArgReturnReg0;
+ fr.loadIncomingStackResultAreaPtr(RegPtr(target));
+ size_t debugFrameOffset =
+ masm.framePushed() - DebugFrame::offsetOfFrame();
+ size_t debugStackResultsPointerOffset =
+ debugFrameOffset + DebugFrame::offsetOfStackResultsPointer();
+ masm.storePtr(target, Address(masm.getStackPointer(),
+ debugStackResultsPointerOffset));
+ }
+ continue;
+ }
+ if (!i->argInRegister()) {
+ continue;
+ }
+ Local& l = localInfo_[args.naturalIndex(i.index())];
+ switch (i.mirType()) {
+ case MIRType::Int32:
+ fr.storeLocalI32(RegI32(i->gpr()), l);
+ break;
+ case MIRType::Int64:
+ fr.storeLocalI64(RegI64(i->gpr64()), l);
+ break;
+ case MIRType::RefOrNull: {
+ DebugOnly<uint32_t> offs = fr.localOffsetFromSp(l);
+ MOZ_ASSERT(0 == (offs % sizeof(void*)));
+ fr.storeLocalRef(RegRef(i->gpr()), l);
+ // We should have just visited this local in the preceding loop.
+ MOZ_ASSERT(stackMapGenerator_.machineStackTracker.isGCPointer(
+ offs / sizeof(void*)));
+ break;
+ }
+ case MIRType::Double:
+ fr.storeLocalF64(RegF64(i->fpu()), l);
+ break;
+ case MIRType::Float32:
+ fr.storeLocalF32(RegF32(i->fpu()), l);
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ fr.storeLocalV128(RegV128(i->fpu()), l);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Function argument type");
+ }
+ }
+
+ fr.zeroLocals(&ra);
+ fr.storeInstancePtr(InstanceReg);
+
+ if (compilerEnv_.debugEnabled()) {
+ insertBreakablePoint(CallSiteDesc::EnterFrame);
+ if (!createStackMap("debug: enter-frame breakpoint")) {
+ return false;
+ }
+ }
+
+ JitSpew(JitSpew_Codegen,
+ "# beginFunction: enter body with masm.framePushed = %u",
+ masm.framePushed());
+ MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isNothing());
+ stackMapGenerator_.framePushedAtEntryToBody.emplace(masm.framePushed());
+
+ return true;
+}
+
+bool BaseCompiler::endFunction() {
+ AutoCreatedBy acb(masm, "(wasm)BaseCompiler::endFunction");
+
+ JitSpew(JitSpew_Codegen, "# endFunction: start of function epilogue");
+
+ // Always branch to returnLabel_.
+ masm.breakpoint();
+
+ // Patch the add in the prologue so that it checks against the correct
+ // frame size. Flush the constant pool in case it needs to be patched.
+ masm.flush();
+
+ // Precondition for patching.
+ if (masm.oom()) {
+ return false;
+ }
+
+ fr.patchCheckStack();
+
+ masm.bind(&returnLabel_);
+
+ ResultType resultType(ResultType::Vector(funcType().results()));
+
+ popStackReturnValues(resultType);
+
+ if (compilerEnv_.debugEnabled()) {
+ // Store and reload the return value from DebugFrame::return so that
+ // it can be clobbered, and/or modified by the debug trap.
+ saveRegisterReturnValues(resultType);
+ insertBreakablePoint(CallSiteDesc::Breakpoint);
+ if (!createStackMap("debug: return-point breakpoint",
+ HasDebugFrameWithLiveRefs::Maybe)) {
+ return false;
+ }
+ insertBreakablePoint(CallSiteDesc::LeaveFrame);
+ if (!createStackMap("debug: leave-frame breakpoint",
+ HasDebugFrameWithLiveRefs::Maybe)) {
+ return false;
+ }
+ restoreRegisterReturnValues(resultType);
+ }
+
+#ifndef RABALDR_PIN_INSTANCE
+ // To satisy instance extent invariant we need to reload InstanceReg because
+ // baseline can clobber it.
+ fr.loadInstancePtr(InstanceReg);
+#endif
+ GenerateFunctionEpilogue(masm, fr.fixedAllocSize(), &offsets_);
+
+#if defined(JS_ION_PERF)
+ // FIXME - profiling code missing. No bug for this.
+
+ // Note the end of the inline code and start of the OOL code.
+ // gen->perfSpewer().noteEndInlineCode(masm);
+#endif
+
+ JitSpew(JitSpew_Codegen, "# endFunction: end of function epilogue");
+ JitSpew(JitSpew_Codegen, "# endFunction: start of OOL code");
+ if (!generateOutOfLineCode()) {
+ return false;
+ }
+ JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code");
+
+ JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code");
+ if (compilerEnv_.debugEnabled()) {
+ JitSpew(JitSpew_Codegen, "# endFunction: start of debug trap stub");
+ insertBreakpointStub();
+ JitSpew(JitSpew_Codegen, "# endFunction: end of debug trap stub");
+ }
+
+ offsets_.end = masm.currentOffset();
+
+ if (!fr.checkStackHeight()) {
+ return decoder_.fail(decoder_.beginOffset(), "stack frame is too large");
+ }
+
+ JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code for index %d",
+ (int)func_.index);
+ return !masm.oom();
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Debugger API.
+
+void BaseCompiler::insertBreakablePoint(CallSiteDesc::Kind kind) {
+#ifndef RABALDR_PIN_INSTANCE
+ fr.loadInstancePtr(InstanceReg);
+#endif
+
+ // The breakpoint code must call the breakpoint handler installed on the
+ // instance if it is not null. There is one breakable point before
+ // every bytecode, and one at the beginning and at the end of the function.
+ //
+ // There are many constraints:
+ //
+ // - Code should be read-only; we do not want to patch
+ // - The breakpoint code should be as dense as possible, given the volume of
+ // breakable points
+ // - The handler-is-null case should be as fast as we can make it
+ //
+ // The scratch register is available here.
+ //
+ // An unconditional callout would be densest but is too slow. The best
+ // balance results from an inline test for null with a conditional call. The
+ // best code sequence is platform-dependent.
+ //
+ // The conditional call goes to a stub attached to the function that performs
+ // further filtering before calling the breakpoint handler.
+#if defined(JS_CODEGEN_X64)
+ // REX 83 MODRM OFFS IB
+ static_assert(Instance::offsetOfDebugTrapHandler() < 128);
+ masm.cmpq(Imm32(0), Operand(Address(InstanceReg,
+ Instance::offsetOfDebugTrapHandler())));
+
+ // 74 OFFS
+ Label L;
+ L.bind(masm.currentOffset() + 7);
+ masm.j(Assembler::Zero, &L);
+
+ // E8 OFFS OFFS OFFS OFFS
+ masm.call(&debugTrapStub_);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
+ CodeOffset(masm.currentOffset()));
+
+ // Branch destination
+ MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset()));
+#elif defined(JS_CODEGEN_X86)
+ // 83 MODRM OFFS IB
+ static_assert(Instance::offsetOfDebugTrapHandler() < 128);
+ masm.cmpl(Imm32(0), Operand(Address(InstanceReg,
+ Instance::offsetOfDebugTrapHandler())));
+
+ // 74 OFFS
+ Label L;
+ L.bind(masm.currentOffset() + 7);
+ masm.j(Assembler::Zero, &L);
+
+ // E8 OFFS OFFS OFFS OFFS
+ masm.call(&debugTrapStub_);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
+ CodeOffset(masm.currentOffset()));
+
+ // Branch destination
+ MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset()));
+#elif defined(JS_CODEGEN_ARM64)
+ ScratchPtr scratch(*this);
+ ARMRegister tmp(scratch, 64);
+ Label L;
+ masm.Ldr(tmp, MemOperand(Address(InstanceReg,
+ Instance::offsetOfDebugTrapHandler())));
+ masm.Cbz(tmp, &L);
+ masm.Bl(&debugTrapStub_);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
+ CodeOffset(masm.currentOffset()));
+ masm.bind(&L);
+#elif defined(JS_CODEGEN_ARM)
+ ScratchPtr scratch(*this);
+ masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()),
+ scratch);
+ masm.ma_orr(scratch, scratch, SetCC);
+ masm.ma_bl(&debugTrapStub_, Assembler::NonZero);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
+ CodeOffset(masm.currentOffset()));
+#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_RISCV64)
+ ScratchPtr scratch(*this);
+ Label L;
+ masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()),
+ scratch);
+ masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), &L);
+ masm.call(&debugTrapStub_);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
+ CodeOffset(masm.currentOffset()));
+ masm.bind(&L);
+#else
+ MOZ_CRASH("BaseCompiler platform hook: insertBreakablePoint");
+#endif
+}
+
+void BaseCompiler::insertBreakpointStub() {
+ // The debug trap stub performs out-of-line filtering before jumping to the
+ // debug trap handler if necessary. The trap handler returns directly to
+ // the breakable point.
+ //
+ // NOTE, the link register is live here on platforms that have LR.
+ //
+ // The scratch register is available here (as it was at the call site).
+ //
+ // It's useful for the debug trap stub to be compact, as every function gets
+ // one.
+
+ Label L;
+ masm.bind(&debugTrapStub_);
+
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ {
+ ScratchPtr scratch(*this);
+
+ // Get the per-instance table of filtering bits.
+ masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()),
+ scratch);
+
+ // Check the filter bit. There is one bit per function in the module.
+ // Table elements are 32-bit because the masm makes that convenient.
+ masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32),
+ Imm32(1 << (func_.index % 32)), &L);
+
+ // Fast path: return to the execution.
+ masm.ret();
+ }
+#elif defined(JS_CODEGEN_ARM64)
+ {
+ ScratchPtr scratch(*this);
+
+ // Logic as above, except abiret to jump to the LR directly
+ masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()),
+ scratch);
+ masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32),
+ Imm32(1 << (func_.index % 32)), &L);
+ masm.abiret();
+ }
+#elif defined(JS_CODEGEN_ARM)
+ {
+ // We must be careful not to use the SecondScratchRegister, which usually
+ // is LR, as LR is live here. This means avoiding masm abstractions such
+ // as branchTest32.
+
+ static_assert(ScratchRegister != lr);
+ static_assert(Instance::offsetOfDebugFilter() < 0x1000);
+
+ ScratchRegisterScope tmp1(masm);
+ ScratchI32 tmp2(*this);
+ masm.ma_ldr(
+ DTRAddr(InstanceReg, DtrOffImm(Instance::offsetOfDebugFilter())), tmp1);
+ masm.ma_mov(Imm32(func_.index / 32), tmp2);
+ masm.ma_ldr(DTRAddr(tmp1, DtrRegImmShift(tmp2, LSL, 0)), tmp2);
+ masm.ma_tst(tmp2, Imm32(1 << func_.index % 32), tmp1, Assembler::Always);
+ masm.ma_bx(lr, Assembler::Zero);
+ }
+#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_RISCV64)
+ {
+ ScratchPtr scratch(*this);
+
+ // Logic same as ARM64.
+ masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()),
+ scratch);
+ masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32),
+ Imm32(1 << (func_.index % 32)), &L);
+ masm.abiret();
+ }
+#else
+ MOZ_CRASH("BaseCompiler platform hook: endFunction");
+#endif
+
+ // Jump to the debug trap handler.
+ masm.bind(&L);
+ masm.jump(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()));
+}
+
+void BaseCompiler::saveRegisterReturnValues(const ResultType& resultType) {
+ MOZ_ASSERT(compilerEnv_.debugEnabled());
+ size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame();
+ size_t registerResultIdx = 0;
+ for (ABIResultIter i(resultType); !i.done(); i.next()) {
+ const ABIResult result = i.cur();
+ if (!result.inRegister()) {
+#ifdef DEBUG
+ for (i.next(); !i.done(); i.next()) {
+ MOZ_ASSERT(!i.cur().inRegister());
+ }
+#endif
+ break;
+ }
+
+ size_t resultOffset = DebugFrame::offsetOfRegisterResult(registerResultIdx);
+ Address dest(masm.getStackPointer(), debugFrameOffset + resultOffset);
+ switch (result.type().kind()) {
+ case ValType::I32:
+ masm.store32(RegI32(result.gpr()), dest);
+ break;
+ case ValType::I64:
+ masm.store64(RegI64(result.gpr64()), dest);
+ break;
+ case ValType::F64:
+ masm.storeDouble(RegF64(result.fpr()), dest);
+ break;
+ case ValType::F32:
+ masm.storeFloat32(RegF32(result.fpr()), dest);
+ break;
+ case ValType::Ref: {
+ uint32_t flag =
+ DebugFrame::hasSpilledRegisterRefResultBitMask(registerResultIdx);
+ // Tell Instance::traceFrame that we have a pointer to trace.
+ masm.or32(Imm32(flag),
+ Address(masm.getStackPointer(),
+ debugFrameOffset + DebugFrame::offsetOfFlags()));
+ masm.storePtr(RegRef(result.gpr()), dest);
+ break;
+ }
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ masm.storeUnalignedSimd128(RegV128(result.fpr()), dest);
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ registerResultIdx++;
+ }
+}
+
+void BaseCompiler::restoreRegisterReturnValues(const ResultType& resultType) {
+ MOZ_ASSERT(compilerEnv_.debugEnabled());
+ size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame();
+ size_t registerResultIdx = 0;
+ for (ABIResultIter i(resultType); !i.done(); i.next()) {
+ const ABIResult result = i.cur();
+ if (!result.inRegister()) {
+#ifdef DEBUG
+ for (i.next(); !i.done(); i.next()) {
+ MOZ_ASSERT(!i.cur().inRegister());
+ }
+#endif
+ break;
+ }
+ size_t resultOffset =
+ DebugFrame::offsetOfRegisterResult(registerResultIdx++);
+ Address src(masm.getStackPointer(), debugFrameOffset + resultOffset);
+ switch (result.type().kind()) {
+ case ValType::I32:
+ masm.load32(src, RegI32(result.gpr()));
+ break;
+ case ValType::I64:
+ masm.load64(src, RegI64(result.gpr64()));
+ break;
+ case ValType::F64:
+ masm.loadDouble(src, RegF64(result.fpr()));
+ break;
+ case ValType::F32:
+ masm.loadFloat32(src, RegF32(result.fpr()));
+ break;
+ case ValType::Ref:
+ masm.loadPtr(src, RegRef(result.gpr()));
+ break;
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ masm.loadUnalignedSimd128(src, RegV128(result.fpr()));
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Results and block parameters
+
+void BaseCompiler::popStackReturnValues(const ResultType& resultType) {
+ uint32_t bytes = ABIResultIter::MeasureStackBytes(resultType);
+ if (bytes == 0) {
+ return;
+ }
+ Register target = ABINonArgReturnReg0;
+ Register temp = ABINonArgReturnReg1;
+ fr.loadIncomingStackResultAreaPtr(RegPtr(target));
+ fr.popStackResultsToMemory(target, bytes, temp);
+}
+
+// TODO / OPTIMIZE (Bug 1316818): At the moment we use the Wasm
+// inter-procedure ABI for block returns, which allocates ReturnReg as the
+// single block result register. It is possible other choices would lead to
+// better register allocation, as ReturnReg is often first in the register set
+// and will be heavily wanted by the register allocator that uses takeFirst().
+//
+// Obvious options:
+// - pick a register at the back of the register set
+// - pick a random register per block (different blocks have
+// different join regs)
+
+void BaseCompiler::popRegisterResults(ABIResultIter& iter) {
+ // Pop register results. Note that in the single-value case, popping to a
+ // register may cause a sync(); for multi-value we sync'd already.
+ for (; !iter.done(); iter.next()) {
+ const ABIResult& result = iter.cur();
+ if (!result.inRegister()) {
+ // TODO / OPTIMIZE: We sync here to avoid solving the general parallel
+ // move problem in popStackResults. However we could avoid syncing the
+ // values that are going to registers anyway, if they are already in
+ // registers.
+ sync();
+ break;
+ }
+ switch (result.type().kind()) {
+ case ValType::I32:
+ popI32(RegI32(result.gpr()));
+ break;
+ case ValType::I64:
+ popI64(RegI64(result.gpr64()));
+ break;
+ case ValType::F32:
+ popF32(RegF32(result.fpr()));
+ break;
+ case ValType::F64:
+ popF64(RegF64(result.fpr()));
+ break;
+ case ValType::Ref:
+ popRef(RegRef(result.gpr()));
+ break;
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ popV128(RegV128(result.fpr()));
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ }
+}
+
+void BaseCompiler::popStackResults(ABIResultIter& iter, StackHeight stackBase) {
+ MOZ_ASSERT(!iter.done());
+
+ // The iterator should be advanced beyond register results, and register
+ // results should be popped already from the value stack.
+ uint32_t alreadyPopped = iter.index();
+
+ // At this point, only stack arguments are remaining. Iterate through them
+ // to measure how much stack space they will take up.
+ for (; !iter.done(); iter.next()) {
+ MOZ_ASSERT(iter.cur().onStack());
+ }
+
+ // Calculate the space needed to store stack results, in bytes.
+ uint32_t stackResultBytes = iter.stackBytesConsumedSoFar();
+ MOZ_ASSERT(stackResultBytes);
+
+ // Compute the stack height including the stack results. Note that it's
+ // possible that this call expands the stack, for example if some of the
+ // results are supplied by constants and so are not already on the machine
+ // stack.
+ uint32_t endHeight = fr.prepareStackResultArea(stackBase, stackResultBytes);
+
+ // Find a free GPR to use when shuffling stack values. If none is
+ // available, push ReturnReg and restore it after we're done.
+ bool saved = false;
+ RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved);
+
+ // The sequence of Stk values is in the same order on the machine stack as
+ // the result locations, but there is a complication: constant values are
+ // not actually pushed on the machine stack. (At this point registers and
+ // locals have been spilled already.) So, moving the Stk values into place
+ // isn't simply a shuffle-down or shuffle-up operation. There is a part of
+ // the Stk sequence that shuffles toward the FP, a part that's already in
+ // place, and a part that shuffles toward the SP. After shuffling, we have
+ // to materialize the constants.
+
+ // Shuffle mem values toward the frame pointer, copying deepest values
+ // first. Stop when we run out of results, get to a register result, or
+ // find a Stk value that is closer to the FP than the result.
+ for (iter.switchToPrev(); !iter.done(); iter.prev()) {
+ const ABIResult& result = iter.cur();
+ if (!result.onStack()) {
+ break;
+ }
+ MOZ_ASSERT(result.stackOffset() < stackResultBytes);
+ uint32_t destHeight = endHeight - result.stackOffset();
+ uint32_t stkBase = stk_.length() - (iter.count() - alreadyPopped);
+ Stk& v = stk_[stkBase + iter.index()];
+ if (v.isMem()) {
+ uint32_t srcHeight = v.offs();
+ if (srcHeight <= destHeight) {
+ break;
+ }
+ fr.shuffleStackResultsTowardFP(srcHeight, destHeight, result.size(),
+ temp);
+ }
+ }
+
+ // Reset iterator and skip register results.
+ for (iter.reset(); !iter.done(); iter.next()) {
+ if (iter.cur().onStack()) {
+ break;
+ }
+ }
+
+ // Revisit top stack values, shuffling mem values toward the stack pointer,
+ // copying shallowest values first.
+ for (; !iter.done(); iter.next()) {
+ const ABIResult& result = iter.cur();
+ MOZ_ASSERT(result.onStack());
+ MOZ_ASSERT(result.stackOffset() < stackResultBytes);
+ uint32_t destHeight = endHeight - result.stackOffset();
+ Stk& v = stk_[stk_.length() - (iter.index() - alreadyPopped) - 1];
+ if (v.isMem()) {
+ uint32_t srcHeight = v.offs();
+ if (srcHeight >= destHeight) {
+ break;
+ }
+ fr.shuffleStackResultsTowardSP(srcHeight, destHeight, result.size(),
+ temp);
+ }
+ }
+
+ // Reset iterator and skip register results, which are already popped off
+ // the value stack.
+ for (iter.reset(); !iter.done(); iter.next()) {
+ if (iter.cur().onStack()) {
+ break;
+ }
+ }
+
+ // Materialize constants and pop the remaining items from the value stack.
+ for (; !iter.done(); iter.next()) {
+ const ABIResult& result = iter.cur();
+ uint32_t resultHeight = endHeight - result.stackOffset();
+ Stk& v = stk_.back();
+ switch (v.kind()) {
+ case Stk::ConstI32:
+#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \
+ defined(JS_CODEGEN_RISCV64)
+ fr.storeImmediatePtrToStack(v.i32val_, resultHeight, temp);
+#else
+ fr.storeImmediatePtrToStack(uint32_t(v.i32val_), resultHeight, temp);
+#endif
+ break;
+ case Stk::ConstF32:
+ fr.storeImmediateF32ToStack(v.f32val_, resultHeight, temp);
+ break;
+ case Stk::ConstI64:
+ fr.storeImmediateI64ToStack(v.i64val_, resultHeight, temp);
+ break;
+ case Stk::ConstF64:
+ fr.storeImmediateF64ToStack(v.f64val_, resultHeight, temp);
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case Stk::ConstV128:
+ fr.storeImmediateV128ToStack(v.v128val_, resultHeight, temp);
+ break;
+#endif
+ case Stk::ConstRef:
+ fr.storeImmediatePtrToStack(v.refval_, resultHeight, temp);
+ break;
+ case Stk::MemRef:
+ // Update bookkeeping as we pop the Stk entry.
+ stackMapGenerator_.memRefsOnStk--;
+ break;
+ default:
+ MOZ_ASSERT(v.isMem());
+ break;
+ }
+ stk_.popBack();
+ }
+
+ ra.freeTempPtr(temp, saved);
+
+ // This will pop the stack if needed.
+ fr.finishStackResultArea(stackBase, stackResultBytes);
+}
+
+void BaseCompiler::popBlockResults(ResultType type, StackHeight stackBase,
+ ContinuationKind kind) {
+ if (!type.empty()) {
+ ABIResultIter iter(type);
+ popRegisterResults(iter);
+ if (!iter.done()) {
+ popStackResults(iter, stackBase);
+ // Because popStackResults might clobber the stack, it leaves the stack
+ // pointer already in the right place for the continuation, whether the
+ // continuation is a jump or fallthrough.
+ return;
+ }
+ }
+ // We get here if there are no stack results. For a fallthrough, the stack
+ // is already at the right height. For a jump, we may need to pop the stack
+ // pointer if the continuation's stack height is lower than the current
+ // stack height.
+ if (kind == ContinuationKind::Jump) {
+ fr.popStackBeforeBranch(stackBase, type);
+ }
+}
+
+// This function is similar to popBlockResults, but additionally handles the
+// implicit exception pointer that is pushed to the value stack on entry to
+// a catch handler by dropping it appropriately.
+void BaseCompiler::popCatchResults(ResultType type, StackHeight stackBase) {
+ if (!type.empty()) {
+ ABIResultIter iter(type);
+ popRegisterResults(iter);
+ if (!iter.done()) {
+ popStackResults(iter, stackBase);
+ // Since popStackResults clobbers the stack, we only need to free the
+ // exception off of the value stack.
+ popValueStackBy(1);
+ } else {
+ // If there are no stack results, we have to adjust the stack by
+ // dropping the exception reference that's now on the stack.
+ dropValue();
+ }
+ } else {
+ dropValue();
+ }
+ fr.popStackBeforeBranch(stackBase, type);
+}
+
+Stk BaseCompiler::captureStackResult(const ABIResult& result,
+ StackHeight resultsBase,
+ uint32_t stackResultBytes) {
+ MOZ_ASSERT(result.onStack());
+ uint32_t offs = fr.locateStackResult(result, resultsBase, stackResultBytes);
+ return Stk::StackResult(result.type(), offs);
+}
+
+// TODO: It may be fruitful to inline the fast path here, as it will be common.
+
+bool BaseCompiler::pushResults(ResultType type, StackHeight resultsBase) {
+ if (type.empty()) {
+ return true;
+ }
+
+ if (type.length() > 1) {
+ // Reserve extra space on the stack for all the values we'll push.
+ // Multi-value push is not accounted for by the pre-sizing of the stack in
+ // the decoding loop.
+ //
+ // Also make sure we leave headroom for other pushes that will occur after
+ // pushing results, just to be safe.
+ if (!stk_.reserve(stk_.length() + type.length() + MaxPushesPerOpcode)) {
+ return false;
+ }
+ }
+
+ // We need to push the results in reverse order, so first iterate through
+ // all results to determine the locations of stack result types.
+ ABIResultIter iter(type);
+ while (!iter.done()) {
+ iter.next();
+ }
+ uint32_t stackResultBytes = iter.stackBytesConsumedSoFar();
+ for (iter.switchToPrev(); !iter.done(); iter.prev()) {
+ const ABIResult& result = iter.cur();
+ if (!result.onStack()) {
+ break;
+ }
+ Stk v = captureStackResult(result, resultsBase, stackResultBytes);
+ push(v);
+ if (v.kind() == Stk::MemRef) {
+ stackMapGenerator_.memRefsOnStk++;
+ }
+ }
+
+ for (; !iter.done(); iter.prev()) {
+ const ABIResult& result = iter.cur();
+ MOZ_ASSERT(result.inRegister());
+ switch (result.type().kind()) {
+ case ValType::I32:
+ pushI32(RegI32(result.gpr()));
+ break;
+ case ValType::I64:
+ pushI64(RegI64(result.gpr64()));
+ break;
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ pushV128(RegV128(result.fpr()));
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ case ValType::F32:
+ pushF32(RegF32(result.fpr()));
+ break;
+ case ValType::F64:
+ pushF64(RegF64(result.fpr()));
+ break;
+ case ValType::Ref:
+ pushRef(RegRef(result.gpr()));
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool BaseCompiler::pushBlockResults(ResultType type) {
+ return pushResults(type, controlItem().stackHeight);
+}
+
+// A combination of popBlockResults + pushBlockResults, used when entering a
+// block with a control-flow join (loops) or split (if) to shuffle the
+// fallthrough block parameters into the locations expected by the
+// continuation.
+bool BaseCompiler::topBlockParams(ResultType type) {
+ // This function should only be called when entering a block with a
+ // control-flow join at the entry, where there are no live temporaries in
+ // the current block.
+ StackHeight base = controlItem().stackHeight;
+ MOZ_ASSERT(fr.stackResultsBase(stackConsumed(type.length())) == base);
+ popBlockResults(type, base, ContinuationKind::Fallthrough);
+ return pushBlockResults(type);
+}
+
+// A combination of popBlockResults + pushBlockResults, used before branches
+// where we don't know the target (br_if / br_table). If and when the branch
+// is taken, the stack results will be shuffled down into place. For br_if
+// that has fallthrough, the parameters for the untaken branch flow through to
+// the continuation.
+bool BaseCompiler::topBranchParams(ResultType type, StackHeight* height) {
+ if (type.empty()) {
+ *height = fr.stackHeight();
+ return true;
+ }
+ // There may be temporary values that need spilling; delay computation of
+ // the stack results base until after the popRegisterResults(), which spills
+ // if needed.
+ ABIResultIter iter(type);
+ popRegisterResults(iter);
+ StackHeight base = fr.stackResultsBase(stackConsumed(iter.remaining()));
+ if (!iter.done()) {
+ popStackResults(iter, base);
+ }
+ if (!pushResults(type, base)) {
+ return false;
+ }
+ *height = base;
+ return true;
+}
+
+// Conditional branches with fallthrough are preceded by a topBranchParams, so
+// we know that there are no stack results that need to be materialized. In
+// that case, we can just shuffle the whole block down before popping the
+// stack.
+void BaseCompiler::shuffleStackResultsBeforeBranch(StackHeight srcHeight,
+ StackHeight destHeight,
+ ResultType type) {
+ uint32_t stackResultBytes = 0;
+
+ if (ABIResultIter::HasStackResults(type)) {
+ MOZ_ASSERT(stk_.length() >= type.length());
+ ABIResultIter iter(type);
+ for (; !iter.done(); iter.next()) {
+#ifdef DEBUG
+ const ABIResult& result = iter.cur();
+ const Stk& v = stk_[stk_.length() - iter.index() - 1];
+ MOZ_ASSERT(v.isMem() == result.onStack());
+#endif
+ }
+
+ stackResultBytes = iter.stackBytesConsumedSoFar();
+ MOZ_ASSERT(stackResultBytes > 0);
+
+ if (srcHeight != destHeight) {
+ // Find a free GPR to use when shuffling stack values. If none
+ // is available, push ReturnReg and restore it after we're done.
+ bool saved = false;
+ RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved);
+ fr.shuffleStackResultsTowardFP(srcHeight, destHeight, stackResultBytes,
+ temp);
+ ra.freeTempPtr(temp, saved);
+ }
+ }
+
+ fr.popStackBeforeBranch(destHeight, stackResultBytes);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Function calls.
+
+void BaseCompiler::beginCall(
+ FunctionCall& call, UseABI useABI,
+ RestoreRegisterStateAndRealm restoreRegisterStateAndRealm) {
+ MOZ_ASSERT_IF(
+ useABI == UseABI::Builtin,
+ restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::False);
+
+ call.restoreRegisterStateAndRealm =
+ restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::True;
+ call.usesSystemAbi = useABI == UseABI::System;
+
+ if (call.usesSystemAbi) {
+ // Call-outs need to use the appropriate system ABI.
+#if defined(JS_CODEGEN_ARM)
+ call.hardFP = UseHardFpABI();
+ call.abi.setUseHardFp(call.hardFP);
+#endif
+ } else {
+#if defined(JS_CODEGEN_ARM)
+ MOZ_ASSERT(call.hardFP, "All private ABIs pass FP arguments in registers");
+#endif
+ }
+
+ // Use masm.framePushed() because the value we want here does not depend
+ // on the height of the frame's stack area, but the actual size of the
+ // allocated frame.
+ call.frameAlignAdjustment = ComputeByteAlignment(
+ masm.framePushed() + sizeof(Frame), JitStackAlignment);
+}
+
+void BaseCompiler::endCall(FunctionCall& call, size_t stackSpace) {
+ size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
+ fr.freeArgAreaAndPopBytes(adjustment, stackSpace);
+
+ MOZ_ASSERT(stackMapGenerator_.framePushedExcludingOutboundCallArgs.isSome());
+ stackMapGenerator_.framePushedExcludingOutboundCallArgs.reset();
+
+ if (call.restoreRegisterStateAndRealm) {
+ // The instance has been clobbered, so always reload
+ fr.loadInstancePtr(InstanceReg);
+ masm.loadWasmPinnedRegsFromInstance();
+ masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
+ } else if (call.usesSystemAbi) {
+ // On x86 there are no pinned registers, so don't waste time
+ // reloading the instance.
+#ifndef JS_CODEGEN_X86
+ // The instance has been clobbered, so always reload
+ fr.loadInstancePtr(InstanceReg);
+ masm.loadWasmPinnedRegsFromInstance();
+#endif
+ }
+}
+
+void BaseCompiler::startCallArgs(size_t stackArgAreaSizeUnaligned,
+ FunctionCall* call) {
+ size_t stackArgAreaSizeAligned =
+ AlignStackArgAreaSize(stackArgAreaSizeUnaligned);
+ MOZ_ASSERT(stackArgAreaSizeUnaligned <= stackArgAreaSizeAligned);
+
+ // Record the masm.framePushed() value at this point, before we push args
+ // for the call, but including the alignment space placed above the args.
+ // This defines the lower limit of the stackmap that will be created for
+ // this call.
+ MOZ_ASSERT(
+ stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing());
+ stackMapGenerator_.framePushedExcludingOutboundCallArgs.emplace(
+ // However much we've pushed so far
+ masm.framePushed() +
+ // Extra space we'll push to get the frame aligned
+ call->frameAlignAdjustment +
+ // Extra space we'll push to get the outbound arg area aligned
+ (stackArgAreaSizeAligned - stackArgAreaSizeUnaligned));
+
+ call->stackArgAreaSize = stackArgAreaSizeAligned;
+
+ size_t adjustment = call->stackArgAreaSize + call->frameAlignAdjustment;
+ fr.allocArgArea(adjustment);
+}
+
+ABIArg BaseCompiler::reservePointerArgument(FunctionCall* call) {
+ return call->abi.next(MIRType::Pointer);
+}
+
+// TODO / OPTIMIZE (Bug 1316821): Note passArg is used only in one place.
+// (Or it was, until Luke wandered through, but that can be fixed again.)
+// I'm not saying we should manually inline it, but we could hoist the
+// dispatch into the caller and have type-specific implementations of
+// passArg: passArgI32(), etc. Then those might be inlined, at least in PGO
+// builds.
+//
+// The bulk of the work here (60%) is in the next() call, though.
+//
+// Notably, since next() is so expensive, StackArgAreaSizeUnaligned()
+// becomes expensive too.
+//
+// Somehow there could be a trick here where the sequence of argument types
+// (read from the input stream) leads to a cached entry for
+// StackArgAreaSizeUnaligned() and for how to pass arguments...
+//
+// But at least we could reduce the cost of StackArgAreaSizeUnaligned() by
+// first reading the argument types into a (reusable) vector, then we have
+// the outgoing size at low cost, and then we can pass args based on the
+// info we read.
+
+void BaseCompiler::passArg(ValType type, const Stk& arg, FunctionCall* call) {
+ switch (type.kind()) {
+ case ValType::I32: {
+ ABIArg argLoc = call->abi.next(MIRType::Int32);
+ if (argLoc.kind() == ABIArg::Stack) {
+ ScratchI32 scratch(*this);
+ loadI32(arg, scratch);
+ masm.store32(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+ } else {
+ loadI32(arg, RegI32(argLoc.gpr()));
+ }
+ break;
+ }
+ case ValType::I64: {
+ ABIArg argLoc = call->abi.next(MIRType::Int64);
+ if (argLoc.kind() == ABIArg::Stack) {
+ ScratchI32 scratch(*this);
+#ifdef JS_PUNBOX64
+ loadI64(arg, fromI32(scratch));
+ masm.storePtr(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+#else
+ loadI64Low(arg, scratch);
+ masm.store32(scratch, LowWord(Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase())));
+ loadI64High(arg, scratch);
+ masm.store32(scratch, HighWord(Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase())));
+#endif
+ } else {
+ loadI64(arg, RegI64(argLoc.gpr64()));
+ }
+ break;
+ }
+ case ValType::V128: {
+#ifdef ENABLE_WASM_SIMD
+ ABIArg argLoc = call->abi.next(MIRType::Simd128);
+ switch (argLoc.kind()) {
+ case ABIArg::Stack: {
+ ScratchV128 scratch(*this);
+ loadV128(arg, scratch);
+ masm.storeUnalignedSimd128(
+ (RegV128)scratch,
+ Address(masm.getStackPointer(), argLoc.offsetFromArgBase()));
+ break;
+ }
+ case ABIArg::GPR: {
+ MOZ_CRASH("Unexpected parameter passing discipline");
+ }
+ case ABIArg::FPU: {
+ loadV128(arg, RegV128(argLoc.fpu()));
+ break;
+ }
+# if defined(JS_CODEGEN_REGISTER_PAIR)
+ case ABIArg::GPR_PAIR: {
+ MOZ_CRASH("Unexpected parameter passing discipline");
+ }
+# endif
+ case ABIArg::Uninitialized:
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ case ValType::F64: {
+ ABIArg argLoc = call->abi.next(MIRType::Double);
+ switch (argLoc.kind()) {
+ case ABIArg::Stack: {
+ ScratchF64 scratch(*this);
+ loadF64(arg, scratch);
+ masm.storeDouble(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+ break;
+ }
+#if defined(JS_CODEGEN_REGISTER_PAIR)
+ case ABIArg::GPR_PAIR: {
+# if defined(JS_CODEGEN_ARM)
+ ScratchF64 scratch(*this);
+ loadF64(arg, scratch);
+ masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr());
+ break;
+# else
+ MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair");
+# endif
+ }
+#endif
+ case ABIArg::FPU: {
+ loadF64(arg, RegF64(argLoc.fpu()));
+ break;
+ }
+ case ABIArg::GPR: {
+ MOZ_CRASH("Unexpected parameter passing discipline");
+ }
+ case ABIArg::Uninitialized:
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+ break;
+ }
+ case ValType::F32: {
+ ABIArg argLoc = call->abi.next(MIRType::Float32);
+ switch (argLoc.kind()) {
+ case ABIArg::Stack: {
+ ScratchF32 scratch(*this);
+ loadF32(arg, scratch);
+ masm.storeFloat32(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+ break;
+ }
+ case ABIArg::GPR: {
+ ScratchF32 scratch(*this);
+ loadF32(arg, scratch);
+ masm.moveFloat32ToGPR(scratch, argLoc.gpr());
+ break;
+ }
+ case ABIArg::FPU: {
+ loadF32(arg, RegF32(argLoc.fpu()));
+ break;
+ }
+#if defined(JS_CODEGEN_REGISTER_PAIR)
+ case ABIArg::GPR_PAIR: {
+ MOZ_CRASH("Unexpected parameter passing discipline");
+ }
+#endif
+ case ABIArg::Uninitialized:
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+ break;
+ }
+ case ValType::Ref: {
+ ABIArg argLoc = call->abi.next(MIRType::RefOrNull);
+ if (argLoc.kind() == ABIArg::Stack) {
+ ScratchRef scratch(*this);
+ loadRef(arg, scratch);
+ masm.storePtr(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+ } else {
+ loadRef(arg, RegRef(argLoc.gpr()));
+ }
+ break;
+ }
+ }
+}
+
+CodeOffset BaseCompiler::callDefinition(uint32_t funcIndex,
+ const FunctionCall& call) {
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Func);
+ return masm.call(desc, funcIndex);
+}
+
+CodeOffset BaseCompiler::callSymbolic(SymbolicAddress callee,
+ const FunctionCall& call) {
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic);
+ return masm.call(desc, callee);
+}
+
+// Precondition: sync()
+
+class OutOfLineAbortingTrap : public OutOfLineCode {
+ Trap trap_;
+ BytecodeOffset off_;
+
+ public:
+ OutOfLineAbortingTrap(Trap trap, BytecodeOffset off)
+ : trap_(trap), off_(off) {}
+
+ virtual void generate(MacroAssembler* masm) override {
+ masm->wasmTrap(trap_, off_);
+ MOZ_ASSERT(!rejoin()->bound());
+ }
+};
+
+bool BaseCompiler::callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
+ const Stk& indexVal, const FunctionCall& call,
+ CodeOffset* fastCallOffset,
+ CodeOffset* slowCallOffset) {
+ CallIndirectId callIndirectId =
+ CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex);
+ MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
+
+ const TableDesc& table = moduleEnv_.tables[tableIndex];
+
+ loadI32(indexVal, RegI32(WasmTableCallIndexReg));
+
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Indirect);
+ CalleeDesc callee =
+ CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId);
+ OutOfLineCode* oob = addOutOfLineCode(
+ new (alloc_) OutOfLineAbortingTrap(Trap::OutOfBounds, bytecodeOffset()));
+ if (!oob) {
+ return false;
+ }
+ Label* nullCheckFailed = nullptr;
+#ifndef WASM_HAS_HEAPREG
+ OutOfLineCode* nullref = addOutOfLineCode(new (alloc_) OutOfLineAbortingTrap(
+ Trap::IndirectCallToNull, bytecodeOffset()));
+ if (!oob) {
+ return false;
+ }
+ nullCheckFailed = nullref->entry();
+#endif
+ masm.wasmCallIndirect(desc, callee, oob->entry(), nullCheckFailed,
+ mozilla::Nothing(), fastCallOffset, slowCallOffset);
+ return true;
+}
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+void BaseCompiler::callRef(const Stk& calleeRef, const FunctionCall& call,
+ CodeOffset* fastCallOffset,
+ CodeOffset* slowCallOffset) {
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::FuncRef);
+ CalleeDesc callee = CalleeDesc::wasmFuncRef();
+
+ loadRef(calleeRef, RegRef(WasmCallRefReg));
+ masm.wasmCallRef(desc, callee, fastCallOffset, slowCallOffset);
+}
+#endif
+
+// Precondition: sync()
+
+CodeOffset BaseCompiler::callImport(unsigned instanceDataOffset,
+ const FunctionCall& call) {
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Import);
+ CalleeDesc callee = CalleeDesc::import(instanceDataOffset);
+ return masm.wasmCallImport(desc, callee);
+}
+
+CodeOffset BaseCompiler::builtinCall(SymbolicAddress builtin,
+ const FunctionCall& call) {
+ return callSymbolic(builtin, call);
+}
+
+CodeOffset BaseCompiler::builtinInstanceMethodCall(
+ const SymbolicAddressSignature& builtin, const ABIArg& instanceArg,
+ const FunctionCall& call) {
+#ifndef RABALDR_PIN_INSTANCE
+ // Builtin method calls assume the instance register has been set.
+ fr.loadInstancePtr(InstanceReg);
+#endif
+ CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic);
+ return masm.wasmCallBuiltinInstanceMethod(desc, instanceArg, builtin.identity,
+ builtin.failureMode);
+}
+
+bool BaseCompiler::pushCallResults(const FunctionCall& call, ResultType type,
+ const StackResultsLoc& loc) {
+#if defined(JS_CODEGEN_ARM)
+ // pushResults currently bypasses special case code in captureReturnedFxx()
+ // that converts GPR results to FPR results for systemABI+softFP. If we
+ // ever start using that combination for calls we need more code. This
+ // assert is stronger than we need - we only care about results in return
+ // registers - but that's OK.
+ MOZ_ASSERT(!call.usesSystemAbi || call.hardFP);
+#endif
+ return pushResults(type, fr.stackResultsBase(loc.bytes()));
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Exception handling
+
+// Abstracted helper for throwing, used for throw, rethrow, and rethrowing
+// at the end of a series of catch blocks (if none matched the exception).
+bool BaseCompiler::throwFrom(RegRef exn) {
+ pushRef(exn);
+
+ // ThrowException invokes a trap, and the rest is dead code.
+ return emitInstanceCall(SASigThrowException);
+}
+
+void BaseCompiler::loadTag(RegPtr instance, uint32_t tagIndex, RegRef tagDst) {
+ size_t offset =
+ Instance::offsetInData(moduleEnv_.offsetOfTagInstanceData(tagIndex));
+ masm.loadPtr(Address(instance, offset), tagDst);
+}
+
+void BaseCompiler::consumePendingException(RegRef* exnDst, RegRef* tagDst) {
+ RegPtr pendingAddr = RegPtr(PreBarrierReg);
+ needPtr(pendingAddr);
+ masm.computeEffectiveAddress(
+ Address(InstanceReg, Instance::offsetOfPendingException()), pendingAddr);
+ *exnDst = needRef();
+ masm.loadPtr(Address(pendingAddr, 0), *exnDst);
+ emitBarrieredClear(pendingAddr);
+
+ *tagDst = needRef();
+ masm.computeEffectiveAddress(
+ Address(InstanceReg, Instance::offsetOfPendingExceptionTag()),
+ pendingAddr);
+ masm.loadPtr(Address(pendingAddr, 0), *tagDst);
+ emitBarrieredClear(pendingAddr);
+ freePtr(pendingAddr);
+}
+
+bool BaseCompiler::startTryNote(size_t* tryNoteIndex) {
+ // Check the previous try note to ensure that we don't share an edge with
+ // it that could lead to ambiguity. Insert a nop, if required.
+ TryNoteVector& tryNotes = masm.tryNotes();
+ if (tryNotes.length() > 0) {
+ const TryNote& previous = tryNotes.back();
+ uint32_t currentOffset = masm.currentOffset();
+ if (previous.tryBodyBegin() == currentOffset ||
+ previous.tryBodyEnd() == currentOffset) {
+ masm.nop();
+ }
+ }
+
+ // Mark the beginning of the try note
+ wasm::TryNote tryNote = wasm::TryNote();
+ tryNote.setTryBodyBegin(masm.currentOffset());
+ return masm.append(tryNote, tryNoteIndex);
+}
+
+void BaseCompiler::finishTryNote(size_t tryNoteIndex) {
+ TryNoteVector& tryNotes = masm.tryNotes();
+ TryNote& tryNote = tryNotes[tryNoteIndex];
+
+ // Disallow zero-length try notes by inserting a no-op
+ if (tryNote.tryBodyBegin() == masm.currentOffset()) {
+ masm.nop();
+ }
+
+ // Check the previous try note to ensure that we don't share an edge with
+ // it that could lead to ambiguity. Insert a nop, if required.
+ if (tryNotes.length() > 0) {
+ const TryNote& previous = tryNotes.back();
+ uint32_t currentOffset = masm.currentOffset();
+ if (previous.tryBodyEnd() == currentOffset) {
+ masm.nop();
+ }
+ }
+
+ // Don't set the end of the try note if we've OOM'ed, as the above nop's may
+ // not have been placed. This is okay as this compilation will be thrown
+ // away.
+ if (masm.oom()) {
+ return;
+ }
+
+ // Mark the end of the try note
+ tryNote.setTryBodyEnd(masm.currentOffset());
+}
+
+////////////////////////////////////////////////////////////
+//
+// Platform-specific popping and register targeting.
+
+// The simple popping methods pop values into targeted registers; the caller
+// can free registers using standard functions. These are always called
+// popXForY where X says something about types and Y something about the
+// operation being targeted.
+
+RegI32 BaseCompiler::needRotate64Temp() {
+#if defined(JS_CODEGEN_X86)
+ return needI32();
+#elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \
+ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64)
+ return RegI32::Invalid();
+#else
+ MOZ_CRASH("BaseCompiler platform hook: needRotate64Temp");
+#endif
+}
+
+void BaseCompiler::popAndAllocateForDivAndRemI32(RegI32* r0, RegI32* r1,
+ RegI32* reserved) {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ // r0 must be eax, and edx will be clobbered.
+ need2xI32(specific_.eax, specific_.edx);
+ *r1 = popI32();
+ *r0 = popI32ToSpecific(specific_.eax);
+ *reserved = specific_.edx;
+#else
+ pop2xI32(r0, r1);
+#endif
+}
+
+static void QuotientI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd,
+ RegI32 reserved, IsUnsigned isUnsigned) {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ masm.quotient32(rs, rsd, reserved, isUnsigned);
+#else
+ masm.quotient32(rs, rsd, isUnsigned);
+#endif
+}
+
+static void RemainderI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd,
+ RegI32 reserved, IsUnsigned isUnsigned) {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ masm.remainder32(rs, rsd, reserved, isUnsigned);
+#else
+ masm.remainder32(rs, rsd, isUnsigned);
+#endif
+}
+
+void BaseCompiler::popAndAllocateForMulI64(RegI64* r0, RegI64* r1,
+ RegI32* temp) {
+#if defined(JS_CODEGEN_X64)
+ pop2xI64(r0, r1);
+#elif defined(JS_CODEGEN_X86)
+ // lhsDest must be edx:eax and rhs must not be that.
+ needI64(specific_.edx_eax);
+ *r1 = popI64();
+ *r0 = popI64ToSpecific(specific_.edx_eax);
+ *temp = needI32();
+#elif defined(JS_CODEGEN_MIPS64)
+ pop2xI64(r0, r1);
+#elif defined(JS_CODEGEN_ARM)
+ pop2xI64(r0, r1);
+ *temp = needI32();
+#elif defined(JS_CODEGEN_ARM64)
+ pop2xI64(r0, r1);
+#elif defined(JS_CODEGEN_LOONG64)
+ pop2xI64(r0, r1);
+#else
+ MOZ_CRASH("BaseCompiler porting interface: popAndAllocateForMulI64");
+#endif
+}
+
+#ifndef RABALDR_INT_DIV_I64_CALLOUT
+
+void BaseCompiler::popAndAllocateForDivAndRemI64(RegI64* r0, RegI64* r1,
+ RegI64* reserved,
+ IsRemainder isRemainder) {
+# if defined(JS_CODEGEN_X64)
+ // r0 must be rax, and rdx will be clobbered.
+ need2xI64(specific_.rax, specific_.rdx);
+ *r1 = popI64();
+ *r0 = popI64ToSpecific(specific_.rax);
+ *reserved = specific_.rdx;
+# elif defined(JS_CODEGEN_ARM64)
+ pop2xI64(r0, r1);
+ if (isRemainder) {
+ *reserved = needI64();
+ }
+# else
+ pop2xI64(r0, r1);
+# endif
+}
+
+static void QuotientI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest,
+ RegI64 reserved, IsUnsigned isUnsigned) {
+# if defined(JS_CODEGEN_X64)
+ // The caller must set up the following situation.
+ MOZ_ASSERT(srcDest.reg == rax);
+ MOZ_ASSERT(reserved.reg == rdx);
+ if (isUnsigned) {
+ masm.xorq(rdx, rdx);
+ masm.udivq(rhs.reg);
+ } else {
+ masm.cqo();
+ masm.idivq(rhs.reg);
+ }
+# elif defined(JS_CODEGEN_MIPS64)
+ MOZ_ASSERT(reserved.isInvalid());
+ if (isUnsigned) {
+ masm.as_ddivu(srcDest.reg, rhs.reg);
+ } else {
+ masm.as_ddiv(srcDest.reg, rhs.reg);
+ }
+ masm.as_mflo(srcDest.reg);
+# elif defined(JS_CODEGEN_ARM64)
+ MOZ_ASSERT(reserved.isInvalid());
+ ARMRegister sd(srcDest.reg, 64);
+ ARMRegister r(rhs.reg, 64);
+ if (isUnsigned) {
+ masm.Udiv(sd, sd, r);
+ } else {
+ masm.Sdiv(sd, sd, r);
+ }
+# elif defined(JS_CODEGEN_LOONG64)
+ if (isUnsigned) {
+ masm.as_div_du(srcDest.reg, srcDest.reg, rhs.reg);
+ } else {
+ masm.as_div_d(srcDest.reg, srcDest.reg, rhs.reg);
+ }
+# else
+ MOZ_CRASH("BaseCompiler platform hook: quotientI64");
+# endif
+}
+
+static void RemainderI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest,
+ RegI64 reserved, IsUnsigned isUnsigned) {
+# if defined(JS_CODEGEN_X64)
+ // The caller must set up the following situation.
+ MOZ_ASSERT(srcDest.reg == rax);
+ MOZ_ASSERT(reserved.reg == rdx);
+
+ if (isUnsigned) {
+ masm.xorq(rdx, rdx);
+ masm.udivq(rhs.reg);
+ } else {
+ masm.cqo();
+ masm.idivq(rhs.reg);
+ }
+ masm.movq(rdx, rax);
+# elif defined(JS_CODEGEN_MIPS64)
+ MOZ_ASSERT(reserved.isInvalid());
+ if (isUnsigned) {
+ masm.as_ddivu(srcDest.reg, rhs.reg);
+ } else {
+ masm.as_ddiv(srcDest.reg, rhs.reg);
+ }
+ masm.as_mfhi(srcDest.reg);
+# elif defined(JS_CODEGEN_ARM64)
+ ARMRegister sd(srcDest.reg, 64);
+ ARMRegister r(rhs.reg, 64);
+ ARMRegister t(reserved.reg, 64);
+ if (isUnsigned) {
+ masm.Udiv(t, sd, r);
+ } else {
+ masm.Sdiv(t, sd, r);
+ }
+ masm.Mul(t, t, r);
+ masm.Sub(sd, sd, t);
+# elif defined(JS_CODEGEN_LOONG64)
+ if (isUnsigned) {
+ masm.as_mod_du(srcDest.reg, srcDest.reg, rhs.reg);
+ } else {
+ masm.as_mod_d(srcDest.reg, srcDest.reg, rhs.reg);
+ }
+# else
+ MOZ_CRASH("BaseCompiler platform hook: remainderI64");
+# endif
+}
+
+#endif // RABALDR_INT_DIV_I64_CALLOUT
+
+RegI32 BaseCompiler::popI32RhsForShift() {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ // r1 must be ecx for a variable shift, unless BMI2 is available.
+ if (!Assembler::HasBMI2()) {
+ return popI32(specific_.ecx);
+ }
+#endif
+ RegI32 r = popI32();
+#if defined(JS_CODEGEN_ARM)
+ masm.and32(Imm32(31), r);
+#endif
+ return r;
+}
+
+RegI32 BaseCompiler::popI32RhsForShiftI64() {
+#if defined(JS_CODEGEN_X86)
+ // A limitation in the x86 masm requires ecx here
+ return popI32(specific_.ecx);
+#elif defined(JS_CODEGEN_X64)
+ if (!Assembler::HasBMI2()) {
+ return popI32(specific_.ecx);
+ }
+ return popI32();
+#else
+ return popI32();
+#endif
+}
+
+RegI64 BaseCompiler::popI64RhsForShift() {
+#if defined(JS_CODEGEN_X86)
+ // r1 must be ecx for a variable shift.
+ needI32(specific_.ecx);
+ return popI64ToSpecific(widenI32(specific_.ecx));
+#else
+# if defined(JS_CODEGEN_X64)
+ // r1 must be rcx for a variable shift, unless BMI2 is available.
+ if (!Assembler::HasBMI2()) {
+ needI64(specific_.rcx);
+ return popI64ToSpecific(specific_.rcx);
+ }
+# endif
+ // No masking is necessary on 64-bit platforms, and on arm32 the masm
+ // implementation masks.
+ return popI64();
+#endif
+}
+
+RegI32 BaseCompiler::popI32RhsForRotate() {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ // r1 must be ecx for a variable rotate.
+ return popI32(specific_.ecx);
+#else
+ return popI32();
+#endif
+}
+
+RegI64 BaseCompiler::popI64RhsForRotate() {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ // r1 must be ecx for a variable rotate.
+ needI32(specific_.ecx);
+ return popI64ToSpecific(widenI32(specific_.ecx));
+#else
+ return popI64();
+#endif
+}
+
+void BaseCompiler::popI32ForSignExtendI64(RegI64* r0) {
+#if defined(JS_CODEGEN_X86)
+ // r0 must be edx:eax for cdq
+ need2xI32(specific_.edx, specific_.eax);
+ *r0 = specific_.edx_eax;
+ popI32ToSpecific(specific_.eax);
+#else
+ *r0 = widenI32(popI32());
+#endif
+}
+
+void BaseCompiler::popI64ForSignExtendI64(RegI64* r0) {
+#if defined(JS_CODEGEN_X86)
+ // r0 must be edx:eax for cdq
+ need2xI32(specific_.edx, specific_.eax);
+ // Low on top, high underneath
+ *r0 = popI64ToSpecific(specific_.edx_eax);
+#else
+ *r0 = popI64();
+#endif
+}
+
+class OutOfLineTruncateCheckF32OrF64ToI32 : public OutOfLineCode {
+ AnyReg src;
+ RegI32 dest;
+ TruncFlags flags;
+ BytecodeOffset off;
+
+ public:
+ OutOfLineTruncateCheckF32OrF64ToI32(AnyReg src, RegI32 dest, TruncFlags flags,
+ BytecodeOffset off)
+ : src(src), dest(dest), flags(flags), off(off) {}
+
+ virtual void generate(MacroAssembler* masm) override {
+ if (src.tag == AnyReg::F32) {
+ masm->oolWasmTruncateCheckF32ToI32(src.f32(), dest, flags, off, rejoin());
+ } else if (src.tag == AnyReg::F64) {
+ masm->oolWasmTruncateCheckF64ToI32(src.f64(), dest, flags, off, rejoin());
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ }
+};
+
+bool BaseCompiler::truncateF32ToI32(RegF32 src, RegI32 dest, TruncFlags flags) {
+ BytecodeOffset off = bytecodeOffset();
+ OutOfLineCode* ool =
+ addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32(
+ AnyReg(src), dest, flags, off));
+ if (!ool) {
+ return false;
+ }
+ bool isSaturating = flags & TRUNC_SATURATING;
+ if (flags & TRUNC_UNSIGNED) {
+ masm.wasmTruncateFloat32ToUInt32(src, dest, isSaturating, ool->entry());
+ } else {
+ masm.wasmTruncateFloat32ToInt32(src, dest, isSaturating, ool->entry());
+ }
+ masm.bind(ool->rejoin());
+ return true;
+}
+
+bool BaseCompiler::truncateF64ToI32(RegF64 src, RegI32 dest, TruncFlags flags) {
+ BytecodeOffset off = bytecodeOffset();
+ OutOfLineCode* ool =
+ addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32(
+ AnyReg(src), dest, flags, off));
+ if (!ool) {
+ return false;
+ }
+ bool isSaturating = flags & TRUNC_SATURATING;
+ if (flags & TRUNC_UNSIGNED) {
+ masm.wasmTruncateDoubleToUInt32(src, dest, isSaturating, ool->entry());
+ } else {
+ masm.wasmTruncateDoubleToInt32(src, dest, isSaturating, ool->entry());
+ }
+ masm.bind(ool->rejoin());
+ return true;
+}
+
+class OutOfLineTruncateCheckF32OrF64ToI64 : public OutOfLineCode {
+ AnyReg src;
+ RegI64 dest;
+ TruncFlags flags;
+ BytecodeOffset off;
+
+ public:
+ OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src, RegI64 dest, TruncFlags flags,
+ BytecodeOffset off)
+ : src(src), dest(dest), flags(flags), off(off) {}
+
+ virtual void generate(MacroAssembler* masm) override {
+ if (src.tag == AnyReg::F32) {
+ masm->oolWasmTruncateCheckF32ToI64(src.f32(), dest, flags, off, rejoin());
+ } else if (src.tag == AnyReg::F64) {
+ masm->oolWasmTruncateCheckF64ToI64(src.f64(), dest, flags, off, rejoin());
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ }
+};
+
+#ifndef RABALDR_FLOAT_TO_I64_CALLOUT
+
+RegF64 BaseCompiler::needTempForFloatingToI64(TruncFlags flags) {
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ if (flags & TRUNC_UNSIGNED) {
+ return needF64();
+ }
+# endif
+ return RegF64::Invalid();
+}
+
+bool BaseCompiler::truncateF32ToI64(RegF32 src, RegI64 dest, TruncFlags flags,
+ RegF64 temp) {
+ OutOfLineCode* ool =
+ addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(
+ AnyReg(src), dest, flags, bytecodeOffset()));
+ if (!ool) {
+ return false;
+ }
+ bool isSaturating = flags & TRUNC_SATURATING;
+ if (flags & TRUNC_UNSIGNED) {
+ masm.wasmTruncateFloat32ToUInt64(src, dest, isSaturating, ool->entry(),
+ ool->rejoin(), temp);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(src, dest, isSaturating, ool->entry(),
+ ool->rejoin(), temp);
+ }
+ return true;
+}
+
+bool BaseCompiler::truncateF64ToI64(RegF64 src, RegI64 dest, TruncFlags flags,
+ RegF64 temp) {
+ OutOfLineCode* ool =
+ addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(
+ AnyReg(src), dest, flags, bytecodeOffset()));
+ if (!ool) {
+ return false;
+ }
+ bool isSaturating = flags & TRUNC_SATURATING;
+ if (flags & TRUNC_UNSIGNED) {
+ masm.wasmTruncateDoubleToUInt64(src, dest, isSaturating, ool->entry(),
+ ool->rejoin(), temp);
+ } else {
+ masm.wasmTruncateDoubleToInt64(src, dest, isSaturating, ool->entry(),
+ ool->rejoin(), temp);
+ }
+ return true;
+}
+
+#endif // RABALDR_FLOAT_TO_I64_CALLOUT
+
+#ifndef RABALDR_I64_TO_FLOAT_CALLOUT
+
+RegI32 BaseCompiler::needConvertI64ToFloatTemp(ValType to, bool isUnsigned) {
+ bool needs = false;
+ if (to == ValType::F64) {
+ needs = isUnsigned && masm.convertUInt64ToDoubleNeedsTemp();
+ } else {
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ needs = true;
+# endif
+ }
+ return needs ? needI32() : RegI32::Invalid();
+}
+
+void BaseCompiler::convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest,
+ RegI32 temp) {
+ if (isUnsigned) {
+ masm.convertUInt64ToFloat32(src, dest, temp);
+ } else {
+ masm.convertInt64ToFloat32(src, dest);
+ }
+}
+
+void BaseCompiler::convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest,
+ RegI32 temp) {
+ if (isUnsigned) {
+ masm.convertUInt64ToDouble(src, dest, temp);
+ } else {
+ masm.convertInt64ToDouble(src, dest);
+ }
+}
+
+#endif // RABALDR_I64_TO_FLOAT_CALLOUT
+
+//////////////////////////////////////////////////////////////////////
+//
+// Global variable access.
+
+Address BaseCompiler::addressOfGlobalVar(const GlobalDesc& global, RegPtr tmp) {
+ uint32_t globalToInstanceOffset = Instance::offsetInData(global.offset());
+#ifdef RABALDR_PIN_INSTANCE
+ movePtr(RegPtr(InstanceReg), tmp);
+#else
+ fr.loadInstancePtr(tmp);
+#endif
+ if (global.isIndirect()) {
+ masm.loadPtr(Address(tmp, globalToInstanceOffset), tmp);
+ return Address(tmp, 0);
+ }
+ return Address(tmp, globalToInstanceOffset);
+}
+
+//////////////////////////////////////////////////////////////////////
+//
+// Table access.
+
+Address BaseCompiler::addressOfTableField(uint32_t tableIndex,
+ uint32_t fieldOffset,
+ RegPtr instance) {
+ uint32_t tableToInstanceOffset = wasm::Instance::offsetInData(
+ moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset);
+ return Address(instance, tableToInstanceOffset);
+}
+
+void BaseCompiler::loadTableLength(uint32_t tableIndex, RegPtr instance,
+ RegI32 length) {
+ masm.load32(addressOfTableField(
+ tableIndex, offsetof(TableInstanceData, length), instance),
+ length);
+}
+
+void BaseCompiler::loadTableElements(uint32_t tableIndex, RegPtr instance,
+ RegPtr elements) {
+ masm.loadPtr(addressOfTableField(
+ tableIndex, offsetof(TableInstanceData, elements), instance),
+ elements);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Basic emitters for simple operators.
+
+static void AddI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.add32(rs, rsd);
+}
+
+static void AddImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.add32(Imm32(c), rsd);
+}
+
+static void SubI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.sub32(rs, rsd);
+}
+
+static void SubImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.sub32(Imm32(c), rsd);
+}
+
+static void MulI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.mul32(rs, rsd);
+}
+
+static void OrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.or32(rs, rsd);
+}
+
+static void OrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.or32(Imm32(c), rsd);
+}
+
+static void AndI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.and32(rs, rsd);
+}
+
+static void AndImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.and32(Imm32(c), rsd);
+}
+
+static void XorI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.xor32(rs, rsd);
+}
+
+static void XorImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.xor32(Imm32(c), rsd);
+}
+
+static void ClzI32(MacroAssembler& masm, RegI32 rsd) {
+ masm.clz32(rsd, rsd, IsKnownNotZero(false));
+}
+
+static void CtzI32(MacroAssembler& masm, RegI32 rsd) {
+ masm.ctz32(rsd, rsd, IsKnownNotZero(false));
+}
+
+// Currently common to PopcntI32 and PopcntI64
+static RegI32 PopcntTemp(BaseCompiler& bc) {
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ return AssemblerX86Shared::HasPOPCNT() ? RegI32::Invalid() : bc.needI32();
+#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64)
+ return bc.needI32();
+#else
+ MOZ_CRASH("BaseCompiler platform hook: PopcntTemp");
+#endif
+}
+
+static void PopcntI32(BaseCompiler& bc, RegI32 rsd, RegI32 temp) {
+ bc.masm.popcnt32(rsd, rsd, temp);
+}
+
+static void ShlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.lshift32(rs, rsd);
+}
+
+static void ShlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.lshift32(Imm32(c & 31), rsd);
+}
+
+static void ShrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.rshift32Arithmetic(rs, rsd);
+}
+
+static void ShrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.rshift32Arithmetic(Imm32(c & 31), rsd);
+}
+
+static void ShrUI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.rshift32(rs, rsd);
+}
+
+static void ShrUImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.rshift32(Imm32(c & 31), rsd);
+}
+
+static void RotlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.rotateLeft(rs, rsd, rsd);
+}
+
+static void RotlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.rotateLeft(Imm32(c & 31), rsd, rsd);
+}
+
+static void RotrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) {
+ masm.rotateRight(rs, rsd, rsd);
+}
+
+static void RotrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) {
+ masm.rotateRight(Imm32(c & 31), rsd, rsd);
+}
+
+static void EqzI32(MacroAssembler& masm, RegI32 rsd) {
+ masm.cmp32Set(Assembler::Equal, rsd, Imm32(0), rsd);
+}
+
+static void WrapI64ToI32(MacroAssembler& masm, RegI64 rs, RegI32 rd) {
+ masm.move64To32(rs, rd);
+}
+
+static void AddI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) {
+ masm.add64(rs, rsd);
+}
+
+static void AddImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.add64(Imm64(c), rsd);
+}
+
+static void SubI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) {
+ masm.sub64(rs, rsd);
+}
+
+static void SubImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.sub64(Imm64(c), rsd);
+}
+
+static void OrI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) {
+ masm.or64(rs, rsd);
+}
+
+static void OrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.or64(Imm64(c), rsd);
+}
+
+static void AndI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) {
+ masm.and64(rs, rsd);
+}
+
+static void AndImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.and64(Imm64(c), rsd);
+}
+
+static void XorI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) {
+ masm.xor64(rs, rsd);
+}
+
+static void XorImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.xor64(Imm64(c), rsd);
+}
+
+static void ClzI64(BaseCompiler& bc, RegI64 rsd) {
+ bc.masm.clz64(rsd, bc.lowPart(rsd));
+ bc.maybeClearHighPart(rsd);
+}
+
+static void CtzI64(BaseCompiler& bc, RegI64 rsd) {
+ bc.masm.ctz64(rsd, bc.lowPart(rsd));
+ bc.maybeClearHighPart(rsd);
+}
+
+static void PopcntI64(BaseCompiler& bc, RegI64 rsd, RegI32 temp) {
+ bc.masm.popcnt64(rsd, rsd, temp);
+}
+
+static void ShlI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) {
+ bc.masm.lshift64(bc.lowPart(rs), rsd);
+}
+
+static void ShlImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.lshift64(Imm32(c & 63), rsd);
+}
+
+static void ShrI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) {
+ bc.masm.rshift64Arithmetic(bc.lowPart(rs), rsd);
+}
+
+static void ShrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.rshift64Arithmetic(Imm32(c & 63), rsd);
+}
+
+static void ShrUI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) {
+ bc.masm.rshift64(bc.lowPart(rs), rsd);
+}
+
+static void ShrUImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) {
+ masm.rshift64(Imm32(c & 63), rsd);
+}
+
+static void EqzI64(MacroAssembler& masm, RegI64 rs, RegI32 rd) {
+#ifdef JS_PUNBOX64
+ masm.cmpPtrSet(Assembler::Equal, rs.reg, ImmWord(0), rd);
+#else
+ MOZ_ASSERT(rs.low == rd);
+ masm.or32(rs.high, rs.low);
+ masm.cmp32Set(Assembler::Equal, rs.low, Imm32(0), rd);
+#endif
+}
+
+static void AddF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) {
+ masm.addDouble(rs, rsd);
+}
+
+static void SubF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) {
+ masm.subDouble(rs, rsd);
+}
+
+static void MulF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) {
+ masm.mulDouble(rs, rsd);
+}
+
+static void DivF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) {
+ masm.divDouble(rs, rsd);
+}
+
+static void MinF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) {
+ // Convert signaling NaN to quiet NaNs.
+ //
+ // TODO / OPTIMIZE (bug 1316824): see comment in MinF32.
+#ifdef RABALDR_SCRATCH_F64
+ ScratchF64 zero(bc.ra);
+#else
+ ScratchF64 zero(bc.masm);
+#endif
+ bc.masm.loadConstantDouble(0, zero);
+ bc.masm.subDouble(zero, rsd);
+ bc.masm.subDouble(zero, rs);
+ bc.masm.minDouble(rs, rsd, HandleNaNSpecially(true));
+}
+
+static void MaxF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) {
+ // Convert signaling NaN to quiet NaNs.
+ //
+ // TODO / OPTIMIZE (bug 1316824): see comment in MinF32.
+#ifdef RABALDR_SCRATCH_F64
+ ScratchF64 zero(bc.ra);
+#else
+ ScratchF64 zero(bc.masm);
+#endif
+ bc.masm.loadConstantDouble(0, zero);
+ bc.masm.subDouble(zero, rsd);
+ bc.masm.subDouble(zero, rs);
+ bc.masm.maxDouble(rs, rsd, HandleNaNSpecially(true));
+}
+
+static void CopysignF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd,
+ RegI64 temp0, RegI64 temp1) {
+ masm.moveDoubleToGPR64(rsd, temp0);
+ masm.moveDoubleToGPR64(rs, temp1);
+ masm.and64(Imm64(INT64_MAX), temp0);
+ masm.and64(Imm64(INT64_MIN), temp1);
+ masm.or64(temp1, temp0);
+ masm.moveGPR64ToDouble(temp0, rsd);
+}
+
+static void AbsF64(MacroAssembler& masm, RegF64 rsd) {
+ masm.absDouble(rsd, rsd);
+}
+
+static void NegateF64(MacroAssembler& masm, RegF64 rsd) {
+ masm.negateDouble(rsd);
+}
+
+static void SqrtF64(MacroAssembler& masm, RegF64 rsd) {
+ masm.sqrtDouble(rsd, rsd);
+}
+
+static void AddF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) {
+ masm.addFloat32(rs, rsd);
+}
+
+static void SubF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) {
+ masm.subFloat32(rs, rsd);
+}
+
+static void MulF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) {
+ masm.mulFloat32(rs, rsd);
+}
+
+static void DivF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) {
+ masm.divFloat32(rs, rsd);
+}
+
+static void MinF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) {
+ // Convert signaling NaN to quiet NaNs.
+ //
+ // TODO / OPTIMIZE (bug 1316824): Don't do this if one of the operands
+ // is known to be a constant.
+#ifdef RABALDR_SCRATCH_F32
+ ScratchF32 zero(bc.ra);
+#else
+ ScratchF32 zero(bc.masm);
+#endif
+ bc.masm.loadConstantFloat32(0.f, zero);
+ bc.masm.subFloat32(zero, rsd);
+ bc.masm.subFloat32(zero, rs);
+ bc.masm.minFloat32(rs, rsd, HandleNaNSpecially(true));
+}
+
+static void MaxF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) {
+ // Convert signaling NaN to quiet NaNs.
+ //
+ // TODO / OPTIMIZE (bug 1316824): see comment in MinF32.
+#ifdef RABALDR_SCRATCH_F32
+ ScratchF32 zero(bc.ra);
+#else
+ ScratchF32 zero(bc.masm);
+#endif
+ bc.masm.loadConstantFloat32(0.f, zero);
+ bc.masm.subFloat32(zero, rsd);
+ bc.masm.subFloat32(zero, rs);
+ bc.masm.maxFloat32(rs, rsd, HandleNaNSpecially(true));
+}
+
+static void CopysignF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd,
+ RegI32 temp0, RegI32 temp1) {
+ masm.moveFloat32ToGPR(rsd, temp0);
+ masm.moveFloat32ToGPR(rs, temp1);
+ masm.and32(Imm32(INT32_MAX), temp0);
+ masm.and32(Imm32(INT32_MIN), temp1);
+ masm.or32(temp1, temp0);
+ masm.moveGPRToFloat32(temp0, rsd);
+}
+
+static void AbsF32(MacroAssembler& masm, RegF32 rsd) {
+ masm.absFloat32(rsd, rsd);
+}
+
+static void NegateF32(MacroAssembler& masm, RegF32 rsd) {
+ masm.negateFloat(rsd);
+}
+
+static void SqrtF32(MacroAssembler& masm, RegF32 rsd) {
+ masm.sqrtFloat32(rsd, rsd);
+}
+
+#ifndef RABALDR_I64_TO_FLOAT_CALLOUT
+static void ConvertI64ToF32(MacroAssembler& masm, RegI64 rs, RegF32 rd) {
+ masm.convertInt64ToFloat32(rs, rd);
+}
+
+static void ConvertI64ToF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) {
+ masm.convertInt64ToDouble(rs, rd);
+}
+#endif
+
+static void ReinterpretF32AsI32(MacroAssembler& masm, RegF32 rs, RegI32 rd) {
+ masm.moveFloat32ToGPR(rs, rd);
+}
+
+static void ReinterpretF64AsI64(MacroAssembler& masm, RegF64 rs, RegI64 rd) {
+ masm.moveDoubleToGPR64(rs, rd);
+}
+
+static void ConvertF64ToF32(MacroAssembler& masm, RegF64 rs, RegF32 rd) {
+ masm.convertDoubleToFloat32(rs, rd);
+}
+
+static void ConvertI32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) {
+ masm.convertInt32ToFloat32(rs, rd);
+}
+
+static void ConvertU32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) {
+ masm.convertUInt32ToFloat32(rs, rd);
+}
+
+static void ConvertF32ToF64(MacroAssembler& masm, RegF32 rs, RegF64 rd) {
+ masm.convertFloat32ToDouble(rs, rd);
+}
+
+static void ConvertI32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) {
+ masm.convertInt32ToDouble(rs, rd);
+}
+
+static void ConvertU32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) {
+ masm.convertUInt32ToDouble(rs, rd);
+}
+
+static void ReinterpretI32AsF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) {
+ masm.moveGPRToFloat32(rs, rd);
+}
+
+static void ReinterpretI64AsF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) {
+ masm.moveGPR64ToDouble(rs, rd);
+}
+
+static void ExtendI32_8(BaseCompiler& bc, RegI32 rsd) {
+#ifdef JS_CODEGEN_X86
+ if (!bc.ra.isSingleByteI32(rsd)) {
+ ScratchI8 scratch(bc.ra);
+ bc.masm.move32(rsd, scratch);
+ bc.masm.move8SignExtend(scratch, rsd);
+ return;
+ }
+#endif
+ bc.masm.move8SignExtend(rsd, rsd);
+}
+
+static void ExtendI32_16(MacroAssembler& masm, RegI32 rsd) {
+ masm.move16SignExtend(rsd, rsd);
+}
+
+void BaseCompiler::emitMultiplyI64() {
+ RegI64 r, rs;
+ RegI32 temp;
+ popAndAllocateForMulI64(&r, &rs, &temp);
+ masm.mul64(rs, r, temp);
+ maybeFree(temp);
+ freeI64(rs);
+ pushI64(r);
+}
+
+template <typename RegType, typename IntType>
+void BaseCompiler::quotientOrRemainder(
+ RegType rs, RegType rsd, RegType reserved, IsUnsigned isUnsigned,
+ ZeroOnOverflow zeroOnOverflow, bool isConst, IntType c,
+ void (*operate)(MacroAssembler& masm, RegType rs, RegType rsd,
+ RegType reserved, IsUnsigned isUnsigned)) {
+ Label done;
+ if (!isConst || c == 0) {
+ checkDivideByZero(rs);
+ }
+ if (!isUnsigned && (!isConst || c == -1)) {
+ checkDivideSignedOverflow(rs, rsd, &done, zeroOnOverflow);
+ }
+ operate(masm, rs, rsd, reserved, isUnsigned);
+ masm.bind(&done);
+}
+
+void BaseCompiler::emitQuotientI32() {
+ int32_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 0)) {
+ if (power != 0) {
+ RegI32 r = popI32();
+ Label positive;
+ masm.branchTest32(Assembler::NotSigned, r, r, &positive);
+ masm.add32(Imm32(c - 1), r);
+ masm.bind(&positive);
+
+ masm.rshift32Arithmetic(Imm32(power & 31), r);
+ pushI32(r);
+ }
+ } else {
+ bool isConst = peekConst(&c);
+ RegI32 r, rs, reserved;
+ popAndAllocateForDivAndRemI32(&r, &rs, &reserved);
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(false),
+ ZeroOnOverflow(false), isConst, c, QuotientI32);
+ maybeFree(reserved);
+ freeI32(rs);
+ pushI32(r);
+ }
+}
+
+void BaseCompiler::emitQuotientU32() {
+ int32_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 0)) {
+ if (power != 0) {
+ RegI32 r = popI32();
+ masm.rshift32(Imm32(power & 31), r);
+ pushI32(r);
+ }
+ } else {
+ bool isConst = peekConst(&c);
+ RegI32 r, rs, reserved;
+ popAndAllocateForDivAndRemI32(&r, &rs, &reserved);
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(true),
+ ZeroOnOverflow(false), isConst, c, QuotientI32);
+ maybeFree(reserved);
+ freeI32(rs);
+ pushI32(r);
+ }
+}
+
+void BaseCompiler::emitRemainderI32() {
+ int32_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 1)) {
+ RegI32 r = popI32();
+ RegI32 temp = needI32();
+ moveI32(r, temp);
+
+ Label positive;
+ masm.branchTest32(Assembler::NotSigned, temp, temp, &positive);
+ masm.add32(Imm32(c - 1), temp);
+ masm.bind(&positive);
+
+ masm.rshift32Arithmetic(Imm32(power & 31), temp);
+ masm.lshift32(Imm32(power & 31), temp);
+ masm.sub32(temp, r);
+ freeI32(temp);
+
+ pushI32(r);
+ } else {
+ bool isConst = peekConst(&c);
+ RegI32 r, rs, reserved;
+ popAndAllocateForDivAndRemI32(&r, &rs, &reserved);
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(false),
+ ZeroOnOverflow(true), isConst, c, RemainderI32);
+ maybeFree(reserved);
+ freeI32(rs);
+ pushI32(r);
+ }
+}
+
+void BaseCompiler::emitRemainderU32() {
+ int32_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 1)) {
+ RegI32 r = popI32();
+ masm.and32(Imm32(c - 1), r);
+ pushI32(r);
+ } else {
+ bool isConst = peekConst(&c);
+ RegI32 r, rs, reserved;
+ popAndAllocateForDivAndRemI32(&r, &rs, &reserved);
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true),
+ isConst, c, RemainderI32);
+ maybeFree(reserved);
+ freeI32(rs);
+ pushI32(r);
+ }
+}
+
+#ifndef RABALDR_INT_DIV_I64_CALLOUT
+void BaseCompiler::emitQuotientI64() {
+ int64_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 0)) {
+ if (power != 0) {
+ RegI64 r = popI64();
+ Label positive;
+ masm.branchTest64(Assembler::NotSigned, r, r, RegI32::Invalid(),
+ &positive);
+ masm.add64(Imm64(c - 1), r);
+ masm.bind(&positive);
+
+ masm.rshift64Arithmetic(Imm32(power & 63), r);
+ pushI64(r);
+ }
+ } else {
+ bool isConst = peekConst(&c);
+ RegI64 r, rs, reserved;
+ popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false));
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(false),
+ ZeroOnOverflow(false), isConst, c, QuotientI64);
+ maybeFree(reserved);
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+
+void BaseCompiler::emitQuotientU64() {
+ int64_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 0)) {
+ if (power != 0) {
+ RegI64 r = popI64();
+ masm.rshift64(Imm32(power & 63), r);
+ pushI64(r);
+ }
+ } else {
+ bool isConst = peekConst(&c);
+ RegI64 r, rs, reserved;
+ popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false));
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(true),
+ ZeroOnOverflow(false), isConst, c, QuotientI64);
+ maybeFree(reserved);
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+
+void BaseCompiler::emitRemainderI64() {
+ int64_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 1)) {
+ RegI64 r = popI64();
+ RegI64 temp = needI64();
+ moveI64(r, temp);
+
+ Label positive;
+ masm.branchTest64(Assembler::NotSigned, temp, temp, RegI32::Invalid(),
+ &positive);
+ masm.add64(Imm64(c - 1), temp);
+ masm.bind(&positive);
+
+ masm.rshift64Arithmetic(Imm32(power & 63), temp);
+ masm.lshift64(Imm32(power & 63), temp);
+ masm.sub64(temp, r);
+ freeI64(temp);
+
+ pushI64(r);
+ } else {
+ bool isConst = peekConst(&c);
+ RegI64 r, rs, reserved;
+ popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true));
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(false),
+ ZeroOnOverflow(true), isConst, c, RemainderI64);
+ maybeFree(reserved);
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+
+void BaseCompiler::emitRemainderU64() {
+ int64_t c;
+ uint_fast8_t power;
+ if (popConstPositivePowerOfTwo(&c, &power, 1)) {
+ RegI64 r = popI64();
+ masm.and64(Imm64(c - 1), r);
+ pushI64(r);
+ } else {
+ bool isConst = peekConst(&c);
+ RegI64 r, rs, reserved;
+ popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true));
+ quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true),
+ isConst, c, RemainderI64);
+ maybeFree(reserved);
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+#endif // RABALDR_INT_DIV_I64_CALLOUT
+
+void BaseCompiler::emitRotrI64() {
+ int64_t c;
+ if (popConst(&c)) {
+ RegI64 r = popI64();
+ RegI32 temp = needRotate64Temp();
+ masm.rotateRight64(Imm32(c & 63), r, r, temp);
+ maybeFree(temp);
+ pushI64(r);
+ } else {
+ RegI64 rs = popI64RhsForRotate();
+ RegI64 r = popI64();
+ masm.rotateRight64(lowPart(rs), r, r, maybeHighPart(rs));
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+
+void BaseCompiler::emitRotlI64() {
+ int64_t c;
+ if (popConst(&c)) {
+ RegI64 r = popI64();
+ RegI32 temp = needRotate64Temp();
+ masm.rotateLeft64(Imm32(c & 63), r, r, temp);
+ maybeFree(temp);
+ pushI64(r);
+ } else {
+ RegI64 rs = popI64RhsForRotate();
+ RegI64 r = popI64();
+ masm.rotateLeft64(lowPart(rs), r, r, maybeHighPart(rs));
+ freeI64(rs);
+ pushI64(r);
+ }
+}
+
+void BaseCompiler::emitEqzI32() {
+ if (sniffConditionalControlEqz(ValType::I32)) {
+ return;
+ }
+ emitUnop(EqzI32);
+}
+
+void BaseCompiler::emitEqzI64() {
+ if (sniffConditionalControlEqz(ValType::I64)) {
+ return;
+ }
+ emitUnop(EqzI64);
+}
+
+template <TruncFlags flags>
+bool BaseCompiler::emitTruncateF32ToI32() {
+ RegF32 rs = popF32();
+ RegI32 rd = needI32();
+ if (!truncateF32ToI32(rs, rd, flags)) {
+ return false;
+ }
+ freeF32(rs);
+ pushI32(rd);
+ return true;
+}
+
+template <TruncFlags flags>
+bool BaseCompiler::emitTruncateF64ToI32() {
+ RegF64 rs = popF64();
+ RegI32 rd = needI32();
+ if (!truncateF64ToI32(rs, rd, flags)) {
+ return false;
+ }
+ freeF64(rs);
+ pushI32(rd);
+ return true;
+}
+
+#ifndef RABALDR_FLOAT_TO_I64_CALLOUT
+template <TruncFlags flags>
+bool BaseCompiler::emitTruncateF32ToI64() {
+ RegF32 rs = popF32();
+ RegI64 rd = needI64();
+ RegF64 temp = needTempForFloatingToI64(flags);
+ if (!truncateF32ToI64(rs, rd, flags, temp)) {
+ return false;
+ }
+ maybeFree(temp);
+ freeF32(rs);
+ pushI64(rd);
+ return true;
+}
+
+template <TruncFlags flags>
+bool BaseCompiler::emitTruncateF64ToI64() {
+ RegF64 rs = popF64();
+ RegI64 rd = needI64();
+ RegF64 temp = needTempForFloatingToI64(flags);
+ if (!truncateF64ToI64(rs, rd, flags, temp)) {
+ return false;
+ }
+ maybeFree(temp);
+ freeF64(rs);
+ pushI64(rd);
+ return true;
+}
+#endif // RABALDR_FLOAT_TO_I64_CALLOUT
+
+void BaseCompiler::emitExtendI64_8() {
+ RegI64 r;
+ popI64ForSignExtendI64(&r);
+ masm.move8To64SignExtend(lowPart(r), r);
+ pushI64(r);
+}
+
+void BaseCompiler::emitExtendI64_16() {
+ RegI64 r;
+ popI64ForSignExtendI64(&r);
+ masm.move16To64SignExtend(lowPart(r), r);
+ pushI64(r);
+}
+
+void BaseCompiler::emitExtendI64_32() {
+ RegI64 r;
+ popI64ForSignExtendI64(&r);
+ masm.move32To64SignExtend(lowPart(r), r);
+ pushI64(r);
+}
+
+void BaseCompiler::emitExtendI32ToI64() {
+ RegI64 r;
+ popI32ForSignExtendI64(&r);
+ masm.move32To64SignExtend(lowPart(r), r);
+ pushI64(r);
+}
+
+void BaseCompiler::emitExtendU32ToI64() {
+ RegI32 rs = popI32();
+ RegI64 rd = widenI32(rs);
+ masm.move32To64ZeroExtend(rs, rd);
+ pushI64(rd);
+}
+
+#ifndef RABALDR_I64_TO_FLOAT_CALLOUT
+void BaseCompiler::emitConvertU64ToF32() {
+ RegI64 rs = popI64();
+ RegF32 rd = needF32();
+ RegI32 temp = needConvertI64ToFloatTemp(ValType::F32, IsUnsigned(true));
+ convertI64ToF32(rs, IsUnsigned(true), rd, temp);
+ maybeFree(temp);
+ freeI64(rs);
+ pushF32(rd);
+}
+
+void BaseCompiler::emitConvertU64ToF64() {
+ RegI64 rs = popI64();
+ RegF64 rd = needF64();
+ RegI32 temp = needConvertI64ToFloatTemp(ValType::F64, IsUnsigned(true));
+ convertI64ToF64(rs, IsUnsigned(true), rd, temp);
+ maybeFree(temp);
+ freeI64(rs);
+ pushF64(rd);
+}
+#endif // RABALDR_I64_TO_FLOAT_CALLOUT
+
+////////////////////////////////////////////////////////////
+//
+// Machinery for optimized conditional branches.
+//
+// To disable this optimization it is enough always to return false from
+// sniffConditionalControl{Cmp,Eqz}.
+
+struct BranchState {
+ union {
+ struct {
+ RegI32 lhs;
+ RegI32 rhs;
+ int32_t imm;
+ bool rhsImm;
+ } i32;
+ struct {
+ RegI64 lhs;
+ RegI64 rhs;
+ int64_t imm;
+ bool rhsImm;
+ } i64;
+ struct {
+ RegF32 lhs;
+ RegF32 rhs;
+ } f32;
+ struct {
+ RegF64 lhs;
+ RegF64 rhs;
+ } f64;
+ };
+
+ Label* const label; // The target of the branch, never NULL
+ const StackHeight stackHeight; // The stack base above which to place
+ // stack-spilled block results, if
+ // hasBlockResults().
+ const bool invertBranch; // If true, invert the sense of the branch
+ const ResultType resultType; // The result propagated along the edges
+
+ explicit BranchState(Label* label)
+ : label(label),
+ stackHeight(StackHeight::Invalid()),
+ invertBranch(false),
+ resultType(ResultType::Empty()) {}
+
+ BranchState(Label* label, bool invertBranch)
+ : label(label),
+ stackHeight(StackHeight::Invalid()),
+ invertBranch(invertBranch),
+ resultType(ResultType::Empty()) {}
+
+ BranchState(Label* label, StackHeight stackHeight, bool invertBranch,
+ ResultType resultType)
+ : label(label),
+ stackHeight(stackHeight),
+ invertBranch(invertBranch),
+ resultType(resultType) {}
+
+ bool hasBlockResults() const { return stackHeight.isValid(); }
+};
+
+void BaseCompiler::setLatentCompare(Assembler::Condition compareOp,
+ ValType operandType) {
+ latentOp_ = LatentOp::Compare;
+ latentType_ = operandType;
+ latentIntCmp_ = compareOp;
+}
+
+void BaseCompiler::setLatentCompare(Assembler::DoubleCondition compareOp,
+ ValType operandType) {
+ latentOp_ = LatentOp::Compare;
+ latentType_ = operandType;
+ latentDoubleCmp_ = compareOp;
+}
+
+void BaseCompiler::setLatentEqz(ValType operandType) {
+ latentOp_ = LatentOp::Eqz;
+ latentType_ = operandType;
+}
+
+bool BaseCompiler::hasLatentOp() const { return latentOp_ != LatentOp::None; }
+
+void BaseCompiler::resetLatentOp() { latentOp_ = LatentOp::None; }
+
+// Emit a conditional branch that optionally and optimally cleans up the CPU
+// stack before we branch.
+//
+// Cond is either Assembler::Condition or Assembler::DoubleCondition.
+//
+// Lhs is RegI32, RegI64, or RegF32, RegF64, or RegRef.
+//
+// Rhs is either the same as Lhs, or an immediate expression compatible with
+// Lhs "when applicable".
+
+template <typename Cond, typename Lhs, typename Rhs>
+bool BaseCompiler::jumpConditionalWithResults(BranchState* b, Cond cond,
+ Lhs lhs, Rhs rhs) {
+ if (b->hasBlockResults()) {
+ StackHeight resultsBase(0);
+ if (!topBranchParams(b->resultType, &resultsBase)) {
+ return false;
+ }
+ if (b->stackHeight != resultsBase) {
+ Label notTaken;
+ branchTo(b->invertBranch ? cond : Assembler::InvertCondition(cond), lhs,
+ rhs, &notTaken);
+
+ // Shuffle stack args.
+ shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight,
+ b->resultType);
+ masm.jump(b->label);
+ masm.bind(&notTaken);
+ return true;
+ }
+ }
+
+ branchTo(b->invertBranch ? Assembler::InvertCondition(cond) : cond, lhs, rhs,
+ b->label);
+ return true;
+}
+
+#ifdef ENABLE_WASM_GC
+bool BaseCompiler::jumpConditionalWithResults(BranchState* b, RegRef object,
+ RefType sourceType,
+ RefType destType,
+ bool onSuccess) {
+ if (b->hasBlockResults()) {
+ StackHeight resultsBase(0);
+ if (!topBranchParams(b->resultType, &resultsBase)) {
+ return false;
+ }
+ if (b->stackHeight != resultsBase) {
+ Label notTaken;
+ // Temporarily take the result registers so that branchGcHeapType doesn't
+ // use them.
+ needIntegerResultRegisters(b->resultType);
+ branchGcRefType(object, sourceType, destType, &notTaken,
+ /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess);
+ freeIntegerResultRegisters(b->resultType);
+
+ // Shuffle stack args.
+ shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight,
+ b->resultType);
+ masm.jump(b->label);
+ masm.bind(&notTaken);
+ return true;
+ }
+ }
+
+ branchGcRefType(object, sourceType, destType, b->label,
+ /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess);
+ return true;
+}
+#endif
+
+// sniffConditionalControl{Cmp,Eqz} may modify the latentWhatever_ state in
+// the BaseCompiler so that a subsequent conditional branch can be compiled
+// optimally. emitBranchSetup() and emitBranchPerform() will consume that
+// state. If the latter methods are not called because deadCode_ is true
+// then the compiler MUST instead call resetLatentOp() to reset the state.
+
+template <typename Cond>
+bool BaseCompiler::sniffConditionalControlCmp(Cond compareOp,
+ ValType operandType) {
+ MOZ_ASSERT(latentOp_ == LatentOp::None,
+ "Latent comparison state not properly reset");
+
+#ifdef JS_CODEGEN_X86
+ // On x86, latent i64 binary comparisons use too many registers: the
+ // reserved join register and the lhs and rhs operands require six, but we
+ // only have five.
+ if (operandType == ValType::I64) {
+ return false;
+ }
+#endif
+
+ // No optimization for pointer compares yet.
+ if (operandType.isRefRepr()) {
+ return false;
+ }
+
+ OpBytes op{};
+ iter_.peekOp(&op);
+ switch (op.b0) {
+ case uint16_t(Op::BrIf):
+ case uint16_t(Op::If):
+ case uint16_t(Op::SelectNumeric):
+ case uint16_t(Op::SelectTyped):
+ setLatentCompare(compareOp, operandType);
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool BaseCompiler::sniffConditionalControlEqz(ValType operandType) {
+ MOZ_ASSERT(latentOp_ == LatentOp::None,
+ "Latent comparison state not properly reset");
+
+ OpBytes op{};
+ iter_.peekOp(&op);
+ switch (op.b0) {
+ case uint16_t(Op::BrIf):
+ case uint16_t(Op::SelectNumeric):
+ case uint16_t(Op::SelectTyped):
+ case uint16_t(Op::If):
+ setLatentEqz(operandType);
+ return true;
+ default:
+ return false;
+ }
+}
+
+void BaseCompiler::emitBranchSetup(BranchState* b) {
+ // Avoid allocating operands to latentOp_ to result registers.
+ if (b->hasBlockResults()) {
+ needResultRegisters(b->resultType);
+ }
+
+ // Set up fields so that emitBranchPerform() need not switch on latentOp_.
+ switch (latentOp_) {
+ case LatentOp::None: {
+ latentIntCmp_ = Assembler::NotEqual;
+ latentType_ = ValType::I32;
+ b->i32.lhs = popI32();
+ b->i32.rhsImm = true;
+ b->i32.imm = 0;
+ break;
+ }
+ case LatentOp::Compare: {
+ switch (latentType_.kind()) {
+ case ValType::I32: {
+ if (popConst(&b->i32.imm)) {
+ b->i32.lhs = popI32();
+ b->i32.rhsImm = true;
+ } else {
+ pop2xI32(&b->i32.lhs, &b->i32.rhs);
+ b->i32.rhsImm = false;
+ }
+ break;
+ }
+ case ValType::I64: {
+ pop2xI64(&b->i64.lhs, &b->i64.rhs);
+ b->i64.rhsImm = false;
+ break;
+ }
+ case ValType::F32: {
+ pop2xF32(&b->f32.lhs, &b->f32.rhs);
+ break;
+ }
+ case ValType::F64: {
+ pop2xF64(&b->f64.lhs, &b->f64.rhs);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unexpected type for LatentOp::Compare");
+ }
+ }
+ break;
+ }
+ case LatentOp::Eqz: {
+ switch (latentType_.kind()) {
+ case ValType::I32: {
+ latentIntCmp_ = Assembler::Equal;
+ b->i32.lhs = popI32();
+ b->i32.rhsImm = true;
+ b->i32.imm = 0;
+ break;
+ }
+ case ValType::I64: {
+ latentIntCmp_ = Assembler::Equal;
+ b->i64.lhs = popI64();
+ b->i64.rhsImm = true;
+ b->i64.imm = 0;
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unexpected type for LatentOp::Eqz");
+ }
+ }
+ break;
+ }
+ }
+
+ if (b->hasBlockResults()) {
+ freeResultRegisters(b->resultType);
+ }
+}
+
+bool BaseCompiler::emitBranchPerform(BranchState* b) {
+ switch (latentType_.kind()) {
+ case ValType::I32: {
+ if (b->i32.rhsImm) {
+ if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs,
+ Imm32(b->i32.imm))) {
+ return false;
+ }
+ } else {
+ if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs,
+ b->i32.rhs)) {
+ return false;
+ }
+ freeI32(b->i32.rhs);
+ }
+ freeI32(b->i32.lhs);
+ break;
+ }
+ case ValType::I64: {
+ if (b->i64.rhsImm) {
+ if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs,
+ Imm64(b->i64.imm))) {
+ return false;
+ }
+ } else {
+ if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs,
+ b->i64.rhs)) {
+ return false;
+ }
+ freeI64(b->i64.rhs);
+ }
+ freeI64(b->i64.lhs);
+ break;
+ }
+ case ValType::F32: {
+ if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f32.lhs,
+ b->f32.rhs)) {
+ return false;
+ }
+ freeF32(b->f32.lhs);
+ freeF32(b->f32.rhs);
+ break;
+ }
+ case ValType::F64: {
+ if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f64.lhs,
+ b->f64.rhs)) {
+ return false;
+ }
+ freeF64(b->f64.lhs);
+ freeF64(b->f64.rhs);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unexpected type for LatentOp::Compare");
+ }
+ }
+ resetLatentOp();
+ return true;
+}
+
+// For blocks and loops and ifs:
+//
+// - Sync the value stack before going into the block in order to simplify exit
+// from the block: all exits from the block can assume that there are no
+// live registers except the one carrying the exit value.
+// - The block can accumulate a number of dead values on the stacks, so when
+// branching out of the block or falling out at the end be sure to
+// pop the appropriate stacks back to where they were on entry, while
+// preserving the exit value.
+// - A continue branch in a loop is much like an exit branch, but the branch
+// value must not be preserved.
+// - The exit value is always in a designated join register (type dependent).
+
+bool BaseCompiler::emitBlock() {
+ ResultType params;
+ if (!iter_.readBlock(&params)) {
+ return false;
+ }
+
+ if (!deadCode_) {
+ sync(); // Simplifies branching out from block
+ }
+
+ initControl(controlItem(), params);
+
+ return true;
+}
+
+bool BaseCompiler::endBlock(ResultType type) {
+ Control& block = controlItem();
+
+ if (deadCode_) {
+ // Block does not fall through; reset stack.
+ fr.resetStackHeight(block.stackHeight, type);
+ popValueStackTo(block.stackSize);
+ } else {
+ // If the block label is used, we have a control join, so we need to shuffle
+ // fallthrough values into place. Otherwise if it's not a control join, we
+ // can leave the value stack alone.
+ MOZ_ASSERT(stk_.length() == block.stackSize + type.length());
+ if (block.label.used()) {
+ popBlockResults(type, block.stackHeight, ContinuationKind::Fallthrough);
+ }
+ block.bceSafeOnExit &= bceSafe_;
+ }
+
+ // Bind after cleanup: branches out will have popped the stack.
+ if (block.label.used()) {
+ masm.bind(&block.label);
+ if (deadCode_) {
+ captureResultRegisters(type);
+ deadCode_ = false;
+ }
+ if (!pushBlockResults(type)) {
+ return false;
+ }
+ }
+
+ bceSafe_ = block.bceSafeOnExit;
+
+ return true;
+}
+
+bool BaseCompiler::emitLoop() {
+ ResultType params;
+ if (!iter_.readLoop(&params)) {
+ return false;
+ }
+
+ if (!deadCode_) {
+ sync(); // Simplifies branching out from block
+ }
+
+ initControl(controlItem(), params);
+ bceSafe_ = 0;
+
+ if (!deadCode_) {
+ // Loop entry is a control join, so shuffle the entry parameters into the
+ // well-known locations.
+ if (!topBlockParams(params)) {
+ return false;
+ }
+ masm.nopAlign(CodeAlignment);
+ masm.bind(&controlItem(0).label);
+ // The interrupt check barfs if there are live registers.
+ sync();
+ if (!addInterruptCheck()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// The bodies of the "then" and "else" arms can be arbitrary sequences
+// of expressions, they push control and increment the nesting and can
+// even be targeted by jumps. A branch to the "if" block branches to
+// the exit of the if, ie, it's like "break". Consider:
+//
+// (func (result i32)
+// (if (i32.const 1)
+// (begin (br 1) (unreachable))
+// (begin (unreachable)))
+// (i32.const 1))
+//
+// The branch causes neither of the unreachable expressions to be
+// evaluated.
+
+bool BaseCompiler::emitIf() {
+ ResultType params;
+ Nothing unused_cond;
+ if (!iter_.readIf(&params, &unused_cond)) {
+ return false;
+ }
+
+ BranchState b(&controlItem().otherLabel, InvertBranch(true));
+ if (!deadCode_) {
+ needResultRegisters(params);
+ emitBranchSetup(&b);
+ freeResultRegisters(params);
+ sync();
+ } else {
+ resetLatentOp();
+ }
+
+ initControl(controlItem(), params);
+
+ if (!deadCode_) {
+ // Because params can flow immediately to results in the case of an empty
+ // "then" or "else" block, and the result of an if/then is a join in
+ // general, we shuffle params eagerly to the result allocations.
+ if (!topBlockParams(params)) {
+ return false;
+ }
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool BaseCompiler::endIfThen(ResultType type) {
+ Control& ifThen = controlItem();
+
+ // The parameters to the "if" logically flow to both the "then" and "else"
+ // blocks, but the "else" block is empty. Since we know that the "if"
+ // type-checks, that means that the "else" parameters are the "else" results,
+ // and that the "if"'s result type is the same as its parameter type.
+
+ if (deadCode_) {
+ // "then" arm does not fall through; reset stack.
+ fr.resetStackHeight(ifThen.stackHeight, type);
+ popValueStackTo(ifThen.stackSize);
+ if (!ifThen.deadOnArrival) {
+ captureResultRegisters(type);
+ }
+ } else {
+ MOZ_ASSERT(stk_.length() == ifThen.stackSize + type.length());
+ // Assume we have a control join, so place results in block result
+ // allocations.
+ popBlockResults(type, ifThen.stackHeight, ContinuationKind::Fallthrough);
+ MOZ_ASSERT(!ifThen.deadOnArrival);
+ }
+
+ if (ifThen.otherLabel.used()) {
+ masm.bind(&ifThen.otherLabel);
+ }
+
+ if (ifThen.label.used()) {
+ masm.bind(&ifThen.label);
+ }
+
+ if (!deadCode_) {
+ ifThen.bceSafeOnExit &= bceSafe_;
+ }
+
+ deadCode_ = ifThen.deadOnArrival;
+ if (!deadCode_) {
+ if (!pushBlockResults(type)) {
+ return false;
+ }
+ }
+
+ bceSafe_ = ifThen.bceSafeOnExit & ifThen.bceSafeOnEntry;
+
+ return true;
+}
+
+bool BaseCompiler::emitElse() {
+ ResultType params, results;
+ BaseNothingVector unused_thenValues{};
+
+ if (!iter_.readElse(&params, &results, &unused_thenValues)) {
+ return false;
+ }
+
+ Control& ifThenElse = controlItem(0);
+
+ // See comment in endIfThenElse, below.
+
+ // Exit the "then" branch.
+
+ ifThenElse.deadThenBranch = deadCode_;
+
+ if (deadCode_) {
+ fr.resetStackHeight(ifThenElse.stackHeight, results);
+ popValueStackTo(ifThenElse.stackSize);
+ } else {
+ MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + results.length());
+ popBlockResults(results, ifThenElse.stackHeight, ContinuationKind::Jump);
+ freeResultRegisters(results);
+ MOZ_ASSERT(!ifThenElse.deadOnArrival);
+ }
+
+ if (!deadCode_) {
+ masm.jump(&ifThenElse.label);
+ }
+
+ if (ifThenElse.otherLabel.used()) {
+ masm.bind(&ifThenElse.otherLabel);
+ }
+
+ // Reset to the "else" branch.
+
+ if (!deadCode_) {
+ ifThenElse.bceSafeOnExit &= bceSafe_;
+ }
+
+ deadCode_ = ifThenElse.deadOnArrival;
+ bceSafe_ = ifThenElse.bceSafeOnEntry;
+
+ fr.resetStackHeight(ifThenElse.stackHeight, params);
+
+ if (!deadCode_) {
+ captureResultRegisters(params);
+ if (!pushBlockResults(params)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool BaseCompiler::endIfThenElse(ResultType type) {
+ Control& ifThenElse = controlItem();
+
+ // The expression type is not a reliable guide to what we'll find
+ // on the stack, we could have (if E (i32.const 1) (unreachable))
+ // in which case the "else" arm is AnyType but the type of the
+ // full expression is I32. So restore whatever's there, not what
+ // we want to find there. The "then" arm has the same constraint.
+
+ if (deadCode_) {
+ // "then" arm does not fall through; reset stack.
+ fr.resetStackHeight(ifThenElse.stackHeight, type);
+ popValueStackTo(ifThenElse.stackSize);
+ } else {
+ MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + type.length());
+ // Assume we have a control join, so place results in block result
+ // allocations.
+ popBlockResults(type, ifThenElse.stackHeight,
+ ContinuationKind::Fallthrough);
+ ifThenElse.bceSafeOnExit &= bceSafe_;
+ MOZ_ASSERT(!ifThenElse.deadOnArrival);
+ }
+
+ if (ifThenElse.label.used()) {
+ masm.bind(&ifThenElse.label);
+ }
+
+ bool joinLive =
+ !ifThenElse.deadOnArrival &&
+ (!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label.bound());
+
+ if (joinLive) {
+ // No values were provided by the "then" path, but capture the values
+ // provided by the "else" path.
+ if (deadCode_) {
+ captureResultRegisters(type);
+ }
+ deadCode_ = false;
+ }
+
+ bceSafe_ = ifThenElse.bceSafeOnExit;
+
+ if (!deadCode_) {
+ if (!pushBlockResults(type)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool BaseCompiler::emitEnd() {
+ LabelKind kind;
+ ResultType type;
+ BaseNothingVector unused_values{};
+ if (!iter_.readEnd(&kind, &type, &unused_values, &unused_values)) {
+ return false;
+ }
+
+ // Every label case is responsible to pop the control item at the appropriate
+ // time for the label case
+ switch (kind) {
+ case LabelKind::Body:
+ if (!endBlock(type)) {
+ return false;
+ }
+ doReturn(ContinuationKind::Fallthrough);
+ // This is emitted here after `doReturn` to avoid being executed in the
+ // normal return path of a function, and instead only when a `delegate`
+ // jumps to it.
+ if (!emitBodyDelegateThrowPad()) {
+ return false;
+ }
+ iter_.popEnd();
+ MOZ_ASSERT(iter_.controlStackEmpty());
+ return iter_.endFunction(iter_.end());
+ case LabelKind::Block:
+ if (!endBlock(type)) {
+ return false;
+ }
+ iter_.popEnd();
+ break;
+ case LabelKind::Loop:
+ // The end of a loop isn't a branch target, so we can just leave its
+ // results on the expression stack to be consumed by the outer block.
+ iter_.popEnd();
+ break;
+ case LabelKind::Then:
+ if (!endIfThen(type)) {
+ return false;
+ }
+ iter_.popEnd();
+ break;
+ case LabelKind::Else:
+ if (!endIfThenElse(type)) {
+ return false;
+ }
+ iter_.popEnd();
+ break;
+ case LabelKind::Try:
+ case LabelKind::Catch:
+ case LabelKind::CatchAll:
+ if (!endTryCatch(type)) {
+ return false;
+ }
+ iter_.popEnd();
+ break;
+ }
+
+ return true;
+}
+
+bool BaseCompiler::emitBr() {
+ uint32_t relativeDepth;
+ ResultType type;
+ BaseNothingVector unused_values{};
+ if (!iter_.readBr(&relativeDepth, &type, &unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ Control& target = controlItem(relativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ // Save any values in the designated join registers, as if the target block
+ // returned normally.
+
+ popBlockResults(type, target.stackHeight, ContinuationKind::Jump);
+ masm.jump(&target.label);
+
+ // The registers holding the join values are free for the remainder of this
+ // block.
+
+ freeResultRegisters(type);
+
+ deadCode_ = true;
+
+ return true;
+}
+
+bool BaseCompiler::emitBrIf() {
+ uint32_t relativeDepth;
+ ResultType type;
+ BaseNothingVector unused_values{};
+ Nothing unused_condition;
+ if (!iter_.readBrIf(&relativeDepth, &type, &unused_values,
+ &unused_condition)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ resetLatentOp();
+ return true;
+ }
+
+ Control& target = controlItem(relativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false), type);
+ emitBranchSetup(&b);
+ return emitBranchPerform(&b);
+}
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+bool BaseCompiler::emitBrOnNull() {
+ MOZ_ASSERT(!hasLatentOp());
+
+ uint32_t relativeDepth;
+ ResultType type;
+ BaseNothingVector unused_values{};
+ Nothing unused_condition;
+ if (!iter_.readBrOnNull(&relativeDepth, &type, &unused_values,
+ &unused_condition)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ Control& target = controlItem(relativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false), type);
+ if (b.hasBlockResults()) {
+ needResultRegisters(b.resultType);
+ }
+ RegRef rp = popRef();
+ if (b.hasBlockResults()) {
+ freeResultRegisters(b.resultType);
+ }
+ if (!jumpConditionalWithResults(&b, Assembler::Equal, rp,
+ ImmWord(NULLREF_VALUE))) {
+ return false;
+ }
+ pushRef(rp);
+
+ return true;
+}
+
+bool BaseCompiler::emitBrOnNonNull() {
+ MOZ_ASSERT(!hasLatentOp());
+
+ uint32_t relativeDepth;
+ ResultType type;
+ BaseNothingVector unused_values{};
+ Nothing unused_condition;
+ if (!iter_.readBrOnNonNull(&relativeDepth, &type, &unused_values,
+ &unused_condition)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ Control& target = controlItem(relativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false), type);
+ MOZ_ASSERT(b.hasBlockResults(), "br_on_non_null has block results");
+
+ // Don't allocate the result register used in the branch
+ needIntegerResultRegisters(b.resultType);
+
+ // Get the ref from the top of the stack
+ RegRef condition = popRef();
+
+ // Create a copy of the ref for passing to the on_non_null label,
+ // the original ref is used in the condition.
+ RegRef rp = needRef();
+ moveRef(condition, rp);
+ pushRef(rp);
+
+ freeIntegerResultRegisters(b.resultType);
+
+ if (!jumpConditionalWithResults(&b, Assembler::NotEqual, condition,
+ ImmWord(NULLREF_VALUE))) {
+ return false;
+ }
+
+ freeRef(condition);
+
+ // Dropping null reference.
+ dropValue();
+
+ return true;
+}
+#endif
+
+bool BaseCompiler::emitBrTable() {
+ Uint32Vector depths;
+ uint32_t defaultDepth;
+ ResultType branchParams;
+ BaseNothingVector unused_values{};
+ Nothing unused_index;
+ // N.B., `branchParams' gets set to the type of the default branch target. In
+ // the presence of subtyping, it could be that the different branch targets
+ // have different types. Here we rely on the assumption that the value
+ // representations (e.g. Stk value types) of all branch target types are the
+ // same, in the baseline compiler. Notably, this means that all Ref types
+ // should be represented the same.
+ if (!iter_.readBrTable(&depths, &defaultDepth, &branchParams, &unused_values,
+ &unused_index)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Don't use param registers for rc
+ needIntegerResultRegisters(branchParams);
+
+ // Table switch value always on top.
+ RegI32 rc = popI32();
+
+ freeIntegerResultRegisters(branchParams);
+
+ StackHeight resultsBase(0);
+ if (!topBranchParams(branchParams, &resultsBase)) {
+ return false;
+ }
+
+ Label dispatchCode;
+ masm.branch32(Assembler::Below, rc, Imm32(depths.length()), &dispatchCode);
+
+ // This is the out-of-range stub. rc is dead here but we don't need it.
+
+ shuffleStackResultsBeforeBranch(
+ resultsBase, controlItem(defaultDepth).stackHeight, branchParams);
+ controlItem(defaultDepth).bceSafeOnExit &= bceSafe_;
+ masm.jump(&controlItem(defaultDepth).label);
+
+ // Emit stubs. rc is dead in all of these but we don't need it.
+ //
+ // The labels in the vector are in the TempAllocator and will
+ // be freed by and by.
+ //
+ // TODO / OPTIMIZE (Bug 1316804): Branch directly to the case code if we
+ // can, don't emit an intermediate stub.
+
+ LabelVector stubs;
+ if (!stubs.reserve(depths.length())) {
+ return false;
+ }
+
+ for (uint32_t depth : depths) {
+ stubs.infallibleEmplaceBack(NonAssertingLabel());
+ masm.bind(&stubs.back());
+ shuffleStackResultsBeforeBranch(resultsBase, controlItem(depth).stackHeight,
+ branchParams);
+ controlItem(depth).bceSafeOnExit &= bceSafe_;
+ masm.jump(&controlItem(depth).label);
+ }
+
+ // Emit table.
+
+ Label theTable;
+ jumpTable(stubs, &theTable);
+
+ // Emit indirect jump. rc is live here.
+
+ tableSwitch(&theTable, rc, &dispatchCode);
+
+ deadCode_ = true;
+
+ // Clean up.
+
+ freeI32(rc);
+ popValueStackBy(branchParams.length());
+
+ return true;
+}
+
+bool BaseCompiler::emitTry() {
+ ResultType params;
+ if (!iter_.readTry(&params)) {
+ return false;
+ }
+
+ if (!deadCode_) {
+ // Simplifies jumping out, but it is also necessary so that control
+ // can re-enter the catch handler without restoring registers.
+ sync();
+ }
+
+ initControl(controlItem(), params);
+
+ if (!deadCode_) {
+ // Be conservative for BCE due to complex control flow in try blocks.
+ controlItem().bceSafeOnExit = 0;
+ if (!startTryNote(&controlItem().tryNoteIndex)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void BaseCompiler::emitCatchSetup(LabelKind kind, Control& tryCatch,
+ const ResultType& resultType) {
+ // Catch ends the try or last catch, so we finish this like endIfThen.
+ if (deadCode_) {
+ fr.resetStackHeight(tryCatch.stackHeight, resultType);
+ popValueStackTo(tryCatch.stackSize);
+ } else {
+ // If the previous block is a catch, we need to handle the extra exception
+ // reference on the stack (for rethrow) and thus the stack size is 1 more.
+ MOZ_ASSERT(stk_.length() == tryCatch.stackSize + resultType.length() +
+ (kind == LabelKind::Try ? 0 : 1));
+ // Try jumps to the end of the try-catch block unless a throw is done.
+ if (kind == LabelKind::Try) {
+ popBlockResults(resultType, tryCatch.stackHeight, ContinuationKind::Jump);
+ } else {
+ popCatchResults(resultType, tryCatch.stackHeight);
+ }
+ MOZ_ASSERT(stk_.length() == tryCatch.stackSize);
+ freeResultRegisters(resultType);
+ MOZ_ASSERT(!tryCatch.deadOnArrival);
+ }
+
+ // Reset to this "catch" branch.
+ deadCode_ = tryCatch.deadOnArrival;
+
+ // We use the empty result type here because catch does *not* take the
+ // try-catch block parameters.
+ fr.resetStackHeight(tryCatch.stackHeight, ResultType::Empty());
+
+ if (deadCode_) {
+ return;
+ }
+
+ bceSafe_ = 0;
+
+ // The end of the previous try/catch jumps to the join point.
+ masm.jump(&tryCatch.label);
+
+ // Note end of try block for finding the catch block target. This needs
+ // to happen after the stack is reset to the correct height.
+ if (kind == LabelKind::Try) {
+ finishTryNote(controlItem().tryNoteIndex);
+ }
+}
+
+bool BaseCompiler::emitCatch() {
+ LabelKind kind;
+ uint32_t tagIndex;
+ ResultType paramType, resultType;
+ BaseNothingVector unused_tryValues{};
+
+ if (!iter_.readCatch(&kind, &tagIndex, &paramType, &resultType,
+ &unused_tryValues)) {
+ return false;
+ }
+
+ Control& tryCatch = controlItem();
+
+ emitCatchSetup(kind, tryCatch, resultType);
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Construct info used for the exception landing pad.
+ CatchInfo catchInfo(tagIndex);
+ if (!tryCatch.catchInfos.emplaceBack(catchInfo)) {
+ return false;
+ }
+
+ masm.bind(&tryCatch.catchInfos.back().label);
+
+ // Extract the arguments in the exception package and push them.
+ const SharedTagType& tagType = moduleEnv_.tags[tagIndex].type;
+ const ValTypeVector& params = tagType->argTypes_;
+ const TagOffsetVector& offsets = tagType->argOffsets_;
+
+ // The landing pad uses the block return protocol to communicate the
+ // exception object pointer to the catch block.
+ ResultType exnResult = ResultType::Single(RefType::extern_());
+ captureResultRegisters(exnResult);
+ if (!pushBlockResults(exnResult)) {
+ return false;
+ }
+ RegRef exn = popRef();
+ RegPtr data = needPtr();
+
+ masm.loadPtr(Address(exn, (int32_t)WasmExceptionObject::offsetOfData()),
+ data);
+
+ // This method can increase stk_.length() by an unbounded amount, so we need
+ // to perform an allocation here to accomodate the variable number of values.
+ // There is enough headroom for the fixed number of values. The general case
+ // is handled in emitBody.
+ if (!stk_.reserve(stk_.length() + params.length() + 1)) {
+ return false;
+ }
+
+ // This reference is pushed onto the stack because a potential rethrow
+ // may need to access it. It is always popped at the end of the block.
+ pushRef(exn);
+
+ for (uint32_t i = 0; i < params.length(); i++) {
+ int32_t offset = offsets[i];
+ switch (params[i].kind()) {
+ case ValType::I32: {
+ RegI32 reg = needI32();
+ masm.load32(Address(data, offset), reg);
+ pushI32(reg);
+ break;
+ }
+ case ValType::I64: {
+ RegI64 reg = needI64();
+ masm.load64(Address(data, offset), reg);
+ pushI64(reg);
+ break;
+ }
+ case ValType::F32: {
+ RegF32 reg = needF32();
+ masm.loadFloat32(Address(data, offset), reg);
+ pushF32(reg);
+ break;
+ }
+ case ValType::F64: {
+ RegF64 reg = needF64();
+ masm.loadDouble(Address(data, offset), reg);
+ pushF64(reg);
+ break;
+ }
+ case ValType::V128: {
+#ifdef ENABLE_WASM_SIMD
+ RegV128 reg = needV128();
+ masm.loadUnalignedSimd128(Address(data, offset), reg);
+ pushV128(reg);
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ case ValType::Ref: {
+ // TODO/AnyRef-boxing: With boxed immediates and strings, this may need
+ // to handle other kinds of values.
+ ASSERT_ANYREF_IS_JSOBJECT;
+ RegRef reg = needRef();
+ masm.loadPtr(Address(data, offset), reg);
+ pushRef(reg);
+ break;
+ }
+ }
+ }
+ freePtr(data);
+
+ return true;
+}
+
+bool BaseCompiler::emitCatchAll() {
+ LabelKind kind;
+ ResultType paramType, resultType;
+ BaseNothingVector unused_tryValues{};
+
+ if (!iter_.readCatchAll(&kind, &paramType, &resultType, &unused_tryValues)) {
+ return false;
+ }
+
+ Control& tryCatch = controlItem();
+
+ emitCatchSetup(kind, tryCatch, resultType);
+
+ if (deadCode_) {
+ return true;
+ }
+
+ CatchInfo catchInfo(CatchAllIndex);
+ if (!tryCatch.catchInfos.emplaceBack(catchInfo)) {
+ return false;
+ }
+
+ masm.bind(&tryCatch.catchInfos.back().label);
+
+ // The landing pad uses the block return protocol to communicate the
+ // exception object pointer to the catch block.
+ ResultType exnResult = ResultType::Single(RefType::extern_());
+ captureResultRegisters(exnResult);
+ // This reference is pushed onto the stack because a potential rethrow
+ // may need to access it. It is always popped at the end of the block.
+ return pushBlockResults(exnResult);
+}
+
+bool BaseCompiler::emitBodyDelegateThrowPad() {
+ Control& block = controlItem();
+
+ // Only emit a landing pad if a `delegate` has generated a jump to here.
+ if (block.otherLabel.used()) {
+ StackHeight savedHeight = fr.stackHeight();
+ fr.setStackHeight(block.stackHeight);
+ masm.bind(&block.otherLabel);
+
+ // A try-delegate jumps immediately to its delegated try block, so we are
+ // responsible to unpack the exception and rethrow it.
+ RegRef exn;
+ RegRef tag;
+ consumePendingException(&exn, &tag);
+ freeRef(tag);
+ if (!throwFrom(exn)) {
+ return false;
+ }
+ fr.setStackHeight(savedHeight);
+ }
+
+ return true;
+}
+
+bool BaseCompiler::emitDelegate() {
+ uint32_t relativeDepth;
+ ResultType resultType;
+ BaseNothingVector unused_tryValues{};
+
+ if (!iter_.readDelegate(&relativeDepth, &resultType, &unused_tryValues)) {
+ return false;
+ }
+
+ Control& tryDelegate = controlItem();
+
+ // End the try branch like a plain catch block without exception ref handling.
+ if (deadCode_) {
+ fr.resetStackHeight(tryDelegate.stackHeight, resultType);
+ popValueStackTo(tryDelegate.stackSize);
+ } else {
+ MOZ_ASSERT(stk_.length() == tryDelegate.stackSize + resultType.length());
+ popBlockResults(resultType, tryDelegate.stackHeight,
+ ContinuationKind::Jump);
+ freeResultRegisters(resultType);
+ masm.jump(&tryDelegate.label);
+ MOZ_ASSERT(!tryDelegate.deadOnArrival);
+ }
+
+ deadCode_ = tryDelegate.deadOnArrival;
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Create an exception landing pad that immediately branches to the landing
+ // pad of the delegated try block.
+ masm.bind(&tryDelegate.otherLabel);
+
+ StackHeight savedHeight = fr.stackHeight();
+ fr.setStackHeight(tryDelegate.stackHeight);
+
+ // Mark the end of the try body. This may insert a nop.
+ finishTryNote(controlItem().tryNoteIndex);
+
+ // The landing pad begins at this point
+ TryNoteVector& tryNotes = masm.tryNotes();
+ TryNote& tryNote = tryNotes[controlItem().tryNoteIndex];
+ tryNote.setLandingPad(masm.currentOffset(), masm.framePushed());
+
+ // Store the Instance that was left in InstanceReg by the exception
+ // handling mechanism, that is this frame's Instance but with the exception
+ // filled in Instance::pendingException.
+ fr.storeInstancePtr(InstanceReg);
+
+ // If the target block is a non-try block, skip over it and find the next
+ // try block or the very last block (to re-throw out of the function).
+ Control& lastBlock = controlOutermost();
+ while (controlKind(relativeDepth) != LabelKind::Try &&
+ &controlItem(relativeDepth) != &lastBlock) {
+ relativeDepth++;
+ }
+ Control& target = controlItem(relativeDepth);
+
+ popBlockResults(ResultType::Empty(), target.stackHeight,
+ ContinuationKind::Jump);
+ masm.jump(&target.otherLabel);
+
+ fr.setStackHeight(savedHeight);
+
+ // Where the try branch jumps to, if it's not dead.
+ if (tryDelegate.label.used()) {
+ masm.bind(&tryDelegate.label);
+ }
+
+ captureResultRegisters(resultType);
+ bceSafe_ = tryDelegate.bceSafeOnExit;
+
+ return pushBlockResults(resultType);
+}
+
+bool BaseCompiler::endTryCatch(ResultType type) {
+ Control& tryCatch = controlItem();
+ LabelKind tryKind = controlKind(0);
+
+ if (deadCode_) {
+ fr.resetStackHeight(tryCatch.stackHeight, type);
+ popValueStackTo(tryCatch.stackSize);
+ } else {
+ // If the previous block is a catch, we must handle the extra exception
+ // reference on the stack (for rethrow) and thus the stack size is 1 more.
+ MOZ_ASSERT(stk_.length() == tryCatch.stackSize + type.length() +
+ (tryKind == LabelKind::Try ? 0 : 1));
+ // Assume we have a control join, so place results in block result
+ // allocations and also handle the implicit exception reference if needed.
+ if (tryKind == LabelKind::Try) {
+ popBlockResults(type, tryCatch.stackHeight, ContinuationKind::Jump);
+ } else {
+ popCatchResults(type, tryCatch.stackHeight);
+ }
+ MOZ_ASSERT(stk_.length() == tryCatch.stackSize);
+ // Since we will emit a landing pad after this and jump over it to get to
+ // the control join, we free these here and re-capture at the join.
+ freeResultRegisters(type);
+ masm.jump(&tryCatch.label);
+ MOZ_ASSERT(!tryCatch.bceSafeOnExit);
+ MOZ_ASSERT(!tryCatch.deadOnArrival);
+ }
+
+ deadCode_ = tryCatch.deadOnArrival;
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Create landing pad for all catch handlers in this block.
+ // When used for a catchless try block, this will generate a landing pad
+ // with no handlers and only the fall-back rethrow.
+ masm.bind(&tryCatch.otherLabel);
+
+ // The stack height also needs to be set not for a block result, but for the
+ // entry to the exception handlers. This is reset again below for the join.
+ StackHeight prePadHeight = fr.stackHeight();
+ fr.setStackHeight(tryCatch.stackHeight);
+
+ // If we are in a catchless try block, then there were no catch blocks to
+ // mark the end of the try note, so we need to end it here.
+ if (tryKind == LabelKind::Try) {
+ // Mark the end of the try body. This may insert a nop.
+ finishTryNote(controlItem().tryNoteIndex);
+ }
+
+ // The landing pad begins at this point
+ TryNoteVector& tryNotes = masm.tryNotes();
+ TryNote& tryNote = tryNotes[controlItem().tryNoteIndex];
+ tryNote.setLandingPad(masm.currentOffset(), masm.framePushed());
+
+ // Store the Instance that was left in InstanceReg by the exception
+ // handling mechanism, that is this frame's Instance but with the exception
+ // filled in Instance::pendingException.
+ fr.storeInstancePtr(InstanceReg);
+
+ // Load exception pointer from Instance and make sure that it is
+ // saved before the following call will clear it.
+ RegRef exn;
+ RegRef tag;
+ consumePendingException(&exn, &tag);
+
+ // Get a register to hold the tags for each catch
+ RegRef catchTag = needRef();
+
+ // Ensure that the exception is assigned to the block return register
+ // before branching to a handler.
+ pushRef(exn);
+ ResultType exnResult = ResultType::Single(RefType::extern_());
+ popBlockResults(exnResult, tryCatch.stackHeight, ContinuationKind::Jump);
+ freeResultRegisters(exnResult);
+
+ bool hasCatchAll = false;
+ for (CatchInfo& info : tryCatch.catchInfos) {
+ if (info.tagIndex != CatchAllIndex) {
+ MOZ_ASSERT(!hasCatchAll);
+ loadTag(RegPtr(InstanceReg), info.tagIndex, catchTag);
+ masm.branchPtr(Assembler::Equal, tag, catchTag, &info.label);
+ } else {
+ masm.jump(&info.label);
+ hasCatchAll = true;
+ }
+ }
+ freeRef(catchTag);
+ freeRef(tag);
+
+ // If none of the tag checks succeed and there is no catch_all,
+ // then we rethrow the exception.
+ if (!hasCatchAll) {
+ captureResultRegisters(exnResult);
+ if (!pushBlockResults(exnResult) || !throwFrom(popRef())) {
+ return false;
+ }
+ }
+
+ // Reset stack height for join.
+ fr.setStackHeight(prePadHeight);
+
+ // Create join point.
+ if (tryCatch.label.used()) {
+ masm.bind(&tryCatch.label);
+ }
+
+ captureResultRegisters(type);
+ deadCode_ = tryCatch.deadOnArrival;
+ bceSafe_ = tryCatch.bceSafeOnExit;
+
+ return pushBlockResults(type);
+}
+
+bool BaseCompiler::emitThrow() {
+ uint32_t tagIndex;
+ BaseNothingVector unused_argValues{};
+
+ if (!iter_.readThrow(&tagIndex, &unused_argValues)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const TagDesc& tagDesc = moduleEnv_.tags[tagIndex];
+ const ResultType& params = tagDesc.type->resultType();
+ const TagOffsetVector& offsets = tagDesc.type->argOffsets_;
+
+ // Load the tag object
+#ifdef RABALDR_PIN_INSTANCE
+ RegPtr instance(InstanceReg);
+#else
+ RegPtr instance = needPtr();
+ fr.loadInstancePtr(instance);
+#endif
+ RegRef tag = needRef();
+ loadTag(instance, tagIndex, tag);
+#ifndef RABALDR_PIN_INSTANCE
+ freePtr(instance);
+#endif
+
+ // Create the new exception object that we will throw.
+ pushRef(tag);
+ if (!emitInstanceCall(SASigExceptionNew)) {
+ return false;
+ }
+
+ // Get registers for exn and data, excluding the prebarrier register
+ needPtr(RegPtr(PreBarrierReg));
+ RegRef exn = popRef();
+ RegPtr data = needPtr();
+ freePtr(RegPtr(PreBarrierReg));
+
+ masm.loadPtr(Address(exn, WasmExceptionObject::offsetOfData()), data);
+
+ for (int32_t i = params.length() - 1; i >= 0; i--) {
+ uint32_t offset = offsets[i];
+ switch (params[i].kind()) {
+ case ValType::I32: {
+ RegI32 reg = popI32();
+ masm.store32(reg, Address(data, offset));
+ freeI32(reg);
+ break;
+ }
+ case ValType::I64: {
+ RegI64 reg = popI64();
+ masm.store64(reg, Address(data, offset));
+ freeI64(reg);
+ break;
+ }
+ case ValType::F32: {
+ RegF32 reg = popF32();
+ masm.storeFloat32(reg, Address(data, offset));
+ freeF32(reg);
+ break;
+ }
+ case ValType::F64: {
+ RegF64 reg = popF64();
+ masm.storeDouble(reg, Address(data, offset));
+ freeF64(reg);
+ break;
+ }
+ case ValType::V128: {
+#ifdef ENABLE_WASM_SIMD
+ RegV128 reg = popV128();
+ masm.storeUnalignedSimd128(reg, Address(data, offset));
+ freeV128(reg);
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ case ValType::Ref: {
+ RegPtr valueAddr(PreBarrierReg);
+ needPtr(valueAddr);
+ masm.computeEffectiveAddress(Address(data, offset), valueAddr);
+ RegRef rv = popRef();
+ pushPtr(data);
+ // emitBarrieredStore preserves exn, rv
+ if (!emitBarrieredStore(Some(exn), valueAddr, rv,
+ PreBarrierKind::Normal,
+ PostBarrierKind::Imprecise)) {
+ return false;
+ }
+ popPtr(data);
+ freeRef(rv);
+ break;
+ }
+ }
+ }
+ freePtr(data);
+
+ deadCode_ = true;
+
+ return throwFrom(exn);
+}
+
+bool BaseCompiler::emitRethrow() {
+ uint32_t relativeDepth;
+ if (!iter_.readRethrow(&relativeDepth)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ Control& tryCatch = controlItem(relativeDepth);
+ RegRef exn = needRef();
+ peekRefAt(tryCatch.stackSize, exn);
+
+ deadCode_ = true;
+
+ return throwFrom(exn);
+}
+
+bool BaseCompiler::emitDrop() {
+ if (!iter_.readDrop()) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ dropValue();
+ return true;
+}
+
+void BaseCompiler::doReturn(ContinuationKind kind) {
+ if (deadCode_) {
+ return;
+ }
+
+ StackHeight height = controlOutermost().stackHeight;
+ ResultType type = ResultType::Vector(funcType().results());
+ popBlockResults(type, height, kind);
+ masm.jump(&returnLabel_);
+ freeResultRegisters(type);
+}
+
+bool BaseCompiler::emitReturn() {
+ BaseNothingVector unused_values{};
+ if (!iter_.readReturn(&unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ doReturn(ContinuationKind::Jump);
+ deadCode_ = true;
+
+ return true;
+}
+
+bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes,
+ const StackResultsLoc& results,
+ FunctionCall* baselineCall,
+ CalleeOnStack calleeOnStack) {
+ MOZ_ASSERT(!deadCode_);
+
+ ArgTypeVector args(argTypes, results.stackResults());
+ uint32_t naturalArgCount = argTypes.length();
+ uint32_t abiArgCount = args.lengthWithStackResults();
+ startCallArgs(StackArgAreaSizeUnaligned(args), baselineCall);
+
+ // Args are deeper on the stack than the stack result area, if any.
+ size_t argsDepth = results.count();
+ // They're deeper than the callee too, for callIndirect.
+ if (calleeOnStack == CalleeOnStack::True) {
+ argsDepth++;
+ }
+
+ for (size_t i = 0; i < abiArgCount; ++i) {
+ if (args.isNaturalArg(i)) {
+ size_t naturalIndex = args.naturalIndex(i);
+ size_t stackIndex = naturalArgCount - 1 - naturalIndex + argsDepth;
+ passArg(argTypes[naturalIndex], peek(stackIndex), baselineCall);
+ } else {
+ // The synthetic stack result area pointer.
+ ABIArg argLoc = baselineCall->abi.next(MIRType::Pointer);
+ if (argLoc.kind() == ABIArg::Stack) {
+ ScratchPtr scratch(*this);
+ fr.computeOutgoingStackResultAreaPtr(results, scratch);
+ masm.storePtr(scratch, Address(masm.getStackPointer(),
+ argLoc.offsetFromArgBase()));
+ } else {
+ fr.computeOutgoingStackResultAreaPtr(results, RegPtr(argLoc.gpr()));
+ }
+ }
+ }
+
+#ifndef RABALDR_PIN_INSTANCE
+ fr.loadInstancePtr(InstanceReg);
+#endif
+ return true;
+}
+
+void BaseCompiler::pushReturnValueOfCall(const FunctionCall& call,
+ MIRType type) {
+ switch (type) {
+ case MIRType::Int32: {
+ RegI32 rv = captureReturnedI32();
+ pushI32(rv);
+ break;
+ }
+ case MIRType::Int64: {
+ RegI64 rv = captureReturnedI64();
+ pushI64(rv);
+ break;
+ }
+ case MIRType::Float32: {
+ RegF32 rv = captureReturnedF32(call);
+ pushF32(rv);
+ break;
+ }
+ case MIRType::Double: {
+ RegF64 rv = captureReturnedF64(call);
+ pushF64(rv);
+ break;
+ }
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128: {
+ RegV128 rv = captureReturnedV128(call);
+ pushV128(rv);
+ break;
+ }
+#endif
+ case MIRType::RefOrNull: {
+ RegRef rv = captureReturnedRef();
+ pushRef(rv);
+ break;
+ }
+ default:
+ // In particular, passing |type| as MIRType::Void or MIRType::Pointer to
+ // this function is an error.
+ MOZ_CRASH("Function return type");
+ }
+}
+
+bool BaseCompiler::pushStackResultsForCall(const ResultType& type, RegPtr temp,
+ StackResultsLoc* loc) {
+ if (!ABIResultIter::HasStackResults(type)) {
+ return true;
+ }
+
+ // This method can increase stk_.length() by an unbounded amount, so we need
+ // to perform an allocation here to accomodate the variable number of values.
+ // There is enough headroom for any fixed number of values. The general case
+ // is handled in emitBody.
+ if (!stk_.reserve(stk_.length() + type.length())) {
+ return false;
+ }
+
+ // Measure stack results.
+ ABIResultIter i(type);
+ size_t count = 0;
+ for (; !i.done(); i.next()) {
+ if (i.cur().onStack()) {
+ count++;
+ }
+ }
+ uint32_t bytes = i.stackBytesConsumedSoFar();
+
+ // Reserve space for the stack results.
+ StackHeight resultsBase = fr.stackHeight();
+ uint32_t height = fr.prepareStackResultArea(resultsBase, bytes);
+
+ // Push Stk values onto the value stack, and zero out Ref values.
+ for (i.switchToPrev(); !i.done(); i.prev()) {
+ const ABIResult& result = i.cur();
+ if (result.onStack()) {
+ Stk v = captureStackResult(result, resultsBase, bytes);
+ push(v);
+ if (v.kind() == Stk::MemRef) {
+ stackMapGenerator_.memRefsOnStk++;
+ fr.storeImmediatePtrToStack(intptr_t(0), v.offs(), temp);
+ }
+ }
+ }
+
+ *loc = StackResultsLoc(bytes, count, height);
+
+ return true;
+}
+
+// After a call, some results may be written to the stack result locations that
+// are pushed on the machine stack after any stack args. If there are stack
+// args and stack results, these results need to be shuffled down, as the args
+// are "consumed" by the call.
+void BaseCompiler::popStackResultsAfterCall(const StackResultsLoc& results,
+ uint32_t stackArgBytes) {
+ if (results.bytes() != 0) {
+ popValueStackBy(results.count());
+ if (stackArgBytes != 0) {
+ uint32_t srcHeight = results.height();
+ MOZ_ASSERT(srcHeight >= stackArgBytes + results.bytes());
+ uint32_t destHeight = srcHeight - stackArgBytes;
+
+ fr.shuffleStackResultsTowardFP(srcHeight, destHeight, results.bytes(),
+ ABINonArgReturnVolatileReg);
+ }
+ }
+}
+
+// For now, always sync() at the beginning of the call to easily save live
+// values.
+//
+// TODO / OPTIMIZE (Bug 1316806): We may be able to avoid a full sync(), since
+// all we want is to save live registers that won't be saved by the callee or
+// that we need for outgoing args - we don't need to sync the locals. We can
+// just push the necessary registers, it'll be like a lightweight sync.
+//
+// Even some of the pushing may be unnecessary if the registers will be consumed
+// by the call, because then what we want is parallel assignment to the argument
+// registers or onto the stack for outgoing arguments. A sync() is just
+// simpler.
+
+bool BaseCompiler::emitCall() {
+ uint32_t funcIndex;
+ BaseNothingVector args_{};
+ if (!iter_.readCall(&funcIndex, &args_)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ sync();
+
+ const FuncType& funcType = *moduleEnv_.funcs[funcIndex].type;
+ bool import = moduleEnv_.funcIsImport(funcIndex);
+
+ uint32_t numArgs = funcType.args().length();
+ size_t stackArgBytes = stackConsumed(numArgs);
+
+ ResultType resultType(ResultType::Vector(funcType.results()));
+ StackResultsLoc results;
+ if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) {
+ return false;
+ }
+
+ FunctionCall baselineCall{};
+ beginCall(baselineCall, UseABI::Wasm,
+ import ? RestoreRegisterStateAndRealm::True
+ : RestoreRegisterStateAndRealm::False);
+
+ if (!emitCallArgs(funcType.args(), results, &baselineCall,
+ CalleeOnStack::False)) {
+ return false;
+ }
+
+ CodeOffset raOffset;
+ if (import) {
+ raOffset = callImport(moduleEnv_.offsetOfFuncImportInstanceData(funcIndex),
+ baselineCall);
+ } else {
+ raOffset = callDefinition(funcIndex, baselineCall);
+ }
+
+ if (!createStackMap("emitCall", raOffset)) {
+ return false;
+ }
+
+ popStackResultsAfterCall(results, stackArgBytes);
+
+ endCall(baselineCall, stackArgBytes);
+
+ popValueStackBy(numArgs);
+
+ captureCallResultRegisters(resultType);
+ return pushCallResults(baselineCall, resultType, results);
+}
+
+bool BaseCompiler::emitCallIndirect() {
+ uint32_t funcTypeIndex;
+ uint32_t tableIndex;
+ Nothing callee_;
+ BaseNothingVector args_{};
+ if (!iter_.readCallIndirect(&funcTypeIndex, &tableIndex, &callee_, &args_)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ sync();
+
+ const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType();
+
+ // Stack: ... arg1 .. argn callee
+
+ uint32_t numArgs = funcType.args().length() + 1;
+ size_t stackArgBytes = stackConsumed(numArgs);
+
+ ResultType resultType(ResultType::Vector(funcType.results()));
+ StackResultsLoc results;
+ if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) {
+ return false;
+ }
+
+ FunctionCall baselineCall{};
+ // State and realm are restored as needed by by callIndirect (really by
+ // MacroAssembler::wasmCallIndirect).
+ beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False);
+
+ if (!emitCallArgs(funcType.args(), results, &baselineCall,
+ CalleeOnStack::True)) {
+ return false;
+ }
+
+ const Stk& callee = peek(results.count());
+ CodeOffset fastCallOffset;
+ CodeOffset slowCallOffset;
+ if (!callIndirect(funcTypeIndex, tableIndex, callee, baselineCall,
+ &fastCallOffset, &slowCallOffset)) {
+ return false;
+ }
+ if (!createStackMap("emitCallIndirect", fastCallOffset)) {
+ return false;
+ }
+ if (!createStackMap("emitCallIndirect", slowCallOffset)) {
+ return false;
+ }
+
+ popStackResultsAfterCall(results, stackArgBytes);
+
+ endCall(baselineCall, stackArgBytes);
+
+ popValueStackBy(numArgs);
+
+ captureCallResultRegisters(resultType);
+ return pushCallResults(baselineCall, resultType, results);
+}
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+bool BaseCompiler::emitCallRef() {
+ const FuncType* funcType;
+ Nothing unused_callee;
+ BaseNothingVector unused_args{};
+ if (!iter_.readCallRef(&funcType, &unused_callee, &unused_args)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ sync();
+
+ // Stack: ... arg1 .. argn callee
+
+ uint32_t numArgs = funcType->args().length() + 1;
+ size_t stackArgBytes = stackConsumed(numArgs);
+
+ ResultType resultType(ResultType::Vector(funcType->results()));
+ StackResultsLoc results;
+ if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) {
+ return false;
+ }
+
+ FunctionCall baselineCall{};
+ // State and realm are restored as needed by by callRef (really by
+ // MacroAssembler::wasmCallRef).
+ beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False);
+
+ if (!emitCallArgs(funcType->args(), results, &baselineCall,
+ CalleeOnStack::True)) {
+ return false;
+ }
+
+ const Stk& callee = peek(results.count());
+ CodeOffset fastCallOffset;
+ CodeOffset slowCallOffset;
+ callRef(callee, baselineCall, &fastCallOffset, &slowCallOffset);
+ if (!createStackMap("emitCallRef", fastCallOffset)) {
+ return false;
+ }
+ if (!createStackMap("emitCallRef", slowCallOffset)) {
+ return false;
+ }
+
+ popStackResultsAfterCall(results, stackArgBytes);
+
+ endCall(baselineCall, stackArgBytes);
+
+ popValueStackBy(numArgs);
+
+ captureCallResultRegisters(resultType);
+ return pushCallResults(baselineCall, resultType, results);
+}
+#endif
+
+void BaseCompiler::emitRound(RoundingMode roundingMode, ValType operandType) {
+ if (operandType == ValType::F32) {
+ RegF32 f0 = popF32();
+ roundF32(roundingMode, f0);
+ pushF32(f0);
+ } else if (operandType == ValType::F64) {
+ RegF64 f0 = popF64();
+ roundF64(roundingMode, f0);
+ pushF64(f0);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+bool BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee,
+ ValType operandType) {
+ Nothing operand_;
+ if (!iter_.readUnary(operandType, &operand_)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RoundingMode roundingMode;
+ if (IsRoundingFunction(callee, &roundingMode) &&
+ supportsRoundInstruction(roundingMode)) {
+ emitRound(roundingMode, operandType);
+ return true;
+ }
+
+ sync();
+
+ ValTypeVector& signature = operandType == ValType::F32 ? SigF_ : SigD_;
+ ValType retType = operandType;
+ uint32_t numArgs = signature.length();
+ size_t stackSpace = stackConsumed(numArgs);
+ StackResultsLoc noStackResults;
+
+ FunctionCall baselineCall{};
+ beginCall(baselineCall, UseABI::Builtin, RestoreRegisterStateAndRealm::False);
+
+ if (!emitCallArgs(signature, noStackResults, &baselineCall,
+ CalleeOnStack::False)) {
+ return false;
+ }
+
+ CodeOffset raOffset = builtinCall(callee, baselineCall);
+ if (!createStackMap("emitUnaryMathBuiltin[..]", raOffset)) {
+ return false;
+ }
+
+ endCall(baselineCall, stackSpace);
+
+ popValueStackBy(numArgs);
+
+ pushReturnValueOfCall(baselineCall, retType.toMIRType());
+
+ return true;
+}
+
+#ifdef RABALDR_INT_DIV_I64_CALLOUT
+bool BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee,
+ ValType operandType) {
+ MOZ_ASSERT(operandType == ValType::I64);
+ MOZ_ASSERT(!deadCode_);
+
+ sync();
+
+ needI64(specific_.abiReturnRegI64);
+
+ RegI64 rhs = popI64();
+ RegI64 srcDest = popI64ToSpecific(specific_.abiReturnRegI64);
+
+ Label done;
+
+ checkDivideByZero(rhs);
+
+ if (callee == SymbolicAddress::DivI64) {
+ checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(false));
+ } else if (callee == SymbolicAddress::ModI64) {
+ checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(true));
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(srcDest.high);
+ masm.passABIArg(srcDest.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+ CodeOffset raOffset = masm.callWithABI(
+ bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset()));
+ if (!createStackMap("emitDivOrModI64Bui[..]", raOffset)) {
+ return false;
+ }
+
+ masm.bind(&done);
+
+ freeI64(rhs);
+ pushI64(srcDest);
+ return true;
+}
+#endif // RABALDR_INT_DIV_I64_CALLOUT
+
+#ifdef RABALDR_I64_TO_FLOAT_CALLOUT
+bool BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee,
+ ValType operandType,
+ ValType resultType) {
+ sync();
+
+ RegI64 input = popI64();
+
+ FunctionCall call{};
+
+ masm.setupWasmABICall();
+# ifdef JS_PUNBOX64
+ MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout");
+# else
+ masm.passABIArg(input.high);
+ masm.passABIArg(input.low);
+# endif
+ CodeOffset raOffset = masm.callWithABI(
+ bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset()),
+ resultType == ValType::F32 ? MoveOp::FLOAT32 : MoveOp::DOUBLE);
+ if (!createStackMap("emitConvertInt64To[..]", raOffset)) {
+ return false;
+ }
+
+ freeI64(input);
+
+ if (resultType == ValType::F32) {
+ pushF32(captureReturnedF32(call));
+ } else {
+ pushF64(captureReturnedF64(call));
+ }
+
+ return true;
+}
+#endif // RABALDR_I64_TO_FLOAT_CALLOUT
+
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+// `Callee` always takes a double, so a float32 input must be converted.
+bool BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee,
+ ValType operandType,
+ ValType resultType) {
+ RegF64 doubleInput;
+ if (operandType == ValType::F32) {
+ doubleInput = needF64();
+ RegF32 input = popF32();
+ masm.convertFloat32ToDouble(input, doubleInput);
+ freeF32(input);
+ } else {
+ doubleInput = popF64();
+ }
+
+ // We may need the value after the call for the ool check.
+ RegF64 otherReg = needF64();
+ moveF64(doubleInput, otherReg);
+ pushF64(otherReg);
+
+ sync();
+
+ FunctionCall call{};
+
+ masm.setupWasmABICall();
+ masm.passABIArg(doubleInput, MoveOp::DOUBLE);
+ CodeOffset raOffset = masm.callWithABI(
+ bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset()));
+ if (!createStackMap("emitConvertFloatin[..]", raOffset)) {
+ return false;
+ }
+
+ freeF64(doubleInput);
+
+ RegI64 rv = captureReturnedI64();
+
+ RegF64 inputVal = popF64();
+
+ TruncFlags flags = 0;
+ if (callee == SymbolicAddress::TruncateDoubleToUint64) {
+ flags |= TRUNC_UNSIGNED;
+ }
+ if (callee == SymbolicAddress::SaturatingTruncateDoubleToInt64 ||
+ callee == SymbolicAddress::SaturatingTruncateDoubleToUint64) {
+ flags |= TRUNC_SATURATING;
+ }
+
+ // If we're saturating, the callout will always produce the final result
+ // value. Otherwise, the callout value will return 0x8000000000000000
+ // and we need to produce traps.
+ OutOfLineCode* ool = nullptr;
+ if (!(flags & TRUNC_SATURATING)) {
+ // The OOL check just succeeds or fails, it does not generate a value.
+ ool = addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(
+ AnyReg(inputVal), rv, flags, bytecodeOffset()));
+ if (!ool) {
+ return false;
+ }
+
+ masm.branch64(Assembler::Equal, rv, Imm64(0x8000000000000000),
+ ool->entry());
+ masm.bind(ool->rejoin());
+ }
+
+ pushI64(rv);
+ freeF64(inputVal);
+
+ return true;
+}
+#endif // RABALDR_FLOAT_TO_I64_CALLOUT
+
+bool BaseCompiler::emitGetLocal() {
+ uint32_t slot;
+ if (!iter_.readGetLocal(locals_, &slot)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Local loads are pushed unresolved, ie, they may be deferred
+ // until needed, until they may be affected by a store, or until a
+ // sync. This is intended to reduce register pressure.
+
+ switch (locals_[slot].kind()) {
+ case ValType::I32:
+ pushLocalI32(slot);
+ break;
+ case ValType::I64:
+ pushLocalI64(slot);
+ break;
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ pushLocalV128(slot);
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ case ValType::F64:
+ pushLocalF64(slot);
+ break;
+ case ValType::F32:
+ pushLocalF32(slot);
+ break;
+ case ValType::Ref:
+ pushLocalRef(slot);
+ break;
+ }
+
+ return true;
+}
+
+template <bool isSetLocal>
+bool BaseCompiler::emitSetOrTeeLocal(uint32_t slot) {
+ if (deadCode_) {
+ return true;
+ }
+
+ bceLocalIsUpdated(slot);
+ switch (locals_[slot].kind()) {
+ case ValType::I32: {
+ RegI32 rv = popI32();
+ syncLocal(slot);
+ fr.storeLocalI32(rv, localFromSlot(slot, MIRType::Int32));
+ if (isSetLocal) {
+ freeI32(rv);
+ } else {
+ pushI32(rv);
+ }
+ break;
+ }
+ case ValType::I64: {
+ RegI64 rv = popI64();
+ syncLocal(slot);
+ fr.storeLocalI64(rv, localFromSlot(slot, MIRType::Int64));
+ if (isSetLocal) {
+ freeI64(rv);
+ } else {
+ pushI64(rv);
+ }
+ break;
+ }
+ case ValType::F64: {
+ RegF64 rv = popF64();
+ syncLocal(slot);
+ fr.storeLocalF64(rv, localFromSlot(slot, MIRType::Double));
+ if (isSetLocal) {
+ freeF64(rv);
+ } else {
+ pushF64(rv);
+ }
+ break;
+ }
+ case ValType::F32: {
+ RegF32 rv = popF32();
+ syncLocal(slot);
+ fr.storeLocalF32(rv, localFromSlot(slot, MIRType::Float32));
+ if (isSetLocal) {
+ freeF32(rv);
+ } else {
+ pushF32(rv);
+ }
+ break;
+ }
+ case ValType::V128: {
+#ifdef ENABLE_WASM_SIMD
+ RegV128 rv = popV128();
+ syncLocal(slot);
+ fr.storeLocalV128(rv, localFromSlot(slot, MIRType::Simd128));
+ if (isSetLocal) {
+ freeV128(rv);
+ } else {
+ pushV128(rv);
+ }
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ }
+ case ValType::Ref: {
+ RegRef rv = popRef();
+ syncLocal(slot);
+ fr.storeLocalRef(rv, localFromSlot(slot, MIRType::RefOrNull));
+ if (isSetLocal) {
+ freeRef(rv);
+ } else {
+ pushRef(rv);
+ }
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool BaseCompiler::emitSetLocal() {
+ uint32_t slot;
+ Nothing unused_value;
+ if (!iter_.readSetLocal(locals_, &slot, &unused_value)) {
+ return false;
+ }
+ return emitSetOrTeeLocal<true>(slot);
+}
+
+bool BaseCompiler::emitTeeLocal() {
+ uint32_t slot;
+ Nothing unused_value;
+ if (!iter_.readTeeLocal(locals_, &slot, &unused_value)) {
+ return false;
+ }
+ return emitSetOrTeeLocal<false>(slot);
+}
+
+bool BaseCompiler::emitGetGlobal() {
+ uint32_t id;
+ if (!iter_.readGetGlobal(&id)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const GlobalDesc& global = moduleEnv_.globals[id];
+
+ if (global.isConstant()) {
+ LitVal value = global.constantValue();
+ switch (value.type().kind()) {
+ case ValType::I32:
+ pushI32(value.i32());
+ break;
+ case ValType::I64:
+ pushI64(value.i64());
+ break;
+ case ValType::F32:
+ pushF32(value.f32());
+ break;
+ case ValType::F64:
+ pushF64(value.f64());
+ break;
+ case ValType::Ref:
+ pushRef(intptr_t(value.ref().forCompiledCode()));
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case ValType::V128:
+ pushV128(value.v128());
+ break;
+#endif
+ default:
+ MOZ_CRASH("Global constant type");
+ }
+ return true;
+ }
+
+ switch (global.type().kind()) {
+ case ValType::I32: {
+ RegI32 rv = needI32();
+ ScratchPtr tmp(*this);
+ masm.load32(addressOfGlobalVar(global, tmp), rv);
+ pushI32(rv);
+ break;
+ }
+ case ValType::I64: {
+ RegI64 rv = needI64();
+ ScratchPtr tmp(*this);
+ masm.load64(addressOfGlobalVar(global, tmp), rv);
+ pushI64(rv);
+ break;
+ }
+ case ValType::F32: {
+ RegF32 rv = needF32();
+ ScratchPtr tmp(*this);
+ masm.loadFloat32(addressOfGlobalVar(global, tmp), rv);
+ pushF32(rv);
+ break;
+ }
+ case ValType::F64: {
+ RegF64 rv = needF64();
+ ScratchPtr tmp(*this);
+ masm.loadDouble(addressOfGlobalVar(global, tmp), rv);
+ pushF64(rv);
+ break;
+ }
+ case ValType::Ref: {
+ RegRef rv = needRef();
+ ScratchPtr tmp(*this);
+ masm.loadPtr(addressOfGlobalVar(global, tmp), rv);
+ pushRef(rv);
+ break;
+ }
+#ifdef ENABLE_WASM_SIMD
+ case ValType::V128: {
+ RegV128 rv = needV128();
+ ScratchPtr tmp(*this);
+ masm.loadUnalignedSimd128(addressOfGlobalVar(global, tmp), rv);
+ pushV128(rv);
+ break;
+ }
+#endif
+ default:
+ MOZ_CRASH("Global variable type");
+ break;
+ }
+ return true;
+}
+
+bool BaseCompiler::emitSetGlobal() {
+ uint32_t id;
+ Nothing unused_value;
+ if (!iter_.readSetGlobal(&id, &unused_value)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const GlobalDesc& global = moduleEnv_.globals[id];
+
+ switch (global.type().kind()) {
+ case ValType::I32: {
+ RegI32 rv = popI32();
+ ScratchPtr tmp(*this);
+ masm.store32(rv, addressOfGlobalVar(global, tmp));
+ freeI32(rv);
+ break;
+ }
+ case ValType::I64: {
+ RegI64 rv = popI64();
+ ScratchPtr tmp(*this);
+ masm.store64(rv, addressOfGlobalVar(global, tmp));
+ freeI64(rv);
+ break;
+ }
+ case ValType::F32: {
+ RegF32 rv = popF32();
+ ScratchPtr tmp(*this);
+ masm.storeFloat32(rv, addressOfGlobalVar(global, tmp));
+ freeF32(rv);
+ break;
+ }
+ case ValType::F64: {
+ RegF64 rv = popF64();
+ ScratchPtr tmp(*this);
+ masm.storeDouble(rv, addressOfGlobalVar(global, tmp));
+ freeF64(rv);
+ break;
+ }
+ case ValType::Ref: {
+ RegPtr valueAddr(PreBarrierReg);
+ needPtr(valueAddr);
+ {
+ ScratchPtr tmp(*this);
+ masm.computeEffectiveAddress(addressOfGlobalVar(global, tmp),
+ valueAddr);
+ }
+ RegRef rv = popRef();
+ // emitBarrieredStore preserves rv
+ if (!emitBarrieredStore(Nothing(), valueAddr, rv, PreBarrierKind::Normal,
+ PostBarrierKind::Imprecise)) {
+ return false;
+ }
+ freeRef(rv);
+ break;
+ }
+#ifdef ENABLE_WASM_SIMD
+ case ValType::V128: {
+ RegV128 rv = popV128();
+ ScratchPtr tmp(*this);
+ masm.storeUnalignedSimd128(rv, addressOfGlobalVar(global, tmp));
+ freeV128(rv);
+ break;
+ }
+#endif
+ default:
+ MOZ_CRASH("Global variable type");
+ break;
+ }
+ return true;
+}
+
+bool BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readLoad(type, Scalar::byteSize(viewType), &addr)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ loadCommon(&access, AccessCheck(), type);
+ return true;
+}
+
+bool BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ Nothing unused_value;
+ if (!iter_.readStore(resultType, Scalar::byteSize(viewType), &addr,
+ &unused_value)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ storeCommon(&access, AccessCheck(), resultType);
+ return true;
+}
+
+bool BaseCompiler::emitSelect(bool typed) {
+ StackType type;
+ Nothing unused_trueValue;
+ Nothing unused_falseValue;
+ Nothing unused_condition;
+ if (!iter_.readSelect(typed, &type, &unused_trueValue, &unused_falseValue,
+ &unused_condition)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ resetLatentOp();
+ return true;
+ }
+
+ // I32 condition on top, then false, then true.
+
+ Label done;
+ BranchState b(&done);
+ emitBranchSetup(&b);
+
+ switch (type.valType().kind()) {
+ case ValType::I32: {
+ RegI32 r, rs;
+ pop2xI32(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveI32(rs, r);
+ masm.bind(&done);
+ freeI32(rs);
+ pushI32(r);
+ break;
+ }
+ case ValType::I64: {
+#ifdef JS_CODEGEN_X86
+ // There may be as many as four Int64 values in registers at a time: two
+ // for the latent branch operands, and two for the true/false values we
+ // normally pop before executing the branch. On x86 this is one value
+ // too many, so we need to generate more complicated code here, and for
+ // simplicity's sake we do so even if the branch operands are not Int64.
+ // However, the resulting control flow diamond is complicated since the
+ // arms of the diamond will have to stay synchronized with respect to
+ // their evaluation stack and regalloc state. To simplify further, we
+ // use a double branch and a temporary boolean value for now.
+ RegI32 temp = needI32();
+ moveImm32(0, temp);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveImm32(1, temp);
+ masm.bind(&done);
+
+ Label trueValue;
+ RegI64 r, rs;
+ pop2xI64(&r, &rs);
+ masm.branch32(Assembler::Equal, temp, Imm32(0), &trueValue);
+ moveI64(rs, r);
+ masm.bind(&trueValue);
+ freeI32(temp);
+ freeI64(rs);
+ pushI64(r);
+#else
+ RegI64 r, rs;
+ pop2xI64(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveI64(rs, r);
+ masm.bind(&done);
+ freeI64(rs);
+ pushI64(r);
+#endif
+ break;
+ }
+ case ValType::F32: {
+ RegF32 r, rs;
+ pop2xF32(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveF32(rs, r);
+ masm.bind(&done);
+ freeF32(rs);
+ pushF32(r);
+ break;
+ }
+ case ValType::F64: {
+ RegF64 r, rs;
+ pop2xF64(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveF64(rs, r);
+ masm.bind(&done);
+ freeF64(rs);
+ pushF64(r);
+ break;
+ }
+#ifdef ENABLE_WASM_SIMD
+ case ValType::V128: {
+ RegV128 r, rs;
+ pop2xV128(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveV128(rs, r);
+ masm.bind(&done);
+ freeV128(rs);
+ pushV128(r);
+ break;
+ }
+#endif
+ case ValType::Ref: {
+ RegRef r, rs;
+ pop2xRef(&r, &rs);
+ if (!emitBranchPerform(&b)) {
+ return false;
+ }
+ moveRef(rs, r);
+ masm.bind(&done);
+ freeRef(rs);
+ pushRef(r);
+ break;
+ }
+ default: {
+ MOZ_CRASH("select type");
+ }
+ }
+
+ return true;
+}
+
+void BaseCompiler::emitCompareI32(Assembler::Condition compareOp,
+ ValType compareType) {
+ MOZ_ASSERT(compareType == ValType::I32);
+
+ if (sniffConditionalControlCmp(compareOp, compareType)) {
+ return;
+ }
+
+ int32_t c;
+ if (popConst(&c)) {
+ RegI32 r = popI32();
+ masm.cmp32Set(compareOp, r, Imm32(c), r);
+ pushI32(r);
+ } else {
+ RegI32 r, rs;
+ pop2xI32(&r, &rs);
+ masm.cmp32Set(compareOp, r, rs, r);
+ freeI32(rs);
+ pushI32(r);
+ }
+}
+
+void BaseCompiler::emitCompareI64(Assembler::Condition compareOp,
+ ValType compareType) {
+ MOZ_ASSERT(compareType == ValType::I64);
+
+ if (sniffConditionalControlCmp(compareOp, compareType)) {
+ return;
+ }
+
+ RegI64 rs0, rs1;
+ pop2xI64(&rs0, &rs1);
+ RegI32 rd(fromI64(rs0));
+ cmp64Set(compareOp, rs0, rs1, rd);
+ freeI64(rs1);
+ freeI64Except(rs0, rd);
+ pushI32(rd);
+}
+
+void BaseCompiler::emitCompareF32(Assembler::DoubleCondition compareOp,
+ ValType compareType) {
+ MOZ_ASSERT(compareType == ValType::F32);
+
+ if (sniffConditionalControlCmp(compareOp, compareType)) {
+ return;
+ }
+
+ Label across;
+ RegF32 rs0, rs1;
+ pop2xF32(&rs0, &rs1);
+ RegI32 rd = needI32();
+ moveImm32(1, rd);
+ masm.branchFloat(compareOp, rs0, rs1, &across);
+ moveImm32(0, rd);
+ masm.bind(&across);
+ freeF32(rs0);
+ freeF32(rs1);
+ pushI32(rd);
+}
+
+void BaseCompiler::emitCompareF64(Assembler::DoubleCondition compareOp,
+ ValType compareType) {
+ MOZ_ASSERT(compareType == ValType::F64);
+
+ if (sniffConditionalControlCmp(compareOp, compareType)) {
+ return;
+ }
+
+ Label across;
+ RegF64 rs0, rs1;
+ pop2xF64(&rs0, &rs1);
+ RegI32 rd = needI32();
+ moveImm32(1, rd);
+ masm.branchDouble(compareOp, rs0, rs1, &across);
+ moveImm32(0, rd);
+ masm.bind(&across);
+ freeF64(rs0);
+ freeF64(rs1);
+ pushI32(rd);
+}
+
+void BaseCompiler::emitCompareRef(Assembler::Condition compareOp,
+ ValType compareType) {
+ MOZ_ASSERT(!sniffConditionalControlCmp(compareOp, compareType));
+
+ RegRef rs1, rs2;
+ pop2xRef(&rs1, &rs2);
+ RegI32 rd = needI32();
+ masm.cmpPtrSet(compareOp, rs1, rs2, rd);
+ freeRef(rs1);
+ freeRef(rs2);
+ pushI32(rd);
+}
+
+bool BaseCompiler::emitInstanceCall(const SymbolicAddressSignature& builtin) {
+ // See declaration (WasmBCClass.h) for info on the relationship between the
+ // compiler's value stack and the argument order for the to-be-called
+ // function.
+ const MIRType* argTypes = builtin.argTypes;
+ MOZ_ASSERT(argTypes[0] == MIRType::Pointer);
+
+ sync();
+
+ uint32_t numNonInstanceArgs = builtin.numArgs - 1 /* instance */;
+ size_t stackSpace = stackConsumed(numNonInstanceArgs);
+
+ FunctionCall baselineCall{};
+ beginCall(baselineCall, UseABI::System, RestoreRegisterStateAndRealm::True);
+
+ ABIArg instanceArg = reservePointerArgument(&baselineCall);
+
+ startCallArgs(StackArgAreaSizeUnaligned(builtin), &baselineCall);
+ for (uint32_t i = 1; i < builtin.numArgs; i++) {
+ ValType t;
+ switch (argTypes[i]) {
+ case MIRType::Int32:
+ t = ValType::I32;
+ break;
+ case MIRType::Int64:
+ t = ValType::I64;
+ break;
+ case MIRType::Float32:
+ t = ValType::F32;
+ break;
+ case MIRType::RefOrNull:
+ t = RefType::extern_();
+ break;
+ case MIRType::Pointer:
+ // Instance function args can now be uninterpreted pointers (eg, for
+ // the cases PostBarrier and PostBarrierFilter) so we simply treat
+ // them like the equivalently sized integer.
+ t = ValType::fromMIRType(TargetWordMIRType());
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+ passArg(t, peek(numNonInstanceArgs - i), &baselineCall);
+ }
+ CodeOffset raOffset =
+ builtinInstanceMethodCall(builtin, instanceArg, baselineCall);
+ if (!createStackMap("emitInstanceCall", raOffset)) {
+ return false;
+ }
+
+ endCall(baselineCall, stackSpace);
+
+ popValueStackBy(numNonInstanceArgs);
+
+ // Note, many clients of emitInstanceCall currently assume that pushing the
+ // result here does not destroy ReturnReg.
+ //
+ // Furthermore, clients assume that if builtin.retType != MIRType::None, the
+ // callee will have returned a result and left it in ReturnReg for us to
+ // find, and that that register will not be destroyed here (or above).
+
+ // For the return type only, MIRType::None is used to indicate that the
+ // call doesn't return a result, that is, returns a C/C++ "void".
+
+ if (builtin.retType != MIRType::None) {
+ pushReturnValueOfCall(baselineCall, builtin.retType);
+ }
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Reference types.
+
+bool BaseCompiler::emitRefFunc() {
+ return emitInstanceCallOp<uint32_t>(SASigRefFunc,
+ [this](uint32_t* funcIndex) -> bool {
+ return iter_.readRefFunc(funcIndex);
+ });
+}
+
+bool BaseCompiler::emitRefNull() {
+ RefType type;
+ if (!iter_.readRefNull(&type)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ pushRef(NULLREF_VALUE);
+ return true;
+}
+
+bool BaseCompiler::emitRefIsNull() {
+ Nothing nothing;
+ if (!iter_.readRefIsNull(&nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RegRef r = popRef();
+ RegI32 rd = narrowRef(r);
+
+ masm.cmpPtrSet(Assembler::Equal, r, ImmWord(NULLREF_VALUE), rd);
+ pushI32(rd);
+ return true;
+}
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+bool BaseCompiler::emitRefAsNonNull() {
+ Nothing nothing;
+ if (!iter_.readRefAsNonNull(&nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RegRef rp = popRef();
+ Label ok;
+ masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok);
+ trap(Trap::NullPointerDereference);
+ masm.bind(&ok);
+ pushRef(rp);
+
+ return true;
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Atomic operations.
+
+bool BaseCompiler::emitAtomicCmpXchg(ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ Nothing unused{};
+ if (!iter_.readAtomicCmpXchg(&addr, type, Scalar::byteSize(viewType), &unused,
+ &unused)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(),
+ Synchronization::Full());
+ atomicCmpXchg(&access, type);
+ return true;
+}
+
+bool BaseCompiler::emitAtomicLoad(ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readAtomicLoad(&addr, type, Scalar::byteSize(viewType))) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(),
+ Synchronization::Load());
+ atomicLoad(&access, type);
+ return true;
+}
+
+bool BaseCompiler::emitAtomicRMW(ValType type, Scalar::Type viewType,
+ AtomicOp op) {
+ LinearMemoryAddress<Nothing> addr;
+ Nothing unused_value;
+ if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType),
+ &unused_value)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(),
+ Synchronization::Full());
+ atomicRMW(&access, type, op);
+ return true;
+}
+
+bool BaseCompiler::emitAtomicStore(ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ Nothing unused_value;
+ if (!iter_.readAtomicStore(&addr, type, Scalar::byteSize(viewType),
+ &unused_value)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(),
+ Synchronization::Store());
+ atomicStore(&access, type);
+ return true;
+}
+
+bool BaseCompiler::emitAtomicXchg(ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ Nothing unused_value;
+ if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType),
+ &unused_value)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(),
+ Synchronization::Full());
+ atomicXchg(&access, type);
+ return true;
+}
+
+bool BaseCompiler::emitWait(ValType type, uint32_t byteSize) {
+ Nothing nothing;
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readWait(&addr, type, byteSize, &nothing, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(
+ type.kind() == ValType::I32 ? Scalar::Int32 : Scalar::Int64, addr.align,
+ addr.offset, bytecodeOffset());
+ return atomicWait(type, &access);
+}
+
+bool BaseCompiler::emitWake() {
+ Nothing nothing;
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readWake(&addr, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(Scalar::Int32, addr.align, addr.offset,
+ bytecodeOffset());
+ return atomicWake(&access);
+}
+
+bool BaseCompiler::emitFence() {
+ if (!iter_.readFence()) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ masm.memoryBarrier(MembarFull);
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Bulk memory operations.
+
+bool BaseCompiler::emitMemoryGrow() {
+ return emitInstanceCallOp(
+ !usesMemory() || isMem32() ? SASigMemoryGrowM32 : SASigMemoryGrowM64,
+ [this]() -> bool {
+ Nothing arg;
+ return iter_.readMemoryGrow(&arg);
+ });
+}
+
+bool BaseCompiler::emitMemorySize() {
+ return emitInstanceCallOp(
+ !usesMemory() || isMem32() ? SASigMemorySizeM32 : SASigMemorySizeM64,
+ [this]() -> bool { return iter_.readMemorySize(); });
+}
+
+bool BaseCompiler::emitMemCopy() {
+ uint32_t dstMemOrTableIndex = 0;
+ uint32_t srcMemOrTableIndex = 0;
+ Nothing nothing;
+ if (!iter_.readMemOrTableCopy(true, &dstMemOrTableIndex, &nothing,
+ &srcMemOrTableIndex, &nothing, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+
+ if (isMem32()) {
+ int32_t signedLength;
+ if (peekConst(&signedLength) && signedLength != 0 &&
+ uint32_t(signedLength) <= MaxInlineMemoryCopyLength) {
+ memCopyInlineM32();
+ return true;
+ }
+ }
+
+ return memCopyCall();
+}
+
+bool BaseCompiler::memCopyCall() {
+ pushHeapBase();
+ return emitInstanceCall(
+ usesSharedMemory()
+ ? (isMem32() ? SASigMemCopySharedM32 : SASigMemCopySharedM64)
+ : (isMem32() ? SASigMemCopyM32 : SASigMemCopyM64));
+}
+
+bool BaseCompiler::emitMemFill() {
+ Nothing nothing;
+ if (!iter_.readMemFill(&nothing, &nothing, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+
+ if (isMem32()) {
+ int32_t signedLength;
+ int32_t signedValue;
+ if (peek2xConst(&signedLength, &signedValue) && signedLength != 0 &&
+ uint32_t(signedLength) <= MaxInlineMemoryFillLength) {
+ memFillInlineM32();
+ return true;
+ }
+ }
+ return memFillCall();
+}
+
+bool BaseCompiler::memFillCall() {
+ pushHeapBase();
+ return emitInstanceCall(
+ usesSharedMemory()
+ ? (isMem32() ? SASigMemFillSharedM32 : SASigMemFillSharedM64)
+ : (isMem32() ? SASigMemFillM32 : SASigMemFillM64));
+}
+
+bool BaseCompiler::emitMemInit() {
+ return emitInstanceCallOp<uint32_t>(
+ (!usesMemory() || isMem32() ? SASigMemInitM32 : SASigMemInitM64),
+ [this](uint32_t* segIndex) -> bool {
+ Nothing nothing;
+ if (iter_.readMemOrTableInit(/*isMem*/ true, segIndex, nullptr,
+ &nothing, &nothing, &nothing)) {
+ return true;
+ }
+ return false;
+ });
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Bulk table operations.
+
+bool BaseCompiler::emitTableCopy() {
+ uint32_t dstMemOrTableIndex = 0;
+ uint32_t srcMemOrTableIndex = 0;
+ Nothing nothing;
+ if (!iter_.readMemOrTableCopy(false, &dstMemOrTableIndex, &nothing,
+ &srcMemOrTableIndex, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ pushI32(dstMemOrTableIndex);
+ pushI32(srcMemOrTableIndex);
+ return emitInstanceCall(SASigTableCopy);
+}
+
+bool BaseCompiler::emitTableInit() {
+ return emitInstanceCallOp<uint32_t, uint32_t>(
+ SASigTableInit,
+ [this](uint32_t* segIndex, uint32_t* dstTableIndex) -> bool {
+ Nothing nothing;
+ return iter_.readMemOrTableInit(/*isMem*/ false, segIndex,
+ dstTableIndex, &nothing, &nothing,
+ &nothing);
+ });
+}
+
+bool BaseCompiler::emitTableFill() {
+ // fill(start:u32, val:ref, len:u32, table:u32) -> void
+ return emitInstanceCallOp<uint32_t>(
+ SASigTableFill, [this](uint32_t* tableIndex) -> bool {
+ Nothing nothing;
+ return iter_.readTableFill(tableIndex, &nothing, &nothing, &nothing);
+ });
+}
+
+bool BaseCompiler::emitMemDiscard() {
+ Nothing nothing;
+ if (!iter_.readMemDiscard(&nothing, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+
+ pushHeapBase();
+ return emitInstanceCall(
+ usesSharedMemory()
+ ? (isMem32() ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64)
+ : (isMem32() ? SASigMemDiscardM32 : SASigMemDiscardM64));
+}
+
+bool BaseCompiler::emitTableGet() {
+ uint32_t tableIndex;
+ Nothing nothing;
+ if (!iter_.readTableGet(&tableIndex, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) {
+ return emitTableGetAnyRef(tableIndex);
+ }
+ pushI32(tableIndex);
+ // get(index:u32, table:u32) -> AnyRef
+ return emitInstanceCall(SASigTableGet);
+}
+
+bool BaseCompiler::emitTableGrow() {
+ // grow(initValue:anyref, delta:u32, table:u32) -> u32
+ return emitInstanceCallOp<uint32_t>(
+ SASigTableGrow, [this](uint32_t* tableIndex) -> bool {
+ Nothing nothing;
+ return iter_.readTableGrow(tableIndex, &nothing, &nothing);
+ });
+}
+
+bool BaseCompiler::emitTableSet() {
+ uint32_t tableIndex;
+ Nothing nothing;
+ if (!iter_.readTableSet(&tableIndex, &nothing, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) {
+ return emitTableSetAnyRef(tableIndex);
+ }
+ pushI32(tableIndex);
+ // set(index:u32, value:ref, table:u32) -> void
+ return emitInstanceCall(SASigTableSet);
+}
+
+bool BaseCompiler::emitTableSize() {
+ uint32_t tableIndex;
+ if (!iter_.readTableSize(&tableIndex)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+
+ RegPtr instance = needPtr();
+ RegI32 length = needI32();
+
+ fr.loadInstancePtr(instance);
+ loadTableLength(tableIndex, instance, length);
+
+ pushI32(length);
+ freePtr(instance);
+ return true;
+}
+
+void BaseCompiler::emitTableBoundsCheck(uint32_t tableIndex, RegI32 index,
+ RegPtr instance) {
+ Label ok;
+ masm.wasmBoundsCheck32(
+ Assembler::Condition::Below, index,
+ addressOfTableField(tableIndex, offsetof(TableInstanceData, length),
+ instance),
+ &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, bytecodeOffset());
+ masm.bind(&ok);
+}
+
+bool BaseCompiler::emitTableGetAnyRef(uint32_t tableIndex) {
+ RegPtr instance = needPtr();
+ RegPtr elements = needPtr();
+ RegI32 index = popI32();
+
+ fr.loadInstancePtr(instance);
+ emitTableBoundsCheck(tableIndex, index, instance);
+ loadTableElements(tableIndex, instance, elements);
+ masm.loadPtr(BaseIndex(elements, index, ScalePointer), elements);
+
+ pushRef(RegRef(elements));
+ freeI32(index);
+ freePtr(instance);
+
+ return true;
+}
+
+bool BaseCompiler::emitTableSetAnyRef(uint32_t tableIndex) {
+ // Create temporaries for valueAddr that is not in the prebarrier register
+ // and can be consumed by the barrier operation
+ RegPtr valueAddr = RegPtr(PreBarrierReg);
+ needPtr(valueAddr);
+
+ RegPtr instance = needPtr();
+ RegPtr elements = needPtr();
+ RegRef value = popRef();
+ RegI32 index = popI32();
+
+ // x86 is one register too short for this operation, shuffle `value` back
+ // onto the stack until it is needed.
+#ifdef JS_CODEGEN_X86
+ pushRef(value);
+#endif
+
+ fr.loadInstancePtr(instance);
+ emitTableBoundsCheck(tableIndex, index, instance);
+ loadTableElements(tableIndex, instance, elements);
+ masm.computeEffectiveAddress(BaseIndex(elements, index, ScalePointer),
+ valueAddr);
+
+ freeI32(index);
+ freePtr(elements);
+ freePtr(instance);
+
+#ifdef JS_CODEGEN_X86
+ value = popRef();
+#endif
+
+ if (!emitBarrieredStore(Nothing(), valueAddr, value, PreBarrierKind::Normal,
+ PostBarrierKind::Precise)) {
+ return false;
+ }
+ freeRef(value);
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Data and element segment management.
+
+bool BaseCompiler::emitDataOrElemDrop(bool isData) {
+ return emitInstanceCallOp<uint32_t>(
+ isData ? SASigDataDrop : SASigElemDrop, [&](uint32_t* segIndex) -> bool {
+ return iter_.readDataOrElemDrop(isData, segIndex);
+ });
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// General object support.
+
+void BaseCompiler::emitPreBarrier(RegPtr valueAddr) {
+ Label skipBarrier;
+ ScratchPtr scratch(*this);
+
+#ifdef RABALDR_PIN_INSTANCE
+ Register instance(InstanceReg);
+#else
+ Register instance(scratch);
+ fr.loadInstancePtr(instance);
+#endif
+
+ EmitWasmPreBarrierGuard(masm, instance, scratch, valueAddr,
+ /*valueOffset=*/0, &skipBarrier, nullptr);
+
+#ifndef RABALDR_PIN_INSTANCE
+ fr.loadInstancePtr(instance);
+#endif
+#ifdef JS_CODEGEN_ARM64
+ // The prebarrier stub assumes the PseudoStackPointer is set up. It is OK
+ // to just move the sp to x28 here because x28 is not being used by the
+ // baseline compiler and need not be saved or restored.
+ MOZ_ASSERT(!GeneralRegisterSet::All().hasRegisterIndex(x28.asUnsized()));
+ masm.Mov(x28, sp);
+#endif
+ // The prebarrier call preserves all volatile registers
+ EmitWasmPreBarrierCall(masm, instance, scratch, valueAddr, /*valueOffset=*/0);
+
+ masm.bind(&skipBarrier);
+}
+
+bool BaseCompiler::emitPostBarrierImprecise(const Maybe<RegRef>& object,
+ RegPtr valueAddr, RegRef value) {
+ // We must force a sync before the guard so that locals are in a consistent
+ // location for whether or not the post-barrier call is taken.
+ sync();
+
+ // Emit a guard to skip the post-barrier call if it is not needed.
+ Label skipBarrier;
+ RegPtr otherScratch = needPtr();
+ EmitWasmPostBarrierGuard(masm, object, otherScratch, value, &skipBarrier);
+ freePtr(otherScratch);
+
+ // Push `object` and `value` to preserve them across the call.
+ if (object) {
+ pushRef(*object);
+ }
+ pushRef(value);
+
+ // The `valueAddr` is a raw pointer to the cell within some GC object or
+ // instance area, and we are careful so that the GC will not run while the
+ // post-barrier call is active, so push a uintptr_t value.
+ pushPtr(valueAddr);
+ if (!emitInstanceCall(SASigPostBarrier)) {
+ return false;
+ }
+
+ // Restore `object` and `value`.
+ popRef(value);
+ if (object) {
+ popRef(*object);
+ }
+
+ masm.bind(&skipBarrier);
+ return true;
+}
+
+bool BaseCompiler::emitPostBarrierPrecise(const Maybe<RegRef>& object,
+ RegPtr valueAddr, RegRef prevValue,
+ RegRef value) {
+ // Push `object` and `value` to preserve them across the call.
+ if (object) {
+ pushRef(*object);
+ }
+ pushRef(value);
+
+ // Push the arguments and call the precise post-barrier
+ pushPtr(valueAddr);
+ pushRef(prevValue);
+ if (!emitInstanceCall(SASigPostBarrierPrecise)) {
+ return false;
+ }
+
+ // Restore `object` and `value`.
+ popRef(value);
+ if (object) {
+ popRef(*object);
+ }
+
+ return true;
+}
+
+bool BaseCompiler::emitBarrieredStore(const Maybe<RegRef>& object,
+ RegPtr valueAddr, RegRef value,
+ PreBarrierKind preBarrierKind,
+ PostBarrierKind postBarrierKind) {
+ // TODO/AnyRef-boxing: With boxed immediates and strings, the write
+ // barrier is going to have to be more complicated.
+ ASSERT_ANYREF_IS_JSOBJECT;
+
+ // The pre-barrier preserves all allocated registers.
+ if (preBarrierKind == PreBarrierKind::Normal) {
+ emitPreBarrier(valueAddr);
+ }
+
+ // The precise post-barrier requires the previous value stored in the field,
+ // in order to know if the previous store buffer entry needs to be removed.
+ RegRef prevValue;
+ if (postBarrierKind == PostBarrierKind::Precise) {
+ prevValue = needRef();
+ masm.loadPtr(Address(valueAddr, 0), prevValue);
+ }
+
+ // Store the value
+ masm.storePtr(value, Address(valueAddr, 0));
+
+ // The post-barrier preserves object and value.
+ if (postBarrierKind == PostBarrierKind::Precise) {
+ return emitPostBarrierPrecise(object, valueAddr, prevValue, value);
+ }
+ return emitPostBarrierImprecise(object, valueAddr, value);
+}
+
+void BaseCompiler::emitBarrieredClear(RegPtr valueAddr) {
+ // TODO/AnyRef-boxing: With boxed immediates and strings, the write
+ // barrier is going to have to be more complicated.
+ ASSERT_ANYREF_IS_JSOBJECT;
+
+ // The pre-barrier preserves all allocated registers.
+ emitPreBarrier(valueAddr);
+
+ // Store null
+ masm.storePtr(ImmWord(0), Address(valueAddr, 0));
+
+ // No post-barrier is needed, as null does not require a store buffer entry
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// GC proposal.
+
+#ifdef ENABLE_WASM_GC
+
+RegPtr BaseCompiler::loadTypeDefInstanceData(uint32_t typeIndex) {
+ RegPtr rp = needPtr();
+ RegPtr instance;
+# ifndef RABALDR_PIN_INSTANCE
+ instance = rp;
+ fr.loadInstancePtr(instance);
+# else
+ // We can use the pinned instance register.
+ instance = RegPtr(InstanceReg);
+# endif
+ masm.computeEffectiveAddress(
+ Address(instance, Instance::offsetInData(
+ moduleEnv_.offsetOfTypeDefInstanceData(typeIndex))),
+ rp);
+ return rp;
+}
+
+RegPtr BaseCompiler::loadSuperTypeVector(uint32_t typeIndex) {
+ RegPtr rp = needPtr();
+ RegPtr instance;
+# ifndef RABALDR_PIN_INSTANCE
+ // We need to load the instance register, but can use the destination
+ // register as a temporary.
+ instance = rp;
+ fr.loadInstancePtr(rp);
+# else
+ // We can use the pinned instance register.
+ instance = RegPtr(InstanceReg);
+# endif
+ masm.loadPtr(
+ Address(instance, Instance::offsetInData(
+ moduleEnv_.offsetOfSuperTypeVector(typeIndex))),
+ rp);
+ return rp;
+}
+
+/* static */
+void BaseCompiler::SignalNullCheck::emitNullCheck(BaseCompiler* bc, RegRef rp) {
+ Label ok;
+ MacroAssembler& masm = bc->masm;
+ masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok);
+ bc->trap(Trap::NullPointerDereference);
+ masm.bind(&ok);
+}
+
+/* static */
+void BaseCompiler::SignalNullCheck::emitTrapSite(BaseCompiler* bc) {
+ wasm::BytecodeOffset trapOffset(bc->bytecodeOffset());
+ MacroAssembler& masm = bc->masm;
+ masm.append(wasm::Trap::NullPointerDereference,
+ wasm::TrapSite(masm.currentOffset(), trapOffset));
+}
+
+RegPtr BaseCompiler::emitGcArrayGetData(RegRef rp) {
+ // `rp` points at a WasmArrayObject. Return a reg holding the value of its
+ // `data_` field.
+ RegPtr rdata = needPtr();
+ masm.loadPtr(Address(rp, WasmArrayObject::offsetOfData()), rdata);
+ return rdata;
+}
+
+template <typename NullCheckPolicy>
+RegI32 BaseCompiler::emitGcArrayGetNumElements(RegRef rp) {
+ // `rp` points at a WasmArrayObject. Return a reg holding the value of its
+ // `numElements_` field.
+ STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32;
+ RegI32 numElements = needI32();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.load32(Address(rp, WasmArrayObject::offsetOfNumElements()), numElements);
+ return numElements;
+}
+
+void BaseCompiler::emitGcArrayBoundsCheck(RegI32 index, RegI32 numElements) {
+ Label inBounds;
+ masm.branch32(Assembler::Below, index, numElements, &inBounds);
+ masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset());
+ masm.bind(&inBounds);
+}
+
+template <typename T, typename NullCheckPolicy>
+void BaseCompiler::emitGcGet(FieldType type, FieldWideningOp wideningOp,
+ const T& src) {
+ switch (type.kind()) {
+ case FieldType::I8: {
+ MOZ_ASSERT(wideningOp != FieldWideningOp::None);
+ RegI32 r = needI32();
+ NullCheckPolicy::emitTrapSite(this);
+ if (wideningOp == FieldWideningOp::Unsigned) {
+ masm.load8ZeroExtend(src, r);
+ } else {
+ masm.load8SignExtend(src, r);
+ }
+ pushI32(r);
+ break;
+ }
+ case FieldType::I16: {
+ MOZ_ASSERT(wideningOp != FieldWideningOp::None);
+ RegI32 r = needI32();
+ NullCheckPolicy::emitTrapSite(this);
+ if (wideningOp == FieldWideningOp::Unsigned) {
+ masm.load16ZeroExtend(src, r);
+ } else {
+ masm.load16SignExtend(src, r);
+ }
+ pushI32(r);
+ break;
+ }
+ case FieldType::I32: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegI32 r = needI32();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.load32(src, r);
+ pushI32(r);
+ break;
+ }
+ case FieldType::I64: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegI64 r = needI64();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.load64(src, r);
+ pushI64(r);
+ break;
+ }
+ case FieldType::F32: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegF32 r = needF32();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.loadFloat32(src, r);
+ pushF32(r);
+ break;
+ }
+ case FieldType::F64: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegF64 r = needF64();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.loadDouble(src, r);
+ pushF64(r);
+ break;
+ }
+# ifdef ENABLE_WASM_SIMD
+ case FieldType::V128: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegV128 r = needV128();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.loadUnalignedSimd128(src, r);
+ pushV128(r);
+ break;
+ }
+# endif
+ case FieldType::Ref: {
+ MOZ_ASSERT(wideningOp == FieldWideningOp::None);
+ RegRef r = needRef();
+ NullCheckPolicy::emitTrapSite(this);
+ masm.loadPtr(src, r);
+ pushRef(r);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unexpected field type");
+ }
+ }
+}
+
+template <typename T, typename NullCheckPolicy>
+void BaseCompiler::emitGcSetScalar(const T& dst, FieldType type, AnyReg value) {
+ NullCheckPolicy::emitTrapSite(this);
+ switch (type.kind()) {
+ case FieldType::I8: {
+ masm.store8(value.i32(), dst);
+ break;
+ }
+ case FieldType::I16: {
+ masm.store16(value.i32(), dst);
+ break;
+ }
+ case FieldType::I32: {
+ masm.store32(value.i32(), dst);
+ break;
+ }
+ case FieldType::I64: {
+ masm.store64(value.i64(), dst);
+ break;
+ }
+ case FieldType::F32: {
+ masm.storeFloat32(value.f32(), dst);
+ break;
+ }
+ case FieldType::F64: {
+ masm.storeDouble(value.f64(), dst);
+ break;
+ }
+# ifdef ENABLE_WASM_SIMD
+ case FieldType::V128: {
+ masm.storeUnalignedSimd128(value.v128(), dst);
+ break;
+ }
+# endif
+ default: {
+ MOZ_CRASH("Unexpected field type");
+ }
+ }
+}
+
+template <typename NullCheckPolicy>
+bool BaseCompiler::emitGcStructSet(RegRef object, RegPtr areaBase,
+ uint32_t areaOffset, FieldType fieldType,
+ AnyReg value,
+ PreBarrierKind preBarrierKind) {
+ // Easy path if the field is a scalar
+ if (!fieldType.isRefRepr()) {
+ emitGcSetScalar<Address, NullCheckPolicy>(Address(areaBase, areaOffset),
+ fieldType, value);
+ freeAny(value);
+ return true;
+ }
+
+ // Create temporary for the valueAddr that is not in the prebarrier register
+ // and can be consumed by the barrier operation
+ RegPtr valueAddr = RegPtr(PreBarrierReg);
+ needPtr(valueAddr);
+ masm.computeEffectiveAddress(Address(areaBase, areaOffset), valueAddr);
+
+ NullCheckPolicy::emitNullCheck(this, object);
+
+ // emitBarrieredStore preserves object and value
+ if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind,
+ PostBarrierKind::Imprecise)) {
+ return false;
+ }
+ freeRef(value.ref());
+
+ return true;
+}
+
+bool BaseCompiler::emitGcArraySet(RegRef object, RegPtr data, RegI32 index,
+ const ArrayType& arrayType, AnyReg value,
+ PreBarrierKind preBarrierKind) {
+ // Try to use a base index store instruction if the field type fits in a
+ // shift immediate. If not we shift the index manually and then unshift
+ // it after the store. We don't use an extra register for this because we
+ // don't have any to spare on x86.
+ uint32_t shift = arrayType.elementType_.indexingShift();
+ Scale scale;
+ bool shiftedIndex = false;
+ if (IsShiftInScaleRange(shift)) {
+ scale = ShiftToScale(shift);
+ } else {
+ masm.lshiftPtr(Imm32(shift), index);
+ scale = TimesOne;
+ shiftedIndex = true;
+ }
+ auto unshiftIndex = mozilla::MakeScopeExit([&] {
+ if (shiftedIndex) {
+ masm.rshiftPtr(Imm32(shift), index);
+ }
+ });
+
+ // Easy path if the field is a scalar
+ if (!arrayType.elementType_.isRefRepr()) {
+ emitGcSetScalar<BaseIndex, NoNullCheck>(BaseIndex(data, index, scale, 0),
+ arrayType.elementType_, value);
+ return true;
+ }
+
+ // Create temporaries for valueAddr that is not in the prebarrier register
+ // and can be consumed by the barrier operation
+ RegPtr valueAddr = RegPtr(PreBarrierReg);
+ needPtr(valueAddr);
+ masm.computeEffectiveAddress(BaseIndex(data, index, scale, 0), valueAddr);
+
+ // Save state for after barriered write
+ pushPtr(data);
+ pushI32(index);
+
+ // emitBarrieredStore preserves object and value
+ if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind,
+ PostBarrierKind::Imprecise)) {
+ return false;
+ }
+
+ // Restore state
+ popI32(index);
+ popPtr(data);
+
+ return true;
+}
+
+bool BaseCompiler::emitStructNew() {
+ uint32_t typeIndex;
+ BaseNothingVector args{};
+ if (!iter_.readStructNew(&typeIndex, &args)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const StructType& structType = (*moduleEnv_.types)[typeIndex].structType();
+
+ // Allocate an uninitialized struct. This requires the type definition
+ // for the struct to be pushed on the stack. This will trap on OOM.
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ if (!emitInstanceCall(SASigStructNewUninit)) {
+ return false;
+ }
+
+ // Optimization opportunity: Iterate backward to pop arguments off the
+ // stack. This will generate more instructions than we want, since we
+ // really only need to pop the stack once at the end, not for every element,
+ // but to do better we need a bit more machinery to load elements off the
+ // stack into registers.
+
+ bool isOutlineStruct = structType.size_ > WasmStructObject_MaxInlineBytes;
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ needPtr(RegPtr(PreBarrierReg));
+
+ RegRef object = popRef();
+ RegPtr outlineBase = isOutlineStruct ? needPtr() : RegPtr();
+
+ // Free the barrier reg after we've allocated all registers
+ freePtr(RegPtr(PreBarrierReg));
+
+ // Optimization opportunity: when the value being stored is a known
+ // zero/null we need store nothing. This case may be somewhat common
+ // because struct.new forces a value to be specified for every field.
+
+ // Optimization opportunity: this loop reestablishes the outline base pointer
+ // every iteration, which really isn't very clever. It would be better to
+ // establish it once before we start, then re-set it if/when we transition
+ // from the out-of-line area back to the in-line area. That would however
+ // require making ::emitGcStructSet preserve that register, which it
+ // currently doesn't.
+
+ uint32_t fieldIndex = structType.fields_.length();
+ while (fieldIndex-- > 0) {
+ const StructField& field = structType.fields_[fieldIndex];
+ FieldType fieldType = field.type;
+ uint32_t fieldOffset = field.offset;
+
+ bool areaIsOutline;
+ uint32_t areaOffset;
+ WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
+ &areaIsOutline, &areaOffset);
+
+ // Reserve the barrier reg if we might need it for this store
+ if (fieldType.isRefRepr()) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+ AnyReg value = popAny();
+ // Free the barrier reg now that we've loaded the value
+ if (fieldType.isRefRepr()) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ if (areaIsOutline) {
+ // Load the outline data pointer
+ masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()),
+ outlineBase);
+
+ // Consumes value and outline data, object is preserved by this call.
+ if (!emitGcStructSet<NoNullCheck>(object, outlineBase, areaOffset,
+ fieldType, value,
+ PreBarrierKind::None)) {
+ return false;
+ }
+ } else {
+ // Consumes value. object is unchanged by this call.
+ if (!emitGcStructSet<NoNullCheck>(
+ object, RegPtr(object),
+ WasmStructObject::offsetOfInlineData() + areaOffset, fieldType,
+ value, PreBarrierKind::None)) {
+ return false;
+ }
+ }
+ }
+
+ if (isOutlineStruct) {
+ freePtr(outlineBase);
+ }
+ pushRef(object);
+
+ return true;
+}
+
+bool BaseCompiler::emitStructNewDefault() {
+ uint32_t typeIndex;
+ if (!iter_.readStructNewDefault(&typeIndex)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+
+ // Allocate a default initialized struct. This requires the type definition
+ // for the struct to be pushed on the stack. This will trap on OOM.
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ return emitInstanceCall(SASigStructNew);
+}
+
+bool BaseCompiler::emitStructGet(FieldWideningOp wideningOp) {
+ uint32_t typeIndex;
+ uint32_t fieldIndex;
+ Nothing nothing;
+ if (!iter_.readStructGet(&typeIndex, &fieldIndex, wideningOp, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const StructType& structType = (*moduleEnv_.types)[typeIndex].structType();
+
+ // Decide whether we're accessing inline or outline, and at what offset
+ FieldType fieldType = structType.fields_[fieldIndex].type;
+ uint32_t fieldOffset = structType.fields_[fieldIndex].offset;
+
+ bool areaIsOutline;
+ uint32_t areaOffset;
+ WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
+ &areaIsOutline, &areaOffset);
+
+ RegRef object = popRef();
+ if (areaIsOutline) {
+ RegPtr outlineBase = needPtr();
+ SignalNullCheck::emitTrapSite(this);
+ masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()),
+ outlineBase);
+ // Load the value
+ emitGcGet<Address, NoNullCheck>(fieldType, wideningOp,
+ Address(outlineBase, areaOffset));
+ freePtr(outlineBase);
+ } else {
+ // Load the value
+ emitGcGet<Address, SignalNullCheck>(
+ fieldType, wideningOp,
+ Address(object, WasmStructObject::offsetOfInlineData() + areaOffset));
+ }
+ freeRef(object);
+
+ return true;
+}
+
+bool BaseCompiler::emitStructSet() {
+ uint32_t typeIndex;
+ uint32_t fieldIndex;
+ Nothing nothing;
+ if (!iter_.readStructSet(&typeIndex, &fieldIndex, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const StructType& structType = (*moduleEnv_.types)[typeIndex].structType();
+ const StructField& structField = structType.fields_[fieldIndex];
+
+ // Decide whether we're accessing inline or outline, and at what offset
+ FieldType fieldType = structType.fields_[fieldIndex].type;
+ uint32_t fieldOffset = structType.fields_[fieldIndex].offset;
+
+ bool areaIsOutline;
+ uint32_t areaOffset;
+ WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
+ &areaIsOutline, &areaOffset);
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ if (structField.type.isRefRepr()) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+
+ RegPtr outlineBase = areaIsOutline ? needPtr() : RegPtr();
+ AnyReg value = popAny();
+ RegRef object = popRef();
+
+ // Free the barrier reg after we've allocated all registers
+ if (structField.type.isRefRepr()) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ // Make outlineBase point at the first byte of the relevant area
+ if (areaIsOutline) {
+ SignalNullCheck::emitTrapSite(this);
+ masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()),
+ outlineBase);
+ if (!emitGcStructSet<NoNullCheck>(object, outlineBase, areaOffset,
+ fieldType, value,
+ PreBarrierKind::Normal)) {
+ return false;
+ }
+ } else {
+ // Consumes value. object is unchanged by this call.
+ if (!emitGcStructSet<SignalNullCheck>(
+ object, RegPtr(object),
+ WasmStructObject::offsetOfInlineData() + areaOffset, fieldType,
+ value, PreBarrierKind::Normal)) {
+ return false;
+ }
+ }
+
+ if (areaIsOutline) {
+ freePtr(outlineBase);
+ }
+ freeRef(object);
+
+ return true;
+}
+
+bool BaseCompiler::emitArrayNew() {
+ uint32_t typeIndex;
+ Nothing nothing;
+ if (!iter_.readArrayNew(&typeIndex, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
+
+ // Allocate an uninitialized array. This requires the type definition
+ // for the array to be pushed on the stack. This will trap on OOM.
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ if (!emitInstanceCall(SASigArrayNewUninit)) {
+ return false;
+ }
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ if (arrayType.elementType_.isRefRepr()) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+
+ RegRef rp = popRef();
+ AnyReg value = popAny();
+
+ // Acquire the data pointer from the object
+ RegPtr rdata = emitGcArrayGetData(rp);
+
+ // Acquire the number of elements
+ RegI32 numElements = emitGcArrayGetNumElements<NoNullCheck>(rp);
+
+ // Free the barrier reg after we've allocated all registers
+ if (arrayType.elementType_.isRefRepr()) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ // Perform an initialization loop using `numElements` as the loop variable,
+ // counting down to zero.
+ Label done;
+ Label loop;
+ // Skip initialization if numElements = 0
+ masm.branch32(Assembler::Equal, numElements, Imm32(0), &done);
+ masm.bind(&loop);
+
+ // Move to the next element
+ masm.sub32(Imm32(1), numElements);
+
+ // Assign value to array[numElements]. All registers are preserved
+ if (!emitGcArraySet(rp, rdata, numElements, arrayType, value,
+ PreBarrierKind::None)) {
+ return false;
+ }
+
+ // Loop back if there are still elements to initialize
+ masm.branch32(Assembler::GreaterThan, numElements, Imm32(0), &loop);
+ masm.bind(&done);
+
+ freeI32(numElements);
+ freeAny(value);
+ freePtr(rdata);
+ pushRef(rp);
+
+ return true;
+}
+
+bool BaseCompiler::emitArrayNewFixed() {
+ uint32_t typeIndex, numElements;
+ BaseNothingVector nothings{};
+ if (!iter_.readArrayNewFixed(&typeIndex, &numElements, &nothings)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
+
+ // At this point, the top section of the value stack contains the values to
+ // be used to initialise the array, with index 0 as the topmost value. Push
+ // the required number of elements and the required type on, since the call
+ // to SASigArrayNew will use them.
+ pushI32(numElements);
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ if (!emitInstanceCall(SASigArrayNew)) {
+ return false;
+ }
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ bool avoidPreBarrierReg = arrayType.elementType_.isRefRepr();
+ if (avoidPreBarrierReg) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+
+ // Get hold of the pointer to the array, as created by SASigArrayNew.
+ RegRef rp = popRef();
+
+ // Acquire the data pointer from the object
+ RegPtr rdata = emitGcArrayGetData(rp);
+
+ // Free the barrier reg if we previously reserved it.
+ if (avoidPreBarrierReg) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ // These together ensure that the max value of `index` in the loop below
+ // remains comfortably below the 2^31 boundary. See comments on equivalent
+ // assertions in EmitArrayNewFixed in WasmIonCompile.cpp for explanation.
+ static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <=
+ MaxArrayPayloadBytes);
+ MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes);
+
+ // Generate straight-line initialization code. We could do better here if
+ // there was a version of ::emitGcArraySet that took `index` as a `uint32_t`
+ // rather than a general value-in-a-reg.
+ for (uint32_t forwardIndex = 0; forwardIndex < numElements; forwardIndex++) {
+ uint32_t reverseIndex = numElements - forwardIndex - 1;
+ if (avoidPreBarrierReg) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+ AnyReg value = popAny();
+ pushI32(reverseIndex);
+ RegI32 index = popI32();
+ if (avoidPreBarrierReg) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+ if (!emitGcArraySet(rp, rdata, index, arrayType, value,
+ PreBarrierKind::None)) {
+ return false;
+ }
+ freeI32(index);
+ freeAny(value);
+ }
+
+ freePtr(rdata);
+
+ pushRef(rp);
+ return true;
+}
+
+bool BaseCompiler::emitArrayNewDefault() {
+ uint32_t typeIndex;
+ Nothing nothing;
+ if (!iter_.readArrayNewDefault(&typeIndex, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // Allocate a default initialized array. This requires the type definition
+ // for the array to be pushed on the stack. This will trap on OOM.
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ return emitInstanceCall(SASigArrayNew);
+}
+
+bool BaseCompiler::emitArrayNewData() {
+ uint32_t typeIndex, segIndex;
+ Nothing nothing;
+ if (!iter_.readArrayNewData(&typeIndex, &segIndex, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ pushI32(int32_t(segIndex));
+
+ // The call removes 4 items from the stack: the segment byte offset and
+ // number of elements (operands to array.new_data), and the type index and
+ // seg index as pushed above.
+ return emitInstanceCall(SASigArrayNewData);
+}
+
+bool BaseCompiler::emitArrayNewElem() {
+ uint32_t typeIndex, segIndex;
+ Nothing nothing;
+ if (!iter_.readArrayNewElem(&typeIndex, &segIndex, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ pushPtr(loadTypeDefInstanceData(typeIndex));
+ pushI32(int32_t(segIndex));
+
+ // The call removes 4 items from the stack: the segment element offset and
+ // number of elements (operands to array.new_elem), and the type index and
+ // seg index as pushed above.
+ return emitInstanceCall(SASigArrayNewElem);
+}
+
+bool BaseCompiler::emitArrayGet(FieldWideningOp wideningOp) {
+ uint32_t typeIndex;
+ Nothing nothing;
+ if (!iter_.readArrayGet(&typeIndex, wideningOp, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
+
+ RegI32 index = popI32();
+ RegRef rp = popRef();
+
+ // Acquire the number of elements
+ RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp);
+
+ // Bounds check the index
+ emitGcArrayBoundsCheck(index, numElements);
+ freeI32(numElements);
+
+ // Acquire the data pointer from the object
+ RegPtr rdata = emitGcArrayGetData(rp);
+
+ // Load the value
+ uint32_t shift = arrayType.elementType_.indexingShift();
+ if (IsShiftInScaleRange(shift)) {
+ emitGcGet<BaseIndex, NoNullCheck>(
+ arrayType.elementType_, wideningOp,
+ BaseIndex(rdata, index, ShiftToScale(shift), 0));
+ } else {
+ masm.lshiftPtr(Imm32(shift), index);
+ emitGcGet<BaseIndex, NoNullCheck>(arrayType.elementType_, wideningOp,
+ BaseIndex(rdata, index, TimesOne, 0));
+ }
+
+ freePtr(rdata);
+ freeRef(rp);
+ freeI32(index);
+
+ return true;
+}
+
+bool BaseCompiler::emitArraySet() {
+ uint32_t typeIndex;
+ Nothing nothing;
+ if (!iter_.readArraySet(&typeIndex, &nothing, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ if (arrayType.elementType_.isRefRepr()) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+
+ AnyReg value = popAny();
+ RegI32 index = popI32();
+ RegRef rp = popRef();
+
+ // Acquire the number of elements
+ RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp);
+
+ // Bounds check the index
+ emitGcArrayBoundsCheck(index, numElements);
+ freeI32(numElements);
+
+ // Acquire the data pointer from the object
+ RegPtr rdata = emitGcArrayGetData(rp);
+
+ // Free the barrier reg after we've allocated all registers
+ if (arrayType.elementType_.isRefRepr()) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ // All registers are preserved. This isn't strictly necessary, as we'll just
+ // be freeing them all after this is done. But this is needed for repeated
+ // assignments used in array.new/new_default.
+ if (!emitGcArraySet(rp, rdata, index, arrayType, value,
+ PreBarrierKind::Normal)) {
+ return false;
+ }
+
+ freePtr(rdata);
+ freeRef(rp);
+ freeI32(index);
+ freeAny(value);
+
+ return true;
+}
+
+bool BaseCompiler::emitArrayLen(bool decodeIgnoredTypeIndex) {
+ Nothing nothing;
+ if (!iter_.readArrayLen(decodeIgnoredTypeIndex, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RegRef rp = popRef();
+
+ // Acquire the number of elements
+ RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp);
+ pushI32(numElements);
+
+ freeRef(rp);
+
+ return true;
+}
+
+bool BaseCompiler::emitArrayCopy() {
+ int32_t elemSize;
+ bool elemsAreRefTyped;
+ Nothing nothing;
+ if (!iter_.readArrayCopy(&elemSize, &elemsAreRefTyped, &nothing, &nothing,
+ &nothing, &nothing, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // readArrayCopy writes the element size in bytes to `elemSize`. Code
+ // generated here needs to hand it onwards to the instance fn, but also
+ // indicate whether the element type is reftyped, which it does by negating
+ // the size.
+ //
+ // The value stack currently looks like this:
+ //
+ // (top) [numElements, srcIndex, srcArray, dstIndex, dstArray, ..]
+ //
+ // So push `(possibly negated) elemSize` on it, giving
+ //
+ // (top) [elemSize, numElements, srcIndex, srcArray, dstIndex, dstArray, ..]
+ //
+ // and generate a call to the helper.
+
+ MOZ_ASSERT_IF(elemsAreRefTyped, elemSize == sizeof(void*));
+ MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 ||
+ elemSize == 4 || elemSize == 8 ||
+ elemSize == 16);
+ bool avoidPreBarrierReg = elemsAreRefTyped;
+
+ // Reserve this register early if we will need it so that it is not taken by
+ // any register used in this function.
+ if (avoidPreBarrierReg) {
+ needPtr(RegPtr(PreBarrierReg));
+ }
+
+ // The helper needs to know the element size.
+ pushI32(elemsAreRefTyped ? -elemSize : elemSize);
+
+ if (avoidPreBarrierReg) {
+ freePtr(RegPtr(PreBarrierReg));
+ }
+
+ return emitInstanceCall(SASigArrayCopy);
+}
+
+void BaseCompiler::emitRefTestCommon(RefType sourceType, RefType destType) {
+ Label success;
+ Label join;
+ RegRef object = popRef();
+ RegI32 result = needI32();
+
+ branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true);
+ masm.xor32(result, result);
+ masm.jump(&join);
+ masm.bind(&success);
+ masm.move32(Imm32(1), result);
+ masm.bind(&join);
+
+ pushI32(result);
+ freeRef(object);
+}
+
+void BaseCompiler::emitRefCastCommon(RefType sourceType, RefType destType) {
+ RegRef object = popRef();
+
+ Label success;
+ branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true);
+ masm.wasmTrap(Trap::BadCast, bytecodeOffset());
+ masm.bind(&success);
+ pushRef(object);
+}
+
+bool BaseCompiler::emitRefTestV5() {
+ Nothing nothing;
+ RefType sourceType;
+ uint32_t typeIndex;
+ if (!iter_.readRefTestV5(&sourceType, &typeIndex, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const TypeDef& typeDef = moduleEnv_.types->type(typeIndex);
+ RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/false);
+ emitRefTestCommon(sourceType, destType);
+
+ return true;
+}
+
+void BaseCompiler::branchGcRefType(RegRef object, RefType sourceType,
+ RefType destType, Label* label,
+ bool onSuccess) {
+ RegPtr superSuperTypeVector;
+ if (MacroAssembler::needSuperSuperTypeVectorForBranchWasmGcRefType(
+ destType)) {
+ uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef());
+ superSuperTypeVector = loadSuperTypeVector(typeIndex);
+ }
+ RegI32 scratch1 = MacroAssembler::needScratch1ForBranchWasmGcRefType(destType)
+ ? needI32()
+ : RegI32::Invalid();
+ RegI32 scratch2 = MacroAssembler::needScratch2ForBranchWasmGcRefType(destType)
+ ? needI32()
+ : RegI32::Invalid();
+
+ masm.branchWasmGcObjectIsRefType(object, sourceType, destType, label,
+ onSuccess, superSuperTypeVector, scratch1,
+ scratch2);
+
+ if (scratch2.isValid()) {
+ freeI32(scratch2);
+ }
+ if (scratch1.isValid()) {
+ freeI32(scratch1);
+ }
+ if (superSuperTypeVector.isValid()) {
+ freePtr(superSuperTypeVector);
+ }
+}
+
+bool BaseCompiler::emitRefCastV5() {
+ Nothing nothing;
+ RefType sourceType;
+ uint32_t typeIndex;
+ if (!iter_.readRefCastV5(&sourceType, &typeIndex, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const TypeDef& typeDef = moduleEnv_.types->type(typeIndex);
+ RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/true);
+ emitRefCastCommon(sourceType, destType);
+
+ return true;
+}
+
+bool BaseCompiler::emitRefTest(bool nullable) {
+ Nothing nothing;
+ RefType sourceType;
+ RefType destType;
+ if (!iter_.readRefTest(nullable, &sourceType, &destType, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ emitRefTestCommon(sourceType, destType);
+
+ return true;
+}
+
+bool BaseCompiler::emitRefCast(bool nullable) {
+ Nothing nothing;
+ RefType sourceType;
+ RefType destType;
+ if (!iter_.readRefCast(nullable, &sourceType, &destType, &nothing)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ emitRefCastCommon(sourceType, destType);
+
+ return true;
+}
+
+bool BaseCompiler::emitBrOnCastCommon(bool onSuccess,
+ uint32_t labelRelativeDepth,
+ const ResultType& labelType,
+ RefType sourceType, RefType destType) {
+ Control& target = controlItem(labelRelativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ // 3. br_if $l : [T*, ref] -> [T*, ref]
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false),
+ labelType);
+
+ // Don't allocate the result register used in the branch
+ if (b.hasBlockResults()) {
+ needIntegerResultRegisters(b.resultType);
+ }
+
+ // Create a copy of the ref for passing to the br_on_cast label,
+ // the original ref is used for casting in the condition.
+ RegRef object = popRef();
+ RegRef objectCondition = needRef();
+ moveRef(object, objectCondition);
+ pushRef(object);
+
+ if (b.hasBlockResults()) {
+ freeIntegerResultRegisters(b.resultType);
+ }
+
+ if (!jumpConditionalWithResults(&b, objectCondition, sourceType, destType,
+ onSuccess)) {
+ return false;
+ }
+ freeRef(objectCondition);
+
+ return true;
+}
+
+bool BaseCompiler::emitBrOnCast() {
+ MOZ_ASSERT(!hasLatentOp());
+
+ bool onSuccess;
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ RefType destType;
+ ResultType labelType;
+ BaseNothingVector unused_values{};
+ if (!iter_.readBrOnCast(&onSuccess, &labelRelativeDepth, &sourceType,
+ &destType, &labelType, &unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType,
+ sourceType, destType);
+}
+
+bool BaseCompiler::emitBrOnCastV5(bool onSuccess) {
+ MOZ_ASSERT(!hasLatentOp());
+
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ uint32_t castTypeIndex;
+ ResultType labelType;
+ BaseNothingVector unused_values{};
+ if (onSuccess
+ ? !iter_.readBrOnCastV5(&labelRelativeDepth, &sourceType,
+ &castTypeIndex, &labelType, &unused_values)
+ : !iter_.readBrOnCastFailV5(&labelRelativeDepth, &sourceType,
+ &castTypeIndex, &labelType,
+ &unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ const TypeDef& typeDef = moduleEnv_.types->type(castTypeIndex);
+ RefType destType = RefType::fromTypeDef(&typeDef, false);
+ return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType,
+ sourceType, destType);
+}
+
+bool BaseCompiler::emitBrOnCastHeapV5(bool onSuccess, bool nullable) {
+ MOZ_ASSERT(!hasLatentOp());
+
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ RefType destType;
+ ResultType labelType;
+ BaseNothingVector unused_values{};
+ if (onSuccess ? !iter_.readBrOnCastHeapV5(nullable, &labelRelativeDepth,
+ &sourceType, &destType, &labelType,
+ &unused_values)
+ : !iter_.readBrOnCastFailHeapV5(nullable, &labelRelativeDepth,
+ &sourceType, &destType,
+ &labelType, &unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType,
+ sourceType, destType);
+}
+
+bool BaseCompiler::emitRefAsStructV5() {
+ Nothing nothing;
+ return iter_.readConversion(ValType(RefType::any()),
+ ValType(RefType::struct_().asNonNullable()),
+ &nothing);
+}
+
+bool BaseCompiler::emitBrOnNonStructV5() {
+ MOZ_ASSERT(!hasLatentOp());
+
+ uint32_t labelRelativeDepth;
+ ResultType labelType;
+ BaseNothingVector unused_values{};
+ if (!iter_.readBrOnNonStructV5(&labelRelativeDepth, &labelType,
+ &unused_values)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ Control& target = controlItem(labelRelativeDepth);
+ target.bceSafeOnExit &= bceSafe_;
+
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false),
+ labelType);
+ if (b.hasBlockResults()) {
+ needResultRegisters(b.resultType);
+ }
+ RegI32 condition = needI32();
+ masm.move32(Imm32(1), condition);
+ if (b.hasBlockResults()) {
+ freeResultRegisters(b.resultType);
+ }
+ if (!jumpConditionalWithResults(&b, Assembler::Equal, condition, Imm32(0))) {
+ return false;
+ }
+ freeI32(condition);
+ return true;
+}
+
+bool BaseCompiler::emitExternInternalize() {
+ // extern.internalize is a no-op because anyref and extern share the same
+ // representation
+ Nothing nothing;
+ return iter_.readRefConversion(RefType::extern_(), RefType::any(), &nothing);
+}
+
+bool BaseCompiler::emitExternExternalize() {
+ // extern.externalize is a no-op because anyref and extern share the same
+ // representation
+ Nothing nothing;
+ return iter_.readRefConversion(RefType::any(), RefType::extern_(), &nothing);
+}
+
+#endif // ENABLE_WASM_GC
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// SIMD and Relaxed SIMD.
+
+#ifdef ENABLE_WASM_SIMD
+
+// Emitter trampolines used by abstracted SIMD operations. Naming here follows
+// the SIMD spec pretty closely.
+
+static void AndV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.bitwiseAndSimd128(rs, rsd);
+}
+
+static void OrV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.bitwiseOrSimd128(rs, rsd);
+}
+
+static void XorV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.bitwiseXorSimd128(rs, rsd);
+}
+
+static void AddI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addInt8x16(rsd, rs, rsd);
+}
+
+static void AddI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addInt16x8(rsd, rs, rsd);
+}
+
+static void AddI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addInt32x4(rsd, rs, rsd);
+}
+
+static void AddF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addFloat32x4(rsd, rs, rsd);
+}
+
+static void AddI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addInt64x2(rsd, rs, rsd);
+}
+
+static void AddF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addFloat64x2(rsd, rs, rsd);
+}
+
+static void AddSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addSatInt8x16(rsd, rs, rsd);
+}
+
+static void AddSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedAddSatInt8x16(rsd, rs, rsd);
+}
+
+static void AddSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.addSatInt16x8(rsd, rs, rsd);
+}
+
+static void AddSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedAddSatInt16x8(rsd, rs, rsd);
+}
+
+static void SubI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subInt8x16(rsd, rs, rsd);
+}
+
+static void SubI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subInt16x8(rsd, rs, rsd);
+}
+
+static void SubI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subInt32x4(rsd, rs, rsd);
+}
+
+static void SubF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subFloat32x4(rsd, rs, rsd);
+}
+
+static void SubI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subInt64x2(rsd, rs, rsd);
+}
+
+static void SubF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subFloat64x2(rsd, rs, rsd);
+}
+
+static void SubSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subSatInt8x16(rsd, rs, rsd);
+}
+
+static void SubSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedSubSatInt8x16(rsd, rs, rsd);
+}
+
+static void SubSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.subSatInt16x8(rsd, rs, rsd);
+}
+
+static void SubSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedSubSatInt16x8(rsd, rs, rsd);
+}
+
+static void MulI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.mulInt16x8(rsd, rs, rsd);
+}
+
+static void MulI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.mulInt32x4(rsd, rs, rsd);
+}
+
+static void MulF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.mulFloat32x4(rsd, rs, rsd);
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp) {
+ masm.mulInt64x2(rsd, rs, rsd, temp);
+}
+# elif defined(JS_CODEGEN_ARM64)
+static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp1, RegV128 temp2) {
+ masm.mulInt64x2(rsd, rs, rsd, temp1, temp2);
+}
+# endif
+
+static void MulF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.mulFloat64x2(rsd, rs, rsd);
+}
+
+static void DivF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.divFloat32x4(rsd, rs, rsd);
+}
+
+static void DivF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.divFloat64x2(rsd, rs, rsd);
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp1, RegV128 temp2) {
+ masm.minFloat32x4(rsd, rs, rsd, temp1, temp2);
+}
+
+static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp1, RegV128 temp2) {
+ masm.minFloat64x2(rsd, rs, rsd, temp1, temp2);
+}
+
+static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp1, RegV128 temp2) {
+ masm.maxFloat32x4(rsd, rs, rsd, temp1, temp2);
+}
+
+static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd,
+ RegV128 temp1, RegV128 temp2) {
+ masm.maxFloat64x2(rsd, rs, rsd, temp1, temp2);
+}
+
+static void PMinF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs,
+ RhsDestOp) {
+ masm.pseudoMinFloat32x4(rsd, rs);
+}
+
+static void PMinF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs,
+ RhsDestOp) {
+ masm.pseudoMinFloat64x2(rsd, rs);
+}
+
+static void PMaxF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs,
+ RhsDestOp) {
+ masm.pseudoMaxFloat32x4(rsd, rs);
+}
+
+static void PMaxF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs,
+ RhsDestOp) {
+ masm.pseudoMaxFloat64x2(rsd, rs);
+}
+# elif defined(JS_CODEGEN_ARM64)
+static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minFloat32x4(rs, rsd);
+}
+
+static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minFloat64x2(rs, rsd);
+}
+
+static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxFloat32x4(rs, rsd);
+}
+
+static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxFloat64x2(rs, rsd);
+}
+
+static void PMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.pseudoMinFloat32x4(rs, rsd);
+}
+
+static void PMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.pseudoMinFloat64x2(rs, rsd);
+}
+
+static void PMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.pseudoMaxFloat32x4(rs, rsd);
+}
+
+static void PMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.pseudoMaxFloat64x2(rs, rsd);
+}
+# endif
+
+static void DotI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.widenDotInt16x8(rsd, rs, rsd);
+}
+
+static void ExtMulLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulLowInt8x16(rsd, rs, rsd);
+}
+
+static void ExtMulHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulHighInt8x16(rsd, rs, rsd);
+}
+
+static void ExtMulLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulLowInt8x16(rsd, rs, rsd);
+}
+
+static void ExtMulHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulHighInt8x16(rsd, rs, rsd);
+}
+
+static void ExtMulLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulLowInt16x8(rsd, rs, rsd);
+}
+
+static void ExtMulHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulHighInt16x8(rsd, rs, rsd);
+}
+
+static void ExtMulLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulLowInt16x8(rsd, rs, rsd);
+}
+
+static void ExtMulHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulHighInt16x8(rsd, rs, rsd);
+}
+
+static void ExtMulLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulLowInt32x4(rsd, rs, rsd);
+}
+
+static void ExtMulHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extMulHighInt32x4(rsd, rs, rsd);
+}
+
+static void ExtMulLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulLowInt32x4(rsd, rs, rsd);
+}
+
+static void ExtMulHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedExtMulHighInt32x4(rsd, rs, rsd);
+}
+
+static void Q15MulrSatS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.q15MulrSatInt16x8(rsd, rs, rsd);
+}
+
+static void CmpI8x16(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt8x16(cond, rs, rsd);
+}
+
+static void CmpI16x8(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt16x8(cond, rs, rsd);
+}
+
+static void CmpI32x4(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt32x4(cond, rs, rsd);
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareForEqualityInt64x2(cond, rsd, rs, rsd);
+}
+
+static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd, RegV128 temp1,
+ RegV128 temp2) {
+ masm.compareForOrderingInt64x2(cond, rsd, rs, rsd, temp1, temp2);
+}
+# else
+static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt64x2(cond, rs, rsd);
+}
+
+static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt64x2(cond, rs, rsd);
+}
+# endif // JS_CODEGEN_X86 || JS_CODEGEN_X64
+
+static void CmpUI8x16(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt8x16(cond, rs, rsd);
+}
+
+static void CmpUI16x8(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt16x8(cond, rs, rsd);
+}
+
+static void CmpUI32x4(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareInt32x4(cond, rs, rsd);
+}
+
+static void CmpF32x4(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareFloat32x4(cond, rs, rsd);
+}
+
+static void CmpF64x2(MacroAssembler& masm, Assembler::Condition cond,
+ RegV128 rs, RegV128 rsd) {
+ masm.compareFloat64x2(cond, rs, rsd);
+}
+
+static void NegI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negInt8x16(rs, rd);
+}
+
+static void NegI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negInt16x8(rs, rd);
+}
+
+static void NegI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negInt32x4(rs, rd);
+}
+
+static void NegI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negInt64x2(rs, rd);
+}
+
+static void NegF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negFloat32x4(rs, rd);
+}
+
+static void NegF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.negFloat64x2(rs, rd);
+}
+
+static void AbsF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absFloat32x4(rs, rd);
+}
+
+static void AbsF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absFloat64x2(rs, rd);
+}
+
+static void SqrtF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.sqrtFloat32x4(rs, rd);
+}
+
+static void SqrtF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.sqrtFloat64x2(rs, rd);
+}
+
+static void CeilF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.ceilFloat32x4(rs, rd);
+}
+
+static void FloorF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.floorFloat32x4(rs, rd);
+}
+
+static void TruncF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.truncFloat32x4(rs, rd);
+}
+
+static void NearestF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.nearestFloat32x4(rs, rd);
+}
+
+static void CeilF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.ceilFloat64x2(rs, rd);
+}
+
+static void FloorF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.floorFloat64x2(rs, rd);
+}
+
+static void TruncF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.truncFloat64x2(rs, rd);
+}
+
+static void NearestF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.nearestFloat64x2(rs, rd);
+}
+
+static void NotV128(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.bitwiseNotSimd128(rs, rd);
+}
+
+static void ExtAddPairwiseI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extAddPairwiseInt8x16(rs, rsd);
+}
+
+static void ExtAddPairwiseUI8x16(MacroAssembler& masm, RegV128 rs,
+ RegV128 rsd) {
+ masm.unsignedExtAddPairwiseInt8x16(rs, rsd);
+}
+
+static void ExtAddPairwiseI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.extAddPairwiseInt16x8(rs, rsd);
+}
+
+static void ExtAddPairwiseUI16x8(MacroAssembler& masm, RegV128 rs,
+ RegV128 rsd) {
+ masm.unsignedExtAddPairwiseInt16x8(rs, rsd);
+}
+
+static void ShiftOpMask(MacroAssembler& masm, SimdOp op, RegI32 in,
+ RegI32 out) {
+ int32_t maskBits;
+
+ masm.mov(in, out);
+ if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) {
+ masm.and32(Imm32(maskBits), out);
+ }
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp1, RegV128 temp2) {
+ ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp1);
+ masm.leftShiftInt8x16(temp1, rsd, temp2);
+}
+
+static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp);
+ masm.leftShiftInt16x8(temp, rsd);
+}
+
+static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp);
+ masm.leftShiftInt32x4(temp, rsd);
+}
+
+static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp);
+ masm.leftShiftInt64x2(temp, rsd);
+}
+
+static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp1, RegV128 temp2) {
+ ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp1);
+ masm.rightShiftInt8x16(temp1, rsd, temp2);
+}
+
+static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp1, RegV128 temp2) {
+ ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp1);
+ masm.unsignedRightShiftInt8x16(temp1, rsd, temp2);
+}
+
+static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp);
+ masm.rightShiftInt16x8(temp, rsd);
+}
+
+static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp);
+ masm.unsignedRightShiftInt16x8(temp, rsd);
+}
+
+static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp);
+ masm.rightShiftInt32x4(temp, rsd);
+}
+
+static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp);
+ masm.unsignedRightShiftInt32x4(temp, rsd);
+}
+
+static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp);
+ masm.unsignedRightShiftInt64x2(temp, rsd);
+}
+# elif defined(JS_CODEGEN_ARM64)
+static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp);
+ masm.leftShiftInt8x16(rsd, temp, rsd);
+}
+
+static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp);
+ masm.leftShiftInt16x8(rsd, temp, rsd);
+}
+
+static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp);
+ masm.leftShiftInt32x4(rsd, temp, rsd);
+}
+
+static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp);
+ masm.leftShiftInt64x2(rsd, temp, rsd);
+}
+
+static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp);
+ masm.rightShiftInt8x16(rsd, temp, rsd);
+}
+
+static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp);
+ masm.unsignedRightShiftInt8x16(rsd, temp, rsd);
+}
+
+static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp);
+ masm.rightShiftInt16x8(rsd, temp, rsd);
+}
+
+static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp);
+ masm.unsignedRightShiftInt16x8(rsd, temp, rsd);
+}
+
+static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp);
+ masm.rightShiftInt32x4(rsd, temp, rsd);
+}
+
+static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp);
+ masm.unsignedRightShiftInt32x4(rsd, temp, rsd);
+}
+
+static void ShiftRightI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I64x2ShrS, rs, temp);
+ masm.rightShiftInt64x2(rsd, temp, rsd);
+}
+
+static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd,
+ RegI32 temp) {
+ ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp);
+ masm.unsignedRightShiftInt64x2(rsd, temp, rsd);
+}
+# endif
+
+static void AverageUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedAverageInt8x16(rsd, rs, rsd);
+}
+
+static void AverageUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedAverageInt16x8(rsd, rs, rsd);
+}
+
+static void MinI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minInt8x16(rsd, rs, rsd);
+}
+
+static void MinUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMinInt8x16(rsd, rs, rsd);
+}
+
+static void MaxI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxInt8x16(rsd, rs, rsd);
+}
+
+static void MaxUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMaxInt8x16(rsd, rs, rsd);
+}
+
+static void MinI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minInt16x8(rsd, rs, rsd);
+}
+
+static void MinUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMinInt16x8(rsd, rs, rsd);
+}
+
+static void MaxI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxInt16x8(rsd, rs, rsd);
+}
+
+static void MaxUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMaxInt16x8(rsd, rs, rsd);
+}
+
+static void MinI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minInt32x4(rsd, rs, rsd);
+}
+
+static void MinUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMinInt32x4(rsd, rs, rsd);
+}
+
+static void MaxI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxInt32x4(rsd, rs, rsd);
+}
+
+static void MaxUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedMaxInt32x4(rsd, rs, rsd);
+}
+
+static void NarrowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.narrowInt16x8(rsd, rs, rsd);
+}
+
+static void NarrowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedNarrowInt16x8(rsd, rs, rsd);
+}
+
+static void NarrowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.narrowInt32x4(rsd, rs, rsd);
+}
+
+static void NarrowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.unsignedNarrowInt32x4(rsd, rs, rsd);
+}
+
+static void WidenLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenLowInt8x16(rs, rd);
+}
+
+static void WidenHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenHighInt8x16(rs, rd);
+}
+
+static void WidenLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenLowInt8x16(rs, rd);
+}
+
+static void WidenHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenHighInt8x16(rs, rd);
+}
+
+static void WidenLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenLowInt16x8(rs, rd);
+}
+
+static void WidenHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenHighInt16x8(rs, rd);
+}
+
+static void WidenLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenLowInt16x8(rs, rd);
+}
+
+static void WidenHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenHighInt16x8(rs, rd);
+}
+
+static void WidenLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenLowInt32x4(rs, rd);
+}
+
+static void WidenHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.widenHighInt32x4(rs, rd);
+}
+
+static void WidenLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenLowInt32x4(rs, rd);
+}
+
+static void WidenHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedWidenHighInt32x4(rs, rd);
+}
+
+# if defined(JS_CODEGEN_ARM64)
+static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.popcntInt8x16(rs, rd);
+}
+# else
+static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd,
+ RegV128 temp) {
+ masm.popcntInt8x16(rs, rd, temp);
+}
+# endif // JS_CODEGEN_ARM64
+
+static void AbsI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absInt8x16(rs, rd);
+}
+
+static void AbsI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absInt16x8(rs, rd);
+}
+
+static void AbsI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absInt32x4(rs, rd);
+}
+
+static void AbsI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.absInt64x2(rs, rd);
+}
+
+static void ExtractLaneI8x16(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI32 rd) {
+ masm.extractLaneInt8x16(laneIndex, rs, rd);
+}
+
+static void ExtractLaneUI8x16(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI32 rd) {
+ masm.unsignedExtractLaneInt8x16(laneIndex, rs, rd);
+}
+
+static void ExtractLaneI16x8(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI32 rd) {
+ masm.extractLaneInt16x8(laneIndex, rs, rd);
+}
+
+static void ExtractLaneUI16x8(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI32 rd) {
+ masm.unsignedExtractLaneInt16x8(laneIndex, rs, rd);
+}
+
+static void ExtractLaneI32x4(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI32 rd) {
+ masm.extractLaneInt32x4(laneIndex, rs, rd);
+}
+
+static void ExtractLaneI64x2(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegI64 rd) {
+ masm.extractLaneInt64x2(laneIndex, rs, rd);
+}
+
+static void ExtractLaneF32x4(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegF32 rd) {
+ masm.extractLaneFloat32x4(laneIndex, rs, rd);
+}
+
+static void ExtractLaneF64x2(MacroAssembler& masm, uint32_t laneIndex,
+ RegV128 rs, RegF64 rd) {
+ masm.extractLaneFloat64x2(laneIndex, rs, rd);
+}
+
+static void ReplaceLaneI8x16(MacroAssembler& masm, uint32_t laneIndex,
+ RegI32 rs, RegV128 rsd) {
+ masm.replaceLaneInt8x16(laneIndex, rs, rsd);
+}
+
+static void ReplaceLaneI16x8(MacroAssembler& masm, uint32_t laneIndex,
+ RegI32 rs, RegV128 rsd) {
+ masm.replaceLaneInt16x8(laneIndex, rs, rsd);
+}
+
+static void ReplaceLaneI32x4(MacroAssembler& masm, uint32_t laneIndex,
+ RegI32 rs, RegV128 rsd) {
+ masm.replaceLaneInt32x4(laneIndex, rs, rsd);
+}
+
+static void ReplaceLaneI64x2(MacroAssembler& masm, uint32_t laneIndex,
+ RegI64 rs, RegV128 rsd) {
+ masm.replaceLaneInt64x2(laneIndex, rs, rsd);
+}
+
+static void ReplaceLaneF32x4(MacroAssembler& masm, uint32_t laneIndex,
+ RegF32 rs, RegV128 rsd) {
+ masm.replaceLaneFloat32x4(laneIndex, rs, rsd);
+}
+
+static void ReplaceLaneF64x2(MacroAssembler& masm, uint32_t laneIndex,
+ RegF64 rs, RegV128 rsd) {
+ masm.replaceLaneFloat64x2(laneIndex, rs, rsd);
+}
+
+static void SplatI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rd) {
+ masm.splatX16(rs, rd);
+}
+
+static void SplatI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rd) {
+ masm.splatX8(rs, rd);
+}
+
+static void SplatI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rd) {
+ masm.splatX4(rs, rd);
+}
+
+static void SplatI64x2(MacroAssembler& masm, RegI64 rs, RegV128 rd) {
+ masm.splatX2(rs, rd);
+}
+
+static void SplatF32x4(MacroAssembler& masm, RegF32 rs, RegV128 rd) {
+ masm.splatX4(rs, rd);
+}
+
+static void SplatF64x2(MacroAssembler& masm, RegF64 rs, RegV128 rd) {
+ masm.splatX2(rs, rd);
+}
+
+// This is the same op independent of lanes: it tests for any nonzero bit.
+static void AnyTrue(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.anyTrueSimd128(rs, rd);
+}
+
+static void AllTrueI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.allTrueInt8x16(rs, rd);
+}
+
+static void AllTrueI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.allTrueInt16x8(rs, rd);
+}
+
+static void AllTrueI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.allTrueInt32x4(rs, rd);
+}
+
+static void AllTrueI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.allTrueInt64x2(rs, rd);
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.bitmaskInt8x16(rs, rd);
+}
+
+static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.bitmaskInt16x8(rs, rd);
+}
+
+static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.bitmaskInt32x4(rs, rd);
+}
+
+static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) {
+ masm.bitmaskInt64x2(rs, rd);
+}
+# elif defined(JS_CODEGEN_ARM64)
+static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd,
+ RegV128 temp) {
+ masm.bitmaskInt8x16(rs, rd, temp);
+}
+
+static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd,
+ RegV128 temp) {
+ masm.bitmaskInt16x8(rs, rd, temp);
+}
+
+static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd,
+ RegV128 temp) {
+ masm.bitmaskInt32x4(rs, rd, temp);
+}
+
+static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd,
+ RegV128 temp) {
+ masm.bitmaskInt64x2(rs, rd, temp);
+}
+# endif
+
+static void Swizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.swizzleInt8x16(rsd, rs, rsd);
+}
+
+static void ConvertI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.convertInt32x4ToFloat32x4(rs, rd);
+}
+
+static void ConvertUI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedConvertInt32x4ToFloat32x4(rs, rd);
+}
+
+static void ConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.truncSatFloat32x4ToInt32x4(rs, rd);
+}
+
+# if defined(JS_CODEGEN_ARM64)
+static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd);
+}
+# else
+static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd,
+ RegV128 temp) {
+ masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd, temp);
+}
+# endif // JS_CODEGEN_ARM64
+
+static void ConvertI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.convertInt32x4ToFloat64x2(rs, rd);
+}
+
+static void ConvertUI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.unsignedConvertInt32x4ToFloat64x2(rs, rd);
+}
+
+static void ConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd,
+ RegV128 temp) {
+ masm.truncSatFloat64x2ToInt32x4(rs, rd, temp);
+}
+
+static void ConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd,
+ RegV128 temp) {
+ masm.unsignedTruncSatFloat64x2ToInt32x4(rs, rd, temp);
+}
+
+static void DemoteF64x2ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.convertFloat64x2ToFloat32x4(rs, rd);
+}
+
+static void PromoteF32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) {
+ masm.convertFloat32x4ToFloat64x2(rs, rd);
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control,
+ RegV128 lhsDest, RegV128 temp) {
+ // Ideally, we would have temp=control, and we can probably get away with
+ // just doing that, but don't worry about it yet.
+ masm.bitwiseSelectSimd128(control, lhsDest, rhs, lhsDest, temp);
+}
+# elif defined(JS_CODEGEN_ARM64)
+static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control,
+ RegV128 lhsDest, RegV128 temp) {
+ // The masm interface is not great for the baseline compiler here, but it's
+ // optimal for Ion, so just work around it.
+ masm.moveSimd128(control, temp);
+ masm.bitwiseSelectSimd128(lhsDest, rhs, temp);
+ masm.moveSimd128(temp, lhsDest);
+}
+# endif
+
+# ifdef ENABLE_WASM_RELAXED_SIMD
+static void RelaxedFmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2,
+ RegV128 rsd) {
+ masm.fmaFloat32x4(rs1, rs2, rsd);
+}
+
+static void RelaxedFnmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2,
+ RegV128 rsd) {
+ masm.fnmaFloat32x4(rs1, rs2, rsd);
+}
+
+static void RelaxedFmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2,
+ RegV128 rsd) {
+ masm.fmaFloat64x2(rs1, rs2, rsd);
+}
+
+static void RelaxedFnmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2,
+ RegV128 rsd) {
+ masm.fnmaFloat64x2(rs1, rs2, rsd);
+}
+
+static void RelaxedSwizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.swizzleInt8x16Relaxed(rsd, rs, rsd);
+}
+
+static void RelaxedMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minFloat32x4Relaxed(rs, rsd);
+}
+
+static void RelaxedMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxFloat32x4Relaxed(rs, rsd);
+}
+
+static void RelaxedMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.minFloat64x2Relaxed(rs, rsd);
+}
+
+static void RelaxedMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.maxFloat64x2Relaxed(rs, rsd);
+}
+
+static void RelaxedConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs,
+ RegV128 rd) {
+ masm.truncFloat32x4ToInt32x4Relaxed(rs, rd);
+}
+
+static void RelaxedConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs,
+ RegV128 rd) {
+ masm.unsignedTruncFloat32x4ToInt32x4Relaxed(rs, rd);
+}
+
+static void RelaxedConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs,
+ RegV128 rd) {
+ masm.truncFloat64x2ToInt32x4Relaxed(rs, rd);
+}
+
+static void RelaxedConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs,
+ RegV128 rd) {
+ masm.unsignedTruncFloat64x2ToInt32x4Relaxed(rs, rd);
+}
+
+static void RelaxedQ15MulrS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.q15MulrInt16x8Relaxed(rsd, rs, rsd);
+}
+
+static void DotI8x16I7x16S(MacroAssembler& masm, RegV128 rs, RegV128 rsd) {
+ masm.dotInt8x16Int7x16(rsd, rs, rsd);
+}
+
+void BaseCompiler::emitDotI8x16I7x16AddS() {
+ RegV128 rsd = popV128();
+ RegV128 rs0, rs1;
+ pop2xV128(&rs0, &rs1);
+# if defined(JS_CODEGEN_ARM64)
+ RegV128 temp = needV128();
+ masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd, temp);
+ freeV128(temp);
+# else
+ masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd);
+# endif
+ freeV128(rs1);
+ freeV128(rs0);
+ pushV128(rsd);
+}
+# endif // ENABLE_WASM_RELAXED_SIMD
+
+void BaseCompiler::emitVectorAndNot() {
+ // We want x & ~y but the available operation is ~x & y, so reverse the
+ // operands.
+ RegV128 r, rs;
+ pop2xV128(&r, &rs);
+ masm.bitwiseNotAndSimd128(r, rs);
+ freeV128(r);
+ pushV128(rs);
+}
+
+bool BaseCompiler::emitLoadSplat(Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ loadSplat(&access);
+ return true;
+}
+
+bool BaseCompiler::emitLoadZero(Scalar::Type viewType) {
+ // LoadZero has the structure of LoadSplat, so reuse the reader.
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ loadZero(&access);
+ return true;
+}
+
+bool BaseCompiler::emitLoadExtend(Scalar::Type viewType) {
+ LinearMemoryAddress<Nothing> addr;
+ if (!iter_.readLoadExtend(&addr)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ MemoryAccessDesc access(Scalar::Int64, addr.align, addr.offset,
+ bytecodeOffset());
+ loadExtend(&access, viewType);
+ return true;
+}
+
+bool BaseCompiler::emitLoadLane(uint32_t laneSize) {
+ Nothing nothing;
+ LinearMemoryAddress<Nothing> addr;
+ uint32_t laneIndex;
+ if (!iter_.readLoadLane(laneSize, &addr, &laneIndex, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ Scalar::Type viewType;
+ switch (laneSize) {
+ case 1:
+ viewType = Scalar::Uint8;
+ break;
+ case 2:
+ viewType = Scalar::Uint16;
+ break;
+ case 4:
+ viewType = Scalar::Int32;
+ break;
+ case 8:
+ viewType = Scalar::Int64;
+ break;
+ default:
+ MOZ_CRASH("unsupported laneSize");
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ loadLane(&access, laneIndex);
+ return true;
+}
+
+bool BaseCompiler::emitStoreLane(uint32_t laneSize) {
+ Nothing nothing;
+ LinearMemoryAddress<Nothing> addr;
+ uint32_t laneIndex;
+ if (!iter_.readStoreLane(laneSize, &addr, &laneIndex, &nothing)) {
+ return false;
+ }
+ if (deadCode_) {
+ return true;
+ }
+ Scalar::Type viewType;
+ switch (laneSize) {
+ case 1:
+ viewType = Scalar::Uint8;
+ break;
+ case 2:
+ viewType = Scalar::Uint16;
+ break;
+ case 4:
+ viewType = Scalar::Int32;
+ break;
+ case 8:
+ viewType = Scalar::Int64;
+ break;
+ default:
+ MOZ_CRASH("unsupported laneSize");
+ }
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset());
+ storeLane(&access, laneIndex);
+ return true;
+}
+
+bool BaseCompiler::emitVectorShuffle() {
+ Nothing unused_a, unused_b;
+ V128 shuffleMask;
+
+ if (!iter_.readVectorShuffle(&unused_a, &unused_b, &shuffleMask)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RegV128 rd, rs;
+ pop2xV128(&rd, &rs);
+
+ masm.shuffleInt8x16(shuffleMask.bytes, rs, rd);
+
+ freeV128(rs);
+ pushV128(rd);
+
+ return true;
+}
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+bool BaseCompiler::emitVectorShiftRightI64x2() {
+ Nothing unused_a, unused_b;
+
+ if (!iter_.readVectorShift(&unused_a, &unused_b)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ RegI32 count = popI32RhsForShiftI64();
+ RegV128 lhsDest = popV128();
+ RegI64 tmp = needI64();
+ masm.and32(Imm32(63), count);
+ masm.extractLaneInt64x2(0, lhsDest, tmp);
+ masm.rshift64Arithmetic(count, tmp);
+ masm.replaceLaneInt64x2(0, tmp, lhsDest);
+ masm.extractLaneInt64x2(1, lhsDest, tmp);
+ masm.rshift64Arithmetic(count, tmp);
+ masm.replaceLaneInt64x2(1, tmp, lhsDest);
+ freeI64(tmp);
+ freeI32(count);
+ pushV128(lhsDest);
+
+ return true;
+}
+# endif
+#endif // ENABLE_WASM_SIMD
+
+#ifdef ENABLE_WASM_RELAXED_SIMD
+bool BaseCompiler::emitVectorLaneSelect() {
+ Nothing unused_a, unused_b, unused_c;
+
+ if (!iter_.readTernary(ValType::V128, &unused_a, &unused_b, &unused_c)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ RegV128 mask = popV128(RegV128(vmm0));
+ RegV128 rhsDest = popV128();
+ RegV128 lhs = popV128();
+ masm.laneSelectSimd128(mask, lhs, rhsDest, rhsDest);
+ freeV128(lhs);
+ freeV128(mask);
+ pushV128(rhsDest);
+# elif defined(JS_CODEGEN_ARM64)
+ RegV128 maskDest = popV128();
+ RegV128 rhs = popV128();
+ RegV128 lhs = popV128();
+ masm.laneSelectSimd128(maskDest, lhs, rhs, maskDest);
+ freeV128(lhs);
+ freeV128(rhs);
+ pushV128(maskDest);
+# endif
+
+ return true;
+}
+#endif // ENABLE_WASM_RELAXED_SIMD
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// "Intrinsics" - magically imported functions for internal use.
+
+bool BaseCompiler::emitIntrinsic() {
+ const Intrinsic* intrinsic;
+
+ BaseNothingVector params;
+ if (!iter_.readIntrinsic(&intrinsic, &params)) {
+ return false;
+ }
+
+ if (deadCode_) {
+ return true;
+ }
+
+ // The final parameter of an intrinsic is implicitly the heap base
+ pushHeapBase();
+
+ // Call the intrinsic
+ return emitInstanceCall(intrinsic->signature);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Function bodies - main opcode dispatch loop.
+
+bool BaseCompiler::emitBody() {
+ AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitBody");
+
+ MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isSome());
+
+ if (!iter_.startFunction(func_.index, locals_)) {
+ return false;
+ }
+
+ initControl(controlItem(), ResultType::Empty());
+
+ for (;;) {
+ Nothing unused_a, unused_b, unused_c;
+ (void)unused_a;
+ (void)unused_b;
+ (void)unused_c;
+
+#ifdef DEBUG
+ performRegisterLeakCheck();
+ assertStackInvariants();
+#endif
+
+#define dispatchBinary0(doEmit, type) \
+ iter_.readBinary(type, &unused_a, &unused_b) && \
+ (deadCode_ || (doEmit(), true))
+
+#define dispatchBinary1(arg1, type) \
+ iter_.readBinary(type, &unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(arg1), true))
+
+#define dispatchBinary2(arg1, arg2, type) \
+ iter_.readBinary(type, &unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(arg1, arg2), true))
+
+#define dispatchBinary3(arg1, arg2, arg3, type) \
+ iter_.readBinary(type, &unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(arg1, arg2, arg3), true))
+
+#define dispatchUnary0(doEmit, type) \
+ iter_.readUnary(type, &unused_a) && (deadCode_ || (doEmit(), true))
+
+#define dispatchUnary1(arg1, type) \
+ iter_.readUnary(type, &unused_a) && (deadCode_ || (emitUnop(arg1), true))
+
+#define dispatchUnary2(arg1, arg2, type) \
+ iter_.readUnary(type, &unused_a) && \
+ (deadCode_ || (emitUnop(arg1, arg2), true))
+
+#define dispatchTernary0(doEmit, type) \
+ iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \
+ (deadCode_ || (doEmit(), true))
+
+#define dispatchTernary1(arg1, type) \
+ iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \
+ (deadCode_ || (emitTernary(arg1), true))
+
+#define dispatchTernary2(arg1, type) \
+ iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \
+ (deadCode_ || (emitTernaryResultLast(arg1), true))
+
+#define dispatchComparison0(doEmit, operandType, compareOp) \
+ iter_.readComparison(operandType, &unused_a, &unused_b) && \
+ (deadCode_ || (doEmit(compareOp, operandType), true))
+
+#define dispatchConversion0(doEmit, inType, outType) \
+ iter_.readConversion(inType, outType, &unused_a) && \
+ (deadCode_ || (doEmit(), true))
+
+#define dispatchConversion1(arg1, inType, outType) \
+ iter_.readConversion(inType, outType, &unused_a) && \
+ (deadCode_ || (emitUnop(arg1), true))
+
+#define dispatchConversionOOM(doEmit, inType, outType) \
+ iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || doEmit())
+
+#define dispatchCalloutConversionOOM(doEmit, symbol, inType, outType) \
+ iter_.readConversion(inType, outType, &unused_a) && \
+ (deadCode_ || doEmit(symbol, inType, outType))
+
+#define dispatchIntDivCallout(doEmit, symbol, type) \
+ iter_.readBinary(type, &unused_a, &unused_b) && \
+ (deadCode_ || doEmit(symbol, type))
+
+#define dispatchVectorBinary(op) \
+ iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(op), true))
+
+#define dispatchVectorUnary(op) \
+ iter_.readUnary(ValType::V128, &unused_a) && \
+ (deadCode_ || (emitUnop(op), true))
+
+#define dispatchVectorComparison(op, compareOp) \
+ iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(compareOp, op), true))
+
+#define dispatchVectorVariableShift(op) \
+ iter_.readVectorShift(&unused_a, &unused_b) && \
+ (deadCode_ || (emitBinop(op), true))
+
+#define dispatchExtractLane(op, outType, laneLimit) \
+ iter_.readExtractLane(outType, laneLimit, &laneIndex, &unused_a) && \
+ (deadCode_ || (emitUnop(laneIndex, op), true))
+
+#define dispatchReplaceLane(op, inType, laneLimit) \
+ iter_.readReplaceLane(inType, laneLimit, &laneIndex, &unused_a, \
+ &unused_b) && \
+ (deadCode_ || (emitBinop(laneIndex, op), true))
+
+#define dispatchSplat(op, inType) \
+ iter_.readConversion(inType, ValType::V128, &unused_a) && \
+ (deadCode_ || (emitUnop(op), true))
+
+#define dispatchVectorReduction(op) \
+ iter_.readConversion(ValType::V128, ValType::I32, &unused_a) && \
+ (deadCode_ || (emitUnop(op), true))
+
+#ifdef DEBUG
+ // Check that the number of ref-typed entries in the operand stack matches
+ // reality.
+# define CHECK_POINTER_COUNT \
+ do { \
+ MOZ_ASSERT(countMemRefsOnStk() == stackMapGenerator_.memRefsOnStk); \
+ } while (0)
+#else
+# define CHECK_POINTER_COUNT \
+ do { \
+ } while (0)
+#endif
+
+#define CHECK(E) \
+ if (!(E)) return false
+#define NEXT() \
+ { \
+ CHECK_POINTER_COUNT; \
+ continue; \
+ }
+#define CHECK_NEXT(E) \
+ if (!(E)) return false; \
+ { \
+ CHECK_POINTER_COUNT; \
+ continue; \
+ }
+
+ // Opcodes that push more than MaxPushesPerOpcode (anything with multiple
+ // results) will perform additional reservation.
+ CHECK(stk_.reserve(stk_.length() + MaxPushesPerOpcode));
+
+ OpBytes op{};
+ CHECK(iter_.readOp(&op));
+
+ // When compilerEnv_.debugEnabled(), some operators get a breakpoint site.
+ if (compilerEnv_.debugEnabled() && op.shouldHaveBreakpoint()) {
+ if (previousBreakablePoint_ != masm.currentOffset()) {
+ // TODO sync only registers that can be clobbered by the exit
+ // prologue/epilogue or disable these registers for use in
+ // baseline compiler when compilerEnv_.debugEnabled() is set.
+ sync();
+
+ insertBreakablePoint(CallSiteDesc::Breakpoint);
+ if (!createStackMap("debug: per-insn breakpoint")) {
+ return false;
+ }
+ previousBreakablePoint_ = masm.currentOffset();
+ }
+ }
+
+ // Going below framePushedAtEntryToBody would imply that we've
+ // popped off the machine stack, part of the frame created by
+ // beginFunction().
+ MOZ_ASSERT(masm.framePushed() >=
+ stackMapGenerator_.framePushedAtEntryToBody.value());
+
+ // At this point we're definitely not generating code for a function call.
+ MOZ_ASSERT(
+ stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing());
+
+ switch (op.b0) {
+ case uint16_t(Op::End):
+ if (!emitEnd()) {
+ return false;
+ }
+ if (iter_.controlStackEmpty()) {
+ return true;
+ }
+ NEXT();
+
+ // Control opcodes
+ case uint16_t(Op::Nop):
+ CHECK_NEXT(iter_.readNop());
+ case uint16_t(Op::Drop):
+ CHECK_NEXT(emitDrop());
+ case uint16_t(Op::Block):
+ CHECK_NEXT(emitBlock());
+ case uint16_t(Op::Loop):
+ CHECK_NEXT(emitLoop());
+ case uint16_t(Op::If):
+ CHECK_NEXT(emitIf());
+ case uint16_t(Op::Else):
+ CHECK_NEXT(emitElse());
+ case uint16_t(Op::Try):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitTry());
+ case uint16_t(Op::Catch):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitCatch());
+ case uint16_t(Op::CatchAll):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitCatchAll());
+ case uint16_t(Op::Delegate):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK(emitDelegate());
+ iter_.popDelegate();
+ NEXT();
+ case uint16_t(Op::Throw):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitThrow());
+ case uint16_t(Op::Rethrow):
+ if (!moduleEnv_.exceptionsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitRethrow());
+ case uint16_t(Op::Br):
+ CHECK_NEXT(emitBr());
+ case uint16_t(Op::BrIf):
+ CHECK_NEXT(emitBrIf());
+ case uint16_t(Op::BrTable):
+ CHECK_NEXT(emitBrTable());
+ case uint16_t(Op::Return):
+ CHECK_NEXT(emitReturn());
+ case uint16_t(Op::Unreachable):
+ CHECK(iter_.readUnreachable());
+ if (!deadCode_) {
+ trap(Trap::Unreachable);
+ deadCode_ = true;
+ }
+ NEXT();
+
+ // Calls
+ case uint16_t(Op::Call):
+ CHECK_NEXT(emitCall());
+ case uint16_t(Op::CallIndirect):
+ CHECK_NEXT(emitCallIndirect());
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+ case uint16_t(Op::CallRef):
+ if (!moduleEnv_.functionReferencesEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitCallRef());
+#endif
+
+ // Locals and globals
+ case uint16_t(Op::LocalGet):
+ CHECK_NEXT(emitGetLocal());
+ case uint16_t(Op::LocalSet):
+ CHECK_NEXT(emitSetLocal());
+ case uint16_t(Op::LocalTee):
+ CHECK_NEXT(emitTeeLocal());
+ case uint16_t(Op::GlobalGet):
+ CHECK_NEXT(emitGetGlobal());
+ case uint16_t(Op::GlobalSet):
+ CHECK_NEXT(emitSetGlobal());
+ case uint16_t(Op::TableGet):
+ CHECK_NEXT(emitTableGet());
+ case uint16_t(Op::TableSet):
+ CHECK_NEXT(emitTableSet());
+
+ // Select
+ case uint16_t(Op::SelectNumeric):
+ CHECK_NEXT(emitSelect(/*typed*/ false));
+ case uint16_t(Op::SelectTyped):
+ CHECK_NEXT(emitSelect(/*typed*/ true));
+
+ // I32
+ case uint16_t(Op::I32Const): {
+ int32_t i32;
+ CHECK(iter_.readI32Const(&i32));
+ if (!deadCode_) {
+ pushI32(i32);
+ }
+ NEXT();
+ }
+ case uint16_t(Op::I32Add):
+ CHECK_NEXT(dispatchBinary2(AddI32, AddImmI32, ValType::I32));
+ case uint16_t(Op::I32Sub):
+ CHECK_NEXT(dispatchBinary2(SubI32, SubImmI32, ValType::I32));
+ case uint16_t(Op::I32Mul):
+ CHECK_NEXT(dispatchBinary1(MulI32, ValType::I32));
+ case uint16_t(Op::I32DivS):
+ CHECK_NEXT(dispatchBinary0(emitQuotientI32, ValType::I32));
+ case uint16_t(Op::I32DivU):
+ CHECK_NEXT(dispatchBinary0(emitQuotientU32, ValType::I32));
+ case uint16_t(Op::I32RemS):
+ CHECK_NEXT(dispatchBinary0(emitRemainderI32, ValType::I32));
+ case uint16_t(Op::I32RemU):
+ CHECK_NEXT(dispatchBinary0(emitRemainderU32, ValType::I32));
+ case uint16_t(Op::I32Eqz):
+ CHECK_NEXT(dispatchConversion0(emitEqzI32, ValType::I32, ValType::I32));
+ case uint16_t(Op::I32TruncF32S):
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32<0>, ValType::F32,
+ ValType::I32));
+ case uint16_t(Op::I32TruncF32U):
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32<TRUNC_UNSIGNED>,
+ ValType::F32, ValType::I32));
+ case uint16_t(Op::I32TruncF64S):
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32<0>, ValType::F64,
+ ValType::I32));
+ case uint16_t(Op::I32TruncF64U):
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32<TRUNC_UNSIGNED>,
+ ValType::F64, ValType::I32));
+ case uint16_t(Op::I32WrapI64):
+ CHECK_NEXT(
+ dispatchConversion1(WrapI64ToI32, ValType::I64, ValType::I32));
+ case uint16_t(Op::I32ReinterpretF32):
+ CHECK_NEXT(dispatchConversion1(ReinterpretF32AsI32, ValType::F32,
+ ValType::I32));
+ case uint16_t(Op::I32Clz):
+ CHECK_NEXT(dispatchUnary1(ClzI32, ValType::I32));
+ case uint16_t(Op::I32Ctz):
+ CHECK_NEXT(dispatchUnary1(CtzI32, ValType::I32));
+ case uint16_t(Op::I32Popcnt):
+ CHECK_NEXT(dispatchUnary2(PopcntI32, PopcntTemp, ValType::I32));
+ case uint16_t(Op::I32Or):
+ CHECK_NEXT(dispatchBinary2(OrI32, OrImmI32, ValType::I32));
+ case uint16_t(Op::I32And):
+ CHECK_NEXT(dispatchBinary2(AndI32, AndImmI32, ValType::I32));
+ case uint16_t(Op::I32Xor):
+ CHECK_NEXT(dispatchBinary2(XorI32, XorImmI32, ValType::I32));
+ case uint16_t(Op::I32Shl):
+ CHECK_NEXT(dispatchBinary3(
+ ShlI32, ShlImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32));
+ case uint16_t(Op::I32ShrS):
+ CHECK_NEXT(dispatchBinary3(
+ ShrI32, ShrImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32));
+ case uint16_t(Op::I32ShrU):
+ CHECK_NEXT(dispatchBinary3(ShrUI32, ShrUImmI32,
+ &BaseCompiler::popI32RhsForShift,
+ ValType::I32));
+ case uint16_t(Op::I32Load8S):
+ CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int8));
+ case uint16_t(Op::I32Load8U):
+ CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint8));
+ case uint16_t(Op::I32Load16S):
+ CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int16));
+ case uint16_t(Op::I32Load16U):
+ CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint16));
+ case uint16_t(Op::I32Load):
+ CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int32));
+ case uint16_t(Op::I32Store8):
+ CHECK_NEXT(emitStore(ValType::I32, Scalar::Int8));
+ case uint16_t(Op::I32Store16):
+ CHECK_NEXT(emitStore(ValType::I32, Scalar::Int16));
+ case uint16_t(Op::I32Store):
+ CHECK_NEXT(emitStore(ValType::I32, Scalar::Int32));
+ case uint16_t(Op::I32Rotr):
+ CHECK_NEXT(dispatchBinary3(RotrI32, RotrImmI32,
+ &BaseCompiler::popI32RhsForRotate,
+ ValType::I32));
+ case uint16_t(Op::I32Rotl):
+ CHECK_NEXT(dispatchBinary3(RotlI32, RotlImmI32,
+ &BaseCompiler::popI32RhsForRotate,
+ ValType::I32));
+
+ // I64
+ case uint16_t(Op::I64Const): {
+ int64_t i64;
+ CHECK(iter_.readI64Const(&i64));
+ if (!deadCode_) {
+ pushI64(i64);
+ }
+ NEXT();
+ }
+ case uint16_t(Op::I64Add):
+ CHECK_NEXT(dispatchBinary2(AddI64, AddImmI64, ValType::I64));
+ case uint16_t(Op::I64Sub):
+ CHECK_NEXT(dispatchBinary2(SubI64, SubImmI64, ValType::I64));
+ case uint16_t(Op::I64Mul):
+ CHECK_NEXT(dispatchBinary0(emitMultiplyI64, ValType::I64));
+ case uint16_t(Op::I64DivS):
+#ifdef RABALDR_INT_DIV_I64_CALLOUT
+ CHECK_NEXT(dispatchIntDivCallout(
+ emitDivOrModI64BuiltinCall, SymbolicAddress::DivI64, ValType::I64));
+#else
+ CHECK_NEXT(dispatchBinary0(emitQuotientI64, ValType::I64));
+#endif
+ case uint16_t(Op::I64DivU):
+#ifdef RABALDR_INT_DIV_I64_CALLOUT
+ CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall,
+ SymbolicAddress::UDivI64,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchBinary0(emitQuotientU64, ValType::I64));
+#endif
+ case uint16_t(Op::I64RemS):
+#ifdef RABALDR_INT_DIV_I64_CALLOUT
+ CHECK_NEXT(dispatchIntDivCallout(
+ emitDivOrModI64BuiltinCall, SymbolicAddress::ModI64, ValType::I64));
+#else
+ CHECK_NEXT(dispatchBinary0(emitRemainderI64, ValType::I64));
+#endif
+ case uint16_t(Op::I64RemU):
+#ifdef RABALDR_INT_DIV_I64_CALLOUT
+ CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall,
+ SymbolicAddress::UModI64,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchBinary0(emitRemainderU64, ValType::I64));
+#endif
+ case uint16_t(Op::I64TruncF32S):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(
+ dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout,
+ SymbolicAddress::TruncateDoubleToInt64,
+ ValType::F32, ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64<0>, ValType::F32,
+ ValType::I64));
+#endif
+ case uint16_t(Op::I64TruncF32U):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::TruncateDoubleToUint64, ValType::F32,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64<TRUNC_UNSIGNED>,
+ ValType::F32, ValType::I64));
+#endif
+ case uint16_t(Op::I64TruncF64S):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(
+ dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout,
+ SymbolicAddress::TruncateDoubleToInt64,
+ ValType::F64, ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64<0>, ValType::F64,
+ ValType::I64));
+#endif
+ case uint16_t(Op::I64TruncF64U):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::TruncateDoubleToUint64, ValType::F64,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64<TRUNC_UNSIGNED>,
+ ValType::F64, ValType::I64));
+#endif
+ case uint16_t(Op::I64ExtendI32S):
+ CHECK_NEXT(dispatchConversion0(emitExtendI32ToI64, ValType::I32,
+ ValType::I64));
+ case uint16_t(Op::I64ExtendI32U):
+ CHECK_NEXT(dispatchConversion0(emitExtendU32ToI64, ValType::I32,
+ ValType::I64));
+ case uint16_t(Op::I64ReinterpretF64):
+ CHECK_NEXT(dispatchConversion1(ReinterpretF64AsI64, ValType::F64,
+ ValType::I64));
+ case uint16_t(Op::I64Or):
+ CHECK_NEXT(dispatchBinary2(OrI64, OrImmI64, ValType::I64));
+ case uint16_t(Op::I64And):
+ CHECK_NEXT(dispatchBinary2(AndI64, AndImmI64, ValType::I64));
+ case uint16_t(Op::I64Xor):
+ CHECK_NEXT(dispatchBinary2(XorI64, XorImmI64, ValType::I64));
+ case uint16_t(Op::I64Shl):
+ CHECK_NEXT(dispatchBinary3(
+ ShlI64, ShlImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64));
+ case uint16_t(Op::I64ShrS):
+ CHECK_NEXT(dispatchBinary3(
+ ShrI64, ShrImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64));
+ case uint16_t(Op::I64ShrU):
+ CHECK_NEXT(dispatchBinary3(ShrUI64, ShrUImmI64,
+ &BaseCompiler::popI64RhsForShift,
+ ValType::I64));
+ case uint16_t(Op::I64Rotr):
+ CHECK_NEXT(dispatchBinary0(emitRotrI64, ValType::I64));
+ case uint16_t(Op::I64Rotl):
+ CHECK_NEXT(dispatchBinary0(emitRotlI64, ValType::I64));
+ case uint16_t(Op::I64Clz):
+ CHECK_NEXT(dispatchUnary1(ClzI64, ValType::I64));
+ case uint16_t(Op::I64Ctz):
+ CHECK_NEXT(dispatchUnary1(CtzI64, ValType::I64));
+ case uint16_t(Op::I64Popcnt):
+ CHECK_NEXT(dispatchUnary2(PopcntI64, PopcntTemp, ValType::I64));
+ case uint16_t(Op::I64Eqz):
+ CHECK_NEXT(dispatchConversion0(emitEqzI64, ValType::I64, ValType::I32));
+ case uint16_t(Op::I64Load8S):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int8));
+ case uint16_t(Op::I64Load16S):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int16));
+ case uint16_t(Op::I64Load32S):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int32));
+ case uint16_t(Op::I64Load8U):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint8));
+ case uint16_t(Op::I64Load16U):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint16));
+ case uint16_t(Op::I64Load32U):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint32));
+ case uint16_t(Op::I64Load):
+ CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int64));
+ case uint16_t(Op::I64Store8):
+ CHECK_NEXT(emitStore(ValType::I64, Scalar::Int8));
+ case uint16_t(Op::I64Store16):
+ CHECK_NEXT(emitStore(ValType::I64, Scalar::Int16));
+ case uint16_t(Op::I64Store32):
+ CHECK_NEXT(emitStore(ValType::I64, Scalar::Int32));
+ case uint16_t(Op::I64Store):
+ CHECK_NEXT(emitStore(ValType::I64, Scalar::Int64));
+
+ // F32
+ case uint16_t(Op::F32Const): {
+ float f32;
+ CHECK(iter_.readF32Const(&f32));
+ if (!deadCode_) {
+ pushF32(f32);
+ }
+ NEXT();
+ }
+ case uint16_t(Op::F32Add):
+ CHECK_NEXT(dispatchBinary1(AddF32, ValType::F32))
+ case uint16_t(Op::F32Sub):
+ CHECK_NEXT(dispatchBinary1(SubF32, ValType::F32));
+ case uint16_t(Op::F32Mul):
+ CHECK_NEXT(dispatchBinary1(MulF32, ValType::F32));
+ case uint16_t(Op::F32Div):
+ CHECK_NEXT(dispatchBinary1(DivF32, ValType::F32));
+ case uint16_t(Op::F32Min):
+ CHECK_NEXT(dispatchBinary1(MinF32, ValType::F32));
+ case uint16_t(Op::F32Max):
+ CHECK_NEXT(dispatchBinary1(MaxF32, ValType::F32));
+ case uint16_t(Op::F32Neg):
+ CHECK_NEXT(dispatchUnary1(NegateF32, ValType::F32));
+ case uint16_t(Op::F32Abs):
+ CHECK_NEXT(dispatchUnary1(AbsF32, ValType::F32));
+ case uint16_t(Op::F32Sqrt):
+ CHECK_NEXT(dispatchUnary1(SqrtF32, ValType::F32));
+ case uint16_t(Op::F32Ceil):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::CeilF, ValType::F32));
+ case uint16_t(Op::F32Floor):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::FloorF, ValType::F32));
+ case uint16_t(Op::F32DemoteF64):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertF64ToF32, ValType::F64, ValType::F32));
+ case uint16_t(Op::F32ConvertI32S):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertI32ToF32, ValType::I32, ValType::F32));
+ case uint16_t(Op::F32ConvertI32U):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertU32ToF32, ValType::I32, ValType::F32));
+ case uint16_t(Op::F32ConvertI64S):
+#ifdef RABALDR_I64_TO_FLOAT_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToFloat32,
+ ValType::I64, ValType::F32));
+#else
+ CHECK_NEXT(
+ dispatchConversion1(ConvertI64ToF32, ValType::I64, ValType::F32));
+#endif
+ case uint16_t(Op::F32ConvertI64U):
+#ifdef RABALDR_I64_TO_FLOAT_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToFloat32,
+ ValType::I64, ValType::F32));
+#else
+ CHECK_NEXT(dispatchConversion0(emitConvertU64ToF32, ValType::I64,
+ ValType::F32));
+#endif
+ case uint16_t(Op::F32ReinterpretI32):
+ CHECK_NEXT(dispatchConversion1(ReinterpretI32AsF32, ValType::I32,
+ ValType::F32));
+ case uint16_t(Op::F32Load):
+ CHECK_NEXT(emitLoad(ValType::F32, Scalar::Float32));
+ case uint16_t(Op::F32Store):
+ CHECK_NEXT(emitStore(ValType::F32, Scalar::Float32));
+ case uint16_t(Op::F32CopySign):
+ CHECK_NEXT(dispatchBinary1(CopysignF32, ValType::F32));
+ case uint16_t(Op::F32Nearest):
+ CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntF,
+ ValType::F32));
+ case uint16_t(Op::F32Trunc):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::TruncF, ValType::F32));
+
+ // F64
+ case uint16_t(Op::F64Const): {
+ double f64;
+ CHECK(iter_.readF64Const(&f64));
+ if (!deadCode_) {
+ pushF64(f64);
+ }
+ NEXT();
+ }
+ case uint16_t(Op::F64Add):
+ CHECK_NEXT(dispatchBinary1(AddF64, ValType::F64))
+ case uint16_t(Op::F64Sub):
+ CHECK_NEXT(dispatchBinary1(SubF64, ValType::F64));
+ case uint16_t(Op::F64Mul):
+ CHECK_NEXT(dispatchBinary1(MulF64, ValType::F64));
+ case uint16_t(Op::F64Div):
+ CHECK_NEXT(dispatchBinary1(DivF64, ValType::F64));
+ case uint16_t(Op::F64Min):
+ CHECK_NEXT(dispatchBinary1(MinF64, ValType::F64));
+ case uint16_t(Op::F64Max):
+ CHECK_NEXT(dispatchBinary1(MaxF64, ValType::F64));
+ case uint16_t(Op::F64Neg):
+ CHECK_NEXT(dispatchUnary1(NegateF64, ValType::F64));
+ case uint16_t(Op::F64Abs):
+ CHECK_NEXT(dispatchUnary1(AbsF64, ValType::F64));
+ case uint16_t(Op::F64Sqrt):
+ CHECK_NEXT(dispatchUnary1(SqrtF64, ValType::F64));
+ case uint16_t(Op::F64Ceil):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::CeilD, ValType::F64));
+ case uint16_t(Op::F64Floor):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::FloorD, ValType::F64));
+ case uint16_t(Op::F64PromoteF32):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertF32ToF64, ValType::F32, ValType::F64));
+ case uint16_t(Op::F64ConvertI32S):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertI32ToF64, ValType::I32, ValType::F64));
+ case uint16_t(Op::F64ConvertI32U):
+ CHECK_NEXT(
+ dispatchConversion1(ConvertU32ToF64, ValType::I32, ValType::F64));
+ case uint16_t(Op::F64ConvertI64S):
+#ifdef RABALDR_I64_TO_FLOAT_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToDouble,
+ ValType::I64, ValType::F64));
+#else
+ CHECK_NEXT(
+ dispatchConversion1(ConvertI64ToF64, ValType::I64, ValType::F64));
+#endif
+ case uint16_t(Op::F64ConvertI64U):
+#ifdef RABALDR_I64_TO_FLOAT_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToDouble,
+ ValType::I64, ValType::F64));
+#else
+ CHECK_NEXT(dispatchConversion0(emitConvertU64ToF64, ValType::I64,
+ ValType::F64));
+#endif
+ case uint16_t(Op::F64Load):
+ CHECK_NEXT(emitLoad(ValType::F64, Scalar::Float64));
+ case uint16_t(Op::F64Store):
+ CHECK_NEXT(emitStore(ValType::F64, Scalar::Float64));
+ case uint16_t(Op::F64ReinterpretI64):
+ CHECK_NEXT(dispatchConversion1(ReinterpretI64AsF64, ValType::I64,
+ ValType::F64));
+ case uint16_t(Op::F64CopySign):
+ CHECK_NEXT(dispatchBinary1(CopysignF64, ValType::F64));
+ case uint16_t(Op::F64Nearest):
+ CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntD,
+ ValType::F64));
+ case uint16_t(Op::F64Trunc):
+ CHECK_NEXT(
+ emitUnaryMathBuiltinCall(SymbolicAddress::TruncD, ValType::F64));
+
+ // Comparisons
+ case uint16_t(Op::I32Eq):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::Equal));
+ case uint16_t(Op::I32Ne):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::NotEqual));
+ case uint16_t(Op::I32LtS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::LessThan));
+ case uint16_t(Op::I32LeS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::LessThanOrEqual));
+ case uint16_t(Op::I32GtS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::GreaterThan));
+ case uint16_t(Op::I32GeS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::GreaterThanOrEqual));
+ case uint16_t(Op::I32LtU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::Below));
+ case uint16_t(Op::I32LeU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::BelowOrEqual));
+ case uint16_t(Op::I32GtU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::Above));
+ case uint16_t(Op::I32GeU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32,
+ Assembler::AboveOrEqual));
+ case uint16_t(Op::I64Eq):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::Equal));
+ case uint16_t(Op::I64Ne):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::NotEqual));
+ case uint16_t(Op::I64LtS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::LessThan));
+ case uint16_t(Op::I64LeS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::LessThanOrEqual));
+ case uint16_t(Op::I64GtS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::GreaterThan));
+ case uint16_t(Op::I64GeS):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::GreaterThanOrEqual));
+ case uint16_t(Op::I64LtU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::Below));
+ case uint16_t(Op::I64LeU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::BelowOrEqual));
+ case uint16_t(Op::I64GtU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::Above));
+ case uint16_t(Op::I64GeU):
+ CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64,
+ Assembler::AboveOrEqual));
+ case uint16_t(Op::F32Eq):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleEqual));
+ case uint16_t(Op::F32Ne):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleNotEqualOrUnordered));
+ case uint16_t(Op::F32Lt):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleLessThan));
+ case uint16_t(Op::F32Le):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleLessThanOrEqual));
+ case uint16_t(Op::F32Gt):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleGreaterThan));
+ case uint16_t(Op::F32Ge):
+ CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32,
+ Assembler::DoubleGreaterThanOrEqual));
+ case uint16_t(Op::F64Eq):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleEqual));
+ case uint16_t(Op::F64Ne):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleNotEqualOrUnordered));
+ case uint16_t(Op::F64Lt):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleLessThan));
+ case uint16_t(Op::F64Le):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleLessThanOrEqual));
+ case uint16_t(Op::F64Gt):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleGreaterThan));
+ case uint16_t(Op::F64Ge):
+ CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64,
+ Assembler::DoubleGreaterThanOrEqual));
+
+ // Sign extensions
+ case uint16_t(Op::I32Extend8S):
+ CHECK_NEXT(
+ dispatchConversion1(ExtendI32_8, ValType::I32, ValType::I32));
+ case uint16_t(Op::I32Extend16S):
+ CHECK_NEXT(
+ dispatchConversion1(ExtendI32_16, ValType::I32, ValType::I32));
+ case uint16_t(Op::I64Extend8S):
+ CHECK_NEXT(
+ dispatchConversion0(emitExtendI64_8, ValType::I64, ValType::I64));
+ case uint16_t(Op::I64Extend16S):
+ CHECK_NEXT(
+ dispatchConversion0(emitExtendI64_16, ValType::I64, ValType::I64));
+ case uint16_t(Op::I64Extend32S):
+ CHECK_NEXT(
+ dispatchConversion0(emitExtendI64_32, ValType::I64, ValType::I64));
+
+ // Memory Related
+ case uint16_t(Op::MemoryGrow):
+ CHECK_NEXT(emitMemoryGrow());
+ case uint16_t(Op::MemorySize):
+ CHECK_NEXT(emitMemorySize());
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+ case uint16_t(Op::RefAsNonNull):
+ if (!moduleEnv_.functionReferencesEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitRefAsNonNull());
+ case uint16_t(Op::BrOnNull):
+ if (!moduleEnv_.functionReferencesEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitBrOnNull());
+ case uint16_t(Op::BrOnNonNull):
+ if (!moduleEnv_.functionReferencesEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitBrOnNonNull());
+#endif
+#ifdef ENABLE_WASM_GC
+ case uint16_t(Op::RefEq):
+ if (!moduleEnv_.gcEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchComparison0(emitCompareRef, RefType::eq(),
+ Assembler::Equal));
+#endif
+ case uint16_t(Op::RefFunc):
+ CHECK_NEXT(emitRefFunc());
+ break;
+ case uint16_t(Op::RefNull):
+ CHECK_NEXT(emitRefNull());
+ break;
+ case uint16_t(Op::RefIsNull):
+ CHECK_NEXT(emitRefIsNull());
+ break;
+
+#ifdef ENABLE_WASM_GC
+ // "GC" operations
+ case uint16_t(Op::GcPrefix): {
+ if (!moduleEnv_.gcEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(GcOp::StructNew):
+ CHECK_NEXT(emitStructNew());
+ case uint32_t(GcOp::StructNewDefault):
+ CHECK_NEXT(emitStructNewDefault());
+ case uint32_t(GcOp::StructGet):
+ CHECK_NEXT(emitStructGet(FieldWideningOp::None));
+ case uint32_t(GcOp::StructGetS):
+ CHECK_NEXT(emitStructGet(FieldWideningOp::Signed));
+ case uint32_t(GcOp::StructGetU):
+ CHECK_NEXT(emitStructGet(FieldWideningOp::Unsigned));
+ case uint32_t(GcOp::StructSet):
+ CHECK_NEXT(emitStructSet());
+ case uint32_t(GcOp::ArrayNew):
+ CHECK_NEXT(emitArrayNew());
+ case uint32_t(GcOp::ArrayNewFixed):
+ CHECK_NEXT(emitArrayNewFixed());
+ case uint32_t(GcOp::ArrayNewDefault):
+ CHECK_NEXT(emitArrayNewDefault());
+ case uint32_t(GcOp::ArrayNewData):
+ CHECK_NEXT(emitArrayNewData());
+ case uint32_t(GcOp::ArrayInitFromElemStaticV5):
+ case uint32_t(GcOp::ArrayNewElem):
+ CHECK_NEXT(emitArrayNewElem());
+ case uint32_t(GcOp::ArrayGet):
+ CHECK_NEXT(emitArrayGet(FieldWideningOp::None));
+ case uint32_t(GcOp::ArrayGetS):
+ CHECK_NEXT(emitArrayGet(FieldWideningOp::Signed));
+ case uint32_t(GcOp::ArrayGetU):
+ CHECK_NEXT(emitArrayGet(FieldWideningOp::Unsigned));
+ case uint32_t(GcOp::ArraySet):
+ CHECK_NEXT(emitArraySet());
+ case uint32_t(GcOp::ArrayLenWithTypeIndex):
+ CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/true));
+ case uint32_t(GcOp::ArrayLen):
+ CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/false));
+ case uint32_t(GcOp::ArrayCopy):
+ CHECK_NEXT(emitArrayCopy());
+ case uint32_t(GcOp::RefTestV5):
+ CHECK_NEXT(emitRefTestV5());
+ case uint32_t(GcOp::RefCastV5):
+ CHECK_NEXT(emitRefCastV5());
+ case uint32_t(GcOp::RefTest):
+ CHECK_NEXT(emitRefTest(/*nullable=*/false));
+ case uint32_t(GcOp::RefTestNull):
+ CHECK_NEXT(emitRefTest(/*nullable=*/true));
+ case uint32_t(GcOp::RefCast):
+ CHECK_NEXT(emitRefCast(/*nullable=*/false));
+ case uint32_t(GcOp::RefCastNull):
+ CHECK_NEXT(emitRefCast(/*nullable=*/true));
+ case uint32_t(GcOp::BrOnCast):
+ CHECK_NEXT(emitBrOnCast());
+ case uint32_t(GcOp::BrOnCastV5):
+ CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/true));
+ case uint32_t(GcOp::BrOnCastFailV5):
+ CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/false));
+ case uint32_t(GcOp::BrOnCastHeapV5):
+ CHECK_NEXT(
+ emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/false));
+ case uint32_t(GcOp::BrOnCastHeapNullV5):
+ CHECK_NEXT(
+ emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/true));
+ case uint32_t(GcOp::BrOnCastFailHeapV5):
+ CHECK_NEXT(
+ emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/false));
+ case uint32_t(GcOp::BrOnCastFailHeapNullV5):
+ CHECK_NEXT(
+ emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/true));
+ case uint32_t(GcOp::RefAsStructV5):
+ CHECK_NEXT(emitRefAsStructV5());
+ case uint32_t(GcOp::BrOnNonStructV5):
+ CHECK_NEXT(emitBrOnNonStructV5());
+ case uint16_t(GcOp::ExternInternalize):
+ CHECK_NEXT(emitExternInternalize());
+ case uint16_t(GcOp::ExternExternalize):
+ CHECK_NEXT(emitExternExternalize());
+ default:
+ break;
+ } // switch (op.b1)
+ return iter_.unrecognizedOpcode(&op);
+ }
+#endif
+
+#ifdef ENABLE_WASM_SIMD
+ // SIMD operations
+ case uint16_t(Op::SimdPrefix): {
+ uint32_t laneIndex;
+ if (!moduleEnv_.simdAvailable()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(SimdOp::I8x16ExtractLaneS):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneI8x16, ValType::I32, 16));
+ case uint32_t(SimdOp::I8x16ExtractLaneU):
+ CHECK_NEXT(
+ dispatchExtractLane(ExtractLaneUI8x16, ValType::I32, 16));
+ case uint32_t(SimdOp::I16x8ExtractLaneS):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneI16x8, ValType::I32, 8));
+ case uint32_t(SimdOp::I16x8ExtractLaneU):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneUI16x8, ValType::I32, 8));
+ case uint32_t(SimdOp::I32x4ExtractLane):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneI32x4, ValType::I32, 4));
+ case uint32_t(SimdOp::I64x2ExtractLane):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneI64x2, ValType::I64, 2));
+ case uint32_t(SimdOp::F32x4ExtractLane):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneF32x4, ValType::F32, 4));
+ case uint32_t(SimdOp::F64x2ExtractLane):
+ CHECK_NEXT(dispatchExtractLane(ExtractLaneF64x2, ValType::F64, 2));
+ case uint32_t(SimdOp::I8x16Splat):
+ CHECK_NEXT(dispatchSplat(SplatI8x16, ValType::I32));
+ case uint32_t(SimdOp::I16x8Splat):
+ CHECK_NEXT(dispatchSplat(SplatI16x8, ValType::I32));
+ case uint32_t(SimdOp::I32x4Splat):
+ CHECK_NEXT(dispatchSplat(SplatI32x4, ValType::I32));
+ case uint32_t(SimdOp::I64x2Splat):
+ CHECK_NEXT(dispatchSplat(SplatI64x2, ValType::I64));
+ case uint32_t(SimdOp::F32x4Splat):
+ CHECK_NEXT(dispatchSplat(SplatF32x4, ValType::F32));
+ case uint32_t(SimdOp::F64x2Splat):
+ CHECK_NEXT(dispatchSplat(SplatF64x2, ValType::F64));
+ case uint32_t(SimdOp::V128AnyTrue):
+ CHECK_NEXT(dispatchVectorReduction(AnyTrue));
+ case uint32_t(SimdOp::I8x16AllTrue):
+ CHECK_NEXT(dispatchVectorReduction(AllTrueI8x16));
+ case uint32_t(SimdOp::I16x8AllTrue):
+ CHECK_NEXT(dispatchVectorReduction(AllTrueI16x8));
+ case uint32_t(SimdOp::I32x4AllTrue):
+ CHECK_NEXT(dispatchVectorReduction(AllTrueI32x4));
+ case uint32_t(SimdOp::I64x2AllTrue):
+ CHECK_NEXT(dispatchVectorReduction(AllTrueI64x2));
+ case uint32_t(SimdOp::I8x16Bitmask):
+ CHECK_NEXT(dispatchVectorReduction(BitmaskI8x16));
+ case uint32_t(SimdOp::I16x8Bitmask):
+ CHECK_NEXT(dispatchVectorReduction(BitmaskI16x8));
+ case uint32_t(SimdOp::I32x4Bitmask):
+ CHECK_NEXT(dispatchVectorReduction(BitmaskI32x4));
+ case uint32_t(SimdOp::I64x2Bitmask):
+ CHECK_NEXT(dispatchVectorReduction(BitmaskI64x2));
+ case uint32_t(SimdOp::I8x16ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI8x16, ValType::I32, 16));
+ case uint32_t(SimdOp::I16x8ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI16x8, ValType::I32, 8));
+ case uint32_t(SimdOp::I32x4ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI32x4, ValType::I32, 4));
+ case uint32_t(SimdOp::I64x2ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI64x2, ValType::I64, 2));
+ case uint32_t(SimdOp::F32x4ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF32x4, ValType::F32, 4));
+ case uint32_t(SimdOp::F64x2ReplaceLane):
+ CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF64x2, ValType::F64, 2));
+ case uint32_t(SimdOp::I8x16Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::Equal));
+ case uint32_t(SimdOp::I8x16Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::NotEqual));
+ case uint32_t(SimdOp::I8x16LtS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::LessThan));
+ case uint32_t(SimdOp::I8x16LtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Below));
+ case uint32_t(SimdOp::I8x16GtS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI8x16, Assembler::GreaterThan));
+ case uint32_t(SimdOp::I8x16GtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Above));
+ case uint32_t(SimdOp::I8x16LeS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI8x16, Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::I8x16LeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI8x16, Assembler::BelowOrEqual));
+ case uint32_t(SimdOp::I8x16GeS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI8x16,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::I8x16GeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI8x16, Assembler::AboveOrEqual));
+ case uint32_t(SimdOp::I16x8Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::Equal));
+ case uint32_t(SimdOp::I16x8Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::NotEqual));
+ case uint32_t(SimdOp::I16x8LtS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::LessThan));
+ case uint32_t(SimdOp::I16x8LtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Below));
+ case uint32_t(SimdOp::I16x8GtS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI16x8, Assembler::GreaterThan));
+ case uint32_t(SimdOp::I16x8GtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Above));
+ case uint32_t(SimdOp::I16x8LeS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI16x8, Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::I16x8LeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI16x8, Assembler::BelowOrEqual));
+ case uint32_t(SimdOp::I16x8GeS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI16x8,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::I16x8GeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI16x8, Assembler::AboveOrEqual));
+ case uint32_t(SimdOp::I32x4Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::Equal));
+ case uint32_t(SimdOp::I32x4Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::NotEqual));
+ case uint32_t(SimdOp::I32x4LtS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::LessThan));
+ case uint32_t(SimdOp::I32x4LtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Below));
+ case uint32_t(SimdOp::I32x4GtS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI32x4, Assembler::GreaterThan));
+ case uint32_t(SimdOp::I32x4GtU):
+ CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Above));
+ case uint32_t(SimdOp::I32x4LeS):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpI32x4, Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::I32x4LeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI32x4, Assembler::BelowOrEqual));
+ case uint32_t(SimdOp::I32x4GeS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI32x4,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::I32x4GeU):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpUI32x4, Assembler::AboveOrEqual));
+ case uint32_t(SimdOp::I64x2Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality,
+ Assembler::Equal));
+ case uint32_t(SimdOp::I64x2Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality,
+ Assembler::NotEqual));
+ case uint32_t(SimdOp::I64x2LtS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering,
+ Assembler::LessThan));
+ case uint32_t(SimdOp::I64x2GtS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering,
+ Assembler::GreaterThan));
+ case uint32_t(SimdOp::I64x2LeS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering,
+ Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::I64x2GeS):
+ CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::F32x4Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::Equal));
+ case uint32_t(SimdOp::F32x4Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::NotEqual));
+ case uint32_t(SimdOp::F32x4Lt):
+ CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::LessThan));
+ case uint32_t(SimdOp::F32x4Gt):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpF32x4, Assembler::GreaterThan));
+ case uint32_t(SimdOp::F32x4Le):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpF32x4, Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::F32x4Ge):
+ CHECK_NEXT(dispatchVectorComparison(CmpF32x4,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::F64x2Eq):
+ CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::Equal));
+ case uint32_t(SimdOp::F64x2Ne):
+ CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::NotEqual));
+ case uint32_t(SimdOp::F64x2Lt):
+ CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::LessThan));
+ case uint32_t(SimdOp::F64x2Gt):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpF64x2, Assembler::GreaterThan));
+ case uint32_t(SimdOp::F64x2Le):
+ CHECK_NEXT(
+ dispatchVectorComparison(CmpF64x2, Assembler::LessThanOrEqual));
+ case uint32_t(SimdOp::F64x2Ge):
+ CHECK_NEXT(dispatchVectorComparison(CmpF64x2,
+ Assembler::GreaterThanOrEqual));
+ case uint32_t(SimdOp::V128And):
+ CHECK_NEXT(dispatchVectorBinary(AndV128));
+ case uint32_t(SimdOp::V128Or):
+ CHECK_NEXT(dispatchVectorBinary(OrV128));
+ case uint32_t(SimdOp::V128Xor):
+ CHECK_NEXT(dispatchVectorBinary(XorV128));
+ case uint32_t(SimdOp::V128AndNot):
+ CHECK_NEXT(dispatchBinary0(emitVectorAndNot, ValType::V128));
+ case uint32_t(SimdOp::I8x16AvgrU):
+ CHECK_NEXT(dispatchVectorBinary(AverageUI8x16));
+ case uint32_t(SimdOp::I16x8AvgrU):
+ CHECK_NEXT(dispatchVectorBinary(AverageUI16x8));
+ case uint32_t(SimdOp::I8x16Add):
+ CHECK_NEXT(dispatchVectorBinary(AddI8x16));
+ case uint32_t(SimdOp::I8x16AddSatS):
+ CHECK_NEXT(dispatchVectorBinary(AddSatI8x16));
+ case uint32_t(SimdOp::I8x16AddSatU):
+ CHECK_NEXT(dispatchVectorBinary(AddSatUI8x16));
+ case uint32_t(SimdOp::I8x16Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubI8x16));
+ case uint32_t(SimdOp::I8x16SubSatS):
+ CHECK_NEXT(dispatchVectorBinary(SubSatI8x16));
+ case uint32_t(SimdOp::I8x16SubSatU):
+ CHECK_NEXT(dispatchVectorBinary(SubSatUI8x16));
+ case uint32_t(SimdOp::I8x16MinS):
+ CHECK_NEXT(dispatchVectorBinary(MinI8x16));
+ case uint32_t(SimdOp::I8x16MinU):
+ CHECK_NEXT(dispatchVectorBinary(MinUI8x16));
+ case uint32_t(SimdOp::I8x16MaxS):
+ CHECK_NEXT(dispatchVectorBinary(MaxI8x16));
+ case uint32_t(SimdOp::I8x16MaxU):
+ CHECK_NEXT(dispatchVectorBinary(MaxUI8x16));
+ case uint32_t(SimdOp::I16x8Add):
+ CHECK_NEXT(dispatchVectorBinary(AddI16x8));
+ case uint32_t(SimdOp::I16x8AddSatS):
+ CHECK_NEXT(dispatchVectorBinary(AddSatI16x8));
+ case uint32_t(SimdOp::I16x8AddSatU):
+ CHECK_NEXT(dispatchVectorBinary(AddSatUI16x8));
+ case uint32_t(SimdOp::I16x8Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubI16x8));
+ case uint32_t(SimdOp::I16x8SubSatS):
+ CHECK_NEXT(dispatchVectorBinary(SubSatI16x8));
+ case uint32_t(SimdOp::I16x8SubSatU):
+ CHECK_NEXT(dispatchVectorBinary(SubSatUI16x8));
+ case uint32_t(SimdOp::I16x8Mul):
+ CHECK_NEXT(dispatchVectorBinary(MulI16x8));
+ case uint32_t(SimdOp::I16x8MinS):
+ CHECK_NEXT(dispatchVectorBinary(MinI16x8));
+ case uint32_t(SimdOp::I16x8MinU):
+ CHECK_NEXT(dispatchVectorBinary(MinUI16x8));
+ case uint32_t(SimdOp::I16x8MaxS):
+ CHECK_NEXT(dispatchVectorBinary(MaxI16x8));
+ case uint32_t(SimdOp::I16x8MaxU):
+ CHECK_NEXT(dispatchVectorBinary(MaxUI16x8));
+ case uint32_t(SimdOp::I32x4Add):
+ CHECK_NEXT(dispatchVectorBinary(AddI32x4));
+ case uint32_t(SimdOp::I32x4Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubI32x4));
+ case uint32_t(SimdOp::I32x4Mul):
+ CHECK_NEXT(dispatchVectorBinary(MulI32x4));
+ case uint32_t(SimdOp::I32x4MinS):
+ CHECK_NEXT(dispatchVectorBinary(MinI32x4));
+ case uint32_t(SimdOp::I32x4MinU):
+ CHECK_NEXT(dispatchVectorBinary(MinUI32x4));
+ case uint32_t(SimdOp::I32x4MaxS):
+ CHECK_NEXT(dispatchVectorBinary(MaxI32x4));
+ case uint32_t(SimdOp::I32x4MaxU):
+ CHECK_NEXT(dispatchVectorBinary(MaxUI32x4));
+ case uint32_t(SimdOp::I64x2Add):
+ CHECK_NEXT(dispatchVectorBinary(AddI64x2));
+ case uint32_t(SimdOp::I64x2Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubI64x2));
+ case uint32_t(SimdOp::I64x2Mul):
+ CHECK_NEXT(dispatchVectorBinary(MulI64x2));
+ case uint32_t(SimdOp::F32x4Add):
+ CHECK_NEXT(dispatchVectorBinary(AddF32x4));
+ case uint32_t(SimdOp::F32x4Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubF32x4));
+ case uint32_t(SimdOp::F32x4Mul):
+ CHECK_NEXT(dispatchVectorBinary(MulF32x4));
+ case uint32_t(SimdOp::F32x4Div):
+ CHECK_NEXT(dispatchVectorBinary(DivF32x4));
+ case uint32_t(SimdOp::F32x4Min):
+ CHECK_NEXT(dispatchVectorBinary(MinF32x4));
+ case uint32_t(SimdOp::F32x4Max):
+ CHECK_NEXT(dispatchVectorBinary(MaxF32x4));
+ case uint32_t(SimdOp::F64x2Add):
+ CHECK_NEXT(dispatchVectorBinary(AddF64x2));
+ case uint32_t(SimdOp::F64x2Sub):
+ CHECK_NEXT(dispatchVectorBinary(SubF64x2));
+ case uint32_t(SimdOp::F64x2Mul):
+ CHECK_NEXT(dispatchVectorBinary(MulF64x2));
+ case uint32_t(SimdOp::F64x2Div):
+ CHECK_NEXT(dispatchVectorBinary(DivF64x2));
+ case uint32_t(SimdOp::F64x2Min):
+ CHECK_NEXT(dispatchVectorBinary(MinF64x2));
+ case uint32_t(SimdOp::F64x2Max):
+ CHECK_NEXT(dispatchVectorBinary(MaxF64x2));
+ case uint32_t(SimdOp::I8x16NarrowI16x8S):
+ CHECK_NEXT(dispatchVectorBinary(NarrowI16x8));
+ case uint32_t(SimdOp::I8x16NarrowI16x8U):
+ CHECK_NEXT(dispatchVectorBinary(NarrowUI16x8));
+ case uint32_t(SimdOp::I16x8NarrowI32x4S):
+ CHECK_NEXT(dispatchVectorBinary(NarrowI32x4));
+ case uint32_t(SimdOp::I16x8NarrowI32x4U):
+ CHECK_NEXT(dispatchVectorBinary(NarrowUI32x4));
+ case uint32_t(SimdOp::I8x16Swizzle):
+ CHECK_NEXT(dispatchVectorBinary(Swizzle));
+ case uint32_t(SimdOp::F32x4PMax):
+ CHECK_NEXT(dispatchVectorBinary(PMaxF32x4));
+ case uint32_t(SimdOp::F32x4PMin):
+ CHECK_NEXT(dispatchVectorBinary(PMinF32x4));
+ case uint32_t(SimdOp::F64x2PMax):
+ CHECK_NEXT(dispatchVectorBinary(PMaxF64x2));
+ case uint32_t(SimdOp::F64x2PMin):
+ CHECK_NEXT(dispatchVectorBinary(PMinF64x2));
+ case uint32_t(SimdOp::I32x4DotI16x8S):
+ CHECK_NEXT(dispatchVectorBinary(DotI16x8));
+ case uint32_t(SimdOp::I16x8ExtmulLowI8x16S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowI8x16));
+ case uint32_t(SimdOp::I16x8ExtmulHighI8x16S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighI8x16));
+ case uint32_t(SimdOp::I16x8ExtmulLowI8x16U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI8x16));
+ case uint32_t(SimdOp::I16x8ExtmulHighI8x16U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI8x16));
+ case uint32_t(SimdOp::I32x4ExtmulLowI16x8S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowI16x8));
+ case uint32_t(SimdOp::I32x4ExtmulHighI16x8S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighI16x8));
+ case uint32_t(SimdOp::I32x4ExtmulLowI16x8U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI16x8));
+ case uint32_t(SimdOp::I32x4ExtmulHighI16x8U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI16x8));
+ case uint32_t(SimdOp::I64x2ExtmulLowI32x4S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowI32x4));
+ case uint32_t(SimdOp::I64x2ExtmulHighI32x4S):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighI32x4));
+ case uint32_t(SimdOp::I64x2ExtmulLowI32x4U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI32x4));
+ case uint32_t(SimdOp::I64x2ExtmulHighI32x4U):
+ CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI32x4));
+ case uint32_t(SimdOp::I16x8Q15MulrSatS):
+ CHECK_NEXT(dispatchVectorBinary(Q15MulrSatS));
+ case uint32_t(SimdOp::I8x16Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegI8x16));
+ case uint32_t(SimdOp::I16x8Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegI16x8));
+ case uint32_t(SimdOp::I16x8ExtendLowI8x16S):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowI8x16));
+ case uint32_t(SimdOp::I16x8ExtendHighI8x16S):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighI8x16));
+ case uint32_t(SimdOp::I16x8ExtendLowI8x16U):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowUI8x16));
+ case uint32_t(SimdOp::I16x8ExtendHighI8x16U):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighUI8x16));
+ case uint32_t(SimdOp::I32x4Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegI32x4));
+ case uint32_t(SimdOp::I32x4ExtendLowI16x8S):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowI16x8));
+ case uint32_t(SimdOp::I32x4ExtendHighI16x8S):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighI16x8));
+ case uint32_t(SimdOp::I32x4ExtendLowI16x8U):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowUI16x8));
+ case uint32_t(SimdOp::I32x4ExtendHighI16x8U):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighUI16x8));
+ case uint32_t(SimdOp::I32x4TruncSatF32x4S):
+ CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToI32x4));
+ case uint32_t(SimdOp::I32x4TruncSatF32x4U):
+ CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToUI32x4));
+ case uint32_t(SimdOp::I64x2Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegI64x2));
+ case uint32_t(SimdOp::I64x2ExtendLowI32x4S):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowI32x4));
+ case uint32_t(SimdOp::I64x2ExtendHighI32x4S):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighI32x4));
+ case uint32_t(SimdOp::I64x2ExtendLowI32x4U):
+ CHECK_NEXT(dispatchVectorUnary(WidenLowUI32x4));
+ case uint32_t(SimdOp::I64x2ExtendHighI32x4U):
+ CHECK_NEXT(dispatchVectorUnary(WidenHighUI32x4));
+ case uint32_t(SimdOp::F32x4Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsF32x4));
+ case uint32_t(SimdOp::F32x4Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegF32x4));
+ case uint32_t(SimdOp::F32x4Sqrt):
+ CHECK_NEXT(dispatchVectorUnary(SqrtF32x4));
+ case uint32_t(SimdOp::F32x4ConvertI32x4S):
+ CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF32x4));
+ case uint32_t(SimdOp::F32x4ConvertI32x4U):
+ CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF32x4));
+ case uint32_t(SimdOp::F32x4DemoteF64x2Zero):
+ CHECK_NEXT(dispatchVectorUnary(DemoteF64x2ToF32x4));
+ case uint32_t(SimdOp::F64x2PromoteLowF32x4):
+ CHECK_NEXT(dispatchVectorUnary(PromoteF32x4ToF64x2));
+ case uint32_t(SimdOp::F64x2ConvertLowI32x4S):
+ CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF64x2));
+ case uint32_t(SimdOp::F64x2ConvertLowI32x4U):
+ CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF64x2));
+ case uint32_t(SimdOp::I32x4TruncSatF64x2SZero):
+ CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToI32x4));
+ case uint32_t(SimdOp::I32x4TruncSatF64x2UZero):
+ CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToUI32x4));
+ case uint32_t(SimdOp::F64x2Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsF64x2));
+ case uint32_t(SimdOp::F64x2Neg):
+ CHECK_NEXT(dispatchVectorUnary(NegF64x2));
+ case uint32_t(SimdOp::F64x2Sqrt):
+ CHECK_NEXT(dispatchVectorUnary(SqrtF64x2));
+ case uint32_t(SimdOp::V128Not):
+ CHECK_NEXT(dispatchVectorUnary(NotV128));
+ case uint32_t(SimdOp::I8x16Popcnt):
+ CHECK_NEXT(dispatchVectorUnary(PopcntI8x16));
+ case uint32_t(SimdOp::I8x16Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsI8x16));
+ case uint32_t(SimdOp::I16x8Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsI16x8));
+ case uint32_t(SimdOp::I32x4Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsI32x4));
+ case uint32_t(SimdOp::I64x2Abs):
+ CHECK_NEXT(dispatchVectorUnary(AbsI64x2));
+ case uint32_t(SimdOp::F32x4Ceil):
+ CHECK_NEXT(dispatchVectorUnary(CeilF32x4));
+ case uint32_t(SimdOp::F32x4Floor):
+ CHECK_NEXT(dispatchVectorUnary(FloorF32x4));
+ case uint32_t(SimdOp::F32x4Trunc):
+ CHECK_NEXT(dispatchVectorUnary(TruncF32x4));
+ case uint32_t(SimdOp::F32x4Nearest):
+ CHECK_NEXT(dispatchVectorUnary(NearestF32x4));
+ case uint32_t(SimdOp::F64x2Ceil):
+ CHECK_NEXT(dispatchVectorUnary(CeilF64x2));
+ case uint32_t(SimdOp::F64x2Floor):
+ CHECK_NEXT(dispatchVectorUnary(FloorF64x2));
+ case uint32_t(SimdOp::F64x2Trunc):
+ CHECK_NEXT(dispatchVectorUnary(TruncF64x2));
+ case uint32_t(SimdOp::F64x2Nearest):
+ CHECK_NEXT(dispatchVectorUnary(NearestF64x2));
+ case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S):
+ CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI8x16));
+ case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U):
+ CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI8x16));
+ case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S):
+ CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI16x8));
+ case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U):
+ CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI16x8));
+ case uint32_t(SimdOp::I8x16Shl):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI8x16));
+ case uint32_t(SimdOp::I8x16ShrS):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI8x16));
+ case uint32_t(SimdOp::I8x16ShrU):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI8x16));
+ case uint32_t(SimdOp::I16x8Shl):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI16x8));
+ case uint32_t(SimdOp::I16x8ShrS):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI16x8));
+ case uint32_t(SimdOp::I16x8ShrU):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI16x8));
+ case uint32_t(SimdOp::I32x4Shl):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI32x4));
+ case uint32_t(SimdOp::I32x4ShrS):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI32x4));
+ case uint32_t(SimdOp::I32x4ShrU):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI32x4));
+ case uint32_t(SimdOp::I64x2Shl):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI64x2));
+ case uint32_t(SimdOp::I64x2ShrS):
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ CHECK_NEXT(emitVectorShiftRightI64x2());
+# else
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI64x2));
+# endif
+ case uint32_t(SimdOp::I64x2ShrU):
+ CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI64x2));
+ case uint32_t(SimdOp::V128Bitselect):
+ CHECK_NEXT(dispatchTernary1(BitselectV128, ValType::V128));
+ case uint32_t(SimdOp::I8x16Shuffle):
+ CHECK_NEXT(emitVectorShuffle());
+ case uint32_t(SimdOp::V128Const): {
+ V128 v128;
+ CHECK(iter_.readV128Const(&v128));
+ if (!deadCode_) {
+ pushV128(v128);
+ }
+ NEXT();
+ }
+ case uint32_t(SimdOp::V128Load):
+ CHECK_NEXT(emitLoad(ValType::V128, Scalar::Simd128));
+ case uint32_t(SimdOp::V128Load8Splat):
+ CHECK_NEXT(emitLoadSplat(Scalar::Uint8));
+ case uint32_t(SimdOp::V128Load16Splat):
+ CHECK_NEXT(emitLoadSplat(Scalar::Uint16));
+ case uint32_t(SimdOp::V128Load32Splat):
+ CHECK_NEXT(emitLoadSplat(Scalar::Uint32));
+ case uint32_t(SimdOp::V128Load64Splat):
+ CHECK_NEXT(emitLoadSplat(Scalar::Int64));
+ case uint32_t(SimdOp::V128Load8x8S):
+ CHECK_NEXT(emitLoadExtend(Scalar::Int8));
+ case uint32_t(SimdOp::V128Load8x8U):
+ CHECK_NEXT(emitLoadExtend(Scalar::Uint8));
+ case uint32_t(SimdOp::V128Load16x4S):
+ CHECK_NEXT(emitLoadExtend(Scalar::Int16));
+ case uint32_t(SimdOp::V128Load16x4U):
+ CHECK_NEXT(emitLoadExtend(Scalar::Uint16));
+ case uint32_t(SimdOp::V128Load32x2S):
+ CHECK_NEXT(emitLoadExtend(Scalar::Int32));
+ case uint32_t(SimdOp::V128Load32x2U):
+ CHECK_NEXT(emitLoadExtend(Scalar::Uint32));
+ case uint32_t(SimdOp::V128Load32Zero):
+ CHECK_NEXT(emitLoadZero(Scalar::Float32));
+ case uint32_t(SimdOp::V128Load64Zero):
+ CHECK_NEXT(emitLoadZero(Scalar::Float64));
+ case uint32_t(SimdOp::V128Store):
+ CHECK_NEXT(emitStore(ValType::V128, Scalar::Simd128));
+ case uint32_t(SimdOp::V128Load8Lane):
+ CHECK_NEXT(emitLoadLane(1));
+ case uint32_t(SimdOp::V128Load16Lane):
+ CHECK_NEXT(emitLoadLane(2));
+ case uint32_t(SimdOp::V128Load32Lane):
+ CHECK_NEXT(emitLoadLane(4));
+ case uint32_t(SimdOp::V128Load64Lane):
+ CHECK_NEXT(emitLoadLane(8));
+ case uint32_t(SimdOp::V128Store8Lane):
+ CHECK_NEXT(emitStoreLane(1));
+ case uint32_t(SimdOp::V128Store16Lane):
+ CHECK_NEXT(emitStoreLane(2));
+ case uint32_t(SimdOp::V128Store32Lane):
+ CHECK_NEXT(emitStoreLane(4));
+ case uint32_t(SimdOp::V128Store64Lane):
+ CHECK_NEXT(emitStoreLane(8));
+# ifdef ENABLE_WASM_RELAXED_SIMD
+ case uint32_t(SimdOp::F32x4RelaxedFma):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchTernary2(RelaxedFmaF32x4, ValType::V128));
+ case uint32_t(SimdOp::F32x4RelaxedFnma):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchTernary2(RelaxedFnmaF32x4, ValType::V128));
+ case uint32_t(SimdOp::F64x2RelaxedFma):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchTernary2(RelaxedFmaF64x2, ValType::V128));
+ case uint32_t(SimdOp::F64x2RelaxedFnma):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchTernary2(RelaxedFnmaF64x2, ValType::V128));
+ break;
+ case uint32_t(SimdOp::I8x16RelaxedLaneSelect):
+ case uint32_t(SimdOp::I16x8RelaxedLaneSelect):
+ case uint32_t(SimdOp::I32x4RelaxedLaneSelect):
+ case uint32_t(SimdOp::I64x2RelaxedLaneSelect):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitVectorLaneSelect());
+ case uint32_t(SimdOp::F32x4RelaxedMin):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedMinF32x4));
+ case uint32_t(SimdOp::F32x4RelaxedMax):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF32x4));
+ case uint32_t(SimdOp::F64x2RelaxedMin):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedMinF64x2));
+ case uint32_t(SimdOp::F64x2RelaxedMax):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF64x2));
+ case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToI32x4));
+ case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToUI32x4));
+ case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToI32x4));
+ case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToUI32x4));
+ case uint32_t(SimdOp::I8x16RelaxedSwizzle):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedSwizzle));
+ case uint32_t(SimdOp::I16x8RelaxedQ15MulrS):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(RelaxedQ15MulrS));
+ case uint32_t(SimdOp::I16x8DotI8x16I7x16S):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchVectorBinary(DotI8x16I7x16S));
+ case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS):
+ if (!moduleEnv_.v128RelaxedEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(dispatchTernary0(emitDotI8x16I7x16AddS, ValType::V128));
+# endif
+ default:
+ break;
+ } // switch (op.b1)
+ return iter_.unrecognizedOpcode(&op);
+ }
+#endif // ENABLE_WASM_SIMD
+
+ // "Miscellaneous" operations
+ case uint16_t(Op::MiscPrefix): {
+ switch (op.b1) {
+ case uint32_t(MiscOp::I32TruncSatF32S):
+ CHECK_NEXT(
+ dispatchConversionOOM(emitTruncateF32ToI32<TRUNC_SATURATING>,
+ ValType::F32, ValType::I32));
+ case uint32_t(MiscOp::I32TruncSatF32U):
+ CHECK_NEXT(dispatchConversionOOM(
+ emitTruncateF32ToI32<TRUNC_UNSIGNED | TRUNC_SATURATING>,
+ ValType::F32, ValType::I32));
+ case uint32_t(MiscOp::I32TruncSatF64S):
+ CHECK_NEXT(
+ dispatchConversionOOM(emitTruncateF64ToI32<TRUNC_SATURATING>,
+ ValType::F64, ValType::I32));
+ case uint32_t(MiscOp::I32TruncSatF64U):
+ CHECK_NEXT(dispatchConversionOOM(
+ emitTruncateF64ToI32<TRUNC_UNSIGNED | TRUNC_SATURATING>,
+ ValType::F64, ValType::I32));
+ case uint32_t(MiscOp::I64TruncSatF32S):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F32,
+ ValType::I64));
+#else
+ CHECK_NEXT(
+ dispatchConversionOOM(emitTruncateF32ToI64<TRUNC_SATURATING>,
+ ValType::F32, ValType::I64));
+#endif
+ case uint32_t(MiscOp::I64TruncSatF32U):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F32,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(
+ emitTruncateF32ToI64<TRUNC_UNSIGNED | TRUNC_SATURATING>,
+ ValType::F32, ValType::I64));
+#endif
+ case uint32_t(MiscOp::I64TruncSatF64S):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F64,
+ ValType::I64));
+#else
+ CHECK_NEXT(
+ dispatchConversionOOM(emitTruncateF64ToI64<TRUNC_SATURATING>,
+ ValType::F64, ValType::I64));
+#endif
+ case uint32_t(MiscOp::I64TruncSatF64U):
+#ifdef RABALDR_FLOAT_TO_I64_CALLOUT
+ CHECK_NEXT(dispatchCalloutConversionOOM(
+ emitConvertFloatingToInt64Callout,
+ SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F64,
+ ValType::I64));
+#else
+ CHECK_NEXT(dispatchConversionOOM(
+ emitTruncateF64ToI64<TRUNC_UNSIGNED | TRUNC_SATURATING>,
+ ValType::F64, ValType::I64));
+#endif
+ case uint32_t(MiscOp::MemoryCopy):
+ CHECK_NEXT(emitMemCopy());
+ case uint32_t(MiscOp::DataDrop):
+ CHECK_NEXT(emitDataOrElemDrop(/*isData=*/true));
+ case uint32_t(MiscOp::MemoryFill):
+ CHECK_NEXT(emitMemFill());
+#ifdef ENABLE_WASM_MEMORY_CONTROL
+ case uint32_t(MiscOp::MemoryDiscard): {
+ if (!moduleEnv_.memoryControlEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ CHECK_NEXT(emitMemDiscard());
+ }
+#endif
+ case uint32_t(MiscOp::MemoryInit):
+ CHECK_NEXT(emitMemInit());
+ case uint32_t(MiscOp::TableCopy):
+ CHECK_NEXT(emitTableCopy());
+ case uint32_t(MiscOp::ElemDrop):
+ CHECK_NEXT(emitDataOrElemDrop(/*isData=*/false));
+ case uint32_t(MiscOp::TableInit):
+ CHECK_NEXT(emitTableInit());
+ case uint32_t(MiscOp::TableFill):
+ CHECK_NEXT(emitTableFill());
+ case uint32_t(MiscOp::TableGrow):
+ CHECK_NEXT(emitTableGrow());
+ case uint32_t(MiscOp::TableSize):
+ CHECK_NEXT(emitTableSize());
+ default:
+ break;
+ } // switch (op.b1)
+ return iter_.unrecognizedOpcode(&op);
+ }
+
+ // Thread operations
+ case uint16_t(Op::ThreadPrefix): {
+ // Though thread ops can be used on nonshared memories, we make them
+ // unavailable if shared memory has been disabled in the prefs, for
+ // maximum predictability and safety and consistency with JS.
+ if (moduleEnv_.sharedMemoryEnabled() == Shareable::False) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(ThreadOp::Wake):
+ CHECK_NEXT(emitWake());
+
+ case uint32_t(ThreadOp::I32Wait):
+ CHECK_NEXT(emitWait(ValType::I32, 4));
+ case uint32_t(ThreadOp::I64Wait):
+ CHECK_NEXT(emitWait(ValType::I64, 8));
+ case uint32_t(ThreadOp::Fence):
+ CHECK_NEXT(emitFence());
+
+ case uint32_t(ThreadOp::I32AtomicLoad):
+ CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicLoad):
+ CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicLoad8U):
+ CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicLoad16U):
+ CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicLoad8U):
+ CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicLoad16U):
+ CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicLoad32U):
+ CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicStore):
+ CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicStore):
+ CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicStore8U):
+ CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicStore16U):
+ CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicStore8U):
+ CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicStore16U):
+ CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicStore32U):
+ CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicAdd):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I32AtomicAdd8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I32AtomicAdd16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd32U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAddOp));
+
+ case uint32_t(ThreadOp::I32AtomicSub):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I32AtomicSub8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I32AtomicSub16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub32U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchSubOp));
+
+ case uint32_t(ThreadOp::I32AtomicAnd):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I32AtomicAnd8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I32AtomicAnd16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd32U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAndOp));
+
+ case uint32_t(ThreadOp::I32AtomicOr):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I32AtomicOr8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I32AtomicOr16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr32U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchOrOp));
+
+ case uint32_t(ThreadOp::I32AtomicXor):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I32AtomicXor8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I32AtomicXor16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor8U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor16U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor32U):
+ CHECK_NEXT(
+ emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchXorOp));
+
+ case uint32_t(ThreadOp::I32AtomicXchg):
+ CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicXchg):
+ CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicXchg8U):
+ CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicXchg16U):
+ CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicXchg8U):
+ CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicXchg16U):
+ CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicXchg32U):
+ CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicCmpXchg):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicCmpXchg8U):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicCmpXchg16U):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg8U):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg16U):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg32U):
+ CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint32));
+
+ default:
+ return iter_.unrecognizedOpcode(&op);
+ }
+ break;
+ }
+
+ // asm.js and other private operations
+ case uint16_t(Op::MozPrefix): {
+ if (op.b1 != uint32_t(MozOp::Intrinsic) ||
+ !moduleEnv_.intrinsicsEnabled()) {
+ return iter_.unrecognizedOpcode(&op);
+ }
+ // private intrinsic operations
+ CHECK_NEXT(emitIntrinsic());
+ }
+
+ default:
+ return iter_.unrecognizedOpcode(&op);
+ }
+
+#undef CHECK
+#undef NEXT
+#undef CHECK_NEXT
+#undef CHECK_POINTER_COUNT
+#undef dispatchBinary0
+#undef dispatchBinary1
+#undef dispatchBinary2
+#undef dispatchBinary3
+#undef dispatchUnary0
+#undef dispatchUnary1
+#undef dispatchUnary2
+#undef dispatchComparison0
+#undef dispatchConversion0
+#undef dispatchConversion1
+#undef dispatchConversionOOM
+#undef dispatchCalloutConversionOOM
+#undef dispatchIntDivCallout
+#undef dispatchVectorBinary
+#undef dispatchVectorUnary
+#undef dispatchVectorComparison
+#undef dispatchExtractLane
+#undef dispatchReplaceLane
+#undef dispatchSplat
+#undef dispatchVectorReduction
+
+ MOZ_CRASH("unreachable");
+ }
+
+ MOZ_CRASH("unreachable");
+}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Various helpers.
+
+void BaseCompiler::assertResultRegistersAvailable(ResultType type) {
+#ifdef DEBUG
+ for (ABIResultIter iter(type); !iter.done(); iter.next()) {
+ ABIResult result = iter.cur();
+ if (!result.inRegister()) {
+ return;
+ }
+ switch (result.type().kind()) {
+ case ValType::I32:
+ MOZ_ASSERT(isAvailableI32(RegI32(result.gpr())));
+ break;
+ case ValType::I64:
+ MOZ_ASSERT(isAvailableI64(RegI64(result.gpr64())));
+ break;
+ case ValType::V128:
+# ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(isAvailableV128(RegV128(result.fpr())));
+ break;
+# else
+ MOZ_CRASH("No SIMD support");
+# endif
+ case ValType::F32:
+ MOZ_ASSERT(isAvailableF32(RegF32(result.fpr())));
+ break;
+ case ValType::F64:
+ MOZ_ASSERT(isAvailableF64(RegF64(result.fpr())));
+ break;
+ case ValType::Ref:
+ MOZ_ASSERT(isAvailableRef(RegRef(result.gpr())));
+ break;
+ }
+ }
+#endif
+}
+
+void BaseCompiler::saveTempPtr(const RegPtr& r) {
+ MOZ_ASSERT(!ra.isAvailablePtr(r));
+ fr.pushGPR(r);
+ ra.freePtr(r);
+ MOZ_ASSERT(ra.isAvailablePtr(r));
+}
+
+void BaseCompiler::restoreTempPtr(const RegPtr& r) {
+ MOZ_ASSERT(ra.isAvailablePtr(r));
+ ra.needPtr(r);
+ fr.popGPR(r);
+ MOZ_ASSERT(!ra.isAvailablePtr(r));
+}
+
+#ifdef DEBUG
+void BaseCompiler::performRegisterLeakCheck() {
+ BaseRegAlloc::LeakCheck check(ra);
+ for (auto& item : stk_) {
+ switch (item.kind_) {
+ case Stk::RegisterI32:
+ check.addKnownI32(item.i32reg());
+ break;
+ case Stk::RegisterI64:
+ check.addKnownI64(item.i64reg());
+ break;
+ case Stk::RegisterF32:
+ check.addKnownF32(item.f32reg());
+ break;
+ case Stk::RegisterF64:
+ check.addKnownF64(item.f64reg());
+ break;
+# ifdef ENABLE_WASM_SIMD
+ case Stk::RegisterV128:
+ check.addKnownV128(item.v128reg());
+ break;
+# endif
+ case Stk::RegisterRef:
+ check.addKnownRef(item.refReg());
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+void BaseCompiler::assertStackInvariants() const {
+ if (deadCode_) {
+ // Nonlocal control flow can pass values in stack locations in a way that
+ // isn't accounted for by the value stack. In dead code, which occurs
+ // after unconditional non-local control flow, there is no invariant to
+ // assert.
+ return;
+ }
+ size_t size = 0;
+ for (const Stk& v : stk_) {
+ switch (v.kind()) {
+ case Stk::MemRef:
+ size += BaseStackFrame::StackSizeOfPtr;
+ break;
+ case Stk::MemI32:
+ size += BaseStackFrame::StackSizeOfPtr;
+ break;
+ case Stk::MemI64:
+ size += BaseStackFrame::StackSizeOfInt64;
+ break;
+ case Stk::MemF64:
+ size += BaseStackFrame::StackSizeOfDouble;
+ break;
+ case Stk::MemF32:
+ size += BaseStackFrame::StackSizeOfFloat;
+ break;
+# ifdef ENABLE_WASM_SIMD
+ case Stk::MemV128:
+ size += BaseStackFrame::StackSizeOfV128;
+ break;
+# endif
+ default:
+ MOZ_ASSERT(!v.isMem());
+ break;
+ }
+ }
+ MOZ_ASSERT(size == fr.dynamicHeight());
+}
+
+void BaseCompiler::showStack(const char* who) const {
+ fprintf(stderr, "Stack at %s {{\n", who);
+ size_t n = 0;
+ for (const Stk& elem : stk_) {
+ fprintf(stderr, " [%zu] ", n++);
+ elem.showStackElem();
+ fprintf(stderr, "\n");
+ }
+ fprintf(stderr, "}}\n");
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Main compilation logic.
+
+bool BaseCompiler::emitFunction() {
+ AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitFunction");
+
+ if (!beginFunction()) {
+ return false;
+ }
+
+ if (!emitBody()) {
+ return false;
+ }
+
+ if (!endFunction()) {
+ return false;
+ }
+
+ return true;
+}
+
+BaseCompiler::BaseCompiler(const ModuleEnvironment& moduleEnv,
+ const CompilerEnvironment& compilerEnv,
+ const FuncCompileInput& func,
+ const ValTypeVector& locals,
+ const RegisterOffsets& trapExitLayout,
+ size_t trapExitLayoutNumWords, Decoder& decoder,
+ StkVector& stkSource, TempAllocator* alloc,
+ MacroAssembler* masm, StackMaps* stackMaps)
+ : // Environment
+ moduleEnv_(moduleEnv),
+ compilerEnv_(compilerEnv),
+ func_(func),
+ locals_(locals),
+ previousBreakablePoint_(UINT32_MAX),
+ stkSource_(stkSource),
+ // Output-only data structures
+ alloc_(alloc->fallible()),
+ masm(*masm),
+ // Compilation state
+ decoder_(decoder),
+ iter_(moduleEnv, decoder),
+ fr(*masm),
+ stackMapGenerator_(stackMaps, trapExitLayout, trapExitLayoutNumWords,
+ *masm),
+ deadCode_(false),
+ bceSafe_(0),
+ latentOp_(LatentOp::None),
+ latentType_(ValType::I32),
+ latentIntCmp_(Assembler::Equal),
+ latentDoubleCmp_(Assembler::DoubleEqual) {
+ // Our caller, BaselineCompileFunctions, will lend us the vector contents to
+ // use for the eval stack. To get hold of those contents, we'll temporarily
+ // installing an empty one in its place.
+ MOZ_ASSERT(stk_.empty());
+ stk_.swap(stkSource_);
+
+ // Assuming that previously processed wasm functions are well formed, the
+ // eval stack should now be empty. But empty it anyway; any non-emptyness
+ // at this point will cause chaos.
+ stk_.clear();
+}
+
+BaseCompiler::~BaseCompiler() {
+ stk_.swap(stkSource_);
+ // We've returned the eval stack vector contents to our caller,
+ // BaselineCompileFunctions. We expect the vector we get in return to be
+ // empty since that's what we swapped for the stack vector in our
+ // constructor.
+ MOZ_ASSERT(stk_.empty());
+}
+
+bool BaseCompiler::init() {
+ // We may lift this restriction in the future.
+ MOZ_ASSERT_IF(usesMemory() && isMem64(), !moduleEnv_.hugeMemoryEnabled());
+ // asm.js is not supported in baseline
+ MOZ_ASSERT(!moduleEnv_.isAsmJS());
+ // Only asm.js modules have call site line numbers
+ MOZ_ASSERT(func_.callSiteLineNums.empty());
+
+ ra.init(this);
+
+ if (!SigD_.append(ValType::F64)) {
+ return false;
+ }
+ if (!SigF_.append(ValType::F32)) {
+ return false;
+ }
+
+ ArgTypeVector args(funcType());
+ return fr.setupLocals(locals_, args, compilerEnv_.debugEnabled(),
+ &localInfo_);
+}
+
+FuncOffsets BaseCompiler::finish() {
+ MOZ_ASSERT(iter_.done(), "all bytes must be consumed");
+ MOZ_ASSERT(stk_.empty());
+ MOZ_ASSERT(stackMapGenerator_.memRefsOnStk == 0);
+
+ masm.flushBuffer();
+
+ return offsets_;
+}
+
+} // namespace wasm
+} // namespace js
+
+bool js::wasm::BaselinePlatformSupport() {
+#if defined(JS_CODEGEN_ARM)
+ // Simplifying assumption: require SDIV and UDIV.
+ //
+ // I have no good data on ARM populations allowing me to say that
+ // X% of devices in the market implement SDIV and UDIV. However,
+ // they are definitely implemented on the Cortex-A7 and Cortex-A15
+ // and on all ARMv8 systems.
+ if (!HasIDIV()) {
+ return false;
+ }
+#endif
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \
+ defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64)
+ return true;
+#else
+ return false;
+#endif
+}
+
+bool js::wasm::BaselineCompileFunctions(const ModuleEnvironment& moduleEnv,
+ const CompilerEnvironment& compilerEnv,
+ LifoAlloc& lifo,
+ const FuncCompileInputVector& inputs,
+ CompiledCode* code,
+ UniqueChars* error) {
+ MOZ_ASSERT(compilerEnv.tier() == Tier::Baseline);
+ MOZ_ASSERT(moduleEnv.kind == ModuleKind::Wasm);
+
+ // The MacroAssembler will sometimes access the jitContext.
+
+ TempAllocator alloc(&lifo);
+ JitContext jitContext;
+ MOZ_ASSERT(IsCompilingWasm());
+ WasmMacroAssembler masm(alloc, moduleEnv);
+
+ // Swap in already-allocated empty vectors to avoid malloc/free.
+ MOZ_ASSERT(code->empty());
+ if (!code->swap(masm)) {
+ return false;
+ }
+
+ // Create a description of the stack layout created by GenerateTrapExit().
+ RegisterOffsets trapExitLayout;
+ size_t trapExitLayoutNumWords;
+ GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords);
+
+ // The compiler's operand stack. We reuse it across all functions so as to
+ // avoid malloc/free. Presize it to 128 elements in the hope of avoiding
+ // reallocation later.
+ StkVector stk;
+ if (!stk.reserve(128)) {
+ return false;
+ }
+
+ for (const FuncCompileInput& func : inputs) {
+ Decoder d(func.begin, func.end, func.lineOrBytecode, error);
+
+ // Build the local types vector.
+
+ ValTypeVector locals;
+ if (!locals.appendAll(moduleEnv.funcs[func.index].type->args())) {
+ return false;
+ }
+ if (!DecodeLocalEntries(d, *moduleEnv.types, moduleEnv.features, &locals)) {
+ return false;
+ }
+
+ // One-pass baseline compilation.
+
+ BaseCompiler f(moduleEnv, compilerEnv, func, locals, trapExitLayout,
+ trapExitLayoutNumWords, d, stk, &alloc, &masm,
+ &code->stackMaps);
+ if (!f.init()) {
+ return false;
+ }
+ if (!f.emitFunction()) {
+ return false;
+ }
+ if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode,
+ f.finish())) {
+ return false;
+ }
+ }
+
+ masm.finish();
+ if (masm.oom()) {
+ return false;
+ }
+
+ return code->swap(masm);
+}