diff options
Diffstat (limited to 'js/src/wasm/WasmBaselineCompile.cpp')
| -rw-r--r-- | js/src/wasm/WasmBaselineCompile.cpp | 11009 |
1 files changed, 11009 insertions, 0 deletions
diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp new file mode 100644 index 0000000000..b277ca22c6 --- /dev/null +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -0,0 +1,11009 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * [SMDOC] WebAssembly baseline compiler (RabaldrMonkey) + * + * For now, see WasmBCClass.h for general comments about the compiler's + * structure. + * + * ---------------- + * + * General assumptions for 32-bit vs 64-bit code: + * + * - A 32-bit register can be extended in-place to a 64-bit register on 64-bit + * systems. + * + * - Code that knows that Register64 has a '.reg' member on 64-bit systems and + * '.high' and '.low' members on 32-bit systems, or knows the implications + * thereof, is #ifdef JS_PUNBOX64. All other code is #if(n)?def JS_64BIT. + * + * Coding standards are a little fluid: + * + * - In "small" code generating functions (eg emitMultiplyF64, emitQuotientI32, + * and surrounding functions; most functions fall into this class) where the + * meaning is obvious: + * + * Old school: + * - if there is a single source + destination register, it is called 'r' + * - if there is one source and a different destination, they are called 'rs' + * and 'rd' + * - if there is one source + destination register and another source register + * they are called 'r' and 'rs' + * - if there are two source registers and a destination register they are + * called 'rs0', 'rs1', and 'rd'. + * + * The new thing: + * - what is called 'r' in the old-school naming scheme is increasingly called + * 'rsd' in source+dest cases. + * + * - Generic temp registers are named /temp[0-9]?/ not /tmp[0-9]?/. + * + * - Registers can be named non-generically for their function ('rp' for the + * 'pointer' register and 'rv' for the 'value' register are typical) and those + * names may or may not have an 'r' prefix. + * + * - "Larger" code generating functions make their own rules. + */ + +#include "wasm/WasmBaselineCompile.h" + +#include "wasm/WasmBCClass.h" +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCFrame.h" +#include "wasm/WasmBCRegDefs.h" +#include "wasm/WasmBCStk.h" + +#include "jit/MacroAssembler-inl.h" +#include "wasm/WasmBCClass-inl.h" +#include "wasm/WasmBCCodegen-inl.h" +#include "wasm/WasmBCRegDefs-inl.h" +#include "wasm/WasmBCRegMgmt-inl.h" +#include "wasm/WasmBCStkMgmt-inl.h" + +namespace js { +namespace wasm { + +using namespace js::jit; + +//////////////////////////////////////////////////////////// +// +// Out of line code management. + +// The baseline compiler will use OOL code more sparingly than Ion since our +// code is not high performance and frills like code density and branch +// prediction friendliness will be less important. +class OutOfLineCode : public TempObject { + private: + NonAssertingLabel entry_; + NonAssertingLabel rejoin_; + StackHeight stackHeight_; + + public: + OutOfLineCode() : stackHeight_(StackHeight::Invalid()) {} + + Label* entry() { return &entry_; } + Label* rejoin() { return &rejoin_; } + + void setStackHeight(StackHeight stackHeight) { + MOZ_ASSERT(!stackHeight_.isValid()); + stackHeight_ = stackHeight; + } + + void bind(BaseStackFrame* fr, MacroAssembler* masm) { + MOZ_ASSERT(stackHeight_.isValid()); + masm->bind(&entry_); + fr->setStackHeight(stackHeight_); + } + + // The generate() method must be careful about register use because it will be + // invoked when there is a register assignment in the BaseCompiler that does + // not correspond to the available registers when the generated OOL code is + // executed. The register allocator *must not* be called. + // + // The best strategy is for the creator of the OOL object to allocate all + // temps that the OOL code will need. + // + // Input, output, and temp registers are embedded in the OOL object and are + // known to the code generator. + // + // Scratch registers are available to use in OOL code. + // + // All other registers must be explicitly saved and restored by the OOL code + // before being used. + + virtual void generate(MacroAssembler* masm) = 0; +}; + +OutOfLineCode* BaseCompiler::addOutOfLineCode(OutOfLineCode* ool) { + if (!ool || !outOfLine_.append(ool)) { + return nullptr; + } + ool->setStackHeight(fr.stackHeight()); + return ool; +} + +bool BaseCompiler::generateOutOfLineCode() { + for (auto* ool : outOfLine_) { + if (!ool->entry()->used()) { + continue; + } + ool->bind(&fr, &masm); + ool->generate(&masm); + } + + return !masm.oom(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Sundry code generation. + +bool BaseCompiler::addInterruptCheck() { +#ifdef RABALDR_PIN_INSTANCE + Register tmp(InstanceReg); +#else + ScratchI32 tmp(*this); + fr.loadInstancePtr(tmp); +#endif + Label ok; + masm.branch32(Assembler::Equal, + Address(tmp, wasm::Instance::offsetOfInterrupt()), Imm32(0), + &ok); + masm.wasmTrap(wasm::Trap::CheckInterrupt, bytecodeOffset()); + masm.bind(&ok); + return createStackMap("addInterruptCheck"); +} + +void BaseCompiler::checkDivideByZero(RegI32 rhs) { + Label nonZero; + masm.branchTest32(Assembler::NonZero, rhs, rhs, &nonZero); + trap(Trap::IntegerDivideByZero); + masm.bind(&nonZero); +} + +void BaseCompiler::checkDivideByZero(RegI64 r) { + Label nonZero; + ScratchI32 scratch(*this); + masm.branchTest64(Assembler::NonZero, r, r, scratch, &nonZero); + trap(Trap::IntegerDivideByZero); + masm.bind(&nonZero); +} + +void BaseCompiler::checkDivideSignedOverflow(RegI32 rhs, RegI32 srcDest, + Label* done, bool zeroOnOverflow) { + Label notMin; + masm.branch32(Assembler::NotEqual, srcDest, Imm32(INT32_MIN), ¬Min); + if (zeroOnOverflow) { + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Min); + moveImm32(0, srcDest); + masm.jump(done); + } else { + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Min); + trap(Trap::IntegerOverflow); + } + masm.bind(¬Min); +} + +void BaseCompiler::checkDivideSignedOverflow(RegI64 rhs, RegI64 srcDest, + Label* done, bool zeroOnOverflow) { + Label notmin; + masm.branch64(Assembler::NotEqual, srcDest, Imm64(INT64_MIN), ¬min); + masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), ¬min); + if (zeroOnOverflow) { + masm.xor64(srcDest, srcDest); + masm.jump(done); + } else { + trap(Trap::IntegerOverflow); + } + masm.bind(¬min); +} + +void BaseCompiler::jumpTable(const LabelVector& labels, Label* theTable) { + // Flush constant pools to ensure that the table is never interrupted by + // constant pool entries. + masm.flush(); + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + // Prevent nop sequences to appear in the jump table. + AutoForbidNops afn(&masm); +#endif + masm.bind(theTable); + + for (const auto& label : labels) { + CodeLabel cl; + masm.writeCodePointer(&cl); + cl.target()->bind(label.offset()); + masm.addCodeLabel(cl); + } +} + +void BaseCompiler::tableSwitch(Label* theTable, RegI32 switchValue, + Label* dispatchCode) { + masm.bind(dispatchCode); + +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + ScratchI32 scratch(*this); + CodeLabel tableCl; + + masm.mov(&tableCl, scratch); + + tableCl.target()->bind(theTable->offset()); + masm.addCodeLabel(tableCl); + + masm.jmp(Operand(scratch, switchValue, ScalePointer)); +#elif defined(JS_CODEGEN_ARM) + // Flush constant pools: offset must reflect the distance from the MOV + // to the start of the table; as the address of the MOV is given by the + // label, nothing must come between the bind() and the ma_mov(). + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 5); + + ScratchI32 scratch(*this); + + // Compute the offset from the ma_mov instruction to the jump table. + Label here; + masm.bind(&here); + uint32_t offset = here.offset() - theTable->offset(); + + // Read PC+8 + masm.ma_mov(pc, scratch); + + // ARM scratch register is required by ma_sub. + ScratchRegisterScope arm_scratch(*this); + + // Compute the absolute table base pointer into `scratch`, offset by 8 + // to account for the fact that ma_mov read PC+8. + masm.ma_sub(Imm32(offset + 8), scratch, arm_scratch); + + // Jump indirect via table element. + masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue, LSL, 2)), pc, Offset, + Assembler::Always); +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + ScratchI32 scratch(*this); + CodeLabel tableCl; + + masm.ma_li(scratch, &tableCl); + + tableCl.target()->bind(theTable->offset()); + masm.addCodeLabel(tableCl); + + masm.branchToComputedAddress(BaseIndex(scratch, switchValue, ScalePointer)); +#elif defined(JS_CODEGEN_ARM64) + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 4); + + ScratchI32 scratch(*this); + + ARMRegister s(scratch, 64); + ARMRegister v(switchValue, 64); + masm.Adr(s, theTable); + masm.Add(s, s, Operand(v, vixl::LSL, 3)); + masm.Ldr(s, MemOperand(s, 0)); + masm.Br(s); +#else + MOZ_CRASH("BaseCompiler platform hook: tableSwitch"); +#endif +} + +#ifdef JS_CODEGEN_X86 +void BaseCompiler::stashI64(RegPtr regForInstance, RegI64 r) { + MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8); + MOZ_ASSERT(regForInstance != r.low && regForInstance != r.high); +# ifdef RABALDR_PIN_INSTANCE +# error "Pinned instance not expected" +# endif + fr.loadInstancePtr(regForInstance); + masm.store32(r.low, + Address(regForInstance, Instance::offsetOfBaselineScratch())); + masm.store32( + r.high, Address(regForInstance, Instance::offsetOfBaselineScratch() + 4)); +} + +void BaseCompiler::unstashI64(RegPtr regForInstance, RegI64 r) { + MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8); +# ifdef RABALDR_PIN_INSTANCE +# error "Pinned instance not expected" +# endif + fr.loadInstancePtr(regForInstance); + if (regForInstance == r.low) { + masm.load32( + Address(regForInstance, Instance::offsetOfBaselineScratch() + 4), + r.high); + masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()), + r.low); + } else { + masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()), + r.low); + masm.load32( + Address(regForInstance, Instance::offsetOfBaselineScratch() + 4), + r.high); + } +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Function entry and exit + +bool BaseCompiler::beginFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::beginFunction"); + + JitSpew(JitSpew_Codegen, "# ========================================"); + JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code"); + JitSpew(JitSpew_Codegen, + "# beginFunction: start of function prologue for index %d", + (int)func_.index); + + // Make a start on the stackmap for this function. Inspect the args so + // as to determine which of them are both in-memory and pointer-typed, and + // add entries to machineStackTracker as appropriate. + + ArgTypeVector args(funcType()); + size_t inboundStackArgBytes = StackArgAreaSizeUnaligned(args); + MOZ_ASSERT(inboundStackArgBytes % sizeof(void*) == 0); + stackMapGenerator_.numStackArgWords = inboundStackArgBytes / sizeof(void*); + + MOZ_ASSERT(stackMapGenerator_.machineStackTracker.length() == 0); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + stackMapGenerator_.numStackArgWords)) { + return false; + } + + // Identify GC-managed pointers passed on the stack. + for (WasmABIArgIter i(args); !i.done(); i++) { + ABIArg argLoc = *i; + if (argLoc.kind() == ABIArg::Stack && + args[i.index()] == MIRType::RefOrNull) { + uint32_t offset = argLoc.offsetFromArgBase(); + MOZ_ASSERT(offset < inboundStackArgBytes); + MOZ_ASSERT(offset % sizeof(void*) == 0); + stackMapGenerator_.machineStackTracker.setGCPointer(offset / + sizeof(void*)); + } + } + + GenerateFunctionPrologue( + masm, CallIndirectId::forFunc(moduleEnv_, func_.index), + compilerEnv_.mode() == CompileMode::Tier1 ? Some(func_.index) : Nothing(), + &offsets_); + + // GenerateFunctionPrologue pushes exactly one wasm::Frame's worth of + // stuff, and none of the values are GC pointers. Hence: + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + sizeof(Frame) / sizeof(void*))) { + return false; + } + + // Initialize DebugFrame fields before the stack overflow trap so that + // we have the invariant that all observable Frames in a debugEnabled + // Module have valid DebugFrames. + if (compilerEnv_.debugEnabled()) { +#ifdef JS_CODEGEN_ARM64 + static_assert(DebugFrame::offsetOfFrame() % WasmStackAlignment == 0, + "aligned"); +#endif + masm.reserveStack(DebugFrame::offsetOfFrame()); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + DebugFrame::offsetOfFrame() / sizeof(void*))) { + return false; + } + + masm.store32(Imm32(func_.index), Address(masm.getStackPointer(), + DebugFrame::offsetOfFuncIndex())); + masm.store32(Imm32(0), + Address(masm.getStackPointer(), DebugFrame::offsetOfFlags())); + + // No need to initialize cachedReturnJSValue_ or any ref-typed spilled + // register results, as they are traced if and only if a corresponding + // flag (hasCachedReturnJSValue or hasSpilledRefRegisterResult) is set. + } + + // Generate a stack-overflow check and its associated stackmap. + + fr.checkStack(ABINonArgReg0, BytecodeOffset(func_.lineOrBytecode)); + + ExitStubMapVector extras; + if (!stackMapGenerator_.generateStackmapEntriesForTrapExit(args, &extras)) { + return false; + } + if (!createStackMap("stack check", extras, masm.currentOffset(), + HasDebugFrameWithLiveRefs::No)) { + return false; + } + + size_t reservedBytes = fr.fixedAllocSize() - masm.framePushed(); + MOZ_ASSERT(0 == (reservedBytes % sizeof(void*))); + + masm.reserveStack(reservedBytes); + fr.onFixedStackAllocated(); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + reservedBytes / sizeof(void*))) { + return false; + } + + // Locals are stack allocated. Mark ref-typed ones in the stackmap + // accordingly. + for (const Local& l : localInfo_) { + // Locals that are stack arguments were already added to the stackmap + // before pushing the frame. + if (l.type == MIRType::RefOrNull && !l.isStackArgument()) { + uint32_t offs = fr.localOffsetFromSp(l); + MOZ_ASSERT(0 == (offs % sizeof(void*))); + stackMapGenerator_.machineStackTracker.setGCPointer(offs / sizeof(void*)); + } + } + + // Copy arguments from registers to stack. + for (WasmABIArgIter i(args); !i.done(); i++) { + if (args.isSyntheticStackResultPointerArg(i.index())) { + // If there are stack results and the pointer to stack results + // was passed in a register, store it to the stack. + if (i->argInRegister()) { + fr.storeIncomingStackResultAreaPtr(RegPtr(i->gpr())); + } + // If we're in a debug frame, copy the stack result pointer arg + // to a well-known place. + if (compilerEnv_.debugEnabled()) { + Register target = ABINonArgReturnReg0; + fr.loadIncomingStackResultAreaPtr(RegPtr(target)); + size_t debugFrameOffset = + masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t debugStackResultsPointerOffset = + debugFrameOffset + DebugFrame::offsetOfStackResultsPointer(); + masm.storePtr(target, Address(masm.getStackPointer(), + debugStackResultsPointerOffset)); + } + continue; + } + if (!i->argInRegister()) { + continue; + } + Local& l = localInfo_[args.naturalIndex(i.index())]; + switch (i.mirType()) { + case MIRType::Int32: + fr.storeLocalI32(RegI32(i->gpr()), l); + break; + case MIRType::Int64: + fr.storeLocalI64(RegI64(i->gpr64()), l); + break; + case MIRType::RefOrNull: { + DebugOnly<uint32_t> offs = fr.localOffsetFromSp(l); + MOZ_ASSERT(0 == (offs % sizeof(void*))); + fr.storeLocalRef(RegRef(i->gpr()), l); + // We should have just visited this local in the preceding loop. + MOZ_ASSERT(stackMapGenerator_.machineStackTracker.isGCPointer( + offs / sizeof(void*))); + break; + } + case MIRType::Double: + fr.storeLocalF64(RegF64(i->fpu()), l); + break; + case MIRType::Float32: + fr.storeLocalF32(RegF32(i->fpu()), l); + break; +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: + fr.storeLocalV128(RegV128(i->fpu()), l); + break; +#endif + default: + MOZ_CRASH("Function argument type"); + } + } + + fr.zeroLocals(&ra); + fr.storeInstancePtr(InstanceReg); + + if (compilerEnv_.debugEnabled()) { + insertBreakablePoint(CallSiteDesc::EnterFrame); + if (!createStackMap("debug: enter-frame breakpoint")) { + return false; + } + } + + JitSpew(JitSpew_Codegen, + "# beginFunction: enter body with masm.framePushed = %u", + masm.framePushed()); + MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isNothing()); + stackMapGenerator_.framePushedAtEntryToBody.emplace(masm.framePushed()); + + return true; +} + +bool BaseCompiler::endFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::endFunction"); + + JitSpew(JitSpew_Codegen, "# endFunction: start of function epilogue"); + + // Always branch to returnLabel_. + masm.breakpoint(); + + // Patch the add in the prologue so that it checks against the correct + // frame size. Flush the constant pool in case it needs to be patched. + masm.flush(); + + // Precondition for patching. + if (masm.oom()) { + return false; + } + + fr.patchCheckStack(); + + masm.bind(&returnLabel_); + + ResultType resultType(ResultType::Vector(funcType().results())); + + popStackReturnValues(resultType); + + if (compilerEnv_.debugEnabled()) { + // Store and reload the return value from DebugFrame::return so that + // it can be clobbered, and/or modified by the debug trap. + saveRegisterReturnValues(resultType); + insertBreakablePoint(CallSiteDesc::Breakpoint); + if (!createStackMap("debug: return-point breakpoint", + HasDebugFrameWithLiveRefs::Maybe)) { + return false; + } + insertBreakablePoint(CallSiteDesc::LeaveFrame); + if (!createStackMap("debug: leave-frame breakpoint", + HasDebugFrameWithLiveRefs::Maybe)) { + return false; + } + restoreRegisterReturnValues(resultType); + } + +#ifndef RABALDR_PIN_INSTANCE + // To satisy instance extent invariant we need to reload InstanceReg because + // baseline can clobber it. + fr.loadInstancePtr(InstanceReg); +#endif + GenerateFunctionEpilogue(masm, fr.fixedAllocSize(), &offsets_); + +#if defined(JS_ION_PERF) + // FIXME - profiling code missing. No bug for this. + + // Note the end of the inline code and start of the OOL code. + // gen->perfSpewer().noteEndInlineCode(masm); +#endif + + JitSpew(JitSpew_Codegen, "# endFunction: end of function epilogue"); + JitSpew(JitSpew_Codegen, "# endFunction: start of OOL code"); + if (!generateOutOfLineCode()) { + return false; + } + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code"); + + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code"); + if (compilerEnv_.debugEnabled()) { + JitSpew(JitSpew_Codegen, "# endFunction: start of debug trap stub"); + insertBreakpointStub(); + JitSpew(JitSpew_Codegen, "# endFunction: end of debug trap stub"); + } + + offsets_.end = masm.currentOffset(); + + if (!fr.checkStackHeight()) { + return decoder_.fail(decoder_.beginOffset(), "stack frame is too large"); + } + + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code for index %d", + (int)func_.index); + return !masm.oom(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Debugger API. + +void BaseCompiler::insertBreakablePoint(CallSiteDesc::Kind kind) { +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(InstanceReg); +#endif + + // The breakpoint code must call the breakpoint handler installed on the + // instance if it is not null. There is one breakable point before + // every bytecode, and one at the beginning and at the end of the function. + // + // There are many constraints: + // + // - Code should be read-only; we do not want to patch + // - The breakpoint code should be as dense as possible, given the volume of + // breakable points + // - The handler-is-null case should be as fast as we can make it + // + // The scratch register is available here. + // + // An unconditional callout would be densest but is too slow. The best + // balance results from an inline test for null with a conditional call. The + // best code sequence is platform-dependent. + // + // The conditional call goes to a stub attached to the function that performs + // further filtering before calling the breakpoint handler. +#if defined(JS_CODEGEN_X64) + // REX 83 MODRM OFFS IB + static_assert(Instance::offsetOfDebugTrapHandler() < 128); + masm.cmpq(Imm32(0), Operand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + + // 74 OFFS + Label L; + L.bind(masm.currentOffset() + 7); + masm.j(Assembler::Zero, &L); + + // E8 OFFS OFFS OFFS OFFS + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + + // Branch destination + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset())); +#elif defined(JS_CODEGEN_X86) + // 83 MODRM OFFS IB + static_assert(Instance::offsetOfDebugTrapHandler() < 128); + masm.cmpl(Imm32(0), Operand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + + // 74 OFFS + Label L; + L.bind(masm.currentOffset() + 7); + masm.j(Assembler::Zero, &L); + + // E8 OFFS OFFS OFFS OFFS + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + + // Branch destination + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset())); +#elif defined(JS_CODEGEN_ARM64) + ScratchPtr scratch(*this); + ARMRegister tmp(scratch, 64); + Label L; + masm.Ldr(tmp, MemOperand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + masm.Cbz(tmp, &L); + masm.Bl(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + masm.bind(&L); +#elif defined(JS_CODEGEN_ARM) + ScratchPtr scratch(*this); + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()), + scratch); + masm.ma_orr(scratch, scratch, SetCC); + masm.ma_bl(&debugTrapStub_, Assembler::NonZero); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); +#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_RISCV64) + ScratchPtr scratch(*this); + Label L; + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()), + scratch); + masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), &L); + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + masm.bind(&L); +#else + MOZ_CRASH("BaseCompiler platform hook: insertBreakablePoint"); +#endif +} + +void BaseCompiler::insertBreakpointStub() { + // The debug trap stub performs out-of-line filtering before jumping to the + // debug trap handler if necessary. The trap handler returns directly to + // the breakable point. + // + // NOTE, the link register is live here on platforms that have LR. + // + // The scratch register is available here (as it was at the call site). + // + // It's useful for the debug trap stub to be compact, as every function gets + // one. + + Label L; + masm.bind(&debugTrapStub_); + +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + { + ScratchPtr scratch(*this); + + // Get the per-instance table of filtering bits. + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + + // Check the filter bit. There is one bit per function in the module. + // Table elements are 32-bit because the masm makes that convenient. + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + + // Fast path: return to the execution. + masm.ret(); + } +#elif defined(JS_CODEGEN_ARM64) + { + ScratchPtr scratch(*this); + + // Logic as above, except abiret to jump to the LR directly + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + masm.abiret(); + } +#elif defined(JS_CODEGEN_ARM) + { + // We must be careful not to use the SecondScratchRegister, which usually + // is LR, as LR is live here. This means avoiding masm abstractions such + // as branchTest32. + + static_assert(ScratchRegister != lr); + static_assert(Instance::offsetOfDebugFilter() < 0x1000); + + ScratchRegisterScope tmp1(masm); + ScratchI32 tmp2(*this); + masm.ma_ldr( + DTRAddr(InstanceReg, DtrOffImm(Instance::offsetOfDebugFilter())), tmp1); + masm.ma_mov(Imm32(func_.index / 32), tmp2); + masm.ma_ldr(DTRAddr(tmp1, DtrRegImmShift(tmp2, LSL, 0)), tmp2); + masm.ma_tst(tmp2, Imm32(1 << func_.index % 32), tmp1, Assembler::Always); + masm.ma_bx(lr, Assembler::Zero); + } +#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_RISCV64) + { + ScratchPtr scratch(*this); + + // Logic same as ARM64. + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + masm.abiret(); + } +#else + MOZ_CRASH("BaseCompiler platform hook: endFunction"); +#endif + + // Jump to the debug trap handler. + masm.bind(&L); + masm.jump(Address(InstanceReg, Instance::offsetOfDebugTrapHandler())); +} + +void BaseCompiler::saveRegisterReturnValues(const ResultType& resultType) { + MOZ_ASSERT(compilerEnv_.debugEnabled()); + size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t registerResultIdx = 0; + for (ABIResultIter i(resultType); !i.done(); i.next()) { + const ABIResult result = i.cur(); + if (!result.inRegister()) { +#ifdef DEBUG + for (i.next(); !i.done(); i.next()) { + MOZ_ASSERT(!i.cur().inRegister()); + } +#endif + break; + } + + size_t resultOffset = DebugFrame::offsetOfRegisterResult(registerResultIdx); + Address dest(masm.getStackPointer(), debugFrameOffset + resultOffset); + switch (result.type().kind()) { + case ValType::I32: + masm.store32(RegI32(result.gpr()), dest); + break; + case ValType::I64: + masm.store64(RegI64(result.gpr64()), dest); + break; + case ValType::F64: + masm.storeDouble(RegF64(result.fpr()), dest); + break; + case ValType::F32: + masm.storeFloat32(RegF32(result.fpr()), dest); + break; + case ValType::Ref: { + uint32_t flag = + DebugFrame::hasSpilledRegisterRefResultBitMask(registerResultIdx); + // Tell Instance::traceFrame that we have a pointer to trace. + masm.or32(Imm32(flag), + Address(masm.getStackPointer(), + debugFrameOffset + DebugFrame::offsetOfFlags())); + masm.storePtr(RegRef(result.gpr()), dest); + break; + } + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + masm.storeUnalignedSimd128(RegV128(result.fpr()), dest); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + registerResultIdx++; + } +} + +void BaseCompiler::restoreRegisterReturnValues(const ResultType& resultType) { + MOZ_ASSERT(compilerEnv_.debugEnabled()); + size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t registerResultIdx = 0; + for (ABIResultIter i(resultType); !i.done(); i.next()) { + const ABIResult result = i.cur(); + if (!result.inRegister()) { +#ifdef DEBUG + for (i.next(); !i.done(); i.next()) { + MOZ_ASSERT(!i.cur().inRegister()); + } +#endif + break; + } + size_t resultOffset = + DebugFrame::offsetOfRegisterResult(registerResultIdx++); + Address src(masm.getStackPointer(), debugFrameOffset + resultOffset); + switch (result.type().kind()) { + case ValType::I32: + masm.load32(src, RegI32(result.gpr())); + break; + case ValType::I64: + masm.load64(src, RegI64(result.gpr64())); + break; + case ValType::F64: + masm.loadDouble(src, RegF64(result.fpr())); + break; + case ValType::F32: + masm.loadFloat32(src, RegF32(result.fpr())); + break; + case ValType::Ref: + masm.loadPtr(src, RegRef(result.gpr())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + masm.loadUnalignedSimd128(src, RegV128(result.fpr())); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + } +} + +////////////////////////////////////////////////////////////////////////////// +// +// Results and block parameters + +void BaseCompiler::popStackReturnValues(const ResultType& resultType) { + uint32_t bytes = ABIResultIter::MeasureStackBytes(resultType); + if (bytes == 0) { + return; + } + Register target = ABINonArgReturnReg0; + Register temp = ABINonArgReturnReg1; + fr.loadIncomingStackResultAreaPtr(RegPtr(target)); + fr.popStackResultsToMemory(target, bytes, temp); +} + +// TODO / OPTIMIZE (Bug 1316818): At the moment we use the Wasm +// inter-procedure ABI for block returns, which allocates ReturnReg as the +// single block result register. It is possible other choices would lead to +// better register allocation, as ReturnReg is often first in the register set +// and will be heavily wanted by the register allocator that uses takeFirst(). +// +// Obvious options: +// - pick a register at the back of the register set +// - pick a random register per block (different blocks have +// different join regs) + +void BaseCompiler::popRegisterResults(ABIResultIter& iter) { + // Pop register results. Note that in the single-value case, popping to a + // register may cause a sync(); for multi-value we sync'd already. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + if (!result.inRegister()) { + // TODO / OPTIMIZE: We sync here to avoid solving the general parallel + // move problem in popStackResults. However we could avoid syncing the + // values that are going to registers anyway, if they are already in + // registers. + sync(); + break; + } + switch (result.type().kind()) { + case ValType::I32: + popI32(RegI32(result.gpr())); + break; + case ValType::I64: + popI64(RegI64(result.gpr64())); + break; + case ValType::F32: + popF32(RegF32(result.fpr())); + break; + case ValType::F64: + popF64(RegF64(result.fpr())); + break; + case ValType::Ref: + popRef(RegRef(result.gpr())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + popV128(RegV128(result.fpr())); +#else + MOZ_CRASH("No SIMD support"); +#endif + } + } +} + +void BaseCompiler::popStackResults(ABIResultIter& iter, StackHeight stackBase) { + MOZ_ASSERT(!iter.done()); + + // The iterator should be advanced beyond register results, and register + // results should be popped already from the value stack. + uint32_t alreadyPopped = iter.index(); + + // At this point, only stack arguments are remaining. Iterate through them + // to measure how much stack space they will take up. + for (; !iter.done(); iter.next()) { + MOZ_ASSERT(iter.cur().onStack()); + } + + // Calculate the space needed to store stack results, in bytes. + uint32_t stackResultBytes = iter.stackBytesConsumedSoFar(); + MOZ_ASSERT(stackResultBytes); + + // Compute the stack height including the stack results. Note that it's + // possible that this call expands the stack, for example if some of the + // results are supplied by constants and so are not already on the machine + // stack. + uint32_t endHeight = fr.prepareStackResultArea(stackBase, stackResultBytes); + + // Find a free GPR to use when shuffling stack values. If none is + // available, push ReturnReg and restore it after we're done. + bool saved = false; + RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved); + + // The sequence of Stk values is in the same order on the machine stack as + // the result locations, but there is a complication: constant values are + // not actually pushed on the machine stack. (At this point registers and + // locals have been spilled already.) So, moving the Stk values into place + // isn't simply a shuffle-down or shuffle-up operation. There is a part of + // the Stk sequence that shuffles toward the FP, a part that's already in + // place, and a part that shuffles toward the SP. After shuffling, we have + // to materialize the constants. + + // Shuffle mem values toward the frame pointer, copying deepest values + // first. Stop when we run out of results, get to a register result, or + // find a Stk value that is closer to the FP than the result. + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + if (!result.onStack()) { + break; + } + MOZ_ASSERT(result.stackOffset() < stackResultBytes); + uint32_t destHeight = endHeight - result.stackOffset(); + uint32_t stkBase = stk_.length() - (iter.count() - alreadyPopped); + Stk& v = stk_[stkBase + iter.index()]; + if (v.isMem()) { + uint32_t srcHeight = v.offs(); + if (srcHeight <= destHeight) { + break; + } + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, result.size(), + temp); + } + } + + // Reset iterator and skip register results. + for (iter.reset(); !iter.done(); iter.next()) { + if (iter.cur().onStack()) { + break; + } + } + + // Revisit top stack values, shuffling mem values toward the stack pointer, + // copying shallowest values first. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + MOZ_ASSERT(result.onStack()); + MOZ_ASSERT(result.stackOffset() < stackResultBytes); + uint32_t destHeight = endHeight - result.stackOffset(); + Stk& v = stk_[stk_.length() - (iter.index() - alreadyPopped) - 1]; + if (v.isMem()) { + uint32_t srcHeight = v.offs(); + if (srcHeight >= destHeight) { + break; + } + fr.shuffleStackResultsTowardSP(srcHeight, destHeight, result.size(), + temp); + } + } + + // Reset iterator and skip register results, which are already popped off + // the value stack. + for (iter.reset(); !iter.done(); iter.next()) { + if (iter.cur().onStack()) { + break; + } + } + + // Materialize constants and pop the remaining items from the value stack. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + uint32_t resultHeight = endHeight - result.stackOffset(); + Stk& v = stk_.back(); + switch (v.kind()) { + case Stk::ConstI32: +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + fr.storeImmediatePtrToStack(v.i32val_, resultHeight, temp); +#else + fr.storeImmediatePtrToStack(uint32_t(v.i32val_), resultHeight, temp); +#endif + break; + case Stk::ConstF32: + fr.storeImmediateF32ToStack(v.f32val_, resultHeight, temp); + break; + case Stk::ConstI64: + fr.storeImmediateI64ToStack(v.i64val_, resultHeight, temp); + break; + case Stk::ConstF64: + fr.storeImmediateF64ToStack(v.f64val_, resultHeight, temp); + break; +#ifdef ENABLE_WASM_SIMD + case Stk::ConstV128: + fr.storeImmediateV128ToStack(v.v128val_, resultHeight, temp); + break; +#endif + case Stk::ConstRef: + fr.storeImmediatePtrToStack(v.refval_, resultHeight, temp); + break; + case Stk::MemRef: + // Update bookkeeping as we pop the Stk entry. + stackMapGenerator_.memRefsOnStk--; + break; + default: + MOZ_ASSERT(v.isMem()); + break; + } + stk_.popBack(); + } + + ra.freeTempPtr(temp, saved); + + // This will pop the stack if needed. + fr.finishStackResultArea(stackBase, stackResultBytes); +} + +void BaseCompiler::popBlockResults(ResultType type, StackHeight stackBase, + ContinuationKind kind) { + if (!type.empty()) { + ABIResultIter iter(type); + popRegisterResults(iter); + if (!iter.done()) { + popStackResults(iter, stackBase); + // Because popStackResults might clobber the stack, it leaves the stack + // pointer already in the right place for the continuation, whether the + // continuation is a jump or fallthrough. + return; + } + } + // We get here if there are no stack results. For a fallthrough, the stack + // is already at the right height. For a jump, we may need to pop the stack + // pointer if the continuation's stack height is lower than the current + // stack height. + if (kind == ContinuationKind::Jump) { + fr.popStackBeforeBranch(stackBase, type); + } +} + +// This function is similar to popBlockResults, but additionally handles the +// implicit exception pointer that is pushed to the value stack on entry to +// a catch handler by dropping it appropriately. +void BaseCompiler::popCatchResults(ResultType type, StackHeight stackBase) { + if (!type.empty()) { + ABIResultIter iter(type); + popRegisterResults(iter); + if (!iter.done()) { + popStackResults(iter, stackBase); + // Since popStackResults clobbers the stack, we only need to free the + // exception off of the value stack. + popValueStackBy(1); + } else { + // If there are no stack results, we have to adjust the stack by + // dropping the exception reference that's now on the stack. + dropValue(); + } + } else { + dropValue(); + } + fr.popStackBeforeBranch(stackBase, type); +} + +Stk BaseCompiler::captureStackResult(const ABIResult& result, + StackHeight resultsBase, + uint32_t stackResultBytes) { + MOZ_ASSERT(result.onStack()); + uint32_t offs = fr.locateStackResult(result, resultsBase, stackResultBytes); + return Stk::StackResult(result.type(), offs); +} + +// TODO: It may be fruitful to inline the fast path here, as it will be common. + +bool BaseCompiler::pushResults(ResultType type, StackHeight resultsBase) { + if (type.empty()) { + return true; + } + + if (type.length() > 1) { + // Reserve extra space on the stack for all the values we'll push. + // Multi-value push is not accounted for by the pre-sizing of the stack in + // the decoding loop. + // + // Also make sure we leave headroom for other pushes that will occur after + // pushing results, just to be safe. + if (!stk_.reserve(stk_.length() + type.length() + MaxPushesPerOpcode)) { + return false; + } + } + + // We need to push the results in reverse order, so first iterate through + // all results to determine the locations of stack result types. + ABIResultIter iter(type); + while (!iter.done()) { + iter.next(); + } + uint32_t stackResultBytes = iter.stackBytesConsumedSoFar(); + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + if (!result.onStack()) { + break; + } + Stk v = captureStackResult(result, resultsBase, stackResultBytes); + push(v); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk++; + } + } + + for (; !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + MOZ_ASSERT(result.inRegister()); + switch (result.type().kind()) { + case ValType::I32: + pushI32(RegI32(result.gpr())); + break; + case ValType::I64: + pushI64(RegI64(result.gpr64())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + pushV128(RegV128(result.fpr())); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F32: + pushF32(RegF32(result.fpr())); + break; + case ValType::F64: + pushF64(RegF64(result.fpr())); + break; + case ValType::Ref: + pushRef(RegRef(result.gpr())); + break; + } + } + + return true; +} + +bool BaseCompiler::pushBlockResults(ResultType type) { + return pushResults(type, controlItem().stackHeight); +} + +// A combination of popBlockResults + pushBlockResults, used when entering a +// block with a control-flow join (loops) or split (if) to shuffle the +// fallthrough block parameters into the locations expected by the +// continuation. +bool BaseCompiler::topBlockParams(ResultType type) { + // This function should only be called when entering a block with a + // control-flow join at the entry, where there are no live temporaries in + // the current block. + StackHeight base = controlItem().stackHeight; + MOZ_ASSERT(fr.stackResultsBase(stackConsumed(type.length())) == base); + popBlockResults(type, base, ContinuationKind::Fallthrough); + return pushBlockResults(type); +} + +// A combination of popBlockResults + pushBlockResults, used before branches +// where we don't know the target (br_if / br_table). If and when the branch +// is taken, the stack results will be shuffled down into place. For br_if +// that has fallthrough, the parameters for the untaken branch flow through to +// the continuation. +bool BaseCompiler::topBranchParams(ResultType type, StackHeight* height) { + if (type.empty()) { + *height = fr.stackHeight(); + return true; + } + // There may be temporary values that need spilling; delay computation of + // the stack results base until after the popRegisterResults(), which spills + // if needed. + ABIResultIter iter(type); + popRegisterResults(iter); + StackHeight base = fr.stackResultsBase(stackConsumed(iter.remaining())); + if (!iter.done()) { + popStackResults(iter, base); + } + if (!pushResults(type, base)) { + return false; + } + *height = base; + return true; +} + +// Conditional branches with fallthrough are preceded by a topBranchParams, so +// we know that there are no stack results that need to be materialized. In +// that case, we can just shuffle the whole block down before popping the +// stack. +void BaseCompiler::shuffleStackResultsBeforeBranch(StackHeight srcHeight, + StackHeight destHeight, + ResultType type) { + uint32_t stackResultBytes = 0; + + if (ABIResultIter::HasStackResults(type)) { + MOZ_ASSERT(stk_.length() >= type.length()); + ABIResultIter iter(type); + for (; !iter.done(); iter.next()) { +#ifdef DEBUG + const ABIResult& result = iter.cur(); + const Stk& v = stk_[stk_.length() - iter.index() - 1]; + MOZ_ASSERT(v.isMem() == result.onStack()); +#endif + } + + stackResultBytes = iter.stackBytesConsumedSoFar(); + MOZ_ASSERT(stackResultBytes > 0); + + if (srcHeight != destHeight) { + // Find a free GPR to use when shuffling stack values. If none + // is available, push ReturnReg and restore it after we're done. + bool saved = false; + RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved); + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, stackResultBytes, + temp); + ra.freeTempPtr(temp, saved); + } + } + + fr.popStackBeforeBranch(destHeight, stackResultBytes); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Function calls. + +void BaseCompiler::beginCall( + FunctionCall& call, UseABI useABI, + RestoreRegisterStateAndRealm restoreRegisterStateAndRealm) { + MOZ_ASSERT_IF( + useABI == UseABI::Builtin, + restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::False); + + call.restoreRegisterStateAndRealm = + restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::True; + call.usesSystemAbi = useABI == UseABI::System; + + if (call.usesSystemAbi) { + // Call-outs need to use the appropriate system ABI. +#if defined(JS_CODEGEN_ARM) + call.hardFP = UseHardFpABI(); + call.abi.setUseHardFp(call.hardFP); +#endif + } else { +#if defined(JS_CODEGEN_ARM) + MOZ_ASSERT(call.hardFP, "All private ABIs pass FP arguments in registers"); +#endif + } + + // Use masm.framePushed() because the value we want here does not depend + // on the height of the frame's stack area, but the actual size of the + // allocated frame. + call.frameAlignAdjustment = ComputeByteAlignment( + masm.framePushed() + sizeof(Frame), JitStackAlignment); +} + +void BaseCompiler::endCall(FunctionCall& call, size_t stackSpace) { + size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment; + fr.freeArgAreaAndPopBytes(adjustment, stackSpace); + + MOZ_ASSERT(stackMapGenerator_.framePushedExcludingOutboundCallArgs.isSome()); + stackMapGenerator_.framePushedExcludingOutboundCallArgs.reset(); + + if (call.restoreRegisterStateAndRealm) { + // The instance has been clobbered, so always reload + fr.loadInstancePtr(InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); + masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1); + } else if (call.usesSystemAbi) { + // On x86 there are no pinned registers, so don't waste time + // reloading the instance. +#ifndef JS_CODEGEN_X86 + // The instance has been clobbered, so always reload + fr.loadInstancePtr(InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); +#endif + } +} + +void BaseCompiler::startCallArgs(size_t stackArgAreaSizeUnaligned, + FunctionCall* call) { + size_t stackArgAreaSizeAligned = + AlignStackArgAreaSize(stackArgAreaSizeUnaligned); + MOZ_ASSERT(stackArgAreaSizeUnaligned <= stackArgAreaSizeAligned); + + // Record the masm.framePushed() value at this point, before we push args + // for the call, but including the alignment space placed above the args. + // This defines the lower limit of the stackmap that will be created for + // this call. + MOZ_ASSERT( + stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing()); + stackMapGenerator_.framePushedExcludingOutboundCallArgs.emplace( + // However much we've pushed so far + masm.framePushed() + + // Extra space we'll push to get the frame aligned + call->frameAlignAdjustment + + // Extra space we'll push to get the outbound arg area aligned + (stackArgAreaSizeAligned - stackArgAreaSizeUnaligned)); + + call->stackArgAreaSize = stackArgAreaSizeAligned; + + size_t adjustment = call->stackArgAreaSize + call->frameAlignAdjustment; + fr.allocArgArea(adjustment); +} + +ABIArg BaseCompiler::reservePointerArgument(FunctionCall* call) { + return call->abi.next(MIRType::Pointer); +} + +// TODO / OPTIMIZE (Bug 1316821): Note passArg is used only in one place. +// (Or it was, until Luke wandered through, but that can be fixed again.) +// I'm not saying we should manually inline it, but we could hoist the +// dispatch into the caller and have type-specific implementations of +// passArg: passArgI32(), etc. Then those might be inlined, at least in PGO +// builds. +// +// The bulk of the work here (60%) is in the next() call, though. +// +// Notably, since next() is so expensive, StackArgAreaSizeUnaligned() +// becomes expensive too. +// +// Somehow there could be a trick here where the sequence of argument types +// (read from the input stream) leads to a cached entry for +// StackArgAreaSizeUnaligned() and for how to pass arguments... +// +// But at least we could reduce the cost of StackArgAreaSizeUnaligned() by +// first reading the argument types into a (reusable) vector, then we have +// the outgoing size at low cost, and then we can pass args based on the +// info we read. + +void BaseCompiler::passArg(ValType type, const Stk& arg, FunctionCall* call) { + switch (type.kind()) { + case ValType::I32: { + ABIArg argLoc = call->abi.next(MIRType::Int32); + if (argLoc.kind() == ABIArg::Stack) { + ScratchI32 scratch(*this); + loadI32(arg, scratch); + masm.store32(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + loadI32(arg, RegI32(argLoc.gpr())); + } + break; + } + case ValType::I64: { + ABIArg argLoc = call->abi.next(MIRType::Int64); + if (argLoc.kind() == ABIArg::Stack) { + ScratchI32 scratch(*this); +#ifdef JS_PUNBOX64 + loadI64(arg, fromI32(scratch)); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); +#else + loadI64Low(arg, scratch); + masm.store32(scratch, LowWord(Address(masm.getStackPointer(), + argLoc.offsetFromArgBase()))); + loadI64High(arg, scratch); + masm.store32(scratch, HighWord(Address(masm.getStackPointer(), + argLoc.offsetFromArgBase()))); +#endif + } else { + loadI64(arg, RegI64(argLoc.gpr64())); + } + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + ABIArg argLoc = call->abi.next(MIRType::Simd128); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchV128 scratch(*this); + loadV128(arg, scratch); + masm.storeUnalignedSimd128( + (RegV128)scratch, + Address(masm.getStackPointer(), argLoc.offsetFromArgBase())); + break; + } + case ABIArg::GPR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } + case ABIArg::FPU: { + loadV128(arg, RegV128(argLoc.fpu())); + break; + } +# if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } +# endif + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::F64: { + ABIArg argLoc = call->abi.next(MIRType::Double); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchF64 scratch(*this); + loadF64(arg, scratch); + masm.storeDouble(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + break; + } +#if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { +# if defined(JS_CODEGEN_ARM) + ScratchF64 scratch(*this); + loadF64(arg, scratch); + masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr()); + break; +# else + MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair"); +# endif + } +#endif + case ABIArg::FPU: { + loadF64(arg, RegF64(argLoc.fpu())); + break; + } + case ABIArg::GPR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; + } + case ValType::F32: { + ABIArg argLoc = call->abi.next(MIRType::Float32); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchF32 scratch(*this); + loadF32(arg, scratch); + masm.storeFloat32(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + break; + } + case ABIArg::GPR: { + ScratchF32 scratch(*this); + loadF32(arg, scratch); + masm.moveFloat32ToGPR(scratch, argLoc.gpr()); + break; + } + case ABIArg::FPU: { + loadF32(arg, RegF32(argLoc.fpu())); + break; + } +#if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } +#endif + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; + } + case ValType::Ref: { + ABIArg argLoc = call->abi.next(MIRType::RefOrNull); + if (argLoc.kind() == ABIArg::Stack) { + ScratchRef scratch(*this); + loadRef(arg, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + loadRef(arg, RegRef(argLoc.gpr())); + } + break; + } + } +} + +CodeOffset BaseCompiler::callDefinition(uint32_t funcIndex, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Func); + return masm.call(desc, funcIndex); +} + +CodeOffset BaseCompiler::callSymbolic(SymbolicAddress callee, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic); + return masm.call(desc, callee); +} + +// Precondition: sync() + +class OutOfLineAbortingTrap : public OutOfLineCode { + Trap trap_; + BytecodeOffset off_; + + public: + OutOfLineAbortingTrap(Trap trap, BytecodeOffset off) + : trap_(trap), off_(off) {} + + virtual void generate(MacroAssembler* masm) override { + masm->wasmTrap(trap_, off_); + MOZ_ASSERT(!rejoin()->bound()); + } +}; + +bool BaseCompiler::callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex, + const Stk& indexVal, const FunctionCall& call, + CodeOffset* fastCallOffset, + CodeOffset* slowCallOffset) { + CallIndirectId callIndirectId = + CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex); + MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS); + + const TableDesc& table = moduleEnv_.tables[tableIndex]; + + loadI32(indexVal, RegI32(WasmTableCallIndexReg)); + + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Indirect); + CalleeDesc callee = + CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId); + OutOfLineCode* oob = addOutOfLineCode( + new (alloc_) OutOfLineAbortingTrap(Trap::OutOfBounds, bytecodeOffset())); + if (!oob) { + return false; + } + Label* nullCheckFailed = nullptr; +#ifndef WASM_HAS_HEAPREG + OutOfLineCode* nullref = addOutOfLineCode(new (alloc_) OutOfLineAbortingTrap( + Trap::IndirectCallToNull, bytecodeOffset())); + if (!oob) { + return false; + } + nullCheckFailed = nullref->entry(); +#endif + masm.wasmCallIndirect(desc, callee, oob->entry(), nullCheckFailed, + mozilla::Nothing(), fastCallOffset, slowCallOffset); + return true; +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +void BaseCompiler::callRef(const Stk& calleeRef, const FunctionCall& call, + CodeOffset* fastCallOffset, + CodeOffset* slowCallOffset) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::FuncRef); + CalleeDesc callee = CalleeDesc::wasmFuncRef(); + + loadRef(calleeRef, RegRef(WasmCallRefReg)); + masm.wasmCallRef(desc, callee, fastCallOffset, slowCallOffset); +} +#endif + +// Precondition: sync() + +CodeOffset BaseCompiler::callImport(unsigned instanceDataOffset, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Import); + CalleeDesc callee = CalleeDesc::import(instanceDataOffset); + return masm.wasmCallImport(desc, callee); +} + +CodeOffset BaseCompiler::builtinCall(SymbolicAddress builtin, + const FunctionCall& call) { + return callSymbolic(builtin, call); +} + +CodeOffset BaseCompiler::builtinInstanceMethodCall( + const SymbolicAddressSignature& builtin, const ABIArg& instanceArg, + const FunctionCall& call) { +#ifndef RABALDR_PIN_INSTANCE + // Builtin method calls assume the instance register has been set. + fr.loadInstancePtr(InstanceReg); +#endif + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic); + return masm.wasmCallBuiltinInstanceMethod(desc, instanceArg, builtin.identity, + builtin.failureMode); +} + +bool BaseCompiler::pushCallResults(const FunctionCall& call, ResultType type, + const StackResultsLoc& loc) { +#if defined(JS_CODEGEN_ARM) + // pushResults currently bypasses special case code in captureReturnedFxx() + // that converts GPR results to FPR results for systemABI+softFP. If we + // ever start using that combination for calls we need more code. This + // assert is stronger than we need - we only care about results in return + // registers - but that's OK. + MOZ_ASSERT(!call.usesSystemAbi || call.hardFP); +#endif + return pushResults(type, fr.stackResultsBase(loc.bytes())); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Exception handling + +// Abstracted helper for throwing, used for throw, rethrow, and rethrowing +// at the end of a series of catch blocks (if none matched the exception). +bool BaseCompiler::throwFrom(RegRef exn) { + pushRef(exn); + + // ThrowException invokes a trap, and the rest is dead code. + return emitInstanceCall(SASigThrowException); +} + +void BaseCompiler::loadTag(RegPtr instance, uint32_t tagIndex, RegRef tagDst) { + size_t offset = + Instance::offsetInData(moduleEnv_.offsetOfTagInstanceData(tagIndex)); + masm.loadPtr(Address(instance, offset), tagDst); +} + +void BaseCompiler::consumePendingException(RegRef* exnDst, RegRef* tagDst) { + RegPtr pendingAddr = RegPtr(PreBarrierReg); + needPtr(pendingAddr); + masm.computeEffectiveAddress( + Address(InstanceReg, Instance::offsetOfPendingException()), pendingAddr); + *exnDst = needRef(); + masm.loadPtr(Address(pendingAddr, 0), *exnDst); + emitBarrieredClear(pendingAddr); + + *tagDst = needRef(); + masm.computeEffectiveAddress( + Address(InstanceReg, Instance::offsetOfPendingExceptionTag()), + pendingAddr); + masm.loadPtr(Address(pendingAddr, 0), *tagDst); + emitBarrieredClear(pendingAddr); + freePtr(pendingAddr); +} + +bool BaseCompiler::startTryNote(size_t* tryNoteIndex) { + // Check the previous try note to ensure that we don't share an edge with + // it that could lead to ambiguity. Insert a nop, if required. + TryNoteVector& tryNotes = masm.tryNotes(); + if (tryNotes.length() > 0) { + const TryNote& previous = tryNotes.back(); + uint32_t currentOffset = masm.currentOffset(); + if (previous.tryBodyBegin() == currentOffset || + previous.tryBodyEnd() == currentOffset) { + masm.nop(); + } + } + + // Mark the beginning of the try note + wasm::TryNote tryNote = wasm::TryNote(); + tryNote.setTryBodyBegin(masm.currentOffset()); + return masm.append(tryNote, tryNoteIndex); +} + +void BaseCompiler::finishTryNote(size_t tryNoteIndex) { + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[tryNoteIndex]; + + // Disallow zero-length try notes by inserting a no-op + if (tryNote.tryBodyBegin() == masm.currentOffset()) { + masm.nop(); + } + + // Check the previous try note to ensure that we don't share an edge with + // it that could lead to ambiguity. Insert a nop, if required. + if (tryNotes.length() > 0) { + const TryNote& previous = tryNotes.back(); + uint32_t currentOffset = masm.currentOffset(); + if (previous.tryBodyEnd() == currentOffset) { + masm.nop(); + } + } + + // Don't set the end of the try note if we've OOM'ed, as the above nop's may + // not have been placed. This is okay as this compilation will be thrown + // away. + if (masm.oom()) { + return; + } + + // Mark the end of the try note + tryNote.setTryBodyEnd(masm.currentOffset()); +} + +//////////////////////////////////////////////////////////// +// +// Platform-specific popping and register targeting. + +// The simple popping methods pop values into targeted registers; the caller +// can free registers using standard functions. These are always called +// popXForY where X says something about types and Y something about the +// operation being targeted. + +RegI32 BaseCompiler::needRotate64Temp() { +#if defined(JS_CODEGEN_X86) + return needI32(); +#elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + return RegI32::Invalid(); +#else + MOZ_CRASH("BaseCompiler platform hook: needRotate64Temp"); +#endif +} + +void BaseCompiler::popAndAllocateForDivAndRemI32(RegI32* r0, RegI32* r1, + RegI32* reserved) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r0 must be eax, and edx will be clobbered. + need2xI32(specific_.eax, specific_.edx); + *r1 = popI32(); + *r0 = popI32ToSpecific(specific_.eax); + *reserved = specific_.edx; +#else + pop2xI32(r0, r1); +#endif +} + +static void QuotientI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd, + RegI32 reserved, IsUnsigned isUnsigned) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + masm.quotient32(rs, rsd, reserved, isUnsigned); +#else + masm.quotient32(rs, rsd, isUnsigned); +#endif +} + +static void RemainderI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd, + RegI32 reserved, IsUnsigned isUnsigned) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + masm.remainder32(rs, rsd, reserved, isUnsigned); +#else + masm.remainder32(rs, rsd, isUnsigned); +#endif +} + +void BaseCompiler::popAndAllocateForMulI64(RegI64* r0, RegI64* r1, + RegI32* temp) { +#if defined(JS_CODEGEN_X64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_X86) + // lhsDest must be edx:eax and rhs must not be that. + needI64(specific_.edx_eax); + *r1 = popI64(); + *r0 = popI64ToSpecific(specific_.edx_eax); + *temp = needI32(); +#elif defined(JS_CODEGEN_MIPS64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_ARM) + pop2xI64(r0, r1); + *temp = needI32(); +#elif defined(JS_CODEGEN_ARM64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_LOONG64) + pop2xI64(r0, r1); +#else + MOZ_CRASH("BaseCompiler porting interface: popAndAllocateForMulI64"); +#endif +} + +#ifndef RABALDR_INT_DIV_I64_CALLOUT + +void BaseCompiler::popAndAllocateForDivAndRemI64(RegI64* r0, RegI64* r1, + RegI64* reserved, + IsRemainder isRemainder) { +# if defined(JS_CODEGEN_X64) + // r0 must be rax, and rdx will be clobbered. + need2xI64(specific_.rax, specific_.rdx); + *r1 = popI64(); + *r0 = popI64ToSpecific(specific_.rax); + *reserved = specific_.rdx; +# elif defined(JS_CODEGEN_ARM64) + pop2xI64(r0, r1); + if (isRemainder) { + *reserved = needI64(); + } +# else + pop2xI64(r0, r1); +# endif +} + +static void QuotientI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest, + RegI64 reserved, IsUnsigned isUnsigned) { +# if defined(JS_CODEGEN_X64) + // The caller must set up the following situation. + MOZ_ASSERT(srcDest.reg == rax); + MOZ_ASSERT(reserved.reg == rdx); + if (isUnsigned) { + masm.xorq(rdx, rdx); + masm.udivq(rhs.reg); + } else { + masm.cqo(); + masm.idivq(rhs.reg); + } +# elif defined(JS_CODEGEN_MIPS64) + MOZ_ASSERT(reserved.isInvalid()); + if (isUnsigned) { + masm.as_ddivu(srcDest.reg, rhs.reg); + } else { + masm.as_ddiv(srcDest.reg, rhs.reg); + } + masm.as_mflo(srcDest.reg); +# elif defined(JS_CODEGEN_ARM64) + MOZ_ASSERT(reserved.isInvalid()); + ARMRegister sd(srcDest.reg, 64); + ARMRegister r(rhs.reg, 64); + if (isUnsigned) { + masm.Udiv(sd, sd, r); + } else { + masm.Sdiv(sd, sd, r); + } +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_div_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_div_d(srcDest.reg, srcDest.reg, rhs.reg); + } +# else + MOZ_CRASH("BaseCompiler platform hook: quotientI64"); +# endif +} + +static void RemainderI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest, + RegI64 reserved, IsUnsigned isUnsigned) { +# if defined(JS_CODEGEN_X64) + // The caller must set up the following situation. + MOZ_ASSERT(srcDest.reg == rax); + MOZ_ASSERT(reserved.reg == rdx); + + if (isUnsigned) { + masm.xorq(rdx, rdx); + masm.udivq(rhs.reg); + } else { + masm.cqo(); + masm.idivq(rhs.reg); + } + masm.movq(rdx, rax); +# elif defined(JS_CODEGEN_MIPS64) + MOZ_ASSERT(reserved.isInvalid()); + if (isUnsigned) { + masm.as_ddivu(srcDest.reg, rhs.reg); + } else { + masm.as_ddiv(srcDest.reg, rhs.reg); + } + masm.as_mfhi(srcDest.reg); +# elif defined(JS_CODEGEN_ARM64) + ARMRegister sd(srcDest.reg, 64); + ARMRegister r(rhs.reg, 64); + ARMRegister t(reserved.reg, 64); + if (isUnsigned) { + masm.Udiv(t, sd, r); + } else { + masm.Sdiv(t, sd, r); + } + masm.Mul(t, t, r); + masm.Sub(sd, sd, t); +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_mod_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_mod_d(srcDest.reg, srcDest.reg, rhs.reg); + } +# else + MOZ_CRASH("BaseCompiler platform hook: remainderI64"); +# endif +} + +#endif // RABALDR_INT_DIV_I64_CALLOUT + +RegI32 BaseCompiler::popI32RhsForShift() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable shift, unless BMI2 is available. + if (!Assembler::HasBMI2()) { + return popI32(specific_.ecx); + } +#endif + RegI32 r = popI32(); +#if defined(JS_CODEGEN_ARM) + masm.and32(Imm32(31), r); +#endif + return r; +} + +RegI32 BaseCompiler::popI32RhsForShiftI64() { +#if defined(JS_CODEGEN_X86) + // A limitation in the x86 masm requires ecx here + return popI32(specific_.ecx); +#elif defined(JS_CODEGEN_X64) + if (!Assembler::HasBMI2()) { + return popI32(specific_.ecx); + } + return popI32(); +#else + return popI32(); +#endif +} + +RegI64 BaseCompiler::popI64RhsForShift() { +#if defined(JS_CODEGEN_X86) + // r1 must be ecx for a variable shift. + needI32(specific_.ecx); + return popI64ToSpecific(widenI32(specific_.ecx)); +#else +# if defined(JS_CODEGEN_X64) + // r1 must be rcx for a variable shift, unless BMI2 is available. + if (!Assembler::HasBMI2()) { + needI64(specific_.rcx); + return popI64ToSpecific(specific_.rcx); + } +# endif + // No masking is necessary on 64-bit platforms, and on arm32 the masm + // implementation masks. + return popI64(); +#endif +} + +RegI32 BaseCompiler::popI32RhsForRotate() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable rotate. + return popI32(specific_.ecx); +#else + return popI32(); +#endif +} + +RegI64 BaseCompiler::popI64RhsForRotate() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable rotate. + needI32(specific_.ecx); + return popI64ToSpecific(widenI32(specific_.ecx)); +#else + return popI64(); +#endif +} + +void BaseCompiler::popI32ForSignExtendI64(RegI64* r0) { +#if defined(JS_CODEGEN_X86) + // r0 must be edx:eax for cdq + need2xI32(specific_.edx, specific_.eax); + *r0 = specific_.edx_eax; + popI32ToSpecific(specific_.eax); +#else + *r0 = widenI32(popI32()); +#endif +} + +void BaseCompiler::popI64ForSignExtendI64(RegI64* r0) { +#if defined(JS_CODEGEN_X86) + // r0 must be edx:eax for cdq + need2xI32(specific_.edx, specific_.eax); + // Low on top, high underneath + *r0 = popI64ToSpecific(specific_.edx_eax); +#else + *r0 = popI64(); +#endif +} + +class OutOfLineTruncateCheckF32OrF64ToI32 : public OutOfLineCode { + AnyReg src; + RegI32 dest; + TruncFlags flags; + BytecodeOffset off; + + public: + OutOfLineTruncateCheckF32OrF64ToI32(AnyReg src, RegI32 dest, TruncFlags flags, + BytecodeOffset off) + : src(src), dest(dest), flags(flags), off(off) {} + + virtual void generate(MacroAssembler* masm) override { + if (src.tag == AnyReg::F32) { + masm->oolWasmTruncateCheckF32ToI32(src.f32(), dest, flags, off, rejoin()); + } else if (src.tag == AnyReg::F64) { + masm->oolWasmTruncateCheckF64ToI32(src.f64(), dest, flags, off, rejoin()); + } else { + MOZ_CRASH("unexpected type"); + } + } +}; + +bool BaseCompiler::truncateF32ToI32(RegF32 src, RegI32 dest, TruncFlags flags) { + BytecodeOffset off = bytecodeOffset(); + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32( + AnyReg(src), dest, flags, off)); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateFloat32ToUInt32(src, dest, isSaturating, ool->entry()); + } else { + masm.wasmTruncateFloat32ToInt32(src, dest, isSaturating, ool->entry()); + } + masm.bind(ool->rejoin()); + return true; +} + +bool BaseCompiler::truncateF64ToI32(RegF64 src, RegI32 dest, TruncFlags flags) { + BytecodeOffset off = bytecodeOffset(); + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32( + AnyReg(src), dest, flags, off)); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateDoubleToUInt32(src, dest, isSaturating, ool->entry()); + } else { + masm.wasmTruncateDoubleToInt32(src, dest, isSaturating, ool->entry()); + } + masm.bind(ool->rejoin()); + return true; +} + +class OutOfLineTruncateCheckF32OrF64ToI64 : public OutOfLineCode { + AnyReg src; + RegI64 dest; + TruncFlags flags; + BytecodeOffset off; + + public: + OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src, RegI64 dest, TruncFlags flags, + BytecodeOffset off) + : src(src), dest(dest), flags(flags), off(off) {} + + virtual void generate(MacroAssembler* masm) override { + if (src.tag == AnyReg::F32) { + masm->oolWasmTruncateCheckF32ToI64(src.f32(), dest, flags, off, rejoin()); + } else if (src.tag == AnyReg::F64) { + masm->oolWasmTruncateCheckF64ToI64(src.f64(), dest, flags, off, rejoin()); + } else { + MOZ_CRASH("unexpected type"); + } + } +}; + +#ifndef RABALDR_FLOAT_TO_I64_CALLOUT + +RegF64 BaseCompiler::needTempForFloatingToI64(TruncFlags flags) { +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + if (flags & TRUNC_UNSIGNED) { + return needF64(); + } +# endif + return RegF64::Invalid(); +} + +bool BaseCompiler::truncateF32ToI64(RegF32 src, RegI64 dest, TruncFlags flags, + RegF64 temp) { + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(src), dest, flags, bytecodeOffset())); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateFloat32ToUInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } else { + masm.wasmTruncateFloat32ToInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } + return true; +} + +bool BaseCompiler::truncateF64ToI64(RegF64 src, RegI64 dest, TruncFlags flags, + RegF64 temp) { + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(src), dest, flags, bytecodeOffset())); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateDoubleToUInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } else { + masm.wasmTruncateDoubleToInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } + return true; +} + +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT + +RegI32 BaseCompiler::needConvertI64ToFloatTemp(ValType to, bool isUnsigned) { + bool needs = false; + if (to == ValType::F64) { + needs = isUnsigned && masm.convertUInt64ToDoubleNeedsTemp(); + } else { +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + needs = true; +# endif + } + return needs ? needI32() : RegI32::Invalid(); +} + +void BaseCompiler::convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, + RegI32 temp) { + if (isUnsigned) { + masm.convertUInt64ToFloat32(src, dest, temp); + } else { + masm.convertInt64ToFloat32(src, dest); + } +} + +void BaseCompiler::convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, + RegI32 temp) { + if (isUnsigned) { + masm.convertUInt64ToDouble(src, dest, temp); + } else { + masm.convertInt64ToDouble(src, dest); + } +} + +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +////////////////////////////////////////////////////////////////////// +// +// Global variable access. + +Address BaseCompiler::addressOfGlobalVar(const GlobalDesc& global, RegPtr tmp) { + uint32_t globalToInstanceOffset = Instance::offsetInData(global.offset()); +#ifdef RABALDR_PIN_INSTANCE + movePtr(RegPtr(InstanceReg), tmp); +#else + fr.loadInstancePtr(tmp); +#endif + if (global.isIndirect()) { + masm.loadPtr(Address(tmp, globalToInstanceOffset), tmp); + return Address(tmp, 0); + } + return Address(tmp, globalToInstanceOffset); +} + +////////////////////////////////////////////////////////////////////// +// +// Table access. + +Address BaseCompiler::addressOfTableField(uint32_t tableIndex, + uint32_t fieldOffset, + RegPtr instance) { + uint32_t tableToInstanceOffset = wasm::Instance::offsetInData( + moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset); + return Address(instance, tableToInstanceOffset); +} + +void BaseCompiler::loadTableLength(uint32_t tableIndex, RegPtr instance, + RegI32 length) { + masm.load32(addressOfTableField( + tableIndex, offsetof(TableInstanceData, length), instance), + length); +} + +void BaseCompiler::loadTableElements(uint32_t tableIndex, RegPtr instance, + RegPtr elements) { + masm.loadPtr(addressOfTableField( + tableIndex, offsetof(TableInstanceData, elements), instance), + elements); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Basic emitters for simple operators. + +static void AddI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.add32(rs, rsd); +} + +static void AddImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.add32(Imm32(c), rsd); +} + +static void SubI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.sub32(rs, rsd); +} + +static void SubImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.sub32(Imm32(c), rsd); +} + +static void MulI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.mul32(rs, rsd); +} + +static void OrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.or32(rs, rsd); +} + +static void OrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.or32(Imm32(c), rsd); +} + +static void AndI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.and32(rs, rsd); +} + +static void AndImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.and32(Imm32(c), rsd); +} + +static void XorI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.xor32(rs, rsd); +} + +static void XorImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.xor32(Imm32(c), rsd); +} + +static void ClzI32(MacroAssembler& masm, RegI32 rsd) { + masm.clz32(rsd, rsd, IsKnownNotZero(false)); +} + +static void CtzI32(MacroAssembler& masm, RegI32 rsd) { + masm.ctz32(rsd, rsd, IsKnownNotZero(false)); +} + +// Currently common to PopcntI32 and PopcntI64 +static RegI32 PopcntTemp(BaseCompiler& bc) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + return AssemblerX86Shared::HasPOPCNT() ? RegI32::Invalid() : bc.needI32(); +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + return bc.needI32(); +#else + MOZ_CRASH("BaseCompiler platform hook: PopcntTemp"); +#endif +} + +static void PopcntI32(BaseCompiler& bc, RegI32 rsd, RegI32 temp) { + bc.masm.popcnt32(rsd, rsd, temp); +} + +static void ShlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.lshift32(rs, rsd); +} + +static void ShlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.lshift32(Imm32(c & 31), rsd); +} + +static void ShrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rshift32Arithmetic(rs, rsd); +} + +static void ShrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rshift32Arithmetic(Imm32(c & 31), rsd); +} + +static void ShrUI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rshift32(rs, rsd); +} + +static void ShrUImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rshift32(Imm32(c & 31), rsd); +} + +static void RotlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rotateLeft(rs, rsd, rsd); +} + +static void RotlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rotateLeft(Imm32(c & 31), rsd, rsd); +} + +static void RotrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rotateRight(rs, rsd, rsd); +} + +static void RotrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rotateRight(Imm32(c & 31), rsd, rsd); +} + +static void EqzI32(MacroAssembler& masm, RegI32 rsd) { + masm.cmp32Set(Assembler::Equal, rsd, Imm32(0), rsd); +} + +static void WrapI64ToI32(MacroAssembler& masm, RegI64 rs, RegI32 rd) { + masm.move64To32(rs, rd); +} + +static void AddI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.add64(rs, rsd); +} + +static void AddImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.add64(Imm64(c), rsd); +} + +static void SubI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.sub64(rs, rsd); +} + +static void SubImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.sub64(Imm64(c), rsd); +} + +static void OrI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.or64(rs, rsd); +} + +static void OrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.or64(Imm64(c), rsd); +} + +static void AndI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.and64(rs, rsd); +} + +static void AndImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.and64(Imm64(c), rsd); +} + +static void XorI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.xor64(rs, rsd); +} + +static void XorImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.xor64(Imm64(c), rsd); +} + +static void ClzI64(BaseCompiler& bc, RegI64 rsd) { + bc.masm.clz64(rsd, bc.lowPart(rsd)); + bc.maybeClearHighPart(rsd); +} + +static void CtzI64(BaseCompiler& bc, RegI64 rsd) { + bc.masm.ctz64(rsd, bc.lowPart(rsd)); + bc.maybeClearHighPart(rsd); +} + +static void PopcntI64(BaseCompiler& bc, RegI64 rsd, RegI32 temp) { + bc.masm.popcnt64(rsd, rsd, temp); +} + +static void ShlI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.lshift64(bc.lowPart(rs), rsd); +} + +static void ShlImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.lshift64(Imm32(c & 63), rsd); +} + +static void ShrI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.rshift64Arithmetic(bc.lowPart(rs), rsd); +} + +static void ShrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.rshift64Arithmetic(Imm32(c & 63), rsd); +} + +static void ShrUI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.rshift64(bc.lowPart(rs), rsd); +} + +static void ShrUImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.rshift64(Imm32(c & 63), rsd); +} + +static void EqzI64(MacroAssembler& masm, RegI64 rs, RegI32 rd) { +#ifdef JS_PUNBOX64 + masm.cmpPtrSet(Assembler::Equal, rs.reg, ImmWord(0), rd); +#else + MOZ_ASSERT(rs.low == rd); + masm.or32(rs.high, rs.low); + masm.cmp32Set(Assembler::Equal, rs.low, Imm32(0), rd); +#endif +} + +static void AddF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.addDouble(rs, rsd); +} + +static void SubF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.subDouble(rs, rsd); +} + +static void MulF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.mulDouble(rs, rsd); +} + +static void DivF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.divDouble(rs, rsd); +} + +static void MinF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F64 + ScratchF64 zero(bc.ra); +#else + ScratchF64 zero(bc.masm); +#endif + bc.masm.loadConstantDouble(0, zero); + bc.masm.subDouble(zero, rsd); + bc.masm.subDouble(zero, rs); + bc.masm.minDouble(rs, rsd, HandleNaNSpecially(true)); +} + +static void MaxF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F64 + ScratchF64 zero(bc.ra); +#else + ScratchF64 zero(bc.masm); +#endif + bc.masm.loadConstantDouble(0, zero); + bc.masm.subDouble(zero, rsd); + bc.masm.subDouble(zero, rs); + bc.masm.maxDouble(rs, rsd, HandleNaNSpecially(true)); +} + +static void CopysignF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd, + RegI64 temp0, RegI64 temp1) { + masm.moveDoubleToGPR64(rsd, temp0); + masm.moveDoubleToGPR64(rs, temp1); + masm.and64(Imm64(INT64_MAX), temp0); + masm.and64(Imm64(INT64_MIN), temp1); + masm.or64(temp1, temp0); + masm.moveGPR64ToDouble(temp0, rsd); +} + +static void AbsF64(MacroAssembler& masm, RegF64 rsd) { + masm.absDouble(rsd, rsd); +} + +static void NegateF64(MacroAssembler& masm, RegF64 rsd) { + masm.negateDouble(rsd); +} + +static void SqrtF64(MacroAssembler& masm, RegF64 rsd) { + masm.sqrtDouble(rsd, rsd); +} + +static void AddF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.addFloat32(rs, rsd); +} + +static void SubF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.subFloat32(rs, rsd); +} + +static void MulF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.mulFloat32(rs, rsd); +} + +static void DivF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.divFloat32(rs, rsd); +} + +static void MinF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): Don't do this if one of the operands + // is known to be a constant. +#ifdef RABALDR_SCRATCH_F32 + ScratchF32 zero(bc.ra); +#else + ScratchF32 zero(bc.masm); +#endif + bc.masm.loadConstantFloat32(0.f, zero); + bc.masm.subFloat32(zero, rsd); + bc.masm.subFloat32(zero, rs); + bc.masm.minFloat32(rs, rsd, HandleNaNSpecially(true)); +} + +static void MaxF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F32 + ScratchF32 zero(bc.ra); +#else + ScratchF32 zero(bc.masm); +#endif + bc.masm.loadConstantFloat32(0.f, zero); + bc.masm.subFloat32(zero, rsd); + bc.masm.subFloat32(zero, rs); + bc.masm.maxFloat32(rs, rsd, HandleNaNSpecially(true)); +} + +static void CopysignF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd, + RegI32 temp0, RegI32 temp1) { + masm.moveFloat32ToGPR(rsd, temp0); + masm.moveFloat32ToGPR(rs, temp1); + masm.and32(Imm32(INT32_MAX), temp0); + masm.and32(Imm32(INT32_MIN), temp1); + masm.or32(temp1, temp0); + masm.moveGPRToFloat32(temp0, rsd); +} + +static void AbsF32(MacroAssembler& masm, RegF32 rsd) { + masm.absFloat32(rsd, rsd); +} + +static void NegateF32(MacroAssembler& masm, RegF32 rsd) { + masm.negateFloat(rsd); +} + +static void SqrtF32(MacroAssembler& masm, RegF32 rsd) { + masm.sqrtFloat32(rsd, rsd); +} + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT +static void ConvertI64ToF32(MacroAssembler& masm, RegI64 rs, RegF32 rd) { + masm.convertInt64ToFloat32(rs, rd); +} + +static void ConvertI64ToF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) { + masm.convertInt64ToDouble(rs, rd); +} +#endif + +static void ReinterpretF32AsI32(MacroAssembler& masm, RegF32 rs, RegI32 rd) { + masm.moveFloat32ToGPR(rs, rd); +} + +static void ReinterpretF64AsI64(MacroAssembler& masm, RegF64 rs, RegI64 rd) { + masm.moveDoubleToGPR64(rs, rd); +} + +static void ConvertF64ToF32(MacroAssembler& masm, RegF64 rs, RegF32 rd) { + masm.convertDoubleToFloat32(rs, rd); +} + +static void ConvertI32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.convertInt32ToFloat32(rs, rd); +} + +static void ConvertU32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.convertUInt32ToFloat32(rs, rd); +} + +static void ConvertF32ToF64(MacroAssembler& masm, RegF32 rs, RegF64 rd) { + masm.convertFloat32ToDouble(rs, rd); +} + +static void ConvertI32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) { + masm.convertInt32ToDouble(rs, rd); +} + +static void ConvertU32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) { + masm.convertUInt32ToDouble(rs, rd); +} + +static void ReinterpretI32AsF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.moveGPRToFloat32(rs, rd); +} + +static void ReinterpretI64AsF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) { + masm.moveGPR64ToDouble(rs, rd); +} + +static void ExtendI32_8(BaseCompiler& bc, RegI32 rsd) { +#ifdef JS_CODEGEN_X86 + if (!bc.ra.isSingleByteI32(rsd)) { + ScratchI8 scratch(bc.ra); + bc.masm.move32(rsd, scratch); + bc.masm.move8SignExtend(scratch, rsd); + return; + } +#endif + bc.masm.move8SignExtend(rsd, rsd); +} + +static void ExtendI32_16(MacroAssembler& masm, RegI32 rsd) { + masm.move16SignExtend(rsd, rsd); +} + +void BaseCompiler::emitMultiplyI64() { + RegI64 r, rs; + RegI32 temp; + popAndAllocateForMulI64(&r, &rs, &temp); + masm.mul64(rs, r, temp); + maybeFree(temp); + freeI64(rs); + pushI64(r); +} + +template <typename RegType, typename IntType> +void BaseCompiler::quotientOrRemainder( + RegType rs, RegType rsd, RegType reserved, IsUnsigned isUnsigned, + ZeroOnOverflow zeroOnOverflow, bool isConst, IntType c, + void (*operate)(MacroAssembler& masm, RegType rs, RegType rsd, + RegType reserved, IsUnsigned isUnsigned)) { + Label done; + if (!isConst || c == 0) { + checkDivideByZero(rs); + } + if (!isUnsigned && (!isConst || c == -1)) { + checkDivideSignedOverflow(rs, rsd, &done, zeroOnOverflow); + } + operate(masm, rs, rsd, reserved, isUnsigned); + masm.bind(&done); +} + +void BaseCompiler::emitQuotientI32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI32 r = popI32(); + Label positive; + masm.branchTest32(Assembler::NotSigned, r, r, &positive); + masm.add32(Imm32(c - 1), r); + masm.bind(&positive); + + masm.rshift32Arithmetic(Imm32(power & 31), r); + pushI32(r); + } + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(false), isConst, c, QuotientI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitQuotientU32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI32 r = popI32(); + masm.rshift32(Imm32(power & 31), r); + pushI32(r); + } + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), + ZeroOnOverflow(false), isConst, c, QuotientI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitRemainderI32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI32 r = popI32(); + RegI32 temp = needI32(); + moveI32(r, temp); + + Label positive; + masm.branchTest32(Assembler::NotSigned, temp, temp, &positive); + masm.add32(Imm32(c - 1), temp); + masm.bind(&positive); + + masm.rshift32Arithmetic(Imm32(power & 31), temp); + masm.lshift32(Imm32(power & 31), temp); + masm.sub32(temp, r); + freeI32(temp); + + pushI32(r); + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(true), isConst, c, RemainderI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitRemainderU32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI32 r = popI32(); + masm.and32(Imm32(c - 1), r); + pushI32(r); + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true), + isConst, c, RemainderI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +#ifndef RABALDR_INT_DIV_I64_CALLOUT +void BaseCompiler::emitQuotientI64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI64 r = popI64(); + Label positive; + masm.branchTest64(Assembler::NotSigned, r, r, RegI32::Invalid(), + &positive); + masm.add64(Imm64(c - 1), r); + masm.bind(&positive); + + masm.rshift64Arithmetic(Imm32(power & 63), r); + pushI64(r); + } + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(false), isConst, c, QuotientI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitQuotientU64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI64 r = popI64(); + masm.rshift64(Imm32(power & 63), r); + pushI64(r); + } + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), + ZeroOnOverflow(false), isConst, c, QuotientI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRemainderI64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI64 r = popI64(); + RegI64 temp = needI64(); + moveI64(r, temp); + + Label positive; + masm.branchTest64(Assembler::NotSigned, temp, temp, RegI32::Invalid(), + &positive); + masm.add64(Imm64(c - 1), temp); + masm.bind(&positive); + + masm.rshift64Arithmetic(Imm32(power & 63), temp); + masm.lshift64(Imm32(power & 63), temp); + masm.sub64(temp, r); + freeI64(temp); + + pushI64(r); + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(true), isConst, c, RemainderI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRemainderU64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI64 r = popI64(); + masm.and64(Imm64(c - 1), r); + pushI64(r); + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true), + isConst, c, RemainderI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} +#endif // RABALDR_INT_DIV_I64_CALLOUT + +void BaseCompiler::emitRotrI64() { + int64_t c; + if (popConst(&c)) { + RegI64 r = popI64(); + RegI32 temp = needRotate64Temp(); + masm.rotateRight64(Imm32(c & 63), r, r, temp); + maybeFree(temp); + pushI64(r); + } else { + RegI64 rs = popI64RhsForRotate(); + RegI64 r = popI64(); + masm.rotateRight64(lowPart(rs), r, r, maybeHighPart(rs)); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRotlI64() { + int64_t c; + if (popConst(&c)) { + RegI64 r = popI64(); + RegI32 temp = needRotate64Temp(); + masm.rotateLeft64(Imm32(c & 63), r, r, temp); + maybeFree(temp); + pushI64(r); + } else { + RegI64 rs = popI64RhsForRotate(); + RegI64 r = popI64(); + masm.rotateLeft64(lowPart(rs), r, r, maybeHighPart(rs)); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitEqzI32() { + if (sniffConditionalControlEqz(ValType::I32)) { + return; + } + emitUnop(EqzI32); +} + +void BaseCompiler::emitEqzI64() { + if (sniffConditionalControlEqz(ValType::I64)) { + return; + } + emitUnop(EqzI64); +} + +template <TruncFlags flags> +bool BaseCompiler::emitTruncateF32ToI32() { + RegF32 rs = popF32(); + RegI32 rd = needI32(); + if (!truncateF32ToI32(rs, rd, flags)) { + return false; + } + freeF32(rs); + pushI32(rd); + return true; +} + +template <TruncFlags flags> +bool BaseCompiler::emitTruncateF64ToI32() { + RegF64 rs = popF64(); + RegI32 rd = needI32(); + if (!truncateF64ToI32(rs, rd, flags)) { + return false; + } + freeF64(rs); + pushI32(rd); + return true; +} + +#ifndef RABALDR_FLOAT_TO_I64_CALLOUT +template <TruncFlags flags> +bool BaseCompiler::emitTruncateF32ToI64() { + RegF32 rs = popF32(); + RegI64 rd = needI64(); + RegF64 temp = needTempForFloatingToI64(flags); + if (!truncateF32ToI64(rs, rd, flags, temp)) { + return false; + } + maybeFree(temp); + freeF32(rs); + pushI64(rd); + return true; +} + +template <TruncFlags flags> +bool BaseCompiler::emitTruncateF64ToI64() { + RegF64 rs = popF64(); + RegI64 rd = needI64(); + RegF64 temp = needTempForFloatingToI64(flags); + if (!truncateF64ToI64(rs, rd, flags, temp)) { + return false; + } + maybeFree(temp); + freeF64(rs); + pushI64(rd); + return true; +} +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +void BaseCompiler::emitExtendI64_8() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move8To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI64_16() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move16To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI64_32() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move32To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI32ToI64() { + RegI64 r; + popI32ForSignExtendI64(&r); + masm.move32To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendU32ToI64() { + RegI32 rs = popI32(); + RegI64 rd = widenI32(rs); + masm.move32To64ZeroExtend(rs, rd); + pushI64(rd); +} + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT +void BaseCompiler::emitConvertU64ToF32() { + RegI64 rs = popI64(); + RegF32 rd = needF32(); + RegI32 temp = needConvertI64ToFloatTemp(ValType::F32, IsUnsigned(true)); + convertI64ToF32(rs, IsUnsigned(true), rd, temp); + maybeFree(temp); + freeI64(rs); + pushF32(rd); +} + +void BaseCompiler::emitConvertU64ToF64() { + RegI64 rs = popI64(); + RegF64 rd = needF64(); + RegI32 temp = needConvertI64ToFloatTemp(ValType::F64, IsUnsigned(true)); + convertI64ToF64(rs, IsUnsigned(true), rd, temp); + maybeFree(temp); + freeI64(rs); + pushF64(rd); +} +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +//////////////////////////////////////////////////////////// +// +// Machinery for optimized conditional branches. +// +// To disable this optimization it is enough always to return false from +// sniffConditionalControl{Cmp,Eqz}. + +struct BranchState { + union { + struct { + RegI32 lhs; + RegI32 rhs; + int32_t imm; + bool rhsImm; + } i32; + struct { + RegI64 lhs; + RegI64 rhs; + int64_t imm; + bool rhsImm; + } i64; + struct { + RegF32 lhs; + RegF32 rhs; + } f32; + struct { + RegF64 lhs; + RegF64 rhs; + } f64; + }; + + Label* const label; // The target of the branch, never NULL + const StackHeight stackHeight; // The stack base above which to place + // stack-spilled block results, if + // hasBlockResults(). + const bool invertBranch; // If true, invert the sense of the branch + const ResultType resultType; // The result propagated along the edges + + explicit BranchState(Label* label) + : label(label), + stackHeight(StackHeight::Invalid()), + invertBranch(false), + resultType(ResultType::Empty()) {} + + BranchState(Label* label, bool invertBranch) + : label(label), + stackHeight(StackHeight::Invalid()), + invertBranch(invertBranch), + resultType(ResultType::Empty()) {} + + BranchState(Label* label, StackHeight stackHeight, bool invertBranch, + ResultType resultType) + : label(label), + stackHeight(stackHeight), + invertBranch(invertBranch), + resultType(resultType) {} + + bool hasBlockResults() const { return stackHeight.isValid(); } +}; + +void BaseCompiler::setLatentCompare(Assembler::Condition compareOp, + ValType operandType) { + latentOp_ = LatentOp::Compare; + latentType_ = operandType; + latentIntCmp_ = compareOp; +} + +void BaseCompiler::setLatentCompare(Assembler::DoubleCondition compareOp, + ValType operandType) { + latentOp_ = LatentOp::Compare; + latentType_ = operandType; + latentDoubleCmp_ = compareOp; +} + +void BaseCompiler::setLatentEqz(ValType operandType) { + latentOp_ = LatentOp::Eqz; + latentType_ = operandType; +} + +bool BaseCompiler::hasLatentOp() const { return latentOp_ != LatentOp::None; } + +void BaseCompiler::resetLatentOp() { latentOp_ = LatentOp::None; } + +// Emit a conditional branch that optionally and optimally cleans up the CPU +// stack before we branch. +// +// Cond is either Assembler::Condition or Assembler::DoubleCondition. +// +// Lhs is RegI32, RegI64, or RegF32, RegF64, or RegRef. +// +// Rhs is either the same as Lhs, or an immediate expression compatible with +// Lhs "when applicable". + +template <typename Cond, typename Lhs, typename Rhs> +bool BaseCompiler::jumpConditionalWithResults(BranchState* b, Cond cond, + Lhs lhs, Rhs rhs) { + if (b->hasBlockResults()) { + StackHeight resultsBase(0); + if (!topBranchParams(b->resultType, &resultsBase)) { + return false; + } + if (b->stackHeight != resultsBase) { + Label notTaken; + branchTo(b->invertBranch ? cond : Assembler::InvertCondition(cond), lhs, + rhs, ¬Taken); + + // Shuffle stack args. + shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight, + b->resultType); + masm.jump(b->label); + masm.bind(¬Taken); + return true; + } + } + + branchTo(b->invertBranch ? Assembler::InvertCondition(cond) : cond, lhs, rhs, + b->label); + return true; +} + +#ifdef ENABLE_WASM_GC +bool BaseCompiler::jumpConditionalWithResults(BranchState* b, RegRef object, + RefType sourceType, + RefType destType, + bool onSuccess) { + if (b->hasBlockResults()) { + StackHeight resultsBase(0); + if (!topBranchParams(b->resultType, &resultsBase)) { + return false; + } + if (b->stackHeight != resultsBase) { + Label notTaken; + // Temporarily take the result registers so that branchGcHeapType doesn't + // use them. + needIntegerResultRegisters(b->resultType); + branchGcRefType(object, sourceType, destType, ¬Taken, + /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess); + freeIntegerResultRegisters(b->resultType); + + // Shuffle stack args. + shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight, + b->resultType); + masm.jump(b->label); + masm.bind(¬Taken); + return true; + } + } + + branchGcRefType(object, sourceType, destType, b->label, + /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess); + return true; +} +#endif + +// sniffConditionalControl{Cmp,Eqz} may modify the latentWhatever_ state in +// the BaseCompiler so that a subsequent conditional branch can be compiled +// optimally. emitBranchSetup() and emitBranchPerform() will consume that +// state. If the latter methods are not called because deadCode_ is true +// then the compiler MUST instead call resetLatentOp() to reset the state. + +template <typename Cond> +bool BaseCompiler::sniffConditionalControlCmp(Cond compareOp, + ValType operandType) { + MOZ_ASSERT(latentOp_ == LatentOp::None, + "Latent comparison state not properly reset"); + +#ifdef JS_CODEGEN_X86 + // On x86, latent i64 binary comparisons use too many registers: the + // reserved join register and the lhs and rhs operands require six, but we + // only have five. + if (operandType == ValType::I64) { + return false; + } +#endif + + // No optimization for pointer compares yet. + if (operandType.isRefRepr()) { + return false; + } + + OpBytes op{}; + iter_.peekOp(&op); + switch (op.b0) { + case uint16_t(Op::BrIf): + case uint16_t(Op::If): + case uint16_t(Op::SelectNumeric): + case uint16_t(Op::SelectTyped): + setLatentCompare(compareOp, operandType); + return true; + default: + return false; + } +} + +bool BaseCompiler::sniffConditionalControlEqz(ValType operandType) { + MOZ_ASSERT(latentOp_ == LatentOp::None, + "Latent comparison state not properly reset"); + + OpBytes op{}; + iter_.peekOp(&op); + switch (op.b0) { + case uint16_t(Op::BrIf): + case uint16_t(Op::SelectNumeric): + case uint16_t(Op::SelectTyped): + case uint16_t(Op::If): + setLatentEqz(operandType); + return true; + default: + return false; + } +} + +void BaseCompiler::emitBranchSetup(BranchState* b) { + // Avoid allocating operands to latentOp_ to result registers. + if (b->hasBlockResults()) { + needResultRegisters(b->resultType); + } + + // Set up fields so that emitBranchPerform() need not switch on latentOp_. + switch (latentOp_) { + case LatentOp::None: { + latentIntCmp_ = Assembler::NotEqual; + latentType_ = ValType::I32; + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + b->i32.imm = 0; + break; + } + case LatentOp::Compare: { + switch (latentType_.kind()) { + case ValType::I32: { + if (popConst(&b->i32.imm)) { + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + } else { + pop2xI32(&b->i32.lhs, &b->i32.rhs); + b->i32.rhsImm = false; + } + break; + } + case ValType::I64: { + pop2xI64(&b->i64.lhs, &b->i64.rhs); + b->i64.rhsImm = false; + break; + } + case ValType::F32: { + pop2xF32(&b->f32.lhs, &b->f32.rhs); + break; + } + case ValType::F64: { + pop2xF64(&b->f64.lhs, &b->f64.rhs); + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Compare"); + } + } + break; + } + case LatentOp::Eqz: { + switch (latentType_.kind()) { + case ValType::I32: { + latentIntCmp_ = Assembler::Equal; + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + b->i32.imm = 0; + break; + } + case ValType::I64: { + latentIntCmp_ = Assembler::Equal; + b->i64.lhs = popI64(); + b->i64.rhsImm = true; + b->i64.imm = 0; + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Eqz"); + } + } + break; + } + } + + if (b->hasBlockResults()) { + freeResultRegisters(b->resultType); + } +} + +bool BaseCompiler::emitBranchPerform(BranchState* b) { + switch (latentType_.kind()) { + case ValType::I32: { + if (b->i32.rhsImm) { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs, + Imm32(b->i32.imm))) { + return false; + } + } else { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs, + b->i32.rhs)) { + return false; + } + freeI32(b->i32.rhs); + } + freeI32(b->i32.lhs); + break; + } + case ValType::I64: { + if (b->i64.rhsImm) { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs, + Imm64(b->i64.imm))) { + return false; + } + } else { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs, + b->i64.rhs)) { + return false; + } + freeI64(b->i64.rhs); + } + freeI64(b->i64.lhs); + break; + } + case ValType::F32: { + if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f32.lhs, + b->f32.rhs)) { + return false; + } + freeF32(b->f32.lhs); + freeF32(b->f32.rhs); + break; + } + case ValType::F64: { + if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f64.lhs, + b->f64.rhs)) { + return false; + } + freeF64(b->f64.lhs); + freeF64(b->f64.rhs); + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Compare"); + } + } + resetLatentOp(); + return true; +} + +// For blocks and loops and ifs: +// +// - Sync the value stack before going into the block in order to simplify exit +// from the block: all exits from the block can assume that there are no +// live registers except the one carrying the exit value. +// - The block can accumulate a number of dead values on the stacks, so when +// branching out of the block or falling out at the end be sure to +// pop the appropriate stacks back to where they were on entry, while +// preserving the exit value. +// - A continue branch in a loop is much like an exit branch, but the branch +// value must not be preserved. +// - The exit value is always in a designated join register (type dependent). + +bool BaseCompiler::emitBlock() { + ResultType params; + if (!iter_.readBlock(¶ms)) { + return false; + } + + if (!deadCode_) { + sync(); // Simplifies branching out from block + } + + initControl(controlItem(), params); + + return true; +} + +bool BaseCompiler::endBlock(ResultType type) { + Control& block = controlItem(); + + if (deadCode_) { + // Block does not fall through; reset stack. + fr.resetStackHeight(block.stackHeight, type); + popValueStackTo(block.stackSize); + } else { + // If the block label is used, we have a control join, so we need to shuffle + // fallthrough values into place. Otherwise if it's not a control join, we + // can leave the value stack alone. + MOZ_ASSERT(stk_.length() == block.stackSize + type.length()); + if (block.label.used()) { + popBlockResults(type, block.stackHeight, ContinuationKind::Fallthrough); + } + block.bceSafeOnExit &= bceSafe_; + } + + // Bind after cleanup: branches out will have popped the stack. + if (block.label.used()) { + masm.bind(&block.label); + if (deadCode_) { + captureResultRegisters(type); + deadCode_ = false; + } + if (!pushBlockResults(type)) { + return false; + } + } + + bceSafe_ = block.bceSafeOnExit; + + return true; +} + +bool BaseCompiler::emitLoop() { + ResultType params; + if (!iter_.readLoop(¶ms)) { + return false; + } + + if (!deadCode_) { + sync(); // Simplifies branching out from block + } + + initControl(controlItem(), params); + bceSafe_ = 0; + + if (!deadCode_) { + // Loop entry is a control join, so shuffle the entry parameters into the + // well-known locations. + if (!topBlockParams(params)) { + return false; + } + masm.nopAlign(CodeAlignment); + masm.bind(&controlItem(0).label); + // The interrupt check barfs if there are live registers. + sync(); + if (!addInterruptCheck()) { + return false; + } + } + + return true; +} + +// The bodies of the "then" and "else" arms can be arbitrary sequences +// of expressions, they push control and increment the nesting and can +// even be targeted by jumps. A branch to the "if" block branches to +// the exit of the if, ie, it's like "break". Consider: +// +// (func (result i32) +// (if (i32.const 1) +// (begin (br 1) (unreachable)) +// (begin (unreachable))) +// (i32.const 1)) +// +// The branch causes neither of the unreachable expressions to be +// evaluated. + +bool BaseCompiler::emitIf() { + ResultType params; + Nothing unused_cond; + if (!iter_.readIf(¶ms, &unused_cond)) { + return false; + } + + BranchState b(&controlItem().otherLabel, InvertBranch(true)); + if (!deadCode_) { + needResultRegisters(params); + emitBranchSetup(&b); + freeResultRegisters(params); + sync(); + } else { + resetLatentOp(); + } + + initControl(controlItem(), params); + + if (!deadCode_) { + // Because params can flow immediately to results in the case of an empty + // "then" or "else" block, and the result of an if/then is a join in + // general, we shuffle params eagerly to the result allocations. + if (!topBlockParams(params)) { + return false; + } + if (!emitBranchPerform(&b)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::endIfThen(ResultType type) { + Control& ifThen = controlItem(); + + // The parameters to the "if" logically flow to both the "then" and "else" + // blocks, but the "else" block is empty. Since we know that the "if" + // type-checks, that means that the "else" parameters are the "else" results, + // and that the "if"'s result type is the same as its parameter type. + + if (deadCode_) { + // "then" arm does not fall through; reset stack. + fr.resetStackHeight(ifThen.stackHeight, type); + popValueStackTo(ifThen.stackSize); + if (!ifThen.deadOnArrival) { + captureResultRegisters(type); + } + } else { + MOZ_ASSERT(stk_.length() == ifThen.stackSize + type.length()); + // Assume we have a control join, so place results in block result + // allocations. + popBlockResults(type, ifThen.stackHeight, ContinuationKind::Fallthrough); + MOZ_ASSERT(!ifThen.deadOnArrival); + } + + if (ifThen.otherLabel.used()) { + masm.bind(&ifThen.otherLabel); + } + + if (ifThen.label.used()) { + masm.bind(&ifThen.label); + } + + if (!deadCode_) { + ifThen.bceSafeOnExit &= bceSafe_; + } + + deadCode_ = ifThen.deadOnArrival; + if (!deadCode_) { + if (!pushBlockResults(type)) { + return false; + } + } + + bceSafe_ = ifThen.bceSafeOnExit & ifThen.bceSafeOnEntry; + + return true; +} + +bool BaseCompiler::emitElse() { + ResultType params, results; + BaseNothingVector unused_thenValues{}; + + if (!iter_.readElse(¶ms, &results, &unused_thenValues)) { + return false; + } + + Control& ifThenElse = controlItem(0); + + // See comment in endIfThenElse, below. + + // Exit the "then" branch. + + ifThenElse.deadThenBranch = deadCode_; + + if (deadCode_) { + fr.resetStackHeight(ifThenElse.stackHeight, results); + popValueStackTo(ifThenElse.stackSize); + } else { + MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + results.length()); + popBlockResults(results, ifThenElse.stackHeight, ContinuationKind::Jump); + freeResultRegisters(results); + MOZ_ASSERT(!ifThenElse.deadOnArrival); + } + + if (!deadCode_) { + masm.jump(&ifThenElse.label); + } + + if (ifThenElse.otherLabel.used()) { + masm.bind(&ifThenElse.otherLabel); + } + + // Reset to the "else" branch. + + if (!deadCode_) { + ifThenElse.bceSafeOnExit &= bceSafe_; + } + + deadCode_ = ifThenElse.deadOnArrival; + bceSafe_ = ifThenElse.bceSafeOnEntry; + + fr.resetStackHeight(ifThenElse.stackHeight, params); + + if (!deadCode_) { + captureResultRegisters(params); + if (!pushBlockResults(params)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::endIfThenElse(ResultType type) { + Control& ifThenElse = controlItem(); + + // The expression type is not a reliable guide to what we'll find + // on the stack, we could have (if E (i32.const 1) (unreachable)) + // in which case the "else" arm is AnyType but the type of the + // full expression is I32. So restore whatever's there, not what + // we want to find there. The "then" arm has the same constraint. + + if (deadCode_) { + // "then" arm does not fall through; reset stack. + fr.resetStackHeight(ifThenElse.stackHeight, type); + popValueStackTo(ifThenElse.stackSize); + } else { + MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + type.length()); + // Assume we have a control join, so place results in block result + // allocations. + popBlockResults(type, ifThenElse.stackHeight, + ContinuationKind::Fallthrough); + ifThenElse.bceSafeOnExit &= bceSafe_; + MOZ_ASSERT(!ifThenElse.deadOnArrival); + } + + if (ifThenElse.label.used()) { + masm.bind(&ifThenElse.label); + } + + bool joinLive = + !ifThenElse.deadOnArrival && + (!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label.bound()); + + if (joinLive) { + // No values were provided by the "then" path, but capture the values + // provided by the "else" path. + if (deadCode_) { + captureResultRegisters(type); + } + deadCode_ = false; + } + + bceSafe_ = ifThenElse.bceSafeOnExit; + + if (!deadCode_) { + if (!pushBlockResults(type)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::emitEnd() { + LabelKind kind; + ResultType type; + BaseNothingVector unused_values{}; + if (!iter_.readEnd(&kind, &type, &unused_values, &unused_values)) { + return false; + } + + // Every label case is responsible to pop the control item at the appropriate + // time for the label case + switch (kind) { + case LabelKind::Body: + if (!endBlock(type)) { + return false; + } + doReturn(ContinuationKind::Fallthrough); + // This is emitted here after `doReturn` to avoid being executed in the + // normal return path of a function, and instead only when a `delegate` + // jumps to it. + if (!emitBodyDelegateThrowPad()) { + return false; + } + iter_.popEnd(); + MOZ_ASSERT(iter_.controlStackEmpty()); + return iter_.endFunction(iter_.end()); + case LabelKind::Block: + if (!endBlock(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Loop: + // The end of a loop isn't a branch target, so we can just leave its + // results on the expression stack to be consumed by the outer block. + iter_.popEnd(); + break; + case LabelKind::Then: + if (!endIfThen(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Else: + if (!endIfThenElse(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Try: + case LabelKind::Catch: + case LabelKind::CatchAll: + if (!endTryCatch(type)) { + return false; + } + iter_.popEnd(); + break; + } + + return true; +} + +bool BaseCompiler::emitBr() { + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + if (!iter_.readBr(&relativeDepth, &type, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + // Save any values in the designated join registers, as if the target block + // returned normally. + + popBlockResults(type, target.stackHeight, ContinuationKind::Jump); + masm.jump(&target.label); + + // The registers holding the join values are free for the remainder of this + // block. + + freeResultRegisters(type); + + deadCode_ = true; + + return true; +} + +bool BaseCompiler::emitBrIf() { + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrIf(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + resetLatentOp(); + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + emitBranchSetup(&b); + return emitBranchPerform(&b); +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitBrOnNull() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrOnNull(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + if (b.hasBlockResults()) { + needResultRegisters(b.resultType); + } + RegRef rp = popRef(); + if (b.hasBlockResults()) { + freeResultRegisters(b.resultType); + } + if (!jumpConditionalWithResults(&b, Assembler::Equal, rp, + ImmWord(NULLREF_VALUE))) { + return false; + } + pushRef(rp); + + return true; +} + +bool BaseCompiler::emitBrOnNonNull() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrOnNonNull(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + MOZ_ASSERT(b.hasBlockResults(), "br_on_non_null has block results"); + + // Don't allocate the result register used in the branch + needIntegerResultRegisters(b.resultType); + + // Get the ref from the top of the stack + RegRef condition = popRef(); + + // Create a copy of the ref for passing to the on_non_null label, + // the original ref is used in the condition. + RegRef rp = needRef(); + moveRef(condition, rp); + pushRef(rp); + + freeIntegerResultRegisters(b.resultType); + + if (!jumpConditionalWithResults(&b, Assembler::NotEqual, condition, + ImmWord(NULLREF_VALUE))) { + return false; + } + + freeRef(condition); + + // Dropping null reference. + dropValue(); + + return true; +} +#endif + +bool BaseCompiler::emitBrTable() { + Uint32Vector depths; + uint32_t defaultDepth; + ResultType branchParams; + BaseNothingVector unused_values{}; + Nothing unused_index; + // N.B., `branchParams' gets set to the type of the default branch target. In + // the presence of subtyping, it could be that the different branch targets + // have different types. Here we rely on the assumption that the value + // representations (e.g. Stk value types) of all branch target types are the + // same, in the baseline compiler. Notably, this means that all Ref types + // should be represented the same. + if (!iter_.readBrTable(&depths, &defaultDepth, &branchParams, &unused_values, + &unused_index)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Don't use param registers for rc + needIntegerResultRegisters(branchParams); + + // Table switch value always on top. + RegI32 rc = popI32(); + + freeIntegerResultRegisters(branchParams); + + StackHeight resultsBase(0); + if (!topBranchParams(branchParams, &resultsBase)) { + return false; + } + + Label dispatchCode; + masm.branch32(Assembler::Below, rc, Imm32(depths.length()), &dispatchCode); + + // This is the out-of-range stub. rc is dead here but we don't need it. + + shuffleStackResultsBeforeBranch( + resultsBase, controlItem(defaultDepth).stackHeight, branchParams); + controlItem(defaultDepth).bceSafeOnExit &= bceSafe_; + masm.jump(&controlItem(defaultDepth).label); + + // Emit stubs. rc is dead in all of these but we don't need it. + // + // The labels in the vector are in the TempAllocator and will + // be freed by and by. + // + // TODO / OPTIMIZE (Bug 1316804): Branch directly to the case code if we + // can, don't emit an intermediate stub. + + LabelVector stubs; + if (!stubs.reserve(depths.length())) { + return false; + } + + for (uint32_t depth : depths) { + stubs.infallibleEmplaceBack(NonAssertingLabel()); + masm.bind(&stubs.back()); + shuffleStackResultsBeforeBranch(resultsBase, controlItem(depth).stackHeight, + branchParams); + controlItem(depth).bceSafeOnExit &= bceSafe_; + masm.jump(&controlItem(depth).label); + } + + // Emit table. + + Label theTable; + jumpTable(stubs, &theTable); + + // Emit indirect jump. rc is live here. + + tableSwitch(&theTable, rc, &dispatchCode); + + deadCode_ = true; + + // Clean up. + + freeI32(rc); + popValueStackBy(branchParams.length()); + + return true; +} + +bool BaseCompiler::emitTry() { + ResultType params; + if (!iter_.readTry(¶ms)) { + return false; + } + + if (!deadCode_) { + // Simplifies jumping out, but it is also necessary so that control + // can re-enter the catch handler without restoring registers. + sync(); + } + + initControl(controlItem(), params); + + if (!deadCode_) { + // Be conservative for BCE due to complex control flow in try blocks. + controlItem().bceSafeOnExit = 0; + if (!startTryNote(&controlItem().tryNoteIndex)) { + return false; + } + } + + return true; +} + +void BaseCompiler::emitCatchSetup(LabelKind kind, Control& tryCatch, + const ResultType& resultType) { + // Catch ends the try or last catch, so we finish this like endIfThen. + if (deadCode_) { + fr.resetStackHeight(tryCatch.stackHeight, resultType); + popValueStackTo(tryCatch.stackSize); + } else { + // If the previous block is a catch, we need to handle the extra exception + // reference on the stack (for rethrow) and thus the stack size is 1 more. + MOZ_ASSERT(stk_.length() == tryCatch.stackSize + resultType.length() + + (kind == LabelKind::Try ? 0 : 1)); + // Try jumps to the end of the try-catch block unless a throw is done. + if (kind == LabelKind::Try) { + popBlockResults(resultType, tryCatch.stackHeight, ContinuationKind::Jump); + } else { + popCatchResults(resultType, tryCatch.stackHeight); + } + MOZ_ASSERT(stk_.length() == tryCatch.stackSize); + freeResultRegisters(resultType); + MOZ_ASSERT(!tryCatch.deadOnArrival); + } + + // Reset to this "catch" branch. + deadCode_ = tryCatch.deadOnArrival; + + // We use the empty result type here because catch does *not* take the + // try-catch block parameters. + fr.resetStackHeight(tryCatch.stackHeight, ResultType::Empty()); + + if (deadCode_) { + return; + } + + bceSafe_ = 0; + + // The end of the previous try/catch jumps to the join point. + masm.jump(&tryCatch.label); + + // Note end of try block for finding the catch block target. This needs + // to happen after the stack is reset to the correct height. + if (kind == LabelKind::Try) { + finishTryNote(controlItem().tryNoteIndex); + } +} + +bool BaseCompiler::emitCatch() { + LabelKind kind; + uint32_t tagIndex; + ResultType paramType, resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readCatch(&kind, &tagIndex, ¶mType, &resultType, + &unused_tryValues)) { + return false; + } + + Control& tryCatch = controlItem(); + + emitCatchSetup(kind, tryCatch, resultType); + + if (deadCode_) { + return true; + } + + // Construct info used for the exception landing pad. + CatchInfo catchInfo(tagIndex); + if (!tryCatch.catchInfos.emplaceBack(catchInfo)) { + return false; + } + + masm.bind(&tryCatch.catchInfos.back().label); + + // Extract the arguments in the exception package and push them. + const SharedTagType& tagType = moduleEnv_.tags[tagIndex].type; + const ValTypeVector& params = tagType->argTypes_; + const TagOffsetVector& offsets = tagType->argOffsets_; + + // The landing pad uses the block return protocol to communicate the + // exception object pointer to the catch block. + ResultType exnResult = ResultType::Single(RefType::extern_()); + captureResultRegisters(exnResult); + if (!pushBlockResults(exnResult)) { + return false; + } + RegRef exn = popRef(); + RegPtr data = needPtr(); + + masm.loadPtr(Address(exn, (int32_t)WasmExceptionObject::offsetOfData()), + data); + + // This method can increase stk_.length() by an unbounded amount, so we need + // to perform an allocation here to accomodate the variable number of values. + // There is enough headroom for the fixed number of values. The general case + // is handled in emitBody. + if (!stk_.reserve(stk_.length() + params.length() + 1)) { + return false; + } + + // This reference is pushed onto the stack because a potential rethrow + // may need to access it. It is always popped at the end of the block. + pushRef(exn); + + for (uint32_t i = 0; i < params.length(); i++) { + int32_t offset = offsets[i]; + switch (params[i].kind()) { + case ValType::I32: { + RegI32 reg = needI32(); + masm.load32(Address(data, offset), reg); + pushI32(reg); + break; + } + case ValType::I64: { + RegI64 reg = needI64(); + masm.load64(Address(data, offset), reg); + pushI64(reg); + break; + } + case ValType::F32: { + RegF32 reg = needF32(); + masm.loadFloat32(Address(data, offset), reg); + pushF32(reg); + break; + } + case ValType::F64: { + RegF64 reg = needF64(); + masm.loadDouble(Address(data, offset), reg); + pushF64(reg); + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 reg = needV128(); + masm.loadUnalignedSimd128(Address(data, offset), reg); + pushV128(reg); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + // TODO/AnyRef-boxing: With boxed immediates and strings, this may need + // to handle other kinds of values. + ASSERT_ANYREF_IS_JSOBJECT; + RegRef reg = needRef(); + masm.loadPtr(Address(data, offset), reg); + pushRef(reg); + break; + } + } + } + freePtr(data); + + return true; +} + +bool BaseCompiler::emitCatchAll() { + LabelKind kind; + ResultType paramType, resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readCatchAll(&kind, ¶mType, &resultType, &unused_tryValues)) { + return false; + } + + Control& tryCatch = controlItem(); + + emitCatchSetup(kind, tryCatch, resultType); + + if (deadCode_) { + return true; + } + + CatchInfo catchInfo(CatchAllIndex); + if (!tryCatch.catchInfos.emplaceBack(catchInfo)) { + return false; + } + + masm.bind(&tryCatch.catchInfos.back().label); + + // The landing pad uses the block return protocol to communicate the + // exception object pointer to the catch block. + ResultType exnResult = ResultType::Single(RefType::extern_()); + captureResultRegisters(exnResult); + // This reference is pushed onto the stack because a potential rethrow + // may need to access it. It is always popped at the end of the block. + return pushBlockResults(exnResult); +} + +bool BaseCompiler::emitBodyDelegateThrowPad() { + Control& block = controlItem(); + + // Only emit a landing pad if a `delegate` has generated a jump to here. + if (block.otherLabel.used()) { + StackHeight savedHeight = fr.stackHeight(); + fr.setStackHeight(block.stackHeight); + masm.bind(&block.otherLabel); + + // A try-delegate jumps immediately to its delegated try block, so we are + // responsible to unpack the exception and rethrow it. + RegRef exn; + RegRef tag; + consumePendingException(&exn, &tag); + freeRef(tag); + if (!throwFrom(exn)) { + return false; + } + fr.setStackHeight(savedHeight); + } + + return true; +} + +bool BaseCompiler::emitDelegate() { + uint32_t relativeDepth; + ResultType resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readDelegate(&relativeDepth, &resultType, &unused_tryValues)) { + return false; + } + + Control& tryDelegate = controlItem(); + + // End the try branch like a plain catch block without exception ref handling. + if (deadCode_) { + fr.resetStackHeight(tryDelegate.stackHeight, resultType); + popValueStackTo(tryDelegate.stackSize); + } else { + MOZ_ASSERT(stk_.length() == tryDelegate.stackSize + resultType.length()); + popBlockResults(resultType, tryDelegate.stackHeight, + ContinuationKind::Jump); + freeResultRegisters(resultType); + masm.jump(&tryDelegate.label); + MOZ_ASSERT(!tryDelegate.deadOnArrival); + } + + deadCode_ = tryDelegate.deadOnArrival; + + if (deadCode_) { + return true; + } + + // Create an exception landing pad that immediately branches to the landing + // pad of the delegated try block. + masm.bind(&tryDelegate.otherLabel); + + StackHeight savedHeight = fr.stackHeight(); + fr.setStackHeight(tryDelegate.stackHeight); + + // Mark the end of the try body. This may insert a nop. + finishTryNote(controlItem().tryNoteIndex); + + // The landing pad begins at this point + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[controlItem().tryNoteIndex]; + tryNote.setLandingPad(masm.currentOffset(), masm.framePushed()); + + // Store the Instance that was left in InstanceReg by the exception + // handling mechanism, that is this frame's Instance but with the exception + // filled in Instance::pendingException. + fr.storeInstancePtr(InstanceReg); + + // If the target block is a non-try block, skip over it and find the next + // try block or the very last block (to re-throw out of the function). + Control& lastBlock = controlOutermost(); + while (controlKind(relativeDepth) != LabelKind::Try && + &controlItem(relativeDepth) != &lastBlock) { + relativeDepth++; + } + Control& target = controlItem(relativeDepth); + + popBlockResults(ResultType::Empty(), target.stackHeight, + ContinuationKind::Jump); + masm.jump(&target.otherLabel); + + fr.setStackHeight(savedHeight); + + // Where the try branch jumps to, if it's not dead. + if (tryDelegate.label.used()) { + masm.bind(&tryDelegate.label); + } + + captureResultRegisters(resultType); + bceSafe_ = tryDelegate.bceSafeOnExit; + + return pushBlockResults(resultType); +} + +bool BaseCompiler::endTryCatch(ResultType type) { + Control& tryCatch = controlItem(); + LabelKind tryKind = controlKind(0); + + if (deadCode_) { + fr.resetStackHeight(tryCatch.stackHeight, type); + popValueStackTo(tryCatch.stackSize); + } else { + // If the previous block is a catch, we must handle the extra exception + // reference on the stack (for rethrow) and thus the stack size is 1 more. + MOZ_ASSERT(stk_.length() == tryCatch.stackSize + type.length() + + (tryKind == LabelKind::Try ? 0 : 1)); + // Assume we have a control join, so place results in block result + // allocations and also handle the implicit exception reference if needed. + if (tryKind == LabelKind::Try) { + popBlockResults(type, tryCatch.stackHeight, ContinuationKind::Jump); + } else { + popCatchResults(type, tryCatch.stackHeight); + } + MOZ_ASSERT(stk_.length() == tryCatch.stackSize); + // Since we will emit a landing pad after this and jump over it to get to + // the control join, we free these here and re-capture at the join. + freeResultRegisters(type); + masm.jump(&tryCatch.label); + MOZ_ASSERT(!tryCatch.bceSafeOnExit); + MOZ_ASSERT(!tryCatch.deadOnArrival); + } + + deadCode_ = tryCatch.deadOnArrival; + + if (deadCode_) { + return true; + } + + // Create landing pad for all catch handlers in this block. + // When used for a catchless try block, this will generate a landing pad + // with no handlers and only the fall-back rethrow. + masm.bind(&tryCatch.otherLabel); + + // The stack height also needs to be set not for a block result, but for the + // entry to the exception handlers. This is reset again below for the join. + StackHeight prePadHeight = fr.stackHeight(); + fr.setStackHeight(tryCatch.stackHeight); + + // If we are in a catchless try block, then there were no catch blocks to + // mark the end of the try note, so we need to end it here. + if (tryKind == LabelKind::Try) { + // Mark the end of the try body. This may insert a nop. + finishTryNote(controlItem().tryNoteIndex); + } + + // The landing pad begins at this point + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[controlItem().tryNoteIndex]; + tryNote.setLandingPad(masm.currentOffset(), masm.framePushed()); + + // Store the Instance that was left in InstanceReg by the exception + // handling mechanism, that is this frame's Instance but with the exception + // filled in Instance::pendingException. + fr.storeInstancePtr(InstanceReg); + + // Load exception pointer from Instance and make sure that it is + // saved before the following call will clear it. + RegRef exn; + RegRef tag; + consumePendingException(&exn, &tag); + + // Get a register to hold the tags for each catch + RegRef catchTag = needRef(); + + // Ensure that the exception is assigned to the block return register + // before branching to a handler. + pushRef(exn); + ResultType exnResult = ResultType::Single(RefType::extern_()); + popBlockResults(exnResult, tryCatch.stackHeight, ContinuationKind::Jump); + freeResultRegisters(exnResult); + + bool hasCatchAll = false; + for (CatchInfo& info : tryCatch.catchInfos) { + if (info.tagIndex != CatchAllIndex) { + MOZ_ASSERT(!hasCatchAll); + loadTag(RegPtr(InstanceReg), info.tagIndex, catchTag); + masm.branchPtr(Assembler::Equal, tag, catchTag, &info.label); + } else { + masm.jump(&info.label); + hasCatchAll = true; + } + } + freeRef(catchTag); + freeRef(tag); + + // If none of the tag checks succeed and there is no catch_all, + // then we rethrow the exception. + if (!hasCatchAll) { + captureResultRegisters(exnResult); + if (!pushBlockResults(exnResult) || !throwFrom(popRef())) { + return false; + } + } + + // Reset stack height for join. + fr.setStackHeight(prePadHeight); + + // Create join point. + if (tryCatch.label.used()) { + masm.bind(&tryCatch.label); + } + + captureResultRegisters(type); + deadCode_ = tryCatch.deadOnArrival; + bceSafe_ = tryCatch.bceSafeOnExit; + + return pushBlockResults(type); +} + +bool BaseCompiler::emitThrow() { + uint32_t tagIndex; + BaseNothingVector unused_argValues{}; + + if (!iter_.readThrow(&tagIndex, &unused_argValues)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TagDesc& tagDesc = moduleEnv_.tags[tagIndex]; + const ResultType& params = tagDesc.type->resultType(); + const TagOffsetVector& offsets = tagDesc.type->argOffsets_; + + // Load the tag object +#ifdef RABALDR_PIN_INSTANCE + RegPtr instance(InstanceReg); +#else + RegPtr instance = needPtr(); + fr.loadInstancePtr(instance); +#endif + RegRef tag = needRef(); + loadTag(instance, tagIndex, tag); +#ifndef RABALDR_PIN_INSTANCE + freePtr(instance); +#endif + + // Create the new exception object that we will throw. + pushRef(tag); + if (!emitInstanceCall(SASigExceptionNew)) { + return false; + } + + // Get registers for exn and data, excluding the prebarrier register + needPtr(RegPtr(PreBarrierReg)); + RegRef exn = popRef(); + RegPtr data = needPtr(); + freePtr(RegPtr(PreBarrierReg)); + + masm.loadPtr(Address(exn, WasmExceptionObject::offsetOfData()), data); + + for (int32_t i = params.length() - 1; i >= 0; i--) { + uint32_t offset = offsets[i]; + switch (params[i].kind()) { + case ValType::I32: { + RegI32 reg = popI32(); + masm.store32(reg, Address(data, offset)); + freeI32(reg); + break; + } + case ValType::I64: { + RegI64 reg = popI64(); + masm.store64(reg, Address(data, offset)); + freeI64(reg); + break; + } + case ValType::F32: { + RegF32 reg = popF32(); + masm.storeFloat32(reg, Address(data, offset)); + freeF32(reg); + break; + } + case ValType::F64: { + RegF64 reg = popF64(); + masm.storeDouble(reg, Address(data, offset)); + freeF64(reg); + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 reg = popV128(); + masm.storeUnalignedSimd128(reg, Address(data, offset)); + freeV128(reg); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + RegPtr valueAddr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(Address(data, offset), valueAddr); + RegRef rv = popRef(); + pushPtr(data); + // emitBarrieredStore preserves exn, rv + if (!emitBarrieredStore(Some(exn), valueAddr, rv, + PreBarrierKind::Normal, + PostBarrierKind::Imprecise)) { + return false; + } + popPtr(data); + freeRef(rv); + break; + } + } + } + freePtr(data); + + deadCode_ = true; + + return throwFrom(exn); +} + +bool BaseCompiler::emitRethrow() { + uint32_t relativeDepth; + if (!iter_.readRethrow(&relativeDepth)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& tryCatch = controlItem(relativeDepth); + RegRef exn = needRef(); + peekRefAt(tryCatch.stackSize, exn); + + deadCode_ = true; + + return throwFrom(exn); +} + +bool BaseCompiler::emitDrop() { + if (!iter_.readDrop()) { + return false; + } + + if (deadCode_) { + return true; + } + + dropValue(); + return true; +} + +void BaseCompiler::doReturn(ContinuationKind kind) { + if (deadCode_) { + return; + } + + StackHeight height = controlOutermost().stackHeight; + ResultType type = ResultType::Vector(funcType().results()); + popBlockResults(type, height, kind); + masm.jump(&returnLabel_); + freeResultRegisters(type); +} + +bool BaseCompiler::emitReturn() { + BaseNothingVector unused_values{}; + if (!iter_.readReturn(&unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + doReturn(ContinuationKind::Jump); + deadCode_ = true; + + return true; +} + +bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes, + const StackResultsLoc& results, + FunctionCall* baselineCall, + CalleeOnStack calleeOnStack) { + MOZ_ASSERT(!deadCode_); + + ArgTypeVector args(argTypes, results.stackResults()); + uint32_t naturalArgCount = argTypes.length(); + uint32_t abiArgCount = args.lengthWithStackResults(); + startCallArgs(StackArgAreaSizeUnaligned(args), baselineCall); + + // Args are deeper on the stack than the stack result area, if any. + size_t argsDepth = results.count(); + // They're deeper than the callee too, for callIndirect. + if (calleeOnStack == CalleeOnStack::True) { + argsDepth++; + } + + for (size_t i = 0; i < abiArgCount; ++i) { + if (args.isNaturalArg(i)) { + size_t naturalIndex = args.naturalIndex(i); + size_t stackIndex = naturalArgCount - 1 - naturalIndex + argsDepth; + passArg(argTypes[naturalIndex], peek(stackIndex), baselineCall); + } else { + // The synthetic stack result area pointer. + ABIArg argLoc = baselineCall->abi.next(MIRType::Pointer); + if (argLoc.kind() == ABIArg::Stack) { + ScratchPtr scratch(*this); + fr.computeOutgoingStackResultAreaPtr(results, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + fr.computeOutgoingStackResultAreaPtr(results, RegPtr(argLoc.gpr())); + } + } + } + +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(InstanceReg); +#endif + return true; +} + +void BaseCompiler::pushReturnValueOfCall(const FunctionCall& call, + MIRType type) { + switch (type) { + case MIRType::Int32: { + RegI32 rv = captureReturnedI32(); + pushI32(rv); + break; + } + case MIRType::Int64: { + RegI64 rv = captureReturnedI64(); + pushI64(rv); + break; + } + case MIRType::Float32: { + RegF32 rv = captureReturnedF32(call); + pushF32(rv); + break; + } + case MIRType::Double: { + RegF64 rv = captureReturnedF64(call); + pushF64(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: { + RegV128 rv = captureReturnedV128(call); + pushV128(rv); + break; + } +#endif + case MIRType::RefOrNull: { + RegRef rv = captureReturnedRef(); + pushRef(rv); + break; + } + default: + // In particular, passing |type| as MIRType::Void or MIRType::Pointer to + // this function is an error. + MOZ_CRASH("Function return type"); + } +} + +bool BaseCompiler::pushStackResultsForCall(const ResultType& type, RegPtr temp, + StackResultsLoc* loc) { + if (!ABIResultIter::HasStackResults(type)) { + return true; + } + + // This method can increase stk_.length() by an unbounded amount, so we need + // to perform an allocation here to accomodate the variable number of values. + // There is enough headroom for any fixed number of values. The general case + // is handled in emitBody. + if (!stk_.reserve(stk_.length() + type.length())) { + return false; + } + + // Measure stack results. + ABIResultIter i(type); + size_t count = 0; + for (; !i.done(); i.next()) { + if (i.cur().onStack()) { + count++; + } + } + uint32_t bytes = i.stackBytesConsumedSoFar(); + + // Reserve space for the stack results. + StackHeight resultsBase = fr.stackHeight(); + uint32_t height = fr.prepareStackResultArea(resultsBase, bytes); + + // Push Stk values onto the value stack, and zero out Ref values. + for (i.switchToPrev(); !i.done(); i.prev()) { + const ABIResult& result = i.cur(); + if (result.onStack()) { + Stk v = captureStackResult(result, resultsBase, bytes); + push(v); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk++; + fr.storeImmediatePtrToStack(intptr_t(0), v.offs(), temp); + } + } + } + + *loc = StackResultsLoc(bytes, count, height); + + return true; +} + +// After a call, some results may be written to the stack result locations that +// are pushed on the machine stack after any stack args. If there are stack +// args and stack results, these results need to be shuffled down, as the args +// are "consumed" by the call. +void BaseCompiler::popStackResultsAfterCall(const StackResultsLoc& results, + uint32_t stackArgBytes) { + if (results.bytes() != 0) { + popValueStackBy(results.count()); + if (stackArgBytes != 0) { + uint32_t srcHeight = results.height(); + MOZ_ASSERT(srcHeight >= stackArgBytes + results.bytes()); + uint32_t destHeight = srcHeight - stackArgBytes; + + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, results.bytes(), + ABINonArgReturnVolatileReg); + } + } +} + +// For now, always sync() at the beginning of the call to easily save live +// values. +// +// TODO / OPTIMIZE (Bug 1316806): We may be able to avoid a full sync(), since +// all we want is to save live registers that won't be saved by the callee or +// that we need for outgoing args - we don't need to sync the locals. We can +// just push the necessary registers, it'll be like a lightweight sync. +// +// Even some of the pushing may be unnecessary if the registers will be consumed +// by the call, because then what we want is parallel assignment to the argument +// registers or onto the stack for outgoing arguments. A sync() is just +// simpler. + +bool BaseCompiler::emitCall() { + uint32_t funcIndex; + BaseNothingVector args_{}; + if (!iter_.readCall(&funcIndex, &args_)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + const FuncType& funcType = *moduleEnv_.funcs[funcIndex].type; + bool import = moduleEnv_.funcIsImport(funcIndex); + + uint32_t numArgs = funcType.args().length(); + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType.results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::Wasm, + import ? RestoreRegisterStateAndRealm::True + : RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType.args(), results, &baselineCall, + CalleeOnStack::False)) { + return false; + } + + CodeOffset raOffset; + if (import) { + raOffset = callImport(moduleEnv_.offsetOfFuncImportInstanceData(funcIndex), + baselineCall); + } else { + raOffset = callDefinition(funcIndex, baselineCall); + } + + if (!createStackMap("emitCall", raOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} + +bool BaseCompiler::emitCallIndirect() { + uint32_t funcTypeIndex; + uint32_t tableIndex; + Nothing callee_; + BaseNothingVector args_{}; + if (!iter_.readCallIndirect(&funcTypeIndex, &tableIndex, &callee_, &args_)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType(); + + // Stack: ... arg1 .. argn callee + + uint32_t numArgs = funcType.args().length() + 1; + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType.results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + // State and realm are restored as needed by by callIndirect (really by + // MacroAssembler::wasmCallIndirect). + beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType.args(), results, &baselineCall, + CalleeOnStack::True)) { + return false; + } + + const Stk& callee = peek(results.count()); + CodeOffset fastCallOffset; + CodeOffset slowCallOffset; + if (!callIndirect(funcTypeIndex, tableIndex, callee, baselineCall, + &fastCallOffset, &slowCallOffset)) { + return false; + } + if (!createStackMap("emitCallIndirect", fastCallOffset)) { + return false; + } + if (!createStackMap("emitCallIndirect", slowCallOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitCallRef() { + const FuncType* funcType; + Nothing unused_callee; + BaseNothingVector unused_args{}; + if (!iter_.readCallRef(&funcType, &unused_callee, &unused_args)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + // Stack: ... arg1 .. argn callee + + uint32_t numArgs = funcType->args().length() + 1; + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType->results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + // State and realm are restored as needed by by callRef (really by + // MacroAssembler::wasmCallRef). + beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType->args(), results, &baselineCall, + CalleeOnStack::True)) { + return false; + } + + const Stk& callee = peek(results.count()); + CodeOffset fastCallOffset; + CodeOffset slowCallOffset; + callRef(callee, baselineCall, &fastCallOffset, &slowCallOffset); + if (!createStackMap("emitCallRef", fastCallOffset)) { + return false; + } + if (!createStackMap("emitCallRef", slowCallOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} +#endif + +void BaseCompiler::emitRound(RoundingMode roundingMode, ValType operandType) { + if (operandType == ValType::F32) { + RegF32 f0 = popF32(); + roundF32(roundingMode, f0); + pushF32(f0); + } else if (operandType == ValType::F64) { + RegF64 f0 = popF64(); + roundF64(roundingMode, f0); + pushF64(f0); + } else { + MOZ_CRASH("unexpected type"); + } +} + +bool BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, + ValType operandType) { + Nothing operand_; + if (!iter_.readUnary(operandType, &operand_)) { + return false; + } + + if (deadCode_) { + return true; + } + + RoundingMode roundingMode; + if (IsRoundingFunction(callee, &roundingMode) && + supportsRoundInstruction(roundingMode)) { + emitRound(roundingMode, operandType); + return true; + } + + sync(); + + ValTypeVector& signature = operandType == ValType::F32 ? SigF_ : SigD_; + ValType retType = operandType; + uint32_t numArgs = signature.length(); + size_t stackSpace = stackConsumed(numArgs); + StackResultsLoc noStackResults; + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::Builtin, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(signature, noStackResults, &baselineCall, + CalleeOnStack::False)) { + return false; + } + + CodeOffset raOffset = builtinCall(callee, baselineCall); + if (!createStackMap("emitUnaryMathBuiltin[..]", raOffset)) { + return false; + } + + endCall(baselineCall, stackSpace); + + popValueStackBy(numArgs); + + pushReturnValueOfCall(baselineCall, retType.toMIRType()); + + return true; +} + +#ifdef RABALDR_INT_DIV_I64_CALLOUT +bool BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee, + ValType operandType) { + MOZ_ASSERT(operandType == ValType::I64); + MOZ_ASSERT(!deadCode_); + + sync(); + + needI64(specific_.abiReturnRegI64); + + RegI64 rhs = popI64(); + RegI64 srcDest = popI64ToSpecific(specific_.abiReturnRegI64); + + Label done; + + checkDivideByZero(rhs); + + if (callee == SymbolicAddress::DivI64) { + checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(false)); + } else if (callee == SymbolicAddress::ModI64) { + checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(true)); + } + + masm.setupWasmABICall(); + masm.passABIArg(srcDest.high); + masm.passABIArg(srcDest.low); + masm.passABIArg(rhs.high); + masm.passABIArg(rhs.low); + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset())); + if (!createStackMap("emitDivOrModI64Bui[..]", raOffset)) { + return false; + } + + masm.bind(&done); + + freeI64(rhs); + pushI64(srcDest); + return true; +} +#endif // RABALDR_INT_DIV_I64_CALLOUT + +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT +bool BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee, + ValType operandType, + ValType resultType) { + sync(); + + RegI64 input = popI64(); + + FunctionCall call{}; + + masm.setupWasmABICall(); +# ifdef JS_PUNBOX64 + MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout"); +# else + masm.passABIArg(input.high); + masm.passABIArg(input.low); +# endif + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset()), + resultType == ValType::F32 ? MoveOp::FLOAT32 : MoveOp::DOUBLE); + if (!createStackMap("emitConvertInt64To[..]", raOffset)) { + return false; + } + + freeI64(input); + + if (resultType == ValType::F32) { + pushF32(captureReturnedF32(call)); + } else { + pushF64(captureReturnedF64(call)); + } + + return true; +} +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT +// `Callee` always takes a double, so a float32 input must be converted. +bool BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee, + ValType operandType, + ValType resultType) { + RegF64 doubleInput; + if (operandType == ValType::F32) { + doubleInput = needF64(); + RegF32 input = popF32(); + masm.convertFloat32ToDouble(input, doubleInput); + freeF32(input); + } else { + doubleInput = popF64(); + } + + // We may need the value after the call for the ool check. + RegF64 otherReg = needF64(); + moveF64(doubleInput, otherReg); + pushF64(otherReg); + + sync(); + + FunctionCall call{}; + + masm.setupWasmABICall(); + masm.passABIArg(doubleInput, MoveOp::DOUBLE); + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset())); + if (!createStackMap("emitConvertFloatin[..]", raOffset)) { + return false; + } + + freeF64(doubleInput); + + RegI64 rv = captureReturnedI64(); + + RegF64 inputVal = popF64(); + + TruncFlags flags = 0; + if (callee == SymbolicAddress::TruncateDoubleToUint64) { + flags |= TRUNC_UNSIGNED; + } + if (callee == SymbolicAddress::SaturatingTruncateDoubleToInt64 || + callee == SymbolicAddress::SaturatingTruncateDoubleToUint64) { + flags |= TRUNC_SATURATING; + } + + // If we're saturating, the callout will always produce the final result + // value. Otherwise, the callout value will return 0x8000000000000000 + // and we need to produce traps. + OutOfLineCode* ool = nullptr; + if (!(flags & TRUNC_SATURATING)) { + // The OOL check just succeeds or fails, it does not generate a value. + ool = addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(inputVal), rv, flags, bytecodeOffset())); + if (!ool) { + return false; + } + + masm.branch64(Assembler::Equal, rv, Imm64(0x8000000000000000), + ool->entry()); + masm.bind(ool->rejoin()); + } + + pushI64(rv); + freeF64(inputVal); + + return true; +} +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +bool BaseCompiler::emitGetLocal() { + uint32_t slot; + if (!iter_.readGetLocal(locals_, &slot)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Local loads are pushed unresolved, ie, they may be deferred + // until needed, until they may be affected by a store, or until a + // sync. This is intended to reduce register pressure. + + switch (locals_[slot].kind()) { + case ValType::I32: + pushLocalI32(slot); + break; + case ValType::I64: + pushLocalI64(slot); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + pushLocalV128(slot); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F64: + pushLocalF64(slot); + break; + case ValType::F32: + pushLocalF32(slot); + break; + case ValType::Ref: + pushLocalRef(slot); + break; + } + + return true; +} + +template <bool isSetLocal> +bool BaseCompiler::emitSetOrTeeLocal(uint32_t slot) { + if (deadCode_) { + return true; + } + + bceLocalIsUpdated(slot); + switch (locals_[slot].kind()) { + case ValType::I32: { + RegI32 rv = popI32(); + syncLocal(slot); + fr.storeLocalI32(rv, localFromSlot(slot, MIRType::Int32)); + if (isSetLocal) { + freeI32(rv); + } else { + pushI32(rv); + } + break; + } + case ValType::I64: { + RegI64 rv = popI64(); + syncLocal(slot); + fr.storeLocalI64(rv, localFromSlot(slot, MIRType::Int64)); + if (isSetLocal) { + freeI64(rv); + } else { + pushI64(rv); + } + break; + } + case ValType::F64: { + RegF64 rv = popF64(); + syncLocal(slot); + fr.storeLocalF64(rv, localFromSlot(slot, MIRType::Double)); + if (isSetLocal) { + freeF64(rv); + } else { + pushF64(rv); + } + break; + } + case ValType::F32: { + RegF32 rv = popF32(); + syncLocal(slot); + fr.storeLocalF32(rv, localFromSlot(slot, MIRType::Float32)); + if (isSetLocal) { + freeF32(rv); + } else { + pushF32(rv); + } + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 rv = popV128(); + syncLocal(slot); + fr.storeLocalV128(rv, localFromSlot(slot, MIRType::Simd128)); + if (isSetLocal) { + freeV128(rv); + } else { + pushV128(rv); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + RegRef rv = popRef(); + syncLocal(slot); + fr.storeLocalRef(rv, localFromSlot(slot, MIRType::RefOrNull)); + if (isSetLocal) { + freeRef(rv); + } else { + pushRef(rv); + } + break; + } + } + + return true; +} + +bool BaseCompiler::emitSetLocal() { + uint32_t slot; + Nothing unused_value; + if (!iter_.readSetLocal(locals_, &slot, &unused_value)) { + return false; + } + return emitSetOrTeeLocal<true>(slot); +} + +bool BaseCompiler::emitTeeLocal() { + uint32_t slot; + Nothing unused_value; + if (!iter_.readTeeLocal(locals_, &slot, &unused_value)) { + return false; + } + return emitSetOrTeeLocal<false>(slot); +} + +bool BaseCompiler::emitGetGlobal() { + uint32_t id; + if (!iter_.readGetGlobal(&id)) { + return false; + } + + if (deadCode_) { + return true; + } + + const GlobalDesc& global = moduleEnv_.globals[id]; + + if (global.isConstant()) { + LitVal value = global.constantValue(); + switch (value.type().kind()) { + case ValType::I32: + pushI32(value.i32()); + break; + case ValType::I64: + pushI64(value.i64()); + break; + case ValType::F32: + pushF32(value.f32()); + break; + case ValType::F64: + pushF64(value.f64()); + break; + case ValType::Ref: + pushRef(intptr_t(value.ref().forCompiledCode())); + break; +#ifdef ENABLE_WASM_SIMD + case ValType::V128: + pushV128(value.v128()); + break; +#endif + default: + MOZ_CRASH("Global constant type"); + } + return true; + } + + switch (global.type().kind()) { + case ValType::I32: { + RegI32 rv = needI32(); + ScratchPtr tmp(*this); + masm.load32(addressOfGlobalVar(global, tmp), rv); + pushI32(rv); + break; + } + case ValType::I64: { + RegI64 rv = needI64(); + ScratchPtr tmp(*this); + masm.load64(addressOfGlobalVar(global, tmp), rv); + pushI64(rv); + break; + } + case ValType::F32: { + RegF32 rv = needF32(); + ScratchPtr tmp(*this); + masm.loadFloat32(addressOfGlobalVar(global, tmp), rv); + pushF32(rv); + break; + } + case ValType::F64: { + RegF64 rv = needF64(); + ScratchPtr tmp(*this); + masm.loadDouble(addressOfGlobalVar(global, tmp), rv); + pushF64(rv); + break; + } + case ValType::Ref: { + RegRef rv = needRef(); + ScratchPtr tmp(*this); + masm.loadPtr(addressOfGlobalVar(global, tmp), rv); + pushRef(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 rv = needV128(); + ScratchPtr tmp(*this); + masm.loadUnalignedSimd128(addressOfGlobalVar(global, tmp), rv); + pushV128(rv); + break; + } +#endif + default: + MOZ_CRASH("Global variable type"); + break; + } + return true; +} + +bool BaseCompiler::emitSetGlobal() { + uint32_t id; + Nothing unused_value; + if (!iter_.readSetGlobal(&id, &unused_value)) { + return false; + } + + if (deadCode_) { + return true; + } + + const GlobalDesc& global = moduleEnv_.globals[id]; + + switch (global.type().kind()) { + case ValType::I32: { + RegI32 rv = popI32(); + ScratchPtr tmp(*this); + masm.store32(rv, addressOfGlobalVar(global, tmp)); + freeI32(rv); + break; + } + case ValType::I64: { + RegI64 rv = popI64(); + ScratchPtr tmp(*this); + masm.store64(rv, addressOfGlobalVar(global, tmp)); + freeI64(rv); + break; + } + case ValType::F32: { + RegF32 rv = popF32(); + ScratchPtr tmp(*this); + masm.storeFloat32(rv, addressOfGlobalVar(global, tmp)); + freeF32(rv); + break; + } + case ValType::F64: { + RegF64 rv = popF64(); + ScratchPtr tmp(*this); + masm.storeDouble(rv, addressOfGlobalVar(global, tmp)); + freeF64(rv); + break; + } + case ValType::Ref: { + RegPtr valueAddr(PreBarrierReg); + needPtr(valueAddr); + { + ScratchPtr tmp(*this); + masm.computeEffectiveAddress(addressOfGlobalVar(global, tmp), + valueAddr); + } + RegRef rv = popRef(); + // emitBarrieredStore preserves rv + if (!emitBarrieredStore(Nothing(), valueAddr, rv, PreBarrierKind::Normal, + PostBarrierKind::Imprecise)) { + return false; + } + freeRef(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 rv = popV128(); + ScratchPtr tmp(*this); + masm.storeUnalignedSimd128(rv, addressOfGlobalVar(global, tmp)); + freeV128(rv); + break; + } +#endif + default: + MOZ_CRASH("Global variable type"); + break; + } + return true; +} + +bool BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + if (!iter_.readLoad(type, Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadCommon(&access, AccessCheck(), type); + return true; +} + +bool BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + Nothing unused_value; + if (!iter_.readStore(resultType, Scalar::byteSize(viewType), &addr, + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + storeCommon(&access, AccessCheck(), resultType); + return true; +} + +bool BaseCompiler::emitSelect(bool typed) { + StackType type; + Nothing unused_trueValue; + Nothing unused_falseValue; + Nothing unused_condition; + if (!iter_.readSelect(typed, &type, &unused_trueValue, &unused_falseValue, + &unused_condition)) { + return false; + } + + if (deadCode_) { + resetLatentOp(); + return true; + } + + // I32 condition on top, then false, then true. + + Label done; + BranchState b(&done); + emitBranchSetup(&b); + + switch (type.valType().kind()) { + case ValType::I32: { + RegI32 r, rs; + pop2xI32(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveI32(rs, r); + masm.bind(&done); + freeI32(rs); + pushI32(r); + break; + } + case ValType::I64: { +#ifdef JS_CODEGEN_X86 + // There may be as many as four Int64 values in registers at a time: two + // for the latent branch operands, and two for the true/false values we + // normally pop before executing the branch. On x86 this is one value + // too many, so we need to generate more complicated code here, and for + // simplicity's sake we do so even if the branch operands are not Int64. + // However, the resulting control flow diamond is complicated since the + // arms of the diamond will have to stay synchronized with respect to + // their evaluation stack and regalloc state. To simplify further, we + // use a double branch and a temporary boolean value for now. + RegI32 temp = needI32(); + moveImm32(0, temp); + if (!emitBranchPerform(&b)) { + return false; + } + moveImm32(1, temp); + masm.bind(&done); + + Label trueValue; + RegI64 r, rs; + pop2xI64(&r, &rs); + masm.branch32(Assembler::Equal, temp, Imm32(0), &trueValue); + moveI64(rs, r); + masm.bind(&trueValue); + freeI32(temp); + freeI64(rs); + pushI64(r); +#else + RegI64 r, rs; + pop2xI64(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveI64(rs, r); + masm.bind(&done); + freeI64(rs); + pushI64(r); +#endif + break; + } + case ValType::F32: { + RegF32 r, rs; + pop2xF32(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveF32(rs, r); + masm.bind(&done); + freeF32(rs); + pushF32(r); + break; + } + case ValType::F64: { + RegF64 r, rs; + pop2xF64(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveF64(rs, r); + masm.bind(&done); + freeF64(rs); + pushF64(r); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 r, rs; + pop2xV128(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveV128(rs, r); + masm.bind(&done); + freeV128(rs); + pushV128(r); + break; + } +#endif + case ValType::Ref: { + RegRef r, rs; + pop2xRef(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveRef(rs, r); + masm.bind(&done); + freeRef(rs); + pushRef(r); + break; + } + default: { + MOZ_CRASH("select type"); + } + } + + return true; +} + +void BaseCompiler::emitCompareI32(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::I32); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + int32_t c; + if (popConst(&c)) { + RegI32 r = popI32(); + masm.cmp32Set(compareOp, r, Imm32(c), r); + pushI32(r); + } else { + RegI32 r, rs; + pop2xI32(&r, &rs); + masm.cmp32Set(compareOp, r, rs, r); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitCompareI64(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::I64); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + RegI64 rs0, rs1; + pop2xI64(&rs0, &rs1); + RegI32 rd(fromI64(rs0)); + cmp64Set(compareOp, rs0, rs1, rd); + freeI64(rs1); + freeI64Except(rs0, rd); + pushI32(rd); +} + +void BaseCompiler::emitCompareF32(Assembler::DoubleCondition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::F32); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + Label across; + RegF32 rs0, rs1; + pop2xF32(&rs0, &rs1); + RegI32 rd = needI32(); + moveImm32(1, rd); + masm.branchFloat(compareOp, rs0, rs1, &across); + moveImm32(0, rd); + masm.bind(&across); + freeF32(rs0); + freeF32(rs1); + pushI32(rd); +} + +void BaseCompiler::emitCompareF64(Assembler::DoubleCondition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::F64); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + Label across; + RegF64 rs0, rs1; + pop2xF64(&rs0, &rs1); + RegI32 rd = needI32(); + moveImm32(1, rd); + masm.branchDouble(compareOp, rs0, rs1, &across); + moveImm32(0, rd); + masm.bind(&across); + freeF64(rs0); + freeF64(rs1); + pushI32(rd); +} + +void BaseCompiler::emitCompareRef(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(!sniffConditionalControlCmp(compareOp, compareType)); + + RegRef rs1, rs2; + pop2xRef(&rs1, &rs2); + RegI32 rd = needI32(); + masm.cmpPtrSet(compareOp, rs1, rs2, rd); + freeRef(rs1); + freeRef(rs2); + pushI32(rd); +} + +bool BaseCompiler::emitInstanceCall(const SymbolicAddressSignature& builtin) { + // See declaration (WasmBCClass.h) for info on the relationship between the + // compiler's value stack and the argument order for the to-be-called + // function. + const MIRType* argTypes = builtin.argTypes; + MOZ_ASSERT(argTypes[0] == MIRType::Pointer); + + sync(); + + uint32_t numNonInstanceArgs = builtin.numArgs - 1 /* instance */; + size_t stackSpace = stackConsumed(numNonInstanceArgs); + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::System, RestoreRegisterStateAndRealm::True); + + ABIArg instanceArg = reservePointerArgument(&baselineCall); + + startCallArgs(StackArgAreaSizeUnaligned(builtin), &baselineCall); + for (uint32_t i = 1; i < builtin.numArgs; i++) { + ValType t; + switch (argTypes[i]) { + case MIRType::Int32: + t = ValType::I32; + break; + case MIRType::Int64: + t = ValType::I64; + break; + case MIRType::Float32: + t = ValType::F32; + break; + case MIRType::RefOrNull: + t = RefType::extern_(); + break; + case MIRType::Pointer: + // Instance function args can now be uninterpreted pointers (eg, for + // the cases PostBarrier and PostBarrierFilter) so we simply treat + // them like the equivalently sized integer. + t = ValType::fromMIRType(TargetWordMIRType()); + break; + default: + MOZ_CRASH("Unexpected type"); + } + passArg(t, peek(numNonInstanceArgs - i), &baselineCall); + } + CodeOffset raOffset = + builtinInstanceMethodCall(builtin, instanceArg, baselineCall); + if (!createStackMap("emitInstanceCall", raOffset)) { + return false; + } + + endCall(baselineCall, stackSpace); + + popValueStackBy(numNonInstanceArgs); + + // Note, many clients of emitInstanceCall currently assume that pushing the + // result here does not destroy ReturnReg. + // + // Furthermore, clients assume that if builtin.retType != MIRType::None, the + // callee will have returned a result and left it in ReturnReg for us to + // find, and that that register will not be destroyed here (or above). + + // For the return type only, MIRType::None is used to indicate that the + // call doesn't return a result, that is, returns a C/C++ "void". + + if (builtin.retType != MIRType::None) { + pushReturnValueOfCall(baselineCall, builtin.retType); + } + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Reference types. + +bool BaseCompiler::emitRefFunc() { + return emitInstanceCallOp<uint32_t>(SASigRefFunc, + [this](uint32_t* funcIndex) -> bool { + return iter_.readRefFunc(funcIndex); + }); +} + +bool BaseCompiler::emitRefNull() { + RefType type; + if (!iter_.readRefNull(&type)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushRef(NULLREF_VALUE); + return true; +} + +bool BaseCompiler::emitRefIsNull() { + Nothing nothing; + if (!iter_.readRefIsNull(¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef r = popRef(); + RegI32 rd = narrowRef(r); + + masm.cmpPtrSet(Assembler::Equal, r, ImmWord(NULLREF_VALUE), rd); + pushI32(rd); + return true; +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitRefAsNonNull() { + Nothing nothing; + if (!iter_.readRefAsNonNull(¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef rp = popRef(); + Label ok; + masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok); + trap(Trap::NullPointerDereference); + masm.bind(&ok); + pushRef(rp); + + return true; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic operations. + +bool BaseCompiler::emitAtomicCmpXchg(ValType type, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + Nothing unused{}; + if (!iter_.readAtomicCmpXchg(&addr, type, Scalar::byteSize(viewType), &unused, + &unused)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicCmpXchg(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicLoad(ValType type, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + if (!iter_.readAtomicLoad(&addr, type, Scalar::byteSize(viewType))) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Load()); + atomicLoad(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicRMW(ValType type, Scalar::Type viewType, + AtomicOp op) { + LinearMemoryAddress<Nothing> addr; + Nothing unused_value; + if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicRMW(&access, type, op); + return true; +} + +bool BaseCompiler::emitAtomicStore(ValType type, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + Nothing unused_value; + if (!iter_.readAtomicStore(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Store()); + atomicStore(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicXchg(ValType type, Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + Nothing unused_value; + if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicXchg(&access, type); + return true; +} + +bool BaseCompiler::emitWait(ValType type, uint32_t byteSize) { + Nothing nothing; + LinearMemoryAddress<Nothing> addr; + if (!iter_.readWait(&addr, type, byteSize, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access( + type.kind() == ValType::I32 ? Scalar::Int32 : Scalar::Int64, addr.align, + addr.offset, bytecodeOffset()); + return atomicWait(type, &access); +} + +bool BaseCompiler::emitWake() { + Nothing nothing; + LinearMemoryAddress<Nothing> addr; + if (!iter_.readWake(&addr, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(Scalar::Int32, addr.align, addr.offset, + bytecodeOffset()); + return atomicWake(&access); +} + +bool BaseCompiler::emitFence() { + if (!iter_.readFence()) { + return false; + } + if (deadCode_) { + return true; + } + masm.memoryBarrier(MembarFull); + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk memory operations. + +bool BaseCompiler::emitMemoryGrow() { + return emitInstanceCallOp( + !usesMemory() || isMem32() ? SASigMemoryGrowM32 : SASigMemoryGrowM64, + [this]() -> bool { + Nothing arg; + return iter_.readMemoryGrow(&arg); + }); +} + +bool BaseCompiler::emitMemorySize() { + return emitInstanceCallOp( + !usesMemory() || isMem32() ? SASigMemorySizeM32 : SASigMemorySizeM64, + [this]() -> bool { return iter_.readMemorySize(); }); +} + +bool BaseCompiler::emitMemCopy() { + uint32_t dstMemOrTableIndex = 0; + uint32_t srcMemOrTableIndex = 0; + Nothing nothing; + if (!iter_.readMemOrTableCopy(true, &dstMemOrTableIndex, ¬hing, + &srcMemOrTableIndex, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + if (isMem32()) { + int32_t signedLength; + if (peekConst(&signedLength) && signedLength != 0 && + uint32_t(signedLength) <= MaxInlineMemoryCopyLength) { + memCopyInlineM32(); + return true; + } + } + + return memCopyCall(); +} + +bool BaseCompiler::memCopyCall() { + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemCopySharedM32 : SASigMemCopySharedM64) + : (isMem32() ? SASigMemCopyM32 : SASigMemCopyM64)); +} + +bool BaseCompiler::emitMemFill() { + Nothing nothing; + if (!iter_.readMemFill(¬hing, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + if (isMem32()) { + int32_t signedLength; + int32_t signedValue; + if (peek2xConst(&signedLength, &signedValue) && signedLength != 0 && + uint32_t(signedLength) <= MaxInlineMemoryFillLength) { + memFillInlineM32(); + return true; + } + } + return memFillCall(); +} + +bool BaseCompiler::memFillCall() { + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemFillSharedM32 : SASigMemFillSharedM64) + : (isMem32() ? SASigMemFillM32 : SASigMemFillM64)); +} + +bool BaseCompiler::emitMemInit() { + return emitInstanceCallOp<uint32_t>( + (!usesMemory() || isMem32() ? SASigMemInitM32 : SASigMemInitM64), + [this](uint32_t* segIndex) -> bool { + Nothing nothing; + if (iter_.readMemOrTableInit(/*isMem*/ true, segIndex, nullptr, + ¬hing, ¬hing, ¬hing)) { + return true; + } + return false; + }); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk table operations. + +bool BaseCompiler::emitTableCopy() { + uint32_t dstMemOrTableIndex = 0; + uint32_t srcMemOrTableIndex = 0; + Nothing nothing; + if (!iter_.readMemOrTableCopy(false, &dstMemOrTableIndex, ¬hing, + &srcMemOrTableIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushI32(dstMemOrTableIndex); + pushI32(srcMemOrTableIndex); + return emitInstanceCall(SASigTableCopy); +} + +bool BaseCompiler::emitTableInit() { + return emitInstanceCallOp<uint32_t, uint32_t>( + SASigTableInit, + [this](uint32_t* segIndex, uint32_t* dstTableIndex) -> bool { + Nothing nothing; + return iter_.readMemOrTableInit(/*isMem*/ false, segIndex, + dstTableIndex, ¬hing, ¬hing, + ¬hing); + }); +} + +bool BaseCompiler::emitTableFill() { + // fill(start:u32, val:ref, len:u32, table:u32) -> void + return emitInstanceCallOp<uint32_t>( + SASigTableFill, [this](uint32_t* tableIndex) -> bool { + Nothing nothing; + return iter_.readTableFill(tableIndex, ¬hing, ¬hing, ¬hing); + }); +} + +bool BaseCompiler::emitMemDiscard() { + Nothing nothing; + if (!iter_.readMemDiscard(¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64) + : (isMem32() ? SASigMemDiscardM32 : SASigMemDiscardM64)); +} + +bool BaseCompiler::emitTableGet() { + uint32_t tableIndex; + Nothing nothing; + if (!iter_.readTableGet(&tableIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) { + return emitTableGetAnyRef(tableIndex); + } + pushI32(tableIndex); + // get(index:u32, table:u32) -> AnyRef + return emitInstanceCall(SASigTableGet); +} + +bool BaseCompiler::emitTableGrow() { + // grow(initValue:anyref, delta:u32, table:u32) -> u32 + return emitInstanceCallOp<uint32_t>( + SASigTableGrow, [this](uint32_t* tableIndex) -> bool { + Nothing nothing; + return iter_.readTableGrow(tableIndex, ¬hing, ¬hing); + }); +} + +bool BaseCompiler::emitTableSet() { + uint32_t tableIndex; + Nothing nothing; + if (!iter_.readTableSet(&tableIndex, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) { + return emitTableSetAnyRef(tableIndex); + } + pushI32(tableIndex); + // set(index:u32, value:ref, table:u32) -> void + return emitInstanceCall(SASigTableSet); +} + +bool BaseCompiler::emitTableSize() { + uint32_t tableIndex; + if (!iter_.readTableSize(&tableIndex)) { + return false; + } + if (deadCode_) { + return true; + } + + RegPtr instance = needPtr(); + RegI32 length = needI32(); + + fr.loadInstancePtr(instance); + loadTableLength(tableIndex, instance, length); + + pushI32(length); + freePtr(instance); + return true; +} + +void BaseCompiler::emitTableBoundsCheck(uint32_t tableIndex, RegI32 index, + RegPtr instance) { + Label ok; + masm.wasmBoundsCheck32( + Assembler::Condition::Below, index, + addressOfTableField(tableIndex, offsetof(TableInstanceData, length), + instance), + &ok); + masm.wasmTrap(wasm::Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&ok); +} + +bool BaseCompiler::emitTableGetAnyRef(uint32_t tableIndex) { + RegPtr instance = needPtr(); + RegPtr elements = needPtr(); + RegI32 index = popI32(); + + fr.loadInstancePtr(instance); + emitTableBoundsCheck(tableIndex, index, instance); + loadTableElements(tableIndex, instance, elements); + masm.loadPtr(BaseIndex(elements, index, ScalePointer), elements); + + pushRef(RegRef(elements)); + freeI32(index); + freePtr(instance); + + return true; +} + +bool BaseCompiler::emitTableSetAnyRef(uint32_t tableIndex) { + // Create temporaries for valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + + RegPtr instance = needPtr(); + RegPtr elements = needPtr(); + RegRef value = popRef(); + RegI32 index = popI32(); + + // x86 is one register too short for this operation, shuffle `value` back + // onto the stack until it is needed. +#ifdef JS_CODEGEN_X86 + pushRef(value); +#endif + + fr.loadInstancePtr(instance); + emitTableBoundsCheck(tableIndex, index, instance); + loadTableElements(tableIndex, instance, elements); + masm.computeEffectiveAddress(BaseIndex(elements, index, ScalePointer), + valueAddr); + + freeI32(index); + freePtr(elements); + freePtr(instance); + +#ifdef JS_CODEGEN_X86 + value = popRef(); +#endif + + if (!emitBarrieredStore(Nothing(), valueAddr, value, PreBarrierKind::Normal, + PostBarrierKind::Precise)) { + return false; + } + freeRef(value); + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Data and element segment management. + +bool BaseCompiler::emitDataOrElemDrop(bool isData) { + return emitInstanceCallOp<uint32_t>( + isData ? SASigDataDrop : SASigElemDrop, [&](uint32_t* segIndex) -> bool { + return iter_.readDataOrElemDrop(isData, segIndex); + }); +} + +////////////////////////////////////////////////////////////////////////////// +// +// General object support. + +void BaseCompiler::emitPreBarrier(RegPtr valueAddr) { + Label skipBarrier; + ScratchPtr scratch(*this); + +#ifdef RABALDR_PIN_INSTANCE + Register instance(InstanceReg); +#else + Register instance(scratch); + fr.loadInstancePtr(instance); +#endif + + EmitWasmPreBarrierGuard(masm, instance, scratch, valueAddr, + /*valueOffset=*/0, &skipBarrier, nullptr); + +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(instance); +#endif +#ifdef JS_CODEGEN_ARM64 + // The prebarrier stub assumes the PseudoStackPointer is set up. It is OK + // to just move the sp to x28 here because x28 is not being used by the + // baseline compiler and need not be saved or restored. + MOZ_ASSERT(!GeneralRegisterSet::All().hasRegisterIndex(x28.asUnsized())); + masm.Mov(x28, sp); +#endif + // The prebarrier call preserves all volatile registers + EmitWasmPreBarrierCall(masm, instance, scratch, valueAddr, /*valueOffset=*/0); + + masm.bind(&skipBarrier); +} + +bool BaseCompiler::emitPostBarrierImprecise(const Maybe<RegRef>& object, + RegPtr valueAddr, RegRef value) { + // We must force a sync before the guard so that locals are in a consistent + // location for whether or not the post-barrier call is taken. + sync(); + + // Emit a guard to skip the post-barrier call if it is not needed. + Label skipBarrier; + RegPtr otherScratch = needPtr(); + EmitWasmPostBarrierGuard(masm, object, otherScratch, value, &skipBarrier); + freePtr(otherScratch); + + // Push `object` and `value` to preserve them across the call. + if (object) { + pushRef(*object); + } + pushRef(value); + + // The `valueAddr` is a raw pointer to the cell within some GC object or + // instance area, and we are careful so that the GC will not run while the + // post-barrier call is active, so push a uintptr_t value. + pushPtr(valueAddr); + if (!emitInstanceCall(SASigPostBarrier)) { + return false; + } + + // Restore `object` and `value`. + popRef(value); + if (object) { + popRef(*object); + } + + masm.bind(&skipBarrier); + return true; +} + +bool BaseCompiler::emitPostBarrierPrecise(const Maybe<RegRef>& object, + RegPtr valueAddr, RegRef prevValue, + RegRef value) { + // Push `object` and `value` to preserve them across the call. + if (object) { + pushRef(*object); + } + pushRef(value); + + // Push the arguments and call the precise post-barrier + pushPtr(valueAddr); + pushRef(prevValue); + if (!emitInstanceCall(SASigPostBarrierPrecise)) { + return false; + } + + // Restore `object` and `value`. + popRef(value); + if (object) { + popRef(*object); + } + + return true; +} + +bool BaseCompiler::emitBarrieredStore(const Maybe<RegRef>& object, + RegPtr valueAddr, RegRef value, + PreBarrierKind preBarrierKind, + PostBarrierKind postBarrierKind) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + + // The pre-barrier preserves all allocated registers. + if (preBarrierKind == PreBarrierKind::Normal) { + emitPreBarrier(valueAddr); + } + + // The precise post-barrier requires the previous value stored in the field, + // in order to know if the previous store buffer entry needs to be removed. + RegRef prevValue; + if (postBarrierKind == PostBarrierKind::Precise) { + prevValue = needRef(); + masm.loadPtr(Address(valueAddr, 0), prevValue); + } + + // Store the value + masm.storePtr(value, Address(valueAddr, 0)); + + // The post-barrier preserves object and value. + if (postBarrierKind == PostBarrierKind::Precise) { + return emitPostBarrierPrecise(object, valueAddr, prevValue, value); + } + return emitPostBarrierImprecise(object, valueAddr, value); +} + +void BaseCompiler::emitBarrieredClear(RegPtr valueAddr) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + + // The pre-barrier preserves all allocated registers. + emitPreBarrier(valueAddr); + + // Store null + masm.storePtr(ImmWord(0), Address(valueAddr, 0)); + + // No post-barrier is needed, as null does not require a store buffer entry +} + +////////////////////////////////////////////////////////////////////////////// +// +// GC proposal. + +#ifdef ENABLE_WASM_GC + +RegPtr BaseCompiler::loadTypeDefInstanceData(uint32_t typeIndex) { + RegPtr rp = needPtr(); + RegPtr instance; +# ifndef RABALDR_PIN_INSTANCE + instance = rp; + fr.loadInstancePtr(instance); +# else + // We can use the pinned instance register. + instance = RegPtr(InstanceReg); +# endif + masm.computeEffectiveAddress( + Address(instance, Instance::offsetInData( + moduleEnv_.offsetOfTypeDefInstanceData(typeIndex))), + rp); + return rp; +} + +RegPtr BaseCompiler::loadSuperTypeVector(uint32_t typeIndex) { + RegPtr rp = needPtr(); + RegPtr instance; +# ifndef RABALDR_PIN_INSTANCE + // We need to load the instance register, but can use the destination + // register as a temporary. + instance = rp; + fr.loadInstancePtr(rp); +# else + // We can use the pinned instance register. + instance = RegPtr(InstanceReg); +# endif + masm.loadPtr( + Address(instance, Instance::offsetInData( + moduleEnv_.offsetOfSuperTypeVector(typeIndex))), + rp); + return rp; +} + +/* static */ +void BaseCompiler::SignalNullCheck::emitNullCheck(BaseCompiler* bc, RegRef rp) { + Label ok; + MacroAssembler& masm = bc->masm; + masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok); + bc->trap(Trap::NullPointerDereference); + masm.bind(&ok); +} + +/* static */ +void BaseCompiler::SignalNullCheck::emitTrapSite(BaseCompiler* bc) { + wasm::BytecodeOffset trapOffset(bc->bytecodeOffset()); + MacroAssembler& masm = bc->masm; + masm.append(wasm::Trap::NullPointerDereference, + wasm::TrapSite(masm.currentOffset(), trapOffset)); +} + +RegPtr BaseCompiler::emitGcArrayGetData(RegRef rp) { + // `rp` points at a WasmArrayObject. Return a reg holding the value of its + // `data_` field. + RegPtr rdata = needPtr(); + masm.loadPtr(Address(rp, WasmArrayObject::offsetOfData()), rdata); + return rdata; +} + +template <typename NullCheckPolicy> +RegI32 BaseCompiler::emitGcArrayGetNumElements(RegRef rp) { + // `rp` points at a WasmArrayObject. Return a reg holding the value of its + // `numElements_` field. + STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; + RegI32 numElements = needI32(); + NullCheckPolicy::emitTrapSite(this); + masm.load32(Address(rp, WasmArrayObject::offsetOfNumElements()), numElements); + return numElements; +} + +void BaseCompiler::emitGcArrayBoundsCheck(RegI32 index, RegI32 numElements) { + Label inBounds; + masm.branch32(Assembler::Below, index, numElements, &inBounds); + masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&inBounds); +} + +template <typename T, typename NullCheckPolicy> +void BaseCompiler::emitGcGet(FieldType type, FieldWideningOp wideningOp, + const T& src) { + switch (type.kind()) { + case FieldType::I8: { + MOZ_ASSERT(wideningOp != FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + if (wideningOp == FieldWideningOp::Unsigned) { + masm.load8ZeroExtend(src, r); + } else { + masm.load8SignExtend(src, r); + } + pushI32(r); + break; + } + case FieldType::I16: { + MOZ_ASSERT(wideningOp != FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + if (wideningOp == FieldWideningOp::Unsigned) { + masm.load16ZeroExtend(src, r); + } else { + masm.load16SignExtend(src, r); + } + pushI32(r); + break; + } + case FieldType::I32: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + masm.load32(src, r); + pushI32(r); + break; + } + case FieldType::I64: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegI64 r = needI64(); + NullCheckPolicy::emitTrapSite(this); + masm.load64(src, r); + pushI64(r); + break; + } + case FieldType::F32: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegF32 r = needF32(); + NullCheckPolicy::emitTrapSite(this); + masm.loadFloat32(src, r); + pushF32(r); + break; + } + case FieldType::F64: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegF64 r = needF64(); + NullCheckPolicy::emitTrapSite(this); + masm.loadDouble(src, r); + pushF64(r); + break; + } +# ifdef ENABLE_WASM_SIMD + case FieldType::V128: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegV128 r = needV128(); + NullCheckPolicy::emitTrapSite(this); + masm.loadUnalignedSimd128(src, r); + pushV128(r); + break; + } +# endif + case FieldType::Ref: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegRef r = needRef(); + NullCheckPolicy::emitTrapSite(this); + masm.loadPtr(src, r); + pushRef(r); + break; + } + default: { + MOZ_CRASH("Unexpected field type"); + } + } +} + +template <typename T, typename NullCheckPolicy> +void BaseCompiler::emitGcSetScalar(const T& dst, FieldType type, AnyReg value) { + NullCheckPolicy::emitTrapSite(this); + switch (type.kind()) { + case FieldType::I8: { + masm.store8(value.i32(), dst); + break; + } + case FieldType::I16: { + masm.store16(value.i32(), dst); + break; + } + case FieldType::I32: { + masm.store32(value.i32(), dst); + break; + } + case FieldType::I64: { + masm.store64(value.i64(), dst); + break; + } + case FieldType::F32: { + masm.storeFloat32(value.f32(), dst); + break; + } + case FieldType::F64: { + masm.storeDouble(value.f64(), dst); + break; + } +# ifdef ENABLE_WASM_SIMD + case FieldType::V128: { + masm.storeUnalignedSimd128(value.v128(), dst); + break; + } +# endif + default: { + MOZ_CRASH("Unexpected field type"); + } + } +} + +template <typename NullCheckPolicy> +bool BaseCompiler::emitGcStructSet(RegRef object, RegPtr areaBase, + uint32_t areaOffset, FieldType fieldType, + AnyReg value, + PreBarrierKind preBarrierKind) { + // Easy path if the field is a scalar + if (!fieldType.isRefRepr()) { + emitGcSetScalar<Address, NullCheckPolicy>(Address(areaBase, areaOffset), + fieldType, value); + freeAny(value); + return true; + } + + // Create temporary for the valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(Address(areaBase, areaOffset), valueAddr); + + NullCheckPolicy::emitNullCheck(this, object); + + // emitBarrieredStore preserves object and value + if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind, + PostBarrierKind::Imprecise)) { + return false; + } + freeRef(value.ref()); + + return true; +} + +bool BaseCompiler::emitGcArraySet(RegRef object, RegPtr data, RegI32 index, + const ArrayType& arrayType, AnyReg value, + PreBarrierKind preBarrierKind) { + // Try to use a base index store instruction if the field type fits in a + // shift immediate. If not we shift the index manually and then unshift + // it after the store. We don't use an extra register for this because we + // don't have any to spare on x86. + uint32_t shift = arrayType.elementType_.indexingShift(); + Scale scale; + bool shiftedIndex = false; + if (IsShiftInScaleRange(shift)) { + scale = ShiftToScale(shift); + } else { + masm.lshiftPtr(Imm32(shift), index); + scale = TimesOne; + shiftedIndex = true; + } + auto unshiftIndex = mozilla::MakeScopeExit([&] { + if (shiftedIndex) { + masm.rshiftPtr(Imm32(shift), index); + } + }); + + // Easy path if the field is a scalar + if (!arrayType.elementType_.isRefRepr()) { + emitGcSetScalar<BaseIndex, NoNullCheck>(BaseIndex(data, index, scale, 0), + arrayType.elementType_, value); + return true; + } + + // Create temporaries for valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(BaseIndex(data, index, scale, 0), valueAddr); + + // Save state for after barriered write + pushPtr(data); + pushI32(index); + + // emitBarrieredStore preserves object and value + if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind, + PostBarrierKind::Imprecise)) { + return false; + } + + // Restore state + popI32(index); + popPtr(data); + + return true; +} + +bool BaseCompiler::emitStructNew() { + uint32_t typeIndex; + BaseNothingVector args{}; + if (!iter_.readStructNew(&typeIndex, &args)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + + // Allocate an uninitialized struct. This requires the type definition + // for the struct to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigStructNewUninit)) { + return false; + } + + // Optimization opportunity: Iterate backward to pop arguments off the + // stack. This will generate more instructions than we want, since we + // really only need to pop the stack once at the end, not for every element, + // but to do better we need a bit more machinery to load elements off the + // stack into registers. + + bool isOutlineStruct = structType.size_ > WasmStructObject_MaxInlineBytes; + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + needPtr(RegPtr(PreBarrierReg)); + + RegRef object = popRef(); + RegPtr outlineBase = isOutlineStruct ? needPtr() : RegPtr(); + + // Free the barrier reg after we've allocated all registers + freePtr(RegPtr(PreBarrierReg)); + + // Optimization opportunity: when the value being stored is a known + // zero/null we need store nothing. This case may be somewhat common + // because struct.new forces a value to be specified for every field. + + // Optimization opportunity: this loop reestablishes the outline base pointer + // every iteration, which really isn't very clever. It would be better to + // establish it once before we start, then re-set it if/when we transition + // from the out-of-line area back to the in-line area. That would however + // require making ::emitGcStructSet preserve that register, which it + // currently doesn't. + + uint32_t fieldIndex = structType.fields_.length(); + while (fieldIndex-- > 0) { + const StructField& field = structType.fields_[fieldIndex]; + FieldType fieldType = field.type; + uint32_t fieldOffset = field.offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Reserve the barrier reg if we might need it for this store + if (fieldType.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + AnyReg value = popAny(); + // Free the barrier reg now that we've loaded the value + if (fieldType.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + if (areaIsOutline) { + // Load the outline data pointer + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + + // Consumes value and outline data, object is preserved by this call. + if (!emitGcStructSet<NoNullCheck>(object, outlineBase, areaOffset, + fieldType, value, + PreBarrierKind::None)) { + return false; + } + } else { + // Consumes value. object is unchanged by this call. + if (!emitGcStructSet<NoNullCheck>( + object, RegPtr(object), + WasmStructObject::offsetOfInlineData() + areaOffset, fieldType, + value, PreBarrierKind::None)) { + return false; + } + } + } + + if (isOutlineStruct) { + freePtr(outlineBase); + } + pushRef(object); + + return true; +} + +bool BaseCompiler::emitStructNewDefault() { + uint32_t typeIndex; + if (!iter_.readStructNewDefault(&typeIndex)) { + return false; + } + if (deadCode_) { + return true; + } + + // Allocate a default initialized struct. This requires the type definition + // for the struct to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + return emitInstanceCall(SASigStructNew); +} + +bool BaseCompiler::emitStructGet(FieldWideningOp wideningOp) { + uint32_t typeIndex; + uint32_t fieldIndex; + Nothing nothing; + if (!iter_.readStructGet(&typeIndex, &fieldIndex, wideningOp, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + + // Decide whether we're accessing inline or outline, and at what offset + FieldType fieldType = structType.fields_[fieldIndex].type; + uint32_t fieldOffset = structType.fields_[fieldIndex].offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + RegRef object = popRef(); + if (areaIsOutline) { + RegPtr outlineBase = needPtr(); + SignalNullCheck::emitTrapSite(this); + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + // Load the value + emitGcGet<Address, NoNullCheck>(fieldType, wideningOp, + Address(outlineBase, areaOffset)); + freePtr(outlineBase); + } else { + // Load the value + emitGcGet<Address, SignalNullCheck>( + fieldType, wideningOp, + Address(object, WasmStructObject::offsetOfInlineData() + areaOffset)); + } + freeRef(object); + + return true; +} + +bool BaseCompiler::emitStructSet() { + uint32_t typeIndex; + uint32_t fieldIndex; + Nothing nothing; + if (!iter_.readStructSet(&typeIndex, &fieldIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + const StructField& structField = structType.fields_[fieldIndex]; + + // Decide whether we're accessing inline or outline, and at what offset + FieldType fieldType = structType.fields_[fieldIndex].type; + uint32_t fieldOffset = structType.fields_[fieldIndex].offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (structField.type.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + RegPtr outlineBase = areaIsOutline ? needPtr() : RegPtr(); + AnyReg value = popAny(); + RegRef object = popRef(); + + // Free the barrier reg after we've allocated all registers + if (structField.type.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // Make outlineBase point at the first byte of the relevant area + if (areaIsOutline) { + SignalNullCheck::emitTrapSite(this); + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + if (!emitGcStructSet<NoNullCheck>(object, outlineBase, areaOffset, + fieldType, value, + PreBarrierKind::Normal)) { + return false; + } + } else { + // Consumes value. object is unchanged by this call. + if (!emitGcStructSet<SignalNullCheck>( + object, RegPtr(object), + WasmStructObject::offsetOfInlineData() + areaOffset, fieldType, + value, PreBarrierKind::Normal)) { + return false; + } + } + + if (areaIsOutline) { + freePtr(outlineBase); + } + freeRef(object); + + return true; +} + +bool BaseCompiler::emitArrayNew() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayNew(&typeIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // Allocate an uninitialized array. This requires the type definition + // for the array to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigArrayNewUninit)) { + return false; + } + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (arrayType.elementType_.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + RegRef rp = popRef(); + AnyReg value = popAny(); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements<NoNullCheck>(rp); + + // Free the barrier reg after we've allocated all registers + if (arrayType.elementType_.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // Perform an initialization loop using `numElements` as the loop variable, + // counting down to zero. + Label done; + Label loop; + // Skip initialization if numElements = 0 + masm.branch32(Assembler::Equal, numElements, Imm32(0), &done); + masm.bind(&loop); + + // Move to the next element + masm.sub32(Imm32(1), numElements); + + // Assign value to array[numElements]. All registers are preserved + if (!emitGcArraySet(rp, rdata, numElements, arrayType, value, + PreBarrierKind::None)) { + return false; + } + + // Loop back if there are still elements to initialize + masm.branch32(Assembler::GreaterThan, numElements, Imm32(0), &loop); + masm.bind(&done); + + freeI32(numElements); + freeAny(value); + freePtr(rdata); + pushRef(rp); + + return true; +} + +bool BaseCompiler::emitArrayNewFixed() { + uint32_t typeIndex, numElements; + BaseNothingVector nothings{}; + if (!iter_.readArrayNewFixed(&typeIndex, &numElements, ¬hings)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // At this point, the top section of the value stack contains the values to + // be used to initialise the array, with index 0 as the topmost value. Push + // the required number of elements and the required type on, since the call + // to SASigArrayNew will use them. + pushI32(numElements); + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigArrayNew)) { + return false; + } + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + bool avoidPreBarrierReg = arrayType.elementType_.isRefRepr(); + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + + // Get hold of the pointer to the array, as created by SASigArrayNew. + RegRef rp = popRef(); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Free the barrier reg if we previously reserved it. + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + + // These together ensure that the max value of `index` in the loop below + // remains comfortably below the 2^31 boundary. See comments on equivalent + // assertions in EmitArrayNewFixed in WasmIonCompile.cpp for explanation. + static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <= + MaxArrayPayloadBytes); + MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes); + + // Generate straight-line initialization code. We could do better here if + // there was a version of ::emitGcArraySet that took `index` as a `uint32_t` + // rather than a general value-in-a-reg. + for (uint32_t forwardIndex = 0; forwardIndex < numElements; forwardIndex++) { + uint32_t reverseIndex = numElements - forwardIndex - 1; + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + AnyReg value = popAny(); + pushI32(reverseIndex); + RegI32 index = popI32(); + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + if (!emitGcArraySet(rp, rdata, index, arrayType, value, + PreBarrierKind::None)) { + return false; + } + freeI32(index); + freeAny(value); + } + + freePtr(rdata); + + pushRef(rp); + return true; +} + +bool BaseCompiler::emitArrayNewDefault() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayNewDefault(&typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Allocate a default initialized array. This requires the type definition + // for the array to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + return emitInstanceCall(SASigArrayNew); +} + +bool BaseCompiler::emitArrayNewData() { + uint32_t typeIndex, segIndex; + Nothing nothing; + if (!iter_.readArrayNewData(&typeIndex, &segIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushPtr(loadTypeDefInstanceData(typeIndex)); + pushI32(int32_t(segIndex)); + + // The call removes 4 items from the stack: the segment byte offset and + // number of elements (operands to array.new_data), and the type index and + // seg index as pushed above. + return emitInstanceCall(SASigArrayNewData); +} + +bool BaseCompiler::emitArrayNewElem() { + uint32_t typeIndex, segIndex; + Nothing nothing; + if (!iter_.readArrayNewElem(&typeIndex, &segIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushPtr(loadTypeDefInstanceData(typeIndex)); + pushI32(int32_t(segIndex)); + + // The call removes 4 items from the stack: the segment element offset and + // number of elements (operands to array.new_elem), and the type index and + // seg index as pushed above. + return emitInstanceCall(SASigArrayNewElem); +} + +bool BaseCompiler::emitArrayGet(FieldWideningOp wideningOp) { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayGet(&typeIndex, wideningOp, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + RegI32 index = popI32(); + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp); + + // Bounds check the index + emitGcArrayBoundsCheck(index, numElements); + freeI32(numElements); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Load the value + uint32_t shift = arrayType.elementType_.indexingShift(); + if (IsShiftInScaleRange(shift)) { + emitGcGet<BaseIndex, NoNullCheck>( + arrayType.elementType_, wideningOp, + BaseIndex(rdata, index, ShiftToScale(shift), 0)); + } else { + masm.lshiftPtr(Imm32(shift), index); + emitGcGet<BaseIndex, NoNullCheck>(arrayType.elementType_, wideningOp, + BaseIndex(rdata, index, TimesOne, 0)); + } + + freePtr(rdata); + freeRef(rp); + freeI32(index); + + return true; +} + +bool BaseCompiler::emitArraySet() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArraySet(&typeIndex, ¬hing, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (arrayType.elementType_.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + AnyReg value = popAny(); + RegI32 index = popI32(); + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp); + + // Bounds check the index + emitGcArrayBoundsCheck(index, numElements); + freeI32(numElements); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Free the barrier reg after we've allocated all registers + if (arrayType.elementType_.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // All registers are preserved. This isn't strictly necessary, as we'll just + // be freeing them all after this is done. But this is needed for repeated + // assignments used in array.new/new_default. + if (!emitGcArraySet(rp, rdata, index, arrayType, value, + PreBarrierKind::Normal)) { + return false; + } + + freePtr(rdata); + freeRef(rp); + freeI32(index); + freeAny(value); + + return true; +} + +bool BaseCompiler::emitArrayLen(bool decodeIgnoredTypeIndex) { + Nothing nothing; + if (!iter_.readArrayLen(decodeIgnoredTypeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements<SignalNullCheck>(rp); + pushI32(numElements); + + freeRef(rp); + + return true; +} + +bool BaseCompiler::emitArrayCopy() { + int32_t elemSize; + bool elemsAreRefTyped; + Nothing nothing; + if (!iter_.readArrayCopy(&elemSize, &elemsAreRefTyped, ¬hing, ¬hing, + ¬hing, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + // readArrayCopy writes the element size in bytes to `elemSize`. Code + // generated here needs to hand it onwards to the instance fn, but also + // indicate whether the element type is reftyped, which it does by negating + // the size. + // + // The value stack currently looks like this: + // + // (top) [numElements, srcIndex, srcArray, dstIndex, dstArray, ..] + // + // So push `(possibly negated) elemSize` on it, giving + // + // (top) [elemSize, numElements, srcIndex, srcArray, dstIndex, dstArray, ..] + // + // and generate a call to the helper. + + MOZ_ASSERT_IF(elemsAreRefTyped, elemSize == sizeof(void*)); + MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 || + elemSize == 4 || elemSize == 8 || + elemSize == 16); + bool avoidPreBarrierReg = elemsAreRefTyped; + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + + // The helper needs to know the element size. + pushI32(elemsAreRefTyped ? -elemSize : elemSize); + + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + + return emitInstanceCall(SASigArrayCopy); +} + +void BaseCompiler::emitRefTestCommon(RefType sourceType, RefType destType) { + Label success; + Label join; + RegRef object = popRef(); + RegI32 result = needI32(); + + branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true); + masm.xor32(result, result); + masm.jump(&join); + masm.bind(&success); + masm.move32(Imm32(1), result); + masm.bind(&join); + + pushI32(result); + freeRef(object); +} + +void BaseCompiler::emitRefCastCommon(RefType sourceType, RefType destType) { + RegRef object = popRef(); + + Label success; + branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true); + masm.wasmTrap(Trap::BadCast, bytecodeOffset()); + masm.bind(&success); + pushRef(object); +} + +bool BaseCompiler::emitRefTestV5() { + Nothing nothing; + RefType sourceType; + uint32_t typeIndex; + if (!iter_.readRefTestV5(&sourceType, &typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/false); + emitRefTestCommon(sourceType, destType); + + return true; +} + +void BaseCompiler::branchGcRefType(RegRef object, RefType sourceType, + RefType destType, Label* label, + bool onSuccess) { + RegPtr superSuperTypeVector; + if (MacroAssembler::needSuperSuperTypeVectorForBranchWasmGcRefType( + destType)) { + uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef()); + superSuperTypeVector = loadSuperTypeVector(typeIndex); + } + RegI32 scratch1 = MacroAssembler::needScratch1ForBranchWasmGcRefType(destType) + ? needI32() + : RegI32::Invalid(); + RegI32 scratch2 = MacroAssembler::needScratch2ForBranchWasmGcRefType(destType) + ? needI32() + : RegI32::Invalid(); + + masm.branchWasmGcObjectIsRefType(object, sourceType, destType, label, + onSuccess, superSuperTypeVector, scratch1, + scratch2); + + if (scratch2.isValid()) { + freeI32(scratch2); + } + if (scratch1.isValid()) { + freeI32(scratch1); + } + if (superSuperTypeVector.isValid()) { + freePtr(superSuperTypeVector); + } +} + +bool BaseCompiler::emitRefCastV5() { + Nothing nothing; + RefType sourceType; + uint32_t typeIndex; + if (!iter_.readRefCastV5(&sourceType, &typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/true); + emitRefCastCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitRefTest(bool nullable) { + Nothing nothing; + RefType sourceType; + RefType destType; + if (!iter_.readRefTest(nullable, &sourceType, &destType, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + emitRefTestCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitRefCast(bool nullable) { + Nothing nothing; + RefType sourceType; + RefType destType; + if (!iter_.readRefCast(nullable, &sourceType, &destType, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + emitRefCastCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitBrOnCastCommon(bool onSuccess, + uint32_t labelRelativeDepth, + const ResultType& labelType, + RefType sourceType, RefType destType) { + Control& target = controlItem(labelRelativeDepth); + target.bceSafeOnExit &= bceSafe_; + + // 3. br_if $l : [T*, ref] -> [T*, ref] + BranchState b(&target.label, target.stackHeight, InvertBranch(false), + labelType); + + // Don't allocate the result register used in the branch + if (b.hasBlockResults()) { + needIntegerResultRegisters(b.resultType); + } + + // Create a copy of the ref for passing to the br_on_cast label, + // the original ref is used for casting in the condition. + RegRef object = popRef(); + RegRef objectCondition = needRef(); + moveRef(object, objectCondition); + pushRef(object); + + if (b.hasBlockResults()) { + freeIntegerResultRegisters(b.resultType); + } + + if (!jumpConditionalWithResults(&b, objectCondition, sourceType, destType, + onSuccess)) { + return false; + } + freeRef(objectCondition); + + return true; +} + +bool BaseCompiler::emitBrOnCast() { + MOZ_ASSERT(!hasLatentOp()); + + bool onSuccess; + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + BaseNothingVector unused_values{}; + if (!iter_.readBrOnCast(&onSuccess, &labelRelativeDepth, &sourceType, + &destType, &labelType, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitBrOnCastV5(bool onSuccess) { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + RefType sourceType; + uint32_t castTypeIndex; + ResultType labelType; + BaseNothingVector unused_values{}; + if (onSuccess + ? !iter_.readBrOnCastV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, &unused_values) + : !iter_.readBrOnCastFailV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, + &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(castTypeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, false); + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitBrOnCastHeapV5(bool onSuccess, bool nullable) { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + BaseNothingVector unused_values{}; + if (onSuccess ? !iter_.readBrOnCastHeapV5(nullable, &labelRelativeDepth, + &sourceType, &destType, &labelType, + &unused_values) + : !iter_.readBrOnCastFailHeapV5(nullable, &labelRelativeDepth, + &sourceType, &destType, + &labelType, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitRefAsStructV5() { + Nothing nothing; + return iter_.readConversion(ValType(RefType::any()), + ValType(RefType::struct_().asNonNullable()), + ¬hing); +} + +bool BaseCompiler::emitBrOnNonStructV5() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + ResultType labelType; + BaseNothingVector unused_values{}; + if (!iter_.readBrOnNonStructV5(&labelRelativeDepth, &labelType, + &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(labelRelativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), + labelType); + if (b.hasBlockResults()) { + needResultRegisters(b.resultType); + } + RegI32 condition = needI32(); + masm.move32(Imm32(1), condition); + if (b.hasBlockResults()) { + freeResultRegisters(b.resultType); + } + if (!jumpConditionalWithResults(&b, Assembler::Equal, condition, Imm32(0))) { + return false; + } + freeI32(condition); + return true; +} + +bool BaseCompiler::emitExternInternalize() { + // extern.internalize is a no-op because anyref and extern share the same + // representation + Nothing nothing; + return iter_.readRefConversion(RefType::extern_(), RefType::any(), ¬hing); +} + +bool BaseCompiler::emitExternExternalize() { + // extern.externalize is a no-op because anyref and extern share the same + // representation + Nothing nothing; + return iter_.readRefConversion(RefType::any(), RefType::extern_(), ¬hing); +} + +#endif // ENABLE_WASM_GC + +////////////////////////////////////////////////////////////////////////////// +// +// SIMD and Relaxed SIMD. + +#ifdef ENABLE_WASM_SIMD + +// Emitter trampolines used by abstracted SIMD operations. Naming here follows +// the SIMD spec pretty closely. + +static void AndV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseAndSimd128(rs, rsd); +} + +static void OrV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseOrSimd128(rs, rsd); +} + +static void XorV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseXorSimd128(rs, rsd); +} + +static void AddI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt8x16(rsd, rs, rsd); +} + +static void AddI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt16x8(rsd, rs, rsd); +} + +static void AddI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt32x4(rsd, rs, rsd); +} + +static void AddF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addFloat32x4(rsd, rs, rsd); +} + +static void AddI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt64x2(rsd, rs, rsd); +} + +static void AddF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addFloat64x2(rsd, rs, rsd); +} + +static void AddSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addSatInt8x16(rsd, rs, rsd); +} + +static void AddSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAddSatInt8x16(rsd, rs, rsd); +} + +static void AddSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addSatInt16x8(rsd, rs, rsd); +} + +static void AddSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAddSatInt16x8(rsd, rs, rsd); +} + +static void SubI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt8x16(rsd, rs, rsd); +} + +static void SubI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt16x8(rsd, rs, rsd); +} + +static void SubI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt32x4(rsd, rs, rsd); +} + +static void SubF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subFloat32x4(rsd, rs, rsd); +} + +static void SubI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt64x2(rsd, rs, rsd); +} + +static void SubF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subFloat64x2(rsd, rs, rsd); +} + +static void SubSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subSatInt8x16(rsd, rs, rsd); +} + +static void SubSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedSubSatInt8x16(rsd, rs, rsd); +} + +static void SubSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subSatInt16x8(rsd, rs, rsd); +} + +static void SubSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedSubSatInt16x8(rsd, rs, rsd); +} + +static void MulI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulInt16x8(rsd, rs, rsd); +} + +static void MulI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulInt32x4(rsd, rs, rsd); +} + +static void MulF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulFloat32x4(rsd, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp) { + masm.mulInt64x2(rsd, rs, rsd, temp); +} +# elif defined(JS_CODEGEN_ARM64) +static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.mulInt64x2(rsd, rs, rsd, temp1, temp2); +} +# endif + +static void MulF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulFloat64x2(rsd, rs, rsd); +} + +static void DivF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.divFloat32x4(rsd, rs, rsd); +} + +static void DivF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.divFloat64x2(rsd, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.minFloat32x4(rsd, rs, rsd, temp1, temp2); +} + +static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.minFloat64x2(rsd, rs, rsd, temp1, temp2); +} + +static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.maxFloat32x4(rsd, rs, rsd, temp1, temp2); +} + +static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.maxFloat64x2(rsd, rs, rsd, temp1, temp2); +} + +static void PMinF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMinFloat32x4(rsd, rs); +} + +static void PMinF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMinFloat64x2(rsd, rs); +} + +static void PMaxF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMaxFloat32x4(rsd, rs); +} + +static void PMaxF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMaxFloat64x2(rsd, rs); +} +# elif defined(JS_CODEGEN_ARM64) +static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat32x4(rs, rsd); +} + +static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat64x2(rs, rsd); +} + +static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat32x4(rs, rsd); +} + +static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat64x2(rs, rsd); +} + +static void PMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMinFloat32x4(rs, rsd); +} + +static void PMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMinFloat64x2(rs, rsd); +} + +static void PMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMaxFloat32x4(rs, rsd); +} + +static void PMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMaxFloat64x2(rs, rsd); +} +# endif + +static void DotI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.widenDotInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt8x16(rsd, rs, rsd); +} + +static void ExtMulHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt8x16(rsd, rs, rsd); +} + +static void ExtMulLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt8x16(rsd, rs, rsd); +} + +static void ExtMulHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt8x16(rsd, rs, rsd); +} + +static void ExtMulLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt16x8(rsd, rs, rsd); +} + +static void ExtMulHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt16x8(rsd, rs, rsd); +} + +static void ExtMulHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt32x4(rsd, rs, rsd); +} + +static void ExtMulHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt32x4(rsd, rs, rsd); +} + +static void ExtMulLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt32x4(rsd, rs, rsd); +} + +static void ExtMulHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt32x4(rsd, rs, rsd); +} + +static void Q15MulrSatS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.q15MulrSatInt16x8(rsd, rs, rsd); +} + +static void CmpI8x16(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt8x16(cond, rs, rsd); +} + +static void CmpI16x8(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt16x8(cond, rs, rsd); +} + +static void CmpI32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt32x4(cond, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareForEqualityInt64x2(cond, rsd, rs, rsd); +} + +static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd, RegV128 temp1, + RegV128 temp2) { + masm.compareForOrderingInt64x2(cond, rsd, rs, rsd, temp1, temp2); +} +# else +static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt64x2(cond, rs, rsd); +} + +static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt64x2(cond, rs, rsd); +} +# endif // JS_CODEGEN_X86 || JS_CODEGEN_X64 + +static void CmpUI8x16(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt8x16(cond, rs, rsd); +} + +static void CmpUI16x8(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt16x8(cond, rs, rsd); +} + +static void CmpUI32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt32x4(cond, rs, rsd); +} + +static void CmpF32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareFloat32x4(cond, rs, rsd); +} + +static void CmpF64x2(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareFloat64x2(cond, rs, rsd); +} + +static void NegI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt8x16(rs, rd); +} + +static void NegI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt16x8(rs, rd); +} + +static void NegI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt32x4(rs, rd); +} + +static void NegI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt64x2(rs, rd); +} + +static void NegF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negFloat32x4(rs, rd); +} + +static void NegF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negFloat64x2(rs, rd); +} + +static void AbsF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absFloat32x4(rs, rd); +} + +static void AbsF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absFloat64x2(rs, rd); +} + +static void SqrtF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.sqrtFloat32x4(rs, rd); +} + +static void SqrtF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.sqrtFloat64x2(rs, rd); +} + +static void CeilF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.ceilFloat32x4(rs, rd); +} + +static void FloorF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.floorFloat32x4(rs, rd); +} + +static void TruncF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncFloat32x4(rs, rd); +} + +static void NearestF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.nearestFloat32x4(rs, rd); +} + +static void CeilF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.ceilFloat64x2(rs, rd); +} + +static void FloorF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.floorFloat64x2(rs, rd); +} + +static void TruncF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncFloat64x2(rs, rd); +} + +static void NearestF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.nearestFloat64x2(rs, rd); +} + +static void NotV128(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.bitwiseNotSimd128(rs, rd); +} + +static void ExtAddPairwiseI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extAddPairwiseInt8x16(rs, rsd); +} + +static void ExtAddPairwiseUI8x16(MacroAssembler& masm, RegV128 rs, + RegV128 rsd) { + masm.unsignedExtAddPairwiseInt8x16(rs, rsd); +} + +static void ExtAddPairwiseI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extAddPairwiseInt16x8(rs, rsd); +} + +static void ExtAddPairwiseUI16x8(MacroAssembler& masm, RegV128 rs, + RegV128 rsd) { + masm.unsignedExtAddPairwiseInt16x8(rs, rsd); +} + +static void ShiftOpMask(MacroAssembler& masm, SimdOp op, RegI32 in, + RegI32 out) { + int32_t maskBits; + + masm.mov(in, out); + if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) { + masm.and32(Imm32(maskBits), out); + } +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp1); + masm.leftShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp); + masm.leftShiftInt16x8(temp, rsd); +} + +static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp); + masm.leftShiftInt32x4(temp, rsd); +} + +static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp); + masm.leftShiftInt64x2(temp, rsd); +} + +static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp1); + masm.rightShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp1); + masm.unsignedRightShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp); + masm.rightShiftInt16x8(temp, rsd); +} + +static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp); + masm.unsignedRightShiftInt16x8(temp, rsd); +} + +static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp); + masm.rightShiftInt32x4(temp, rsd); +} + +static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp); + masm.unsignedRightShiftInt32x4(temp, rsd); +} + +static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp); + masm.unsignedRightShiftInt64x2(temp, rsd); +} +# elif defined(JS_CODEGEN_ARM64) +static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp); + masm.leftShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp); + masm.leftShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp); + masm.leftShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp); + masm.leftShiftInt64x2(rsd, temp, rsd); +} + +static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp); + masm.rightShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp); + masm.unsignedRightShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp); + masm.rightShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp); + masm.unsignedRightShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp); + masm.rightShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp); + masm.unsignedRightShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftRightI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrS, rs, temp); + masm.rightShiftInt64x2(rsd, temp, rsd); +} + +static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp); + masm.unsignedRightShiftInt64x2(rsd, temp, rsd); +} +# endif + +static void AverageUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAverageInt8x16(rsd, rs, rsd); +} + +static void AverageUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAverageInt16x8(rsd, rs, rsd); +} + +static void MinI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt8x16(rsd, rs, rsd); +} + +static void MinUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt8x16(rsd, rs, rsd); +} + +static void MaxI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt8x16(rsd, rs, rsd); +} + +static void MaxUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt8x16(rsd, rs, rsd); +} + +static void MinI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt16x8(rsd, rs, rsd); +} + +static void MinUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt16x8(rsd, rs, rsd); +} + +static void MaxI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt16x8(rsd, rs, rsd); +} + +static void MaxUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt16x8(rsd, rs, rsd); +} + +static void MinI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt32x4(rsd, rs, rsd); +} + +static void MinUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt32x4(rsd, rs, rsd); +} + +static void MaxI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt32x4(rsd, rs, rsd); +} + +static void MaxUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt32x4(rsd, rs, rsd); +} + +static void NarrowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.narrowInt16x8(rsd, rs, rsd); +} + +static void NarrowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedNarrowInt16x8(rsd, rs, rsd); +} + +static void NarrowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.narrowInt32x4(rsd, rs, rsd); +} + +static void NarrowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedNarrowInt32x4(rsd, rs, rsd); +} + +static void WidenLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt8x16(rs, rd); +} + +static void WidenHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt8x16(rs, rd); +} + +static void WidenLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt8x16(rs, rd); +} + +static void WidenHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt8x16(rs, rd); +} + +static void WidenLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt16x8(rs, rd); +} + +static void WidenHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt16x8(rs, rd); +} + +static void WidenLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt16x8(rs, rd); +} + +static void WidenHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt16x8(rs, rd); +} + +static void WidenLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt32x4(rs, rd); +} + +static void WidenHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt32x4(rs, rd); +} + +static void WidenLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt32x4(rs, rd); +} + +static void WidenHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt32x4(rs, rd); +} + +# if defined(JS_CODEGEN_ARM64) +static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.popcntInt8x16(rs, rd); +} +# else +static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.popcntInt8x16(rs, rd, temp); +} +# endif // JS_CODEGEN_ARM64 + +static void AbsI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt8x16(rs, rd); +} + +static void AbsI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt16x8(rs, rd); +} + +static void AbsI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt32x4(rs, rd); +} + +static void AbsI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt64x2(rs, rd); +} + +static void ExtractLaneI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt8x16(laneIndex, rs, rd); +} + +static void ExtractLaneUI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.unsignedExtractLaneInt8x16(laneIndex, rs, rd); +} + +static void ExtractLaneI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt16x8(laneIndex, rs, rd); +} + +static void ExtractLaneUI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.unsignedExtractLaneInt16x8(laneIndex, rs, rd); +} + +static void ExtractLaneI32x4(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt32x4(laneIndex, rs, rd); +} + +static void ExtractLaneI64x2(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI64 rd) { + masm.extractLaneInt64x2(laneIndex, rs, rd); +} + +static void ExtractLaneF32x4(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegF32 rd) { + masm.extractLaneFloat32x4(laneIndex, rs, rd); +} + +static void ExtractLaneF64x2(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegF64 rd) { + masm.extractLaneFloat64x2(laneIndex, rs, rd); +} + +static void ReplaceLaneI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt8x16(laneIndex, rs, rsd); +} + +static void ReplaceLaneI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt16x8(laneIndex, rs, rsd); +} + +static void ReplaceLaneI32x4(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt32x4(laneIndex, rs, rsd); +} + +static void ReplaceLaneI64x2(MacroAssembler& masm, uint32_t laneIndex, + RegI64 rs, RegV128 rsd) { + masm.replaceLaneInt64x2(laneIndex, rs, rsd); +} + +static void ReplaceLaneF32x4(MacroAssembler& masm, uint32_t laneIndex, + RegF32 rs, RegV128 rsd) { + masm.replaceLaneFloat32x4(laneIndex, rs, rsd); +} + +static void ReplaceLaneF64x2(MacroAssembler& masm, uint32_t laneIndex, + RegF64 rs, RegV128 rsd) { + masm.replaceLaneFloat64x2(laneIndex, rs, rsd); +} + +static void SplatI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX16(rs, rd); +} + +static void SplatI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX8(rs, rd); +} + +static void SplatI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX4(rs, rd); +} + +static void SplatI64x2(MacroAssembler& masm, RegI64 rs, RegV128 rd) { + masm.splatX2(rs, rd); +} + +static void SplatF32x4(MacroAssembler& masm, RegF32 rs, RegV128 rd) { + masm.splatX4(rs, rd); +} + +static void SplatF64x2(MacroAssembler& masm, RegF64 rs, RegV128 rd) { + masm.splatX2(rs, rd); +} + +// This is the same op independent of lanes: it tests for any nonzero bit. +static void AnyTrue(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.anyTrueSimd128(rs, rd); +} + +static void AllTrueI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt8x16(rs, rd); +} + +static void AllTrueI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt16x8(rs, rd); +} + +static void AllTrueI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt32x4(rs, rd); +} + +static void AllTrueI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt64x2(rs, rd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt8x16(rs, rd); +} + +static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt16x8(rs, rd); +} + +static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt32x4(rs, rd); +} + +static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt64x2(rs, rd); +} +# elif defined(JS_CODEGEN_ARM64) +static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt8x16(rs, rd, temp); +} + +static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt16x8(rs, rd, temp); +} + +static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt32x4(rs, rd, temp); +} + +static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt64x2(rs, rd, temp); +} +# endif + +static void Swizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.swizzleInt8x16(rsd, rs, rsd); +} + +static void ConvertI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertInt32x4ToFloat32x4(rs, rd); +} + +static void ConvertUI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedConvertInt32x4ToFloat32x4(rs, rd); +} + +static void ConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncSatFloat32x4ToInt32x4(rs, rd); +} + +# if defined(JS_CODEGEN_ARM64) +static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd); +} +# else +static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd, temp); +} +# endif // JS_CODEGEN_ARM64 + +static void ConvertI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertInt32x4ToFloat64x2(rs, rd); +} + +static void ConvertUI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedConvertInt32x4ToFloat64x2(rs, rd); +} + +static void ConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.truncSatFloat64x2ToInt32x4(rs, rd, temp); +} + +static void ConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.unsignedTruncSatFloat64x2ToInt32x4(rs, rd, temp); +} + +static void DemoteF64x2ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertFloat64x2ToFloat32x4(rs, rd); +} + +static void PromoteF32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertFloat32x4ToFloat64x2(rs, rd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control, + RegV128 lhsDest, RegV128 temp) { + // Ideally, we would have temp=control, and we can probably get away with + // just doing that, but don't worry about it yet. + masm.bitwiseSelectSimd128(control, lhsDest, rhs, lhsDest, temp); +} +# elif defined(JS_CODEGEN_ARM64) +static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control, + RegV128 lhsDest, RegV128 temp) { + // The masm interface is not great for the baseline compiler here, but it's + // optimal for Ion, so just work around it. + masm.moveSimd128(control, temp); + masm.bitwiseSelectSimd128(lhsDest, rhs, temp); + masm.moveSimd128(temp, lhsDest); +} +# endif + +# ifdef ENABLE_WASM_RELAXED_SIMD +static void RelaxedFmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fmaFloat32x4(rs1, rs2, rsd); +} + +static void RelaxedFnmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fnmaFloat32x4(rs1, rs2, rsd); +} + +static void RelaxedFmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fmaFloat64x2(rs1, rs2, rsd); +} + +static void RelaxedFnmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fnmaFloat64x2(rs1, rs2, rsd); +} + +static void RelaxedSwizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.swizzleInt8x16Relaxed(rsd, rs, rsd); +} + +static void RelaxedMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat32x4Relaxed(rs, rsd); +} + +static void RelaxedMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat32x4Relaxed(rs, rsd); +} + +static void RelaxedMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat64x2Relaxed(rs, rsd); +} + +static void RelaxedMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat64x2Relaxed(rs, rsd); +} + +static void RelaxedConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.truncFloat32x4ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.unsignedTruncFloat32x4ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.truncFloat64x2ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.unsignedTruncFloat64x2ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedQ15MulrS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.q15MulrInt16x8Relaxed(rsd, rs, rsd); +} + +static void DotI8x16I7x16S(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.dotInt8x16Int7x16(rsd, rs, rsd); +} + +void BaseCompiler::emitDotI8x16I7x16AddS() { + RegV128 rsd = popV128(); + RegV128 rs0, rs1; + pop2xV128(&rs0, &rs1); +# if defined(JS_CODEGEN_ARM64) + RegV128 temp = needV128(); + masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd, temp); + freeV128(temp); +# else + masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd); +# endif + freeV128(rs1); + freeV128(rs0); + pushV128(rsd); +} +# endif // ENABLE_WASM_RELAXED_SIMD + +void BaseCompiler::emitVectorAndNot() { + // We want x & ~y but the available operation is ~x & y, so reverse the + // operands. + RegV128 r, rs; + pop2xV128(&r, &rs); + masm.bitwiseNotAndSimd128(r, rs); + freeV128(r); + pushV128(rs); +} + +bool BaseCompiler::emitLoadSplat(Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadSplat(&access); + return true; +} + +bool BaseCompiler::emitLoadZero(Scalar::Type viewType) { + // LoadZero has the structure of LoadSplat, so reuse the reader. + LinearMemoryAddress<Nothing> addr; + if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadZero(&access); + return true; +} + +bool BaseCompiler::emitLoadExtend(Scalar::Type viewType) { + LinearMemoryAddress<Nothing> addr; + if (!iter_.readLoadExtend(&addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(Scalar::Int64, addr.align, addr.offset, + bytecodeOffset()); + loadExtend(&access, viewType); + return true; +} + +bool BaseCompiler::emitLoadLane(uint32_t laneSize) { + Nothing nothing; + LinearMemoryAddress<Nothing> addr; + uint32_t laneIndex; + if (!iter_.readLoadLane(laneSize, &addr, &laneIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + Scalar::Type viewType; + switch (laneSize) { + case 1: + viewType = Scalar::Uint8; + break; + case 2: + viewType = Scalar::Uint16; + break; + case 4: + viewType = Scalar::Int32; + break; + case 8: + viewType = Scalar::Int64; + break; + default: + MOZ_CRASH("unsupported laneSize"); + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadLane(&access, laneIndex); + return true; +} + +bool BaseCompiler::emitStoreLane(uint32_t laneSize) { + Nothing nothing; + LinearMemoryAddress<Nothing> addr; + uint32_t laneIndex; + if (!iter_.readStoreLane(laneSize, &addr, &laneIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + Scalar::Type viewType; + switch (laneSize) { + case 1: + viewType = Scalar::Uint8; + break; + case 2: + viewType = Scalar::Uint16; + break; + case 4: + viewType = Scalar::Int32; + break; + case 8: + viewType = Scalar::Int64; + break; + default: + MOZ_CRASH("unsupported laneSize"); + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + storeLane(&access, laneIndex); + return true; +} + +bool BaseCompiler::emitVectorShuffle() { + Nothing unused_a, unused_b; + V128 shuffleMask; + + if (!iter_.readVectorShuffle(&unused_a, &unused_b, &shuffleMask)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegV128 rd, rs; + pop2xV128(&rd, &rs); + + masm.shuffleInt8x16(shuffleMask.bytes, rs, rd); + + freeV128(rs); + pushV128(rd); + + return true; +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +bool BaseCompiler::emitVectorShiftRightI64x2() { + Nothing unused_a, unused_b; + + if (!iter_.readVectorShift(&unused_a, &unused_b)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegI32 count = popI32RhsForShiftI64(); + RegV128 lhsDest = popV128(); + RegI64 tmp = needI64(); + masm.and32(Imm32(63), count); + masm.extractLaneInt64x2(0, lhsDest, tmp); + masm.rshift64Arithmetic(count, tmp); + masm.replaceLaneInt64x2(0, tmp, lhsDest); + masm.extractLaneInt64x2(1, lhsDest, tmp); + masm.rshift64Arithmetic(count, tmp); + masm.replaceLaneInt64x2(1, tmp, lhsDest); + freeI64(tmp); + freeI32(count); + pushV128(lhsDest); + + return true; +} +# endif +#endif // ENABLE_WASM_SIMD + +#ifdef ENABLE_WASM_RELAXED_SIMD +bool BaseCompiler::emitVectorLaneSelect() { + Nothing unused_a, unused_b, unused_c; + + if (!iter_.readTernary(ValType::V128, &unused_a, &unused_b, &unused_c)) { + return false; + } + + if (deadCode_) { + return true; + } + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + RegV128 mask = popV128(RegV128(vmm0)); + RegV128 rhsDest = popV128(); + RegV128 lhs = popV128(); + masm.laneSelectSimd128(mask, lhs, rhsDest, rhsDest); + freeV128(lhs); + freeV128(mask); + pushV128(rhsDest); +# elif defined(JS_CODEGEN_ARM64) + RegV128 maskDest = popV128(); + RegV128 rhs = popV128(); + RegV128 lhs = popV128(); + masm.laneSelectSimd128(maskDest, lhs, rhs, maskDest); + freeV128(lhs); + freeV128(rhs); + pushV128(maskDest); +# endif + + return true; +} +#endif // ENABLE_WASM_RELAXED_SIMD + +////////////////////////////////////////////////////////////////////////////// +// +// "Intrinsics" - magically imported functions for internal use. + +bool BaseCompiler::emitIntrinsic() { + const Intrinsic* intrinsic; + + BaseNothingVector params; + if (!iter_.readIntrinsic(&intrinsic, ¶ms)) { + return false; + } + + if (deadCode_) { + return true; + } + + // The final parameter of an intrinsic is implicitly the heap base + pushHeapBase(); + + // Call the intrinsic + return emitInstanceCall(intrinsic->signature); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Function bodies - main opcode dispatch loop. + +bool BaseCompiler::emitBody() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitBody"); + + MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isSome()); + + if (!iter_.startFunction(func_.index, locals_)) { + return false; + } + + initControl(controlItem(), ResultType::Empty()); + + for (;;) { + Nothing unused_a, unused_b, unused_c; + (void)unused_a; + (void)unused_b; + (void)unused_c; + +#ifdef DEBUG + performRegisterLeakCheck(); + assertStackInvariants(); +#endif + +#define dispatchBinary0(doEmit, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchBinary1(arg1, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1), true)) + +#define dispatchBinary2(arg1, arg2, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1, arg2), true)) + +#define dispatchBinary3(arg1, arg2, arg3, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1, arg2, arg3), true)) + +#define dispatchUnary0(doEmit, type) \ + iter_.readUnary(type, &unused_a) && (deadCode_ || (doEmit(), true)) + +#define dispatchUnary1(arg1, type) \ + iter_.readUnary(type, &unused_a) && (deadCode_ || (emitUnop(arg1), true)) + +#define dispatchUnary2(arg1, arg2, type) \ + iter_.readUnary(type, &unused_a) && \ + (deadCode_ || (emitUnop(arg1, arg2), true)) + +#define dispatchTernary0(doEmit, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchTernary1(arg1, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (emitTernary(arg1), true)) + +#define dispatchTernary2(arg1, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (emitTernaryResultLast(arg1), true)) + +#define dispatchComparison0(doEmit, operandType, compareOp) \ + iter_.readComparison(operandType, &unused_a, &unused_b) && \ + (deadCode_ || (doEmit(compareOp, operandType), true)) + +#define dispatchConversion0(doEmit, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchConversion1(arg1, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || (emitUnop(arg1), true)) + +#define dispatchConversionOOM(doEmit, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || doEmit()) + +#define dispatchCalloutConversionOOM(doEmit, symbol, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || doEmit(symbol, inType, outType)) + +#define dispatchIntDivCallout(doEmit, symbol, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || doEmit(symbol, type)) + +#define dispatchVectorBinary(op) \ + iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(op), true)) + +#define dispatchVectorUnary(op) \ + iter_.readUnary(ValType::V128, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#define dispatchVectorComparison(op, compareOp) \ + iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(compareOp, op), true)) + +#define dispatchVectorVariableShift(op) \ + iter_.readVectorShift(&unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(op), true)) + +#define dispatchExtractLane(op, outType, laneLimit) \ + iter_.readExtractLane(outType, laneLimit, &laneIndex, &unused_a) && \ + (deadCode_ || (emitUnop(laneIndex, op), true)) + +#define dispatchReplaceLane(op, inType, laneLimit) \ + iter_.readReplaceLane(inType, laneLimit, &laneIndex, &unused_a, \ + &unused_b) && \ + (deadCode_ || (emitBinop(laneIndex, op), true)) + +#define dispatchSplat(op, inType) \ + iter_.readConversion(inType, ValType::V128, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#define dispatchVectorReduction(op) \ + iter_.readConversion(ValType::V128, ValType::I32, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#ifdef DEBUG + // Check that the number of ref-typed entries in the operand stack matches + // reality. +# define CHECK_POINTER_COUNT \ + do { \ + MOZ_ASSERT(countMemRefsOnStk() == stackMapGenerator_.memRefsOnStk); \ + } while (0) +#else +# define CHECK_POINTER_COUNT \ + do { \ + } while (0) +#endif + +#define CHECK(E) \ + if (!(E)) return false +#define NEXT() \ + { \ + CHECK_POINTER_COUNT; \ + continue; \ + } +#define CHECK_NEXT(E) \ + if (!(E)) return false; \ + { \ + CHECK_POINTER_COUNT; \ + continue; \ + } + + // Opcodes that push more than MaxPushesPerOpcode (anything with multiple + // results) will perform additional reservation. + CHECK(stk_.reserve(stk_.length() + MaxPushesPerOpcode)); + + OpBytes op{}; + CHECK(iter_.readOp(&op)); + + // When compilerEnv_.debugEnabled(), some operators get a breakpoint site. + if (compilerEnv_.debugEnabled() && op.shouldHaveBreakpoint()) { + if (previousBreakablePoint_ != masm.currentOffset()) { + // TODO sync only registers that can be clobbered by the exit + // prologue/epilogue or disable these registers for use in + // baseline compiler when compilerEnv_.debugEnabled() is set. + sync(); + + insertBreakablePoint(CallSiteDesc::Breakpoint); + if (!createStackMap("debug: per-insn breakpoint")) { + return false; + } + previousBreakablePoint_ = masm.currentOffset(); + } + } + + // Going below framePushedAtEntryToBody would imply that we've + // popped off the machine stack, part of the frame created by + // beginFunction(). + MOZ_ASSERT(masm.framePushed() >= + stackMapGenerator_.framePushedAtEntryToBody.value()); + + // At this point we're definitely not generating code for a function call. + MOZ_ASSERT( + stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing()); + + switch (op.b0) { + case uint16_t(Op::End): + if (!emitEnd()) { + return false; + } + if (iter_.controlStackEmpty()) { + return true; + } + NEXT(); + + // Control opcodes + case uint16_t(Op::Nop): + CHECK_NEXT(iter_.readNop()); + case uint16_t(Op::Drop): + CHECK_NEXT(emitDrop()); + case uint16_t(Op::Block): + CHECK_NEXT(emitBlock()); + case uint16_t(Op::Loop): + CHECK_NEXT(emitLoop()); + case uint16_t(Op::If): + CHECK_NEXT(emitIf()); + case uint16_t(Op::Else): + CHECK_NEXT(emitElse()); + case uint16_t(Op::Try): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitTry()); + case uint16_t(Op::Catch): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCatch()); + case uint16_t(Op::CatchAll): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCatchAll()); + case uint16_t(Op::Delegate): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK(emitDelegate()); + iter_.popDelegate(); + NEXT(); + case uint16_t(Op::Throw): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitThrow()); + case uint16_t(Op::Rethrow): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitRethrow()); + case uint16_t(Op::Br): + CHECK_NEXT(emitBr()); + case uint16_t(Op::BrIf): + CHECK_NEXT(emitBrIf()); + case uint16_t(Op::BrTable): + CHECK_NEXT(emitBrTable()); + case uint16_t(Op::Return): + CHECK_NEXT(emitReturn()); + case uint16_t(Op::Unreachable): + CHECK(iter_.readUnreachable()); + if (!deadCode_) { + trap(Trap::Unreachable); + deadCode_ = true; + } + NEXT(); + + // Calls + case uint16_t(Op::Call): + CHECK_NEXT(emitCall()); + case uint16_t(Op::CallIndirect): + CHECK_NEXT(emitCallIndirect()); +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::CallRef): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCallRef()); +#endif + + // Locals and globals + case uint16_t(Op::LocalGet): + CHECK_NEXT(emitGetLocal()); + case uint16_t(Op::LocalSet): + CHECK_NEXT(emitSetLocal()); + case uint16_t(Op::LocalTee): + CHECK_NEXT(emitTeeLocal()); + case uint16_t(Op::GlobalGet): + CHECK_NEXT(emitGetGlobal()); + case uint16_t(Op::GlobalSet): + CHECK_NEXT(emitSetGlobal()); + case uint16_t(Op::TableGet): + CHECK_NEXT(emitTableGet()); + case uint16_t(Op::TableSet): + CHECK_NEXT(emitTableSet()); + + // Select + case uint16_t(Op::SelectNumeric): + CHECK_NEXT(emitSelect(/*typed*/ false)); + case uint16_t(Op::SelectTyped): + CHECK_NEXT(emitSelect(/*typed*/ true)); + + // I32 + case uint16_t(Op::I32Const): { + int32_t i32; + CHECK(iter_.readI32Const(&i32)); + if (!deadCode_) { + pushI32(i32); + } + NEXT(); + } + case uint16_t(Op::I32Add): + CHECK_NEXT(dispatchBinary2(AddI32, AddImmI32, ValType::I32)); + case uint16_t(Op::I32Sub): + CHECK_NEXT(dispatchBinary2(SubI32, SubImmI32, ValType::I32)); + case uint16_t(Op::I32Mul): + CHECK_NEXT(dispatchBinary1(MulI32, ValType::I32)); + case uint16_t(Op::I32DivS): + CHECK_NEXT(dispatchBinary0(emitQuotientI32, ValType::I32)); + case uint16_t(Op::I32DivU): + CHECK_NEXT(dispatchBinary0(emitQuotientU32, ValType::I32)); + case uint16_t(Op::I32RemS): + CHECK_NEXT(dispatchBinary0(emitRemainderI32, ValType::I32)); + case uint16_t(Op::I32RemU): + CHECK_NEXT(dispatchBinary0(emitRemainderU32, ValType::I32)); + case uint16_t(Op::I32Eqz): + CHECK_NEXT(dispatchConversion0(emitEqzI32, ValType::I32, ValType::I32)); + case uint16_t(Op::I32TruncF32S): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32<0>, ValType::F32, + ValType::I32)); + case uint16_t(Op::I32TruncF32U): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32<TRUNC_UNSIGNED>, + ValType::F32, ValType::I32)); + case uint16_t(Op::I32TruncF64S): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32<0>, ValType::F64, + ValType::I32)); + case uint16_t(Op::I32TruncF64U): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32<TRUNC_UNSIGNED>, + ValType::F64, ValType::I32)); + case uint16_t(Op::I32WrapI64): + CHECK_NEXT( + dispatchConversion1(WrapI64ToI32, ValType::I64, ValType::I32)); + case uint16_t(Op::I32ReinterpretF32): + CHECK_NEXT(dispatchConversion1(ReinterpretF32AsI32, ValType::F32, + ValType::I32)); + case uint16_t(Op::I32Clz): + CHECK_NEXT(dispatchUnary1(ClzI32, ValType::I32)); + case uint16_t(Op::I32Ctz): + CHECK_NEXT(dispatchUnary1(CtzI32, ValType::I32)); + case uint16_t(Op::I32Popcnt): + CHECK_NEXT(dispatchUnary2(PopcntI32, PopcntTemp, ValType::I32)); + case uint16_t(Op::I32Or): + CHECK_NEXT(dispatchBinary2(OrI32, OrImmI32, ValType::I32)); + case uint16_t(Op::I32And): + CHECK_NEXT(dispatchBinary2(AndI32, AndImmI32, ValType::I32)); + case uint16_t(Op::I32Xor): + CHECK_NEXT(dispatchBinary2(XorI32, XorImmI32, ValType::I32)); + case uint16_t(Op::I32Shl): + CHECK_NEXT(dispatchBinary3( + ShlI32, ShlImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32)); + case uint16_t(Op::I32ShrS): + CHECK_NEXT(dispatchBinary3( + ShrI32, ShrImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32)); + case uint16_t(Op::I32ShrU): + CHECK_NEXT(dispatchBinary3(ShrUI32, ShrUImmI32, + &BaseCompiler::popI32RhsForShift, + ValType::I32)); + case uint16_t(Op::I32Load8S): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Load8U): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint8)); + case uint16_t(Op::I32Load16S): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int16)); + case uint16_t(Op::I32Load16U): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint16)); + case uint16_t(Op::I32Load): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int32)); + case uint16_t(Op::I32Store8): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Store16): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int16)); + case uint16_t(Op::I32Store): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int32)); + case uint16_t(Op::I32Rotr): + CHECK_NEXT(dispatchBinary3(RotrI32, RotrImmI32, + &BaseCompiler::popI32RhsForRotate, + ValType::I32)); + case uint16_t(Op::I32Rotl): + CHECK_NEXT(dispatchBinary3(RotlI32, RotlImmI32, + &BaseCompiler::popI32RhsForRotate, + ValType::I32)); + + // I64 + case uint16_t(Op::I64Const): { + int64_t i64; + CHECK(iter_.readI64Const(&i64)); + if (!deadCode_) { + pushI64(i64); + } + NEXT(); + } + case uint16_t(Op::I64Add): + CHECK_NEXT(dispatchBinary2(AddI64, AddImmI64, ValType::I64)); + case uint16_t(Op::I64Sub): + CHECK_NEXT(dispatchBinary2(SubI64, SubImmI64, ValType::I64)); + case uint16_t(Op::I64Mul): + CHECK_NEXT(dispatchBinary0(emitMultiplyI64, ValType::I64)); + case uint16_t(Op::I64DivS): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout( + emitDivOrModI64BuiltinCall, SymbolicAddress::DivI64, ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitQuotientI64, ValType::I64)); +#endif + case uint16_t(Op::I64DivU): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall, + SymbolicAddress::UDivI64, + ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitQuotientU64, ValType::I64)); +#endif + case uint16_t(Op::I64RemS): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout( + emitDivOrModI64BuiltinCall, SymbolicAddress::ModI64, ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitRemainderI64, ValType::I64)); +#endif + case uint16_t(Op::I64RemU): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall, + SymbolicAddress::UModI64, + ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitRemainderU64, ValType::I64)); +#endif + case uint16_t(Op::I64TruncF32S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT( + dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToInt64, + ValType::F32, ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64<0>, ValType::F32, + ValType::I64)); +#endif + case uint16_t(Op::I64TruncF32U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToUint64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64<TRUNC_UNSIGNED>, + ValType::F32, ValType::I64)); +#endif + case uint16_t(Op::I64TruncF64S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT( + dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToInt64, + ValType::F64, ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64<0>, ValType::F64, + ValType::I64)); +#endif + case uint16_t(Op::I64TruncF64U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToUint64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64<TRUNC_UNSIGNED>, + ValType::F64, ValType::I64)); +#endif + case uint16_t(Op::I64ExtendI32S): + CHECK_NEXT(dispatchConversion0(emitExtendI32ToI64, ValType::I32, + ValType::I64)); + case uint16_t(Op::I64ExtendI32U): + CHECK_NEXT(dispatchConversion0(emitExtendU32ToI64, ValType::I32, + ValType::I64)); + case uint16_t(Op::I64ReinterpretF64): + CHECK_NEXT(dispatchConversion1(ReinterpretF64AsI64, ValType::F64, + ValType::I64)); + case uint16_t(Op::I64Or): + CHECK_NEXT(dispatchBinary2(OrI64, OrImmI64, ValType::I64)); + case uint16_t(Op::I64And): + CHECK_NEXT(dispatchBinary2(AndI64, AndImmI64, ValType::I64)); + case uint16_t(Op::I64Xor): + CHECK_NEXT(dispatchBinary2(XorI64, XorImmI64, ValType::I64)); + case uint16_t(Op::I64Shl): + CHECK_NEXT(dispatchBinary3( + ShlI64, ShlImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64)); + case uint16_t(Op::I64ShrS): + CHECK_NEXT(dispatchBinary3( + ShrI64, ShrImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64)); + case uint16_t(Op::I64ShrU): + CHECK_NEXT(dispatchBinary3(ShrUI64, ShrUImmI64, + &BaseCompiler::popI64RhsForShift, + ValType::I64)); + case uint16_t(Op::I64Rotr): + CHECK_NEXT(dispatchBinary0(emitRotrI64, ValType::I64)); + case uint16_t(Op::I64Rotl): + CHECK_NEXT(dispatchBinary0(emitRotlI64, ValType::I64)); + case uint16_t(Op::I64Clz): + CHECK_NEXT(dispatchUnary1(ClzI64, ValType::I64)); + case uint16_t(Op::I64Ctz): + CHECK_NEXT(dispatchUnary1(CtzI64, ValType::I64)); + case uint16_t(Op::I64Popcnt): + CHECK_NEXT(dispatchUnary2(PopcntI64, PopcntTemp, ValType::I64)); + case uint16_t(Op::I64Eqz): + CHECK_NEXT(dispatchConversion0(emitEqzI64, ValType::I64, ValType::I32)); + case uint16_t(Op::I64Load8S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Load16S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Load32S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int32)); + case uint16_t(Op::I64Load8U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint8)); + case uint16_t(Op::I64Load16U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint16)); + case uint16_t(Op::I64Load32U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint32)); + case uint16_t(Op::I64Load): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int64)); + case uint16_t(Op::I64Store8): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Store16): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Store32): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int32)); + case uint16_t(Op::I64Store): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int64)); + + // F32 + case uint16_t(Op::F32Const): { + float f32; + CHECK(iter_.readF32Const(&f32)); + if (!deadCode_) { + pushF32(f32); + } + NEXT(); + } + case uint16_t(Op::F32Add): + CHECK_NEXT(dispatchBinary1(AddF32, ValType::F32)) + case uint16_t(Op::F32Sub): + CHECK_NEXT(dispatchBinary1(SubF32, ValType::F32)); + case uint16_t(Op::F32Mul): + CHECK_NEXT(dispatchBinary1(MulF32, ValType::F32)); + case uint16_t(Op::F32Div): + CHECK_NEXT(dispatchBinary1(DivF32, ValType::F32)); + case uint16_t(Op::F32Min): + CHECK_NEXT(dispatchBinary1(MinF32, ValType::F32)); + case uint16_t(Op::F32Max): + CHECK_NEXT(dispatchBinary1(MaxF32, ValType::F32)); + case uint16_t(Op::F32Neg): + CHECK_NEXT(dispatchUnary1(NegateF32, ValType::F32)); + case uint16_t(Op::F32Abs): + CHECK_NEXT(dispatchUnary1(AbsF32, ValType::F32)); + case uint16_t(Op::F32Sqrt): + CHECK_NEXT(dispatchUnary1(SqrtF32, ValType::F32)); + case uint16_t(Op::F32Ceil): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::CeilF, ValType::F32)); + case uint16_t(Op::F32Floor): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::FloorF, ValType::F32)); + case uint16_t(Op::F32DemoteF64): + CHECK_NEXT( + dispatchConversion1(ConvertF64ToF32, ValType::F64, ValType::F32)); + case uint16_t(Op::F32ConvertI32S): + CHECK_NEXT( + dispatchConversion1(ConvertI32ToF32, ValType::I32, ValType::F32)); + case uint16_t(Op::F32ConvertI32U): + CHECK_NEXT( + dispatchConversion1(ConvertU32ToF32, ValType::I32, ValType::F32)); + case uint16_t(Op::F32ConvertI64S): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToFloat32, + ValType::I64, ValType::F32)); +#else + CHECK_NEXT( + dispatchConversion1(ConvertI64ToF32, ValType::I64, ValType::F32)); +#endif + case uint16_t(Op::F32ConvertI64U): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToFloat32, + ValType::I64, ValType::F32)); +#else + CHECK_NEXT(dispatchConversion0(emitConvertU64ToF32, ValType::I64, + ValType::F32)); +#endif + case uint16_t(Op::F32ReinterpretI32): + CHECK_NEXT(dispatchConversion1(ReinterpretI32AsF32, ValType::I32, + ValType::F32)); + case uint16_t(Op::F32Load): + CHECK_NEXT(emitLoad(ValType::F32, Scalar::Float32)); + case uint16_t(Op::F32Store): + CHECK_NEXT(emitStore(ValType::F32, Scalar::Float32)); + case uint16_t(Op::F32CopySign): + CHECK_NEXT(dispatchBinary1(CopysignF32, ValType::F32)); + case uint16_t(Op::F32Nearest): + CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntF, + ValType::F32)); + case uint16_t(Op::F32Trunc): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::TruncF, ValType::F32)); + + // F64 + case uint16_t(Op::F64Const): { + double f64; + CHECK(iter_.readF64Const(&f64)); + if (!deadCode_) { + pushF64(f64); + } + NEXT(); + } + case uint16_t(Op::F64Add): + CHECK_NEXT(dispatchBinary1(AddF64, ValType::F64)) + case uint16_t(Op::F64Sub): + CHECK_NEXT(dispatchBinary1(SubF64, ValType::F64)); + case uint16_t(Op::F64Mul): + CHECK_NEXT(dispatchBinary1(MulF64, ValType::F64)); + case uint16_t(Op::F64Div): + CHECK_NEXT(dispatchBinary1(DivF64, ValType::F64)); + case uint16_t(Op::F64Min): + CHECK_NEXT(dispatchBinary1(MinF64, ValType::F64)); + case uint16_t(Op::F64Max): + CHECK_NEXT(dispatchBinary1(MaxF64, ValType::F64)); + case uint16_t(Op::F64Neg): + CHECK_NEXT(dispatchUnary1(NegateF64, ValType::F64)); + case uint16_t(Op::F64Abs): + CHECK_NEXT(dispatchUnary1(AbsF64, ValType::F64)); + case uint16_t(Op::F64Sqrt): + CHECK_NEXT(dispatchUnary1(SqrtF64, ValType::F64)); + case uint16_t(Op::F64Ceil): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::CeilD, ValType::F64)); + case uint16_t(Op::F64Floor): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::FloorD, ValType::F64)); + case uint16_t(Op::F64PromoteF32): + CHECK_NEXT( + dispatchConversion1(ConvertF32ToF64, ValType::F32, ValType::F64)); + case uint16_t(Op::F64ConvertI32S): + CHECK_NEXT( + dispatchConversion1(ConvertI32ToF64, ValType::I32, ValType::F64)); + case uint16_t(Op::F64ConvertI32U): + CHECK_NEXT( + dispatchConversion1(ConvertU32ToF64, ValType::I32, ValType::F64)); + case uint16_t(Op::F64ConvertI64S): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToDouble, + ValType::I64, ValType::F64)); +#else + CHECK_NEXT( + dispatchConversion1(ConvertI64ToF64, ValType::I64, ValType::F64)); +#endif + case uint16_t(Op::F64ConvertI64U): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToDouble, + ValType::I64, ValType::F64)); +#else + CHECK_NEXT(dispatchConversion0(emitConvertU64ToF64, ValType::I64, + ValType::F64)); +#endif + case uint16_t(Op::F64Load): + CHECK_NEXT(emitLoad(ValType::F64, Scalar::Float64)); + case uint16_t(Op::F64Store): + CHECK_NEXT(emitStore(ValType::F64, Scalar::Float64)); + case uint16_t(Op::F64ReinterpretI64): + CHECK_NEXT(dispatchConversion1(ReinterpretI64AsF64, ValType::I64, + ValType::F64)); + case uint16_t(Op::F64CopySign): + CHECK_NEXT(dispatchBinary1(CopysignF64, ValType::F64)); + case uint16_t(Op::F64Nearest): + CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntD, + ValType::F64)); + case uint16_t(Op::F64Trunc): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::TruncD, ValType::F64)); + + // Comparisons + case uint16_t(Op::I32Eq): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Equal)); + case uint16_t(Op::I32Ne): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::NotEqual)); + case uint16_t(Op::I32LtS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::LessThan)); + case uint16_t(Op::I32LeS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::LessThanOrEqual)); + case uint16_t(Op::I32GtS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::GreaterThan)); + case uint16_t(Op::I32GeS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::GreaterThanOrEqual)); + case uint16_t(Op::I32LtU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Below)); + case uint16_t(Op::I32LeU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::BelowOrEqual)); + case uint16_t(Op::I32GtU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Above)); + case uint16_t(Op::I32GeU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::AboveOrEqual)); + case uint16_t(Op::I64Eq): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Equal)); + case uint16_t(Op::I64Ne): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::NotEqual)); + case uint16_t(Op::I64LtS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::LessThan)); + case uint16_t(Op::I64LeS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::LessThanOrEqual)); + case uint16_t(Op::I64GtS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::GreaterThan)); + case uint16_t(Op::I64GeS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::GreaterThanOrEqual)); + case uint16_t(Op::I64LtU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Below)); + case uint16_t(Op::I64LeU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::BelowOrEqual)); + case uint16_t(Op::I64GtU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Above)); + case uint16_t(Op::I64GeU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::AboveOrEqual)); + case uint16_t(Op::F32Eq): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleEqual)); + case uint16_t(Op::F32Ne): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleNotEqualOrUnordered)); + case uint16_t(Op::F32Lt): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleLessThan)); + case uint16_t(Op::F32Le): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleLessThanOrEqual)); + case uint16_t(Op::F32Gt): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleGreaterThan)); + case uint16_t(Op::F32Ge): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleGreaterThanOrEqual)); + case uint16_t(Op::F64Eq): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleEqual)); + case uint16_t(Op::F64Ne): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleNotEqualOrUnordered)); + case uint16_t(Op::F64Lt): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleLessThan)); + case uint16_t(Op::F64Le): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleLessThanOrEqual)); + case uint16_t(Op::F64Gt): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleGreaterThan)); + case uint16_t(Op::F64Ge): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleGreaterThanOrEqual)); + + // Sign extensions + case uint16_t(Op::I32Extend8S): + CHECK_NEXT( + dispatchConversion1(ExtendI32_8, ValType::I32, ValType::I32)); + case uint16_t(Op::I32Extend16S): + CHECK_NEXT( + dispatchConversion1(ExtendI32_16, ValType::I32, ValType::I32)); + case uint16_t(Op::I64Extend8S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_8, ValType::I64, ValType::I64)); + case uint16_t(Op::I64Extend16S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_16, ValType::I64, ValType::I64)); + case uint16_t(Op::I64Extend32S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_32, ValType::I64, ValType::I64)); + + // Memory Related + case uint16_t(Op::MemoryGrow): + CHECK_NEXT(emitMemoryGrow()); + case uint16_t(Op::MemorySize): + CHECK_NEXT(emitMemorySize()); + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::RefAsNonNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitRefAsNonNull()); + case uint16_t(Op::BrOnNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitBrOnNull()); + case uint16_t(Op::BrOnNonNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitBrOnNonNull()); +#endif +#ifdef ENABLE_WASM_GC + case uint16_t(Op::RefEq): + if (!moduleEnv_.gcEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchComparison0(emitCompareRef, RefType::eq(), + Assembler::Equal)); +#endif + case uint16_t(Op::RefFunc): + CHECK_NEXT(emitRefFunc()); + break; + case uint16_t(Op::RefNull): + CHECK_NEXT(emitRefNull()); + break; + case uint16_t(Op::RefIsNull): + CHECK_NEXT(emitRefIsNull()); + break; + +#ifdef ENABLE_WASM_GC + // "GC" operations + case uint16_t(Op::GcPrefix): { + if (!moduleEnv_.gcEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(GcOp::StructNew): + CHECK_NEXT(emitStructNew()); + case uint32_t(GcOp::StructNewDefault): + CHECK_NEXT(emitStructNewDefault()); + case uint32_t(GcOp::StructGet): + CHECK_NEXT(emitStructGet(FieldWideningOp::None)); + case uint32_t(GcOp::StructGetS): + CHECK_NEXT(emitStructGet(FieldWideningOp::Signed)); + case uint32_t(GcOp::StructGetU): + CHECK_NEXT(emitStructGet(FieldWideningOp::Unsigned)); + case uint32_t(GcOp::StructSet): + CHECK_NEXT(emitStructSet()); + case uint32_t(GcOp::ArrayNew): + CHECK_NEXT(emitArrayNew()); + case uint32_t(GcOp::ArrayNewFixed): + CHECK_NEXT(emitArrayNewFixed()); + case uint32_t(GcOp::ArrayNewDefault): + CHECK_NEXT(emitArrayNewDefault()); + case uint32_t(GcOp::ArrayNewData): + CHECK_NEXT(emitArrayNewData()); + case uint32_t(GcOp::ArrayInitFromElemStaticV5): + case uint32_t(GcOp::ArrayNewElem): + CHECK_NEXT(emitArrayNewElem()); + case uint32_t(GcOp::ArrayGet): + CHECK_NEXT(emitArrayGet(FieldWideningOp::None)); + case uint32_t(GcOp::ArrayGetS): + CHECK_NEXT(emitArrayGet(FieldWideningOp::Signed)); + case uint32_t(GcOp::ArrayGetU): + CHECK_NEXT(emitArrayGet(FieldWideningOp::Unsigned)); + case uint32_t(GcOp::ArraySet): + CHECK_NEXT(emitArraySet()); + case uint32_t(GcOp::ArrayLenWithTypeIndex): + CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/true)); + case uint32_t(GcOp::ArrayLen): + CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/false)); + case uint32_t(GcOp::ArrayCopy): + CHECK_NEXT(emitArrayCopy()); + case uint32_t(GcOp::RefTestV5): + CHECK_NEXT(emitRefTestV5()); + case uint32_t(GcOp::RefCastV5): + CHECK_NEXT(emitRefCastV5()); + case uint32_t(GcOp::RefTest): + CHECK_NEXT(emitRefTest(/*nullable=*/false)); + case uint32_t(GcOp::RefTestNull): + CHECK_NEXT(emitRefTest(/*nullable=*/true)); + case uint32_t(GcOp::RefCast): + CHECK_NEXT(emitRefCast(/*nullable=*/false)); + case uint32_t(GcOp::RefCastNull): + CHECK_NEXT(emitRefCast(/*nullable=*/true)); + case uint32_t(GcOp::BrOnCast): + CHECK_NEXT(emitBrOnCast()); + case uint32_t(GcOp::BrOnCastV5): + CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/true)); + case uint32_t(GcOp::BrOnCastFailV5): + CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/false)); + case uint32_t(GcOp::BrOnCastHeapV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastHeapNullV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/true)); + case uint32_t(GcOp::BrOnCastFailHeapV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastFailHeapNullV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/true)); + case uint32_t(GcOp::RefAsStructV5): + CHECK_NEXT(emitRefAsStructV5()); + case uint32_t(GcOp::BrOnNonStructV5): + CHECK_NEXT(emitBrOnNonStructV5()); + case uint16_t(GcOp::ExternInternalize): + CHECK_NEXT(emitExternInternalize()); + case uint16_t(GcOp::ExternExternalize): + CHECK_NEXT(emitExternExternalize()); + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } +#endif + +#ifdef ENABLE_WASM_SIMD + // SIMD operations + case uint16_t(Op::SimdPrefix): { + uint32_t laneIndex; + if (!moduleEnv_.simdAvailable()) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(SimdOp::I8x16ExtractLaneS): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I8x16ExtractLaneU): + CHECK_NEXT( + dispatchExtractLane(ExtractLaneUI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I16x8ExtractLaneS): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I16x8ExtractLaneU): + CHECK_NEXT(dispatchExtractLane(ExtractLaneUI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I32x4ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI32x4, ValType::I32, 4)); + case uint32_t(SimdOp::I64x2ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI64x2, ValType::I64, 2)); + case uint32_t(SimdOp::F32x4ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneF32x4, ValType::F32, 4)); + case uint32_t(SimdOp::F64x2ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneF64x2, ValType::F64, 2)); + case uint32_t(SimdOp::I8x16Splat): + CHECK_NEXT(dispatchSplat(SplatI8x16, ValType::I32)); + case uint32_t(SimdOp::I16x8Splat): + CHECK_NEXT(dispatchSplat(SplatI16x8, ValType::I32)); + case uint32_t(SimdOp::I32x4Splat): + CHECK_NEXT(dispatchSplat(SplatI32x4, ValType::I32)); + case uint32_t(SimdOp::I64x2Splat): + CHECK_NEXT(dispatchSplat(SplatI64x2, ValType::I64)); + case uint32_t(SimdOp::F32x4Splat): + CHECK_NEXT(dispatchSplat(SplatF32x4, ValType::F32)); + case uint32_t(SimdOp::F64x2Splat): + CHECK_NEXT(dispatchSplat(SplatF64x2, ValType::F64)); + case uint32_t(SimdOp::V128AnyTrue): + CHECK_NEXT(dispatchVectorReduction(AnyTrue)); + case uint32_t(SimdOp::I8x16AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI8x16)); + case uint32_t(SimdOp::I16x8AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI16x8)); + case uint32_t(SimdOp::I32x4AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI32x4)); + case uint32_t(SimdOp::I64x2AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI64x2)); + case uint32_t(SimdOp::I8x16Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI8x16)); + case uint32_t(SimdOp::I16x8Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI16x8)); + case uint32_t(SimdOp::I32x4Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI32x4)); + case uint32_t(SimdOp::I64x2Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI64x2)); + case uint32_t(SimdOp::I8x16ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I16x8ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I32x4ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI32x4, ValType::I32, 4)); + case uint32_t(SimdOp::I64x2ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI64x2, ValType::I64, 2)); + case uint32_t(SimdOp::F32x4ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF32x4, ValType::F32, 4)); + case uint32_t(SimdOp::F64x2ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF64x2, ValType::F64, 2)); + case uint32_t(SimdOp::I8x16Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::Equal)); + case uint32_t(SimdOp::I8x16Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::NotEqual)); + case uint32_t(SimdOp::I8x16LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::LessThan)); + case uint32_t(SimdOp::I8x16LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Below)); + case uint32_t(SimdOp::I8x16GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI8x16, Assembler::GreaterThan)); + case uint32_t(SimdOp::I8x16GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Above)); + case uint32_t(SimdOp::I8x16LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI8x16, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I8x16LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI8x16, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I8x16GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I8x16GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI8x16, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I16x8Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::Equal)); + case uint32_t(SimdOp::I16x8Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::NotEqual)); + case uint32_t(SimdOp::I16x8LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::LessThan)); + case uint32_t(SimdOp::I16x8LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Below)); + case uint32_t(SimdOp::I16x8GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI16x8, Assembler::GreaterThan)); + case uint32_t(SimdOp::I16x8GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Above)); + case uint32_t(SimdOp::I16x8LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI16x8, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I16x8LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI16x8, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I16x8GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I16x8GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI16x8, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I32x4Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::Equal)); + case uint32_t(SimdOp::I32x4Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::NotEqual)); + case uint32_t(SimdOp::I32x4LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::LessThan)); + case uint32_t(SimdOp::I32x4LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Below)); + case uint32_t(SimdOp::I32x4GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI32x4, Assembler::GreaterThan)); + case uint32_t(SimdOp::I32x4GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Above)); + case uint32_t(SimdOp::I32x4LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI32x4, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I32x4LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI32x4, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I32x4GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I32x4GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI32x4, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I64x2Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality, + Assembler::Equal)); + case uint32_t(SimdOp::I64x2Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality, + Assembler::NotEqual)); + case uint32_t(SimdOp::I64x2LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::LessThan)); + case uint32_t(SimdOp::I64x2GtS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::GreaterThan)); + case uint32_t(SimdOp::I64x2LeS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I64x2GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::F32x4Eq): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::Equal)); + case uint32_t(SimdOp::F32x4Ne): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::NotEqual)); + case uint32_t(SimdOp::F32x4Lt): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::LessThan)); + case uint32_t(SimdOp::F32x4Gt): + CHECK_NEXT( + dispatchVectorComparison(CmpF32x4, Assembler::GreaterThan)); + case uint32_t(SimdOp::F32x4Le): + CHECK_NEXT( + dispatchVectorComparison(CmpF32x4, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::F32x4Ge): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::F64x2Eq): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::Equal)); + case uint32_t(SimdOp::F64x2Ne): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::NotEqual)); + case uint32_t(SimdOp::F64x2Lt): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::LessThan)); + case uint32_t(SimdOp::F64x2Gt): + CHECK_NEXT( + dispatchVectorComparison(CmpF64x2, Assembler::GreaterThan)); + case uint32_t(SimdOp::F64x2Le): + CHECK_NEXT( + dispatchVectorComparison(CmpF64x2, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::F64x2Ge): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::V128And): + CHECK_NEXT(dispatchVectorBinary(AndV128)); + case uint32_t(SimdOp::V128Or): + CHECK_NEXT(dispatchVectorBinary(OrV128)); + case uint32_t(SimdOp::V128Xor): + CHECK_NEXT(dispatchVectorBinary(XorV128)); + case uint32_t(SimdOp::V128AndNot): + CHECK_NEXT(dispatchBinary0(emitVectorAndNot, ValType::V128)); + case uint32_t(SimdOp::I8x16AvgrU): + CHECK_NEXT(dispatchVectorBinary(AverageUI8x16)); + case uint32_t(SimdOp::I16x8AvgrU): + CHECK_NEXT(dispatchVectorBinary(AverageUI16x8)); + case uint32_t(SimdOp::I8x16Add): + CHECK_NEXT(dispatchVectorBinary(AddI8x16)); + case uint32_t(SimdOp::I8x16AddSatS): + CHECK_NEXT(dispatchVectorBinary(AddSatI8x16)); + case uint32_t(SimdOp::I8x16AddSatU): + CHECK_NEXT(dispatchVectorBinary(AddSatUI8x16)); + case uint32_t(SimdOp::I8x16Sub): + CHECK_NEXT(dispatchVectorBinary(SubI8x16)); + case uint32_t(SimdOp::I8x16SubSatS): + CHECK_NEXT(dispatchVectorBinary(SubSatI8x16)); + case uint32_t(SimdOp::I8x16SubSatU): + CHECK_NEXT(dispatchVectorBinary(SubSatUI8x16)); + case uint32_t(SimdOp::I8x16MinS): + CHECK_NEXT(dispatchVectorBinary(MinI8x16)); + case uint32_t(SimdOp::I8x16MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI8x16)); + case uint32_t(SimdOp::I8x16MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI8x16)); + case uint32_t(SimdOp::I8x16MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI8x16)); + case uint32_t(SimdOp::I16x8Add): + CHECK_NEXT(dispatchVectorBinary(AddI16x8)); + case uint32_t(SimdOp::I16x8AddSatS): + CHECK_NEXT(dispatchVectorBinary(AddSatI16x8)); + case uint32_t(SimdOp::I16x8AddSatU): + CHECK_NEXT(dispatchVectorBinary(AddSatUI16x8)); + case uint32_t(SimdOp::I16x8Sub): + CHECK_NEXT(dispatchVectorBinary(SubI16x8)); + case uint32_t(SimdOp::I16x8SubSatS): + CHECK_NEXT(dispatchVectorBinary(SubSatI16x8)); + case uint32_t(SimdOp::I16x8SubSatU): + CHECK_NEXT(dispatchVectorBinary(SubSatUI16x8)); + case uint32_t(SimdOp::I16x8Mul): + CHECK_NEXT(dispatchVectorBinary(MulI16x8)); + case uint32_t(SimdOp::I16x8MinS): + CHECK_NEXT(dispatchVectorBinary(MinI16x8)); + case uint32_t(SimdOp::I16x8MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI16x8)); + case uint32_t(SimdOp::I16x8MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI16x8)); + case uint32_t(SimdOp::I16x8MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI16x8)); + case uint32_t(SimdOp::I32x4Add): + CHECK_NEXT(dispatchVectorBinary(AddI32x4)); + case uint32_t(SimdOp::I32x4Sub): + CHECK_NEXT(dispatchVectorBinary(SubI32x4)); + case uint32_t(SimdOp::I32x4Mul): + CHECK_NEXT(dispatchVectorBinary(MulI32x4)); + case uint32_t(SimdOp::I32x4MinS): + CHECK_NEXT(dispatchVectorBinary(MinI32x4)); + case uint32_t(SimdOp::I32x4MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI32x4)); + case uint32_t(SimdOp::I32x4MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI32x4)); + case uint32_t(SimdOp::I32x4MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI32x4)); + case uint32_t(SimdOp::I64x2Add): + CHECK_NEXT(dispatchVectorBinary(AddI64x2)); + case uint32_t(SimdOp::I64x2Sub): + CHECK_NEXT(dispatchVectorBinary(SubI64x2)); + case uint32_t(SimdOp::I64x2Mul): + CHECK_NEXT(dispatchVectorBinary(MulI64x2)); + case uint32_t(SimdOp::F32x4Add): + CHECK_NEXT(dispatchVectorBinary(AddF32x4)); + case uint32_t(SimdOp::F32x4Sub): + CHECK_NEXT(dispatchVectorBinary(SubF32x4)); + case uint32_t(SimdOp::F32x4Mul): + CHECK_NEXT(dispatchVectorBinary(MulF32x4)); + case uint32_t(SimdOp::F32x4Div): + CHECK_NEXT(dispatchVectorBinary(DivF32x4)); + case uint32_t(SimdOp::F32x4Min): + CHECK_NEXT(dispatchVectorBinary(MinF32x4)); + case uint32_t(SimdOp::F32x4Max): + CHECK_NEXT(dispatchVectorBinary(MaxF32x4)); + case uint32_t(SimdOp::F64x2Add): + CHECK_NEXT(dispatchVectorBinary(AddF64x2)); + case uint32_t(SimdOp::F64x2Sub): + CHECK_NEXT(dispatchVectorBinary(SubF64x2)); + case uint32_t(SimdOp::F64x2Mul): + CHECK_NEXT(dispatchVectorBinary(MulF64x2)); + case uint32_t(SimdOp::F64x2Div): + CHECK_NEXT(dispatchVectorBinary(DivF64x2)); + case uint32_t(SimdOp::F64x2Min): + CHECK_NEXT(dispatchVectorBinary(MinF64x2)); + case uint32_t(SimdOp::F64x2Max): + CHECK_NEXT(dispatchVectorBinary(MaxF64x2)); + case uint32_t(SimdOp::I8x16NarrowI16x8S): + CHECK_NEXT(dispatchVectorBinary(NarrowI16x8)); + case uint32_t(SimdOp::I8x16NarrowI16x8U): + CHECK_NEXT(dispatchVectorBinary(NarrowUI16x8)); + case uint32_t(SimdOp::I16x8NarrowI32x4S): + CHECK_NEXT(dispatchVectorBinary(NarrowI32x4)); + case uint32_t(SimdOp::I16x8NarrowI32x4U): + CHECK_NEXT(dispatchVectorBinary(NarrowUI32x4)); + case uint32_t(SimdOp::I8x16Swizzle): + CHECK_NEXT(dispatchVectorBinary(Swizzle)); + case uint32_t(SimdOp::F32x4PMax): + CHECK_NEXT(dispatchVectorBinary(PMaxF32x4)); + case uint32_t(SimdOp::F32x4PMin): + CHECK_NEXT(dispatchVectorBinary(PMinF32x4)); + case uint32_t(SimdOp::F64x2PMax): + CHECK_NEXT(dispatchVectorBinary(PMaxF64x2)); + case uint32_t(SimdOp::F64x2PMin): + CHECK_NEXT(dispatchVectorBinary(PMinF64x2)); + case uint32_t(SimdOp::I32x4DotI16x8S): + CHECK_NEXT(dispatchVectorBinary(DotI16x8)); + case uint32_t(SimdOp::I16x8ExtmulLowI8x16S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI8x16)); + case uint32_t(SimdOp::I16x8ExtmulHighI8x16S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI8x16)); + case uint32_t(SimdOp::I16x8ExtmulLowI8x16U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI8x16)); + case uint32_t(SimdOp::I16x8ExtmulHighI8x16U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI8x16)); + case uint32_t(SimdOp::I32x4ExtmulLowI16x8S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI16x8)); + case uint32_t(SimdOp::I32x4ExtmulHighI16x8S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI16x8)); + case uint32_t(SimdOp::I32x4ExtmulLowI16x8U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI16x8)); + case uint32_t(SimdOp::I32x4ExtmulHighI16x8U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI16x8)); + case uint32_t(SimdOp::I64x2ExtmulLowI32x4S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI32x4)); + case uint32_t(SimdOp::I64x2ExtmulHighI32x4S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI32x4)); + case uint32_t(SimdOp::I64x2ExtmulLowI32x4U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI32x4)); + case uint32_t(SimdOp::I64x2ExtmulHighI32x4U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI32x4)); + case uint32_t(SimdOp::I16x8Q15MulrSatS): + CHECK_NEXT(dispatchVectorBinary(Q15MulrSatS)); + case uint32_t(SimdOp::I8x16Neg): + CHECK_NEXT(dispatchVectorUnary(NegI8x16)); + case uint32_t(SimdOp::I16x8Neg): + CHECK_NEXT(dispatchVectorUnary(NegI16x8)); + case uint32_t(SimdOp::I16x8ExtendLowI8x16S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI8x16)); + case uint32_t(SimdOp::I16x8ExtendHighI8x16S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI8x16)); + case uint32_t(SimdOp::I16x8ExtendLowI8x16U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI8x16)); + case uint32_t(SimdOp::I16x8ExtendHighI8x16U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI8x16)); + case uint32_t(SimdOp::I32x4Neg): + CHECK_NEXT(dispatchVectorUnary(NegI32x4)); + case uint32_t(SimdOp::I32x4ExtendLowI16x8S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI16x8)); + case uint32_t(SimdOp::I32x4ExtendHighI16x8S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI16x8)); + case uint32_t(SimdOp::I32x4ExtendLowI16x8U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI16x8)); + case uint32_t(SimdOp::I32x4ExtendHighI16x8U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI16x8)); + case uint32_t(SimdOp::I32x4TruncSatF32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToI32x4)); + case uint32_t(SimdOp::I32x4TruncSatF32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToUI32x4)); + case uint32_t(SimdOp::I64x2Neg): + CHECK_NEXT(dispatchVectorUnary(NegI64x2)); + case uint32_t(SimdOp::I64x2ExtendLowI32x4S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI32x4)); + case uint32_t(SimdOp::I64x2ExtendHighI32x4S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI32x4)); + case uint32_t(SimdOp::I64x2ExtendLowI32x4U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI32x4)); + case uint32_t(SimdOp::I64x2ExtendHighI32x4U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI32x4)); + case uint32_t(SimdOp::F32x4Abs): + CHECK_NEXT(dispatchVectorUnary(AbsF32x4)); + case uint32_t(SimdOp::F32x4Neg): + CHECK_NEXT(dispatchVectorUnary(NegF32x4)); + case uint32_t(SimdOp::F32x4Sqrt): + CHECK_NEXT(dispatchVectorUnary(SqrtF32x4)); + case uint32_t(SimdOp::F32x4ConvertI32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF32x4)); + case uint32_t(SimdOp::F32x4ConvertI32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF32x4)); + case uint32_t(SimdOp::F32x4DemoteF64x2Zero): + CHECK_NEXT(dispatchVectorUnary(DemoteF64x2ToF32x4)); + case uint32_t(SimdOp::F64x2PromoteLowF32x4): + CHECK_NEXT(dispatchVectorUnary(PromoteF32x4ToF64x2)); + case uint32_t(SimdOp::F64x2ConvertLowI32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF64x2)); + case uint32_t(SimdOp::F64x2ConvertLowI32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF64x2)); + case uint32_t(SimdOp::I32x4TruncSatF64x2SZero): + CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToI32x4)); + case uint32_t(SimdOp::I32x4TruncSatF64x2UZero): + CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToUI32x4)); + case uint32_t(SimdOp::F64x2Abs): + CHECK_NEXT(dispatchVectorUnary(AbsF64x2)); + case uint32_t(SimdOp::F64x2Neg): + CHECK_NEXT(dispatchVectorUnary(NegF64x2)); + case uint32_t(SimdOp::F64x2Sqrt): + CHECK_NEXT(dispatchVectorUnary(SqrtF64x2)); + case uint32_t(SimdOp::V128Not): + CHECK_NEXT(dispatchVectorUnary(NotV128)); + case uint32_t(SimdOp::I8x16Popcnt): + CHECK_NEXT(dispatchVectorUnary(PopcntI8x16)); + case uint32_t(SimdOp::I8x16Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI8x16)); + case uint32_t(SimdOp::I16x8Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI16x8)); + case uint32_t(SimdOp::I32x4Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI32x4)); + case uint32_t(SimdOp::I64x2Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI64x2)); + case uint32_t(SimdOp::F32x4Ceil): + CHECK_NEXT(dispatchVectorUnary(CeilF32x4)); + case uint32_t(SimdOp::F32x4Floor): + CHECK_NEXT(dispatchVectorUnary(FloorF32x4)); + case uint32_t(SimdOp::F32x4Trunc): + CHECK_NEXT(dispatchVectorUnary(TruncF32x4)); + case uint32_t(SimdOp::F32x4Nearest): + CHECK_NEXT(dispatchVectorUnary(NearestF32x4)); + case uint32_t(SimdOp::F64x2Ceil): + CHECK_NEXT(dispatchVectorUnary(CeilF64x2)); + case uint32_t(SimdOp::F64x2Floor): + CHECK_NEXT(dispatchVectorUnary(FloorF64x2)); + case uint32_t(SimdOp::F64x2Trunc): + CHECK_NEXT(dispatchVectorUnary(TruncF64x2)); + case uint32_t(SimdOp::F64x2Nearest): + CHECK_NEXT(dispatchVectorUnary(NearestF64x2)); + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI8x16)); + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI8x16)); + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI16x8)); + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI16x8)); + case uint32_t(SimdOp::I8x16Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI8x16)); + case uint32_t(SimdOp::I8x16ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI8x16)); + case uint32_t(SimdOp::I8x16ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI8x16)); + case uint32_t(SimdOp::I16x8Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI16x8)); + case uint32_t(SimdOp::I16x8ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI16x8)); + case uint32_t(SimdOp::I16x8ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI16x8)); + case uint32_t(SimdOp::I32x4Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI32x4)); + case uint32_t(SimdOp::I32x4ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI32x4)); + case uint32_t(SimdOp::I32x4ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI32x4)); + case uint32_t(SimdOp::I64x2Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI64x2)); + case uint32_t(SimdOp::I64x2ShrS): +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + CHECK_NEXT(emitVectorShiftRightI64x2()); +# else + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI64x2)); +# endif + case uint32_t(SimdOp::I64x2ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI64x2)); + case uint32_t(SimdOp::V128Bitselect): + CHECK_NEXT(dispatchTernary1(BitselectV128, ValType::V128)); + case uint32_t(SimdOp::I8x16Shuffle): + CHECK_NEXT(emitVectorShuffle()); + case uint32_t(SimdOp::V128Const): { + V128 v128; + CHECK(iter_.readV128Const(&v128)); + if (!deadCode_) { + pushV128(v128); + } + NEXT(); + } + case uint32_t(SimdOp::V128Load): + CHECK_NEXT(emitLoad(ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128Load8Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint8)); + case uint32_t(SimdOp::V128Load16Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint16)); + case uint32_t(SimdOp::V128Load32Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint32)); + case uint32_t(SimdOp::V128Load64Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Int64)); + case uint32_t(SimdOp::V128Load8x8S): + CHECK_NEXT(emitLoadExtend(Scalar::Int8)); + case uint32_t(SimdOp::V128Load8x8U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint8)); + case uint32_t(SimdOp::V128Load16x4S): + CHECK_NEXT(emitLoadExtend(Scalar::Int16)); + case uint32_t(SimdOp::V128Load16x4U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint16)); + case uint32_t(SimdOp::V128Load32x2S): + CHECK_NEXT(emitLoadExtend(Scalar::Int32)); + case uint32_t(SimdOp::V128Load32x2U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint32)); + case uint32_t(SimdOp::V128Load32Zero): + CHECK_NEXT(emitLoadZero(Scalar::Float32)); + case uint32_t(SimdOp::V128Load64Zero): + CHECK_NEXT(emitLoadZero(Scalar::Float64)); + case uint32_t(SimdOp::V128Store): + CHECK_NEXT(emitStore(ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128Load8Lane): + CHECK_NEXT(emitLoadLane(1)); + case uint32_t(SimdOp::V128Load16Lane): + CHECK_NEXT(emitLoadLane(2)); + case uint32_t(SimdOp::V128Load32Lane): + CHECK_NEXT(emitLoadLane(4)); + case uint32_t(SimdOp::V128Load64Lane): + CHECK_NEXT(emitLoadLane(8)); + case uint32_t(SimdOp::V128Store8Lane): + CHECK_NEXT(emitStoreLane(1)); + case uint32_t(SimdOp::V128Store16Lane): + CHECK_NEXT(emitStoreLane(2)); + case uint32_t(SimdOp::V128Store32Lane): + CHECK_NEXT(emitStoreLane(4)); + case uint32_t(SimdOp::V128Store64Lane): + CHECK_NEXT(emitStoreLane(8)); +# ifdef ENABLE_WASM_RELAXED_SIMD + case uint32_t(SimdOp::F32x4RelaxedFma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFmaF32x4, ValType::V128)); + case uint32_t(SimdOp::F32x4RelaxedFnma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFnmaF32x4, ValType::V128)); + case uint32_t(SimdOp::F64x2RelaxedFma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFmaF64x2, ValType::V128)); + case uint32_t(SimdOp::F64x2RelaxedFnma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFnmaF64x2, ValType::V128)); + break; + case uint32_t(SimdOp::I8x16RelaxedLaneSelect): + case uint32_t(SimdOp::I16x8RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4RelaxedLaneSelect): + case uint32_t(SimdOp::I64x2RelaxedLaneSelect): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitVectorLaneSelect()); + case uint32_t(SimdOp::F32x4RelaxedMin): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMinF32x4)); + case uint32_t(SimdOp::F32x4RelaxedMax): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF32x4)); + case uint32_t(SimdOp::F64x2RelaxedMin): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMinF64x2)); + case uint32_t(SimdOp::F64x2RelaxedMax): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF64x2)); + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToUI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToUI32x4)); + case uint32_t(SimdOp::I8x16RelaxedSwizzle): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedSwizzle)); + case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedQ15MulrS)); + case uint32_t(SimdOp::I16x8DotI8x16I7x16S): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(DotI8x16I7x16S)); + case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary0(emitDotI8x16I7x16AddS, ValType::V128)); +# endif + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } +#endif // ENABLE_WASM_SIMD + + // "Miscellaneous" operations + case uint16_t(Op::MiscPrefix): { + switch (op.b1) { + case uint32_t(MiscOp::I32TruncSatF32S): + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF32ToI32<TRUNC_SATURATING>, + ValType::F32, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF32U): + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF32ToI32<TRUNC_UNSIGNED | TRUNC_SATURATING>, + ValType::F32, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF64S): + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF64ToI32<TRUNC_SATURATING>, + ValType::F64, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF64U): + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF64ToI32<TRUNC_UNSIGNED | TRUNC_SATURATING>, + ValType::F64, ValType::I32)); + case uint32_t(MiscOp::I64TruncSatF32S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF32ToI64<TRUNC_SATURATING>, + ValType::F32, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF32U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF32ToI64<TRUNC_UNSIGNED | TRUNC_SATURATING>, + ValType::F32, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF64S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF64ToI64<TRUNC_SATURATING>, + ValType::F64, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF64U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF64ToI64<TRUNC_UNSIGNED | TRUNC_SATURATING>, + ValType::F64, ValType::I64)); +#endif + case uint32_t(MiscOp::MemoryCopy): + CHECK_NEXT(emitMemCopy()); + case uint32_t(MiscOp::DataDrop): + CHECK_NEXT(emitDataOrElemDrop(/*isData=*/true)); + case uint32_t(MiscOp::MemoryFill): + CHECK_NEXT(emitMemFill()); +#ifdef ENABLE_WASM_MEMORY_CONTROL + case uint32_t(MiscOp::MemoryDiscard): { + if (!moduleEnv_.memoryControlEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitMemDiscard()); + } +#endif + case uint32_t(MiscOp::MemoryInit): + CHECK_NEXT(emitMemInit()); + case uint32_t(MiscOp::TableCopy): + CHECK_NEXT(emitTableCopy()); + case uint32_t(MiscOp::ElemDrop): + CHECK_NEXT(emitDataOrElemDrop(/*isData=*/false)); + case uint32_t(MiscOp::TableInit): + CHECK_NEXT(emitTableInit()); + case uint32_t(MiscOp::TableFill): + CHECK_NEXT(emitTableFill()); + case uint32_t(MiscOp::TableGrow): + CHECK_NEXT(emitTableGrow()); + case uint32_t(MiscOp::TableSize): + CHECK_NEXT(emitTableSize()); + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } + + // Thread operations + case uint16_t(Op::ThreadPrefix): { + // Though thread ops can be used on nonshared memories, we make them + // unavailable if shared memory has been disabled in the prefs, for + // maximum predictability and safety and consistency with JS. + if (moduleEnv_.sharedMemoryEnabled() == Shareable::False) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(ThreadOp::Wake): + CHECK_NEXT(emitWake()); + + case uint32_t(ThreadOp::I32Wait): + CHECK_NEXT(emitWait(ValType::I32, 4)); + case uint32_t(ThreadOp::I64Wait): + CHECK_NEXT(emitWait(ValType::I64, 8)); + case uint32_t(ThreadOp::Fence): + CHECK_NEXT(emitFence()); + + case uint32_t(ThreadOp::I32AtomicLoad): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicLoad): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicLoad8U): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicLoad16U): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad8U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicLoad16U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad32U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicStore): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicStore): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicStore8U): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicStore16U): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore8U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicStore16U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore32U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicAdd): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAddOp)); + + case uint32_t(ThreadOp::I32AtomicSub): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchSubOp)); + + case uint32_t(ThreadOp::I32AtomicAnd): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAndOp)); + + case uint32_t(ThreadOp::I32AtomicOr): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchOrOp)); + + case uint32_t(ThreadOp::I32AtomicXor): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchXorOp)); + + case uint32_t(ThreadOp::I32AtomicXchg): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicXchg): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicXchg8U): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicXchg16U): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg8U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicXchg16U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg32U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicCmpXchg): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicCmpXchg): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicCmpXchg8U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicCmpXchg16U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg8U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicCmpXchg16U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg32U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint32)); + + default: + return iter_.unrecognizedOpcode(&op); + } + break; + } + + // asm.js and other private operations + case uint16_t(Op::MozPrefix): { + if (op.b1 != uint32_t(MozOp::Intrinsic) || + !moduleEnv_.intrinsicsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + // private intrinsic operations + CHECK_NEXT(emitIntrinsic()); + } + + default: + return iter_.unrecognizedOpcode(&op); + } + +#undef CHECK +#undef NEXT +#undef CHECK_NEXT +#undef CHECK_POINTER_COUNT +#undef dispatchBinary0 +#undef dispatchBinary1 +#undef dispatchBinary2 +#undef dispatchBinary3 +#undef dispatchUnary0 +#undef dispatchUnary1 +#undef dispatchUnary2 +#undef dispatchComparison0 +#undef dispatchConversion0 +#undef dispatchConversion1 +#undef dispatchConversionOOM +#undef dispatchCalloutConversionOOM +#undef dispatchIntDivCallout +#undef dispatchVectorBinary +#undef dispatchVectorUnary +#undef dispatchVectorComparison +#undef dispatchExtractLane +#undef dispatchReplaceLane +#undef dispatchSplat +#undef dispatchVectorReduction + + MOZ_CRASH("unreachable"); + } + + MOZ_CRASH("unreachable"); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Various helpers. + +void BaseCompiler::assertResultRegistersAvailable(ResultType type) { +#ifdef DEBUG + for (ABIResultIter iter(type); !iter.done(); iter.next()) { + ABIResult result = iter.cur(); + if (!result.inRegister()) { + return; + } + switch (result.type().kind()) { + case ValType::I32: + MOZ_ASSERT(isAvailableI32(RegI32(result.gpr()))); + break; + case ValType::I64: + MOZ_ASSERT(isAvailableI64(RegI64(result.gpr64()))); + break; + case ValType::V128: +# ifdef ENABLE_WASM_SIMD + MOZ_ASSERT(isAvailableV128(RegV128(result.fpr()))); + break; +# else + MOZ_CRASH("No SIMD support"); +# endif + case ValType::F32: + MOZ_ASSERT(isAvailableF32(RegF32(result.fpr()))); + break; + case ValType::F64: + MOZ_ASSERT(isAvailableF64(RegF64(result.fpr()))); + break; + case ValType::Ref: + MOZ_ASSERT(isAvailableRef(RegRef(result.gpr()))); + break; + } + } +#endif +} + +void BaseCompiler::saveTempPtr(const RegPtr& r) { + MOZ_ASSERT(!ra.isAvailablePtr(r)); + fr.pushGPR(r); + ra.freePtr(r); + MOZ_ASSERT(ra.isAvailablePtr(r)); +} + +void BaseCompiler::restoreTempPtr(const RegPtr& r) { + MOZ_ASSERT(ra.isAvailablePtr(r)); + ra.needPtr(r); + fr.popGPR(r); + MOZ_ASSERT(!ra.isAvailablePtr(r)); +} + +#ifdef DEBUG +void BaseCompiler::performRegisterLeakCheck() { + BaseRegAlloc::LeakCheck check(ra); + for (auto& item : stk_) { + switch (item.kind_) { + case Stk::RegisterI32: + check.addKnownI32(item.i32reg()); + break; + case Stk::RegisterI64: + check.addKnownI64(item.i64reg()); + break; + case Stk::RegisterF32: + check.addKnownF32(item.f32reg()); + break; + case Stk::RegisterF64: + check.addKnownF64(item.f64reg()); + break; +# ifdef ENABLE_WASM_SIMD + case Stk::RegisterV128: + check.addKnownV128(item.v128reg()); + break; +# endif + case Stk::RegisterRef: + check.addKnownRef(item.refReg()); + break; + default: + break; + } + } +} + +void BaseCompiler::assertStackInvariants() const { + if (deadCode_) { + // Nonlocal control flow can pass values in stack locations in a way that + // isn't accounted for by the value stack. In dead code, which occurs + // after unconditional non-local control flow, there is no invariant to + // assert. + return; + } + size_t size = 0; + for (const Stk& v : stk_) { + switch (v.kind()) { + case Stk::MemRef: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI32: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI64: + size += BaseStackFrame::StackSizeOfInt64; + break; + case Stk::MemF64: + size += BaseStackFrame::StackSizeOfDouble; + break; + case Stk::MemF32: + size += BaseStackFrame::StackSizeOfFloat; + break; +# ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + size += BaseStackFrame::StackSizeOfV128; + break; +# endif + default: + MOZ_ASSERT(!v.isMem()); + break; + } + } + MOZ_ASSERT(size == fr.dynamicHeight()); +} + +void BaseCompiler::showStack(const char* who) const { + fprintf(stderr, "Stack at %s {{\n", who); + size_t n = 0; + for (const Stk& elem : stk_) { + fprintf(stderr, " [%zu] ", n++); + elem.showStackElem(); + fprintf(stderr, "\n"); + } + fprintf(stderr, "}}\n"); +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Main compilation logic. + +bool BaseCompiler::emitFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitFunction"); + + if (!beginFunction()) { + return false; + } + + if (!emitBody()) { + return false; + } + + if (!endFunction()) { + return false; + } + + return true; +} + +BaseCompiler::BaseCompiler(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + const FuncCompileInput& func, + const ValTypeVector& locals, + const RegisterOffsets& trapExitLayout, + size_t trapExitLayoutNumWords, Decoder& decoder, + StkVector& stkSource, TempAllocator* alloc, + MacroAssembler* masm, StackMaps* stackMaps) + : // Environment + moduleEnv_(moduleEnv), + compilerEnv_(compilerEnv), + func_(func), + locals_(locals), + previousBreakablePoint_(UINT32_MAX), + stkSource_(stkSource), + // Output-only data structures + alloc_(alloc->fallible()), + masm(*masm), + // Compilation state + decoder_(decoder), + iter_(moduleEnv, decoder), + fr(*masm), + stackMapGenerator_(stackMaps, trapExitLayout, trapExitLayoutNumWords, + *masm), + deadCode_(false), + bceSafe_(0), + latentOp_(LatentOp::None), + latentType_(ValType::I32), + latentIntCmp_(Assembler::Equal), + latentDoubleCmp_(Assembler::DoubleEqual) { + // Our caller, BaselineCompileFunctions, will lend us the vector contents to + // use for the eval stack. To get hold of those contents, we'll temporarily + // installing an empty one in its place. + MOZ_ASSERT(stk_.empty()); + stk_.swap(stkSource_); + + // Assuming that previously processed wasm functions are well formed, the + // eval stack should now be empty. But empty it anyway; any non-emptyness + // at this point will cause chaos. + stk_.clear(); +} + +BaseCompiler::~BaseCompiler() { + stk_.swap(stkSource_); + // We've returned the eval stack vector contents to our caller, + // BaselineCompileFunctions. We expect the vector we get in return to be + // empty since that's what we swapped for the stack vector in our + // constructor. + MOZ_ASSERT(stk_.empty()); +} + +bool BaseCompiler::init() { + // We may lift this restriction in the future. + MOZ_ASSERT_IF(usesMemory() && isMem64(), !moduleEnv_.hugeMemoryEnabled()); + // asm.js is not supported in baseline + MOZ_ASSERT(!moduleEnv_.isAsmJS()); + // Only asm.js modules have call site line numbers + MOZ_ASSERT(func_.callSiteLineNums.empty()); + + ra.init(this); + + if (!SigD_.append(ValType::F64)) { + return false; + } + if (!SigF_.append(ValType::F32)) { + return false; + } + + ArgTypeVector args(funcType()); + return fr.setupLocals(locals_, args, compilerEnv_.debugEnabled(), + &localInfo_); +} + +FuncOffsets BaseCompiler::finish() { + MOZ_ASSERT(iter_.done(), "all bytes must be consumed"); + MOZ_ASSERT(stk_.empty()); + MOZ_ASSERT(stackMapGenerator_.memRefsOnStk == 0); + + masm.flushBuffer(); + + return offsets_; +} + +} // namespace wasm +} // namespace js + +bool js::wasm::BaselinePlatformSupport() { +#if defined(JS_CODEGEN_ARM) + // Simplifying assumption: require SDIV and UDIV. + // + // I have no good data on ARM populations allowing me to say that + // X% of devices in the market implement SDIV and UDIV. However, + // they are definitely implemented on the Cortex-A7 and Cortex-A15 + // and on all ARMv8 systems. + if (!HasIDIV()) { + return false; + } +#endif +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + return true; +#else + return false; +#endif +} + +bool js::wasm::BaselineCompileFunctions(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + LifoAlloc& lifo, + const FuncCompileInputVector& inputs, + CompiledCode* code, + UniqueChars* error) { + MOZ_ASSERT(compilerEnv.tier() == Tier::Baseline); + MOZ_ASSERT(moduleEnv.kind == ModuleKind::Wasm); + + // The MacroAssembler will sometimes access the jitContext. + + TempAllocator alloc(&lifo); + JitContext jitContext; + MOZ_ASSERT(IsCompilingWasm()); + WasmMacroAssembler masm(alloc, moduleEnv); + + // Swap in already-allocated empty vectors to avoid malloc/free. + MOZ_ASSERT(code->empty()); + if (!code->swap(masm)) { + return false; + } + + // Create a description of the stack layout created by GenerateTrapExit(). + RegisterOffsets trapExitLayout; + size_t trapExitLayoutNumWords; + GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords); + + // The compiler's operand stack. We reuse it across all functions so as to + // avoid malloc/free. Presize it to 128 elements in the hope of avoiding + // reallocation later. + StkVector stk; + if (!stk.reserve(128)) { + return false; + } + + for (const FuncCompileInput& func : inputs) { + Decoder d(func.begin, func.end, func.lineOrBytecode, error); + + // Build the local types vector. + + ValTypeVector locals; + if (!locals.appendAll(moduleEnv.funcs[func.index].type->args())) { + return false; + } + if (!DecodeLocalEntries(d, *moduleEnv.types, moduleEnv.features, &locals)) { + return false; + } + + // One-pass baseline compilation. + + BaseCompiler f(moduleEnv, compilerEnv, func, locals, trapExitLayout, + trapExitLayoutNumWords, d, stk, &alloc, &masm, + &code->stackMaps); + if (!f.init()) { + return false; + } + if (!f.emitFunction()) { + return false; + } + if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode, + f.finish())) { + return false; + } + } + + masm.finish(); + if (masm.oom()) { + return false; + } + + return code->swap(masm); +} |