summaryrefslogtreecommitdiffstats
path: root/js/src/wasm/WasmIonCompile.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/wasm/WasmIonCompile.cpp')
-rw-r--r--js/src/wasm/WasmIonCompile.cpp8691
1 files changed, 8691 insertions, 0 deletions
diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp
new file mode 100644
index 0000000000..2f5e3f7233
--- /dev/null
+++ b/js/src/wasm/WasmIonCompile.cpp
@@ -0,0 +1,8691 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright 2015 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "wasm/WasmIonCompile.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+
+#include "jit/ABIArgGenerator.h"
+#include "jit/CodeGenerator.h"
+#include "jit/CompileInfo.h"
+#include "jit/Ion.h"
+#include "jit/IonOptimizationLevels.h"
+#include "jit/MIR.h"
+#include "jit/ShuffleAnalysis.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "wasm/WasmBaselineCompile.h"
+#include "wasm/WasmBuiltins.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmGC.h"
+#include "wasm/WasmGcObject.h"
+#include "wasm/WasmGenerator.h"
+#include "wasm/WasmIntrinsic.h"
+#include "wasm/WasmOpIter.h"
+#include "wasm/WasmSignalHandlers.h"
+#include "wasm/WasmStubs.h"
+#include "wasm/WasmValidate.h"
+
+using namespace js;
+using namespace js::jit;
+using namespace js::wasm;
+
+using mozilla::IsPowerOfTwo;
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+
+namespace {
+
+using BlockVector = Vector<MBasicBlock*, 8, SystemAllocPolicy>;
+using DefVector = Vector<MDefinition*, 8, SystemAllocPolicy>;
+
+// To compile try-catch blocks, we extend the IonCompilePolicy's ControlItem
+// from being just an MBasicBlock* to a Control structure collecting additional
+// information.
+using ControlInstructionVector =
+ Vector<MControlInstruction*, 8, SystemAllocPolicy>;
+
+struct Control {
+ MBasicBlock* block;
+ // For a try-catch ControlItem, when its block's Labelkind is Try, this
+ // collects branches to later bind and create the try's landing pad.
+ ControlInstructionVector tryPadPatches;
+
+ Control() : block(nullptr) {}
+
+ explicit Control(MBasicBlock* block) : block(block) {}
+
+ public:
+ void setBlock(MBasicBlock* newBlock) { block = newBlock; }
+};
+
+// [SMDOC] WebAssembly Exception Handling in Ion
+// =======================================================
+//
+// ## Throwing instructions
+//
+// Wasm exceptions can be thrown by either a throw instruction (local throw),
+// or by a wasm call.
+//
+// ## The "catching try control"
+//
+// We know we are in try-code if there is a surrounding ControlItem with
+// LabelKind::Try. The innermost such control is called the
+// "catching try control".
+//
+// ## Throws without a catching try control
+//
+// Such throws are implemented with an instance call that triggers the exception
+// unwinding runtime. The exception unwinding runtime will not return to the
+// function.
+//
+// ## "landing pad" and "pre-pad" blocks
+//
+// When an exception is thrown, the unwinder will search for the nearest
+// enclosing try block and redirect control flow to it. The code that executes
+// before any catch blocks is called the 'landing pad'. The 'landing pad' is
+// responsible to:
+// 1. Consume the pending exception state from
+// Instance::pendingException(Tag)
+// 2. Branch to the correct catch block, or else rethrow
+//
+// There is one landing pad for each try block. The immediate predecessors of
+// the landing pad are called 'pre-pad' blocks. There is one pre-pad block per
+// throwing instruction.
+//
+// ## Creating pre-pad blocks
+//
+// There are two possible sorts of pre-pad blocks, depending on whether we
+// are branching after a local throw instruction, or after a wasm call:
+//
+// - If we encounter a local throw, we create the exception and tag objects,
+// store them to Instance::pendingException(Tag), and then jump to the
+// landing pad.
+//
+// - If we encounter a wasm call, we construct a MWasmCallCatchable which is a
+// control instruction with either a branch to a fallthrough block or
+// to a pre-pad block.
+//
+// The pre-pad block for a wasm call is empty except for a jump to the
+// landing pad. It only exists to avoid critical edges which when split would
+// violate the invariants of MWasmCallCatchable. The pending exception state
+// is taken care of by the unwinder.
+//
+// Each pre-pad ends with a pending jump to the landing pad. The pending jumps
+// to the landing pad are tracked in `tryPadPatches`. These are called
+// "pad patches".
+//
+// ## Creating the landing pad
+//
+// When we exit try-code, we check if tryPadPatches has captured any control
+// instructions (pad patches). If not, we don't compile any catches and we mark
+// the rest as dead code.
+//
+// If there are pre-pad blocks, we join them to create a landing pad (or just
+// "pad"). The pad's last two slots are the caught exception, and the
+// exception's tag object.
+//
+// There are three different forms of try-catch/catch_all Wasm instructions,
+// which result in different form of landing pad.
+//
+// 1. A catchless try, so a Wasm instruction of the form "try ... end".
+// - In this case, we end the pad by rethrowing the caught exception.
+//
+// 2. A single catch_all after a try.
+// - If the first catch after a try is a catch_all, then there won't be
+// any more catches, but we need the exception and its tag object, in
+// case the code in a catch_all contains "rethrow" instructions.
+// - The Wasm instruction "rethrow", gets the exception and tag object to
+// rethrow from the last two slots of the landing pad which, due to
+// validation, is the l'th surrounding ControlItem.
+// - We immediately GoTo to a new block after the pad and pop both the
+// exception and tag object, as we don't need them anymore in this case.
+//
+// 3. Otherwise, there is one or more catch code blocks following.
+// - In this case, we construct the landing pad by creating a sequence
+// of compare and branch blocks that compare the pending exception tag
+// object to the tag object of the current tagged catch block. This is
+// done incrementally as we visit each tagged catch block in the bytecode
+// stream. At every step, we update the ControlItem's block to point to
+// the next block to be created in the landing pad sequence. The final
+// block will either be a rethrow, if there is no catch_all, or else a
+// jump to a catch_all block.
+
+struct IonCompilePolicy {
+ // We store SSA definitions in the value stack.
+ using Value = MDefinition*;
+ using ValueVector = DefVector;
+
+ // We store loop headers and then/else blocks in the control flow stack.
+ // In the case of try-catch control blocks, we collect additional information
+ // regarding the possible paths from throws and calls to a landing pad, as
+ // well as information on the landing pad's handlers (its catches).
+ using ControlItem = Control;
+};
+
+using IonOpIter = OpIter<IonCompilePolicy>;
+
+class FunctionCompiler;
+
+// CallCompileState describes a call that is being compiled.
+
+class CallCompileState {
+ // A generator object that is passed each argument as it is compiled.
+ WasmABIArgGenerator abi_;
+
+ // Accumulates the register arguments while compiling arguments.
+ MWasmCallBase::Args regArgs_;
+
+ // Reserved argument for passing Instance* to builtin instance method calls.
+ ABIArg instanceArg_;
+
+ // The stack area in which the callee will write stack return values, or
+ // nullptr if no stack results.
+ MWasmStackResultArea* stackResultArea_ = nullptr;
+
+ // Only FunctionCompiler should be directly manipulating CallCompileState.
+ friend class FunctionCompiler;
+};
+
+// Encapsulates the compilation of a single function in an asm.js module. The
+// function compiler handles the creation and final backend compilation of the
+// MIR graph.
+class FunctionCompiler {
+ struct ControlFlowPatch {
+ MControlInstruction* ins;
+ uint32_t index;
+ ControlFlowPatch(MControlInstruction* ins, uint32_t index)
+ : ins(ins), index(index) {}
+ };
+
+ using ControlFlowPatchVector = Vector<ControlFlowPatch, 0, SystemAllocPolicy>;
+ using ControlFlowPatchVectorVector =
+ Vector<ControlFlowPatchVector, 0, SystemAllocPolicy>;
+
+ const ModuleEnvironment& moduleEnv_;
+ IonOpIter iter_;
+ const FuncCompileInput& func_;
+ const ValTypeVector& locals_;
+ size_t lastReadCallSite_;
+
+ TempAllocator& alloc_;
+ MIRGraph& graph_;
+ const CompileInfo& info_;
+ MIRGenerator& mirGen_;
+
+ MBasicBlock* curBlock_;
+ uint32_t maxStackArgBytes_;
+
+ uint32_t loopDepth_;
+ uint32_t blockDepth_;
+ ControlFlowPatchVectorVector blockPatches_;
+
+ // Instance pointer argument to the current function.
+ MWasmParameter* instancePointer_;
+ MWasmParameter* stackResultPointer_;
+
+ // Reference to masm.tryNotes_
+ wasm::TryNoteVector& tryNotes_;
+
+ public:
+ FunctionCompiler(const ModuleEnvironment& moduleEnv, Decoder& decoder,
+ const FuncCompileInput& func, const ValTypeVector& locals,
+ MIRGenerator& mirGen, TryNoteVector& tryNotes)
+ : moduleEnv_(moduleEnv),
+ iter_(moduleEnv, decoder),
+ func_(func),
+ locals_(locals),
+ lastReadCallSite_(0),
+ alloc_(mirGen.alloc()),
+ graph_(mirGen.graph()),
+ info_(mirGen.outerInfo()),
+ mirGen_(mirGen),
+ curBlock_(nullptr),
+ maxStackArgBytes_(0),
+ loopDepth_(0),
+ blockDepth_(0),
+ instancePointer_(nullptr),
+ stackResultPointer_(nullptr),
+ tryNotes_(tryNotes) {}
+
+ const ModuleEnvironment& moduleEnv() const { return moduleEnv_; }
+
+ IonOpIter& iter() { return iter_; }
+ TempAllocator& alloc() const { return alloc_; }
+ // FIXME(1401675): Replace with BlockType.
+ uint32_t funcIndex() const { return func_.index; }
+ const FuncType& funcType() const {
+ return *moduleEnv_.funcs[func_.index].type;
+ }
+
+ BytecodeOffset bytecodeOffset() const { return iter_.bytecodeOffset(); }
+ BytecodeOffset bytecodeIfNotAsmJS() const {
+ return moduleEnv_.isAsmJS() ? BytecodeOffset() : iter_.bytecodeOffset();
+ }
+
+ [[nodiscard]] bool init() {
+ // Prepare the entry block for MIR generation:
+
+ const ArgTypeVector args(funcType());
+
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+ if (!newBlock(/* prev */ nullptr, &curBlock_)) {
+ return false;
+ }
+
+ for (WasmABIArgIter i(args); !i.done(); i++) {
+ MWasmParameter* ins = MWasmParameter::New(alloc(), *i, i.mirType());
+ curBlock_->add(ins);
+ if (args.isSyntheticStackResultPointerArg(i.index())) {
+ MOZ_ASSERT(stackResultPointer_ == nullptr);
+ stackResultPointer_ = ins;
+ } else {
+ curBlock_->initSlot(info().localSlot(args.naturalIndex(i.index())),
+ ins);
+ }
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+ }
+
+ // Set up a parameter that receives the hidden instance pointer argument.
+ instancePointer_ =
+ MWasmParameter::New(alloc(), ABIArg(InstanceReg), MIRType::Pointer);
+ curBlock_->add(instancePointer_);
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+
+ for (size_t i = args.lengthWithoutStackResults(); i < locals_.length();
+ i++) {
+ ValType slotValType = locals_[i];
+#ifndef ENABLE_WASM_SIMD
+ if (slotValType == ValType::V128) {
+ return iter().fail("Ion has no SIMD support yet");
+ }
+#endif
+ MDefinition* zero = constantZeroOfValType(slotValType);
+ curBlock_->initSlot(info().localSlot(i), zero);
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ void finish() {
+ mirGen().initWasmMaxStackArgBytes(maxStackArgBytes_);
+
+ MOZ_ASSERT(loopDepth_ == 0);
+ MOZ_ASSERT(blockDepth_ == 0);
+#ifdef DEBUG
+ for (ControlFlowPatchVector& patches : blockPatches_) {
+ MOZ_ASSERT(patches.empty());
+ }
+#endif
+ MOZ_ASSERT(inDeadCode());
+ MOZ_ASSERT(done(), "all bytes must be consumed");
+ MOZ_ASSERT(func_.callSiteLineNums.length() == lastReadCallSite_);
+ }
+
+ /************************* Read-only interface (after local scope setup) */
+
+ MIRGenerator& mirGen() const { return mirGen_; }
+ MIRGraph& mirGraph() const { return graph_; }
+ const CompileInfo& info() const { return info_; }
+
+ MDefinition* getLocalDef(unsigned slot) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ return curBlock_->getSlot(info().localSlot(slot));
+ }
+
+ const ValTypeVector& locals() const { return locals_; }
+
+ /*********************************************************** Constants ***/
+
+ MDefinition* constantF32(float f) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* cst = MWasmFloatConstant::NewFloat32(alloc(), f);
+ curBlock_->add(cst);
+ return cst;
+ }
+ // Hide all other overloads, to guarantee no implicit argument conversion.
+ template <typename T>
+ MDefinition* constantF32(T) = delete;
+
+ MDefinition* constantF64(double d) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* cst = MWasmFloatConstant::NewDouble(alloc(), d);
+ curBlock_->add(cst);
+ return cst;
+ }
+ template <typename T>
+ MDefinition* constantF64(T) = delete;
+
+ MDefinition* constantI32(int32_t i) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ MConstant* constant =
+ MConstant::New(alloc(), Int32Value(i), MIRType::Int32);
+ curBlock_->add(constant);
+ return constant;
+ }
+ template <typename T>
+ MDefinition* constantI32(T) = delete;
+
+ MDefinition* constantI64(int64_t i) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ MConstant* constant = MConstant::NewInt64(alloc(), i);
+ curBlock_->add(constant);
+ return constant;
+ }
+ template <typename T>
+ MDefinition* constantI64(T) = delete;
+
+ // Produce an MConstant of the machine's target int type (Int32 or Int64).
+ MDefinition* constantTargetWord(intptr_t n) {
+ return targetIs64Bit() ? constantI64(int64_t(n)) : constantI32(int32_t(n));
+ }
+ template <typename T>
+ MDefinition* constantTargetWord(T) = delete;
+
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* constantV128(V128 v) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ MWasmFloatConstant* constant = MWasmFloatConstant::NewSimd128(
+ alloc(), SimdConstant::CreateSimd128((int8_t*)v.bytes));
+ curBlock_->add(constant);
+ return constant;
+ }
+ template <typename T>
+ MDefinition* constantV128(T) = delete;
+#endif
+
+ MDefinition* constantNullRef() {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ // MConstant has a lot of baggage so we don't use that here.
+ MWasmNullConstant* constant = MWasmNullConstant::New(alloc());
+ curBlock_->add(constant);
+ return constant;
+ }
+
+ // Produce a zero constant for the specified ValType.
+ MDefinition* constantZeroOfValType(ValType valType) {
+ switch (valType.kind()) {
+ case ValType::I32:
+ return constantI32(0);
+ case ValType::I64:
+ return constantI64(int64_t(0));
+#ifdef ENABLE_WASM_SIMD
+ case ValType::V128:
+ return constantV128(V128(0));
+#endif
+ case ValType::F32:
+ return constantF32(0.0f);
+ case ValType::F64:
+ return constantF64(0.0);
+ case ValType::Ref:
+ return constantNullRef();
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ /***************************** Code generation (after local scope setup) */
+
+ void fence() {
+ if (inDeadCode()) {
+ return;
+ }
+ MWasmFence* ins = MWasmFence::New(alloc());
+ curBlock_->add(ins);
+ }
+
+ template <class T>
+ MDefinition* unary(MDefinition* op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ T* ins = T::New(alloc(), op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ template <class T>
+ MDefinition* unary(MDefinition* op, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ T* ins = T::New(alloc(), op, type);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ template <class T>
+ MDefinition* binary(MDefinition* lhs, MDefinition* rhs) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ T* ins = T::New(alloc(), lhs, rhs);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ template <class T>
+ MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ T* ins = T::New(alloc(), lhs, rhs, type);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ template <class T>
+ MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type,
+ MWasmBinaryBitwise::SubOpcode subOpc) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ T* ins = T::New(alloc(), lhs, rhs, type, subOpc);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* ursh(MDefinition* lhs, MDefinition* rhs, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MUrsh::NewWasm(alloc(), lhs, rhs, type);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* add(MDefinition* lhs, MDefinition* rhs, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MAdd::NewWasm(alloc(), lhs, rhs, type);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ bool mustPreserveNaN(MIRType type) {
+ return IsFloatingPointType(type) && !moduleEnv().isAsmJS();
+ }
+
+ MDefinition* sub(MDefinition* lhs, MDefinition* rhs, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ // wasm can't fold x - 0.0 because of NaN with custom payloads.
+ MSub* ins = MSub::NewWasm(alloc(), lhs, rhs, type, mustPreserveNaN(type));
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* nearbyInt(MDefinition* input, RoundingMode roundingMode) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ auto* ins = MNearbyInt::New(alloc(), input, input->type(), roundingMode);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* minMax(MDefinition* lhs, MDefinition* rhs, MIRType type,
+ bool isMax) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ if (mustPreserveNaN(type)) {
+ // Convert signaling NaN to quiet NaNs.
+ MDefinition* zero = constantZeroOfValType(ValType::fromMIRType(type));
+ lhs = sub(lhs, zero, type);
+ rhs = sub(rhs, zero, type);
+ }
+
+ MMinMax* ins = MMinMax::NewWasm(alloc(), lhs, rhs, type, isMax);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* mul(MDefinition* lhs, MDefinition* rhs, MIRType type,
+ MMul::Mode mode) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ // wasm can't fold x * 1.0 because of NaN with custom payloads.
+ auto* ins =
+ MMul::NewWasm(alloc(), lhs, rhs, type, mode, mustPreserveNaN(type));
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* div(MDefinition* lhs, MDefinition* rhs, MIRType type,
+ bool unsignd) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ bool trapOnError = !moduleEnv().isAsmJS();
+ if (!unsignd && type == MIRType::Int32) {
+ // Enforce the signedness of the operation by coercing the operands
+ // to signed. Otherwise, operands that "look" unsigned to Ion but
+ // are not unsigned to Baldr (eg, unsigned right shifts) may lead to
+ // the operation being executed unsigned. Applies to mod() as well.
+ //
+ // Do this for Int32 only since Int64 is not subject to the same
+ // issues.
+ //
+ // Note the offsets passed to MWasmBuiltinTruncateToInt32 are wrong here,
+ // but it doesn't matter: they're not codegen'd to calls since inputs
+ // already are int32.
+ auto* lhs2 = createTruncateToInt32(lhs);
+ curBlock_->add(lhs2);
+ lhs = lhs2;
+ auto* rhs2 = createTruncateToInt32(rhs);
+ curBlock_->add(rhs2);
+ rhs = rhs2;
+ }
+
+ // For x86 and arm we implement i64 div via c++ builtin.
+ // A call to c++ builtin requires instance pointer.
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
+ if (type == MIRType::Int64) {
+ auto* ins =
+ MWasmBuiltinDivI64::New(alloc(), lhs, rhs, instancePointer_, unsignd,
+ trapOnError, bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+#endif
+
+ auto* ins = MDiv::New(alloc(), lhs, rhs, type, unsignd, trapOnError,
+ bytecodeOffset(), mustPreserveNaN(type));
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MInstruction* createTruncateToInt32(MDefinition* op) {
+ if (op->type() == MIRType::Double || op->type() == MIRType::Float32) {
+ return MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_);
+ }
+
+ return MTruncateToInt32::New(alloc(), op);
+ }
+
+ MDefinition* mod(MDefinition* lhs, MDefinition* rhs, MIRType type,
+ bool unsignd) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ bool trapOnError = !moduleEnv().isAsmJS();
+ if (!unsignd && type == MIRType::Int32) {
+ // See block comment in div().
+ auto* lhs2 = createTruncateToInt32(lhs);
+ curBlock_->add(lhs2);
+ lhs = lhs2;
+ auto* rhs2 = createTruncateToInt32(rhs);
+ curBlock_->add(rhs2);
+ rhs = rhs2;
+ }
+
+ // For x86 and arm we implement i64 mod via c++ builtin.
+ // A call to c++ builtin requires instance pointer.
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
+ if (type == MIRType::Int64) {
+ auto* ins =
+ MWasmBuiltinModI64::New(alloc(), lhs, rhs, instancePointer_, unsignd,
+ trapOnError, bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+#endif
+
+ // Should be handled separately because we call BuiltinThunk for this case
+ // and so, need to add the dependency from instancePointer.
+ if (type == MIRType::Double) {
+ auto* ins = MWasmBuiltinModD::New(alloc(), lhs, rhs, instancePointer_,
+ type, bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ auto* ins = MMod::New(alloc(), lhs, rhs, type, unsignd, trapOnError,
+ bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* bitnot(MDefinition* op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MBitNot::New(alloc(), op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* select(MDefinition* trueExpr, MDefinition* falseExpr,
+ MDefinition* condExpr) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MWasmSelect::New(alloc(), trueExpr, falseExpr, condExpr);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* extendI32(MDefinition* op, bool isUnsigned) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MExtendInt32ToInt64::New(alloc(), op, isUnsigned);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* signExtend(MDefinition* op, uint32_t srcSize,
+ uint32_t targetSize) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ MInstruction* ins;
+ switch (targetSize) {
+ case 4: {
+ MSignExtendInt32::Mode mode;
+ switch (srcSize) {
+ case 1:
+ mode = MSignExtendInt32::Byte;
+ break;
+ case 2:
+ mode = MSignExtendInt32::Half;
+ break;
+ default:
+ MOZ_CRASH("Bad sign extension");
+ }
+ ins = MSignExtendInt32::New(alloc(), op, mode);
+ break;
+ }
+ case 8: {
+ MSignExtendInt64::Mode mode;
+ switch (srcSize) {
+ case 1:
+ mode = MSignExtendInt64::Byte;
+ break;
+ case 2:
+ mode = MSignExtendInt64::Half;
+ break;
+ case 4:
+ mode = MSignExtendInt64::Word;
+ break;
+ default:
+ MOZ_CRASH("Bad sign extension");
+ }
+ ins = MSignExtendInt64::New(alloc(), op, mode);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Bad sign extension");
+ }
+ }
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* convertI64ToFloatingPoint(MDefinition* op, MIRType type,
+ bool isUnsigned) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+#if defined(JS_CODEGEN_ARM)
+ auto* ins = MBuiltinInt64ToFloatingPoint::New(
+ alloc(), op, instancePointer_, type, bytecodeOffset(), isUnsigned);
+#else
+ auto* ins = MInt64ToFloatingPoint::New(alloc(), op, type, bytecodeOffset(),
+ isUnsigned);
+#endif
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ MDefinition* rotate(MDefinition* input, MDefinition* count, MIRType type,
+ bool left) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MRotate::New(alloc(), input, count, type, left);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ template <class T>
+ MDefinition* truncate(MDefinition* op, TruncFlags flags) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = T::New(alloc(), op, flags, bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+
+#if defined(JS_CODEGEN_ARM)
+ MDefinition* truncateWithInstance(MDefinition* op, TruncFlags flags) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MWasmBuiltinTruncateToInt64::New(alloc(), op, instancePointer_,
+ flags, bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+ }
+#endif
+
+ MDefinition* compare(MDefinition* lhs, MDefinition* rhs, JSOp op,
+ MCompare::CompareType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MCompare::NewWasm(alloc(), lhs, rhs, op, type);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ void assign(unsigned slot, MDefinition* def) {
+ if (inDeadCode()) {
+ return;
+ }
+ curBlock_->setSlot(info().localSlot(slot), def);
+ }
+
+ MDefinition* compareIsNull(MDefinition* value, JSOp compareOp) {
+ MDefinition* nullVal = constantNullRef();
+ if (!nullVal) {
+ return nullptr;
+ }
+ return compare(value, nullVal, compareOp, MCompare::Compare_RefOrNull);
+ }
+
+ [[nodiscard]] bool refAsNonNull(MDefinition* value) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ auto* ins = MWasmTrapIfNull::New(
+ alloc(), value, wasm::Trap::NullPointerDereference, bytecodeOffset());
+
+ curBlock_->add(ins);
+ return true;
+ }
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+ [[nodiscard]] bool brOnNull(uint32_t relativeDepth, const DefVector& values,
+ const ResultType& type, MDefinition* condition) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* fallthroughBlock = nullptr;
+ if (!newBlock(curBlock_, &fallthroughBlock)) {
+ return false;
+ }
+
+ MDefinition* check = compareIsNull(condition, JSOp::Eq);
+ if (!check) {
+ return false;
+ }
+ MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock);
+ if (!test ||
+ !addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) {
+ return false;
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(test);
+ curBlock_ = fallthroughBlock;
+ return true;
+ }
+
+ [[nodiscard]] bool brOnNonNull(uint32_t relativeDepth,
+ const DefVector& values,
+ const ResultType& type,
+ MDefinition* condition) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* fallthroughBlock = nullptr;
+ if (!newBlock(curBlock_, &fallthroughBlock)) {
+ return false;
+ }
+
+ MDefinition* check = compareIsNull(condition, JSOp::Ne);
+ if (!check) {
+ return false;
+ }
+ MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock);
+ if (!test ||
+ !addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) {
+ return false;
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(test);
+ curBlock_ = fallthroughBlock;
+ return true;
+ }
+
+#endif // ENABLE_WASM_FUNCTION_REFERENCES
+
+#ifdef ENABLE_WASM_SIMD
+ // About Wasm SIMD as supported by Ion:
+ //
+ // The expectation is that Ion will only ever support SIMD on x86 and x64,
+ // since ARMv7 will cease to be a tier-1 platform soon, and MIPS64 will never
+ // implement SIMD.
+ //
+ // The division of the operations into MIR nodes reflects that expectation,
+ // and is a good fit for x86/x64. Should the expectation change we'll
+ // possibly want to re-architect the SIMD support to be a little more general.
+ //
+ // Most SIMD operations map directly to a single MIR node that ultimately ends
+ // up being expanded in the macroassembler.
+ //
+ // Some SIMD operations that do have a complete macroassembler expansion are
+ // open-coded into multiple MIR nodes here; in some cases that's just
+ // convenience, in other cases it may also allow them to benefit from Ion
+ // optimizations. The reason for the expansions will be documented by a
+ // comment.
+
+ // (v128,v128) -> v128 effect-free binary operations
+ MDefinition* binarySimd128(MDefinition* lhs, MDefinition* rhs,
+ bool commutative, SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
+ rhs->type() == MIRType::Simd128);
+
+ auto* ins = MWasmBinarySimd128::New(alloc(), lhs, rhs, commutative, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128,i32) -> v128 effect-free shift operations
+ MDefinition* shiftSimd128(MDefinition* lhs, MDefinition* rhs, SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
+ rhs->type() == MIRType::Int32);
+
+ int32_t maskBits;
+ if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) {
+ MDefinition* mask = constantI32(maskBits);
+ auto* rhs2 = MBitAnd::New(alloc(), rhs, mask, MIRType::Int32);
+ curBlock_->add(rhs2);
+ rhs = rhs2;
+ }
+
+ auto* ins = MWasmShiftSimd128::New(alloc(), lhs, rhs, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128,scalar,imm) -> v128
+ MDefinition* replaceLaneSimd128(MDefinition* lhs, MDefinition* rhs,
+ uint32_t laneIndex, SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+
+ auto* ins = MWasmReplaceLaneSimd128::New(alloc(), lhs, rhs, laneIndex, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (scalar) -> v128 effect-free unary operations
+ MDefinition* scalarToSimd128(MDefinition* src, SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ auto* ins = MWasmScalarToSimd128::New(alloc(), src, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128) -> v128 effect-free unary operations
+ MDefinition* unarySimd128(MDefinition* src, SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(src->type() == MIRType::Simd128);
+ auto* ins = MWasmUnarySimd128::New(alloc(), src, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128, imm) -> scalar effect-free unary operations
+ MDefinition* reduceSimd128(MDefinition* src, SimdOp op, ValType outType,
+ uint32_t imm = 0) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(src->type() == MIRType::Simd128);
+ auto* ins =
+ MWasmReduceSimd128::New(alloc(), src, op, outType.toMIRType(), imm);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128, v128, v128) -> v128 effect-free operations
+ MDefinition* ternarySimd128(MDefinition* v0, MDefinition* v1, MDefinition* v2,
+ SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(v0->type() == MIRType::Simd128 &&
+ v1->type() == MIRType::Simd128 &&
+ v2->type() == MIRType::Simd128);
+
+ auto* ins = MWasmTernarySimd128::New(alloc(), v0, v1, v2, op);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // (v128, v128, imm_v128) -> v128 effect-free operations
+ MDefinition* shuffleSimd128(MDefinition* v1, MDefinition* v2, V128 control) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(v1->type() == MIRType::Simd128);
+ MOZ_ASSERT(v2->type() == MIRType::Simd128);
+ auto* ins = BuildWasmShuffleSimd128(
+ alloc(), reinterpret_cast<int8_t*>(control.bytes), v1, v2);
+ curBlock_->add(ins);
+ return ins;
+ }
+
+ // Also see below for SIMD memory references
+
+#endif // ENABLE_WASM_SIMD
+
+ /************************************************ Linear memory accesses */
+
+ // For detailed information about memory accesses, see "Linear memory
+ // addresses and bounds checking" in WasmMemory.cpp.
+
+ private:
+ // If the platform does not have a HeapReg, load the memory base from
+ // instance.
+ MWasmLoadInstance* maybeLoadMemoryBase() {
+ MWasmLoadInstance* load = nullptr;
+#ifdef JS_CODEGEN_X86
+ AliasSet aliases = !moduleEnv_.memory->canMovingGrow()
+ ? AliasSet::None()
+ : AliasSet::Load(AliasSet::WasmHeapMeta);
+ load = MWasmLoadInstance::New(alloc(), instancePointer_,
+ wasm::Instance::offsetOfMemoryBase(),
+ MIRType::Pointer, aliases);
+ curBlock_->add(load);
+#endif
+ return load;
+ }
+
+ public:
+ // A value holding the memory base, whether that's HeapReg or some other
+ // register.
+ MWasmHeapBase* memoryBase() {
+ MWasmHeapBase* base = nullptr;
+ AliasSet aliases = !moduleEnv_.memory->canMovingGrow()
+ ? AliasSet::None()
+ : AliasSet::Load(AliasSet::WasmHeapMeta);
+ base = MWasmHeapBase::New(alloc(), instancePointer_, aliases);
+ curBlock_->add(base);
+ return base;
+ }
+
+ private:
+ // If the bounds checking strategy requires it, load the bounds check limit
+ // from the instance.
+ MWasmLoadInstance* maybeLoadBoundsCheckLimit(MIRType type) {
+ MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64);
+ if (moduleEnv_.hugeMemoryEnabled()) {
+ return nullptr;
+ }
+ AliasSet aliases = !moduleEnv_.memory->canMovingGrow()
+ ? AliasSet::None()
+ : AliasSet::Load(AliasSet::WasmHeapMeta);
+ auto* load = MWasmLoadInstance::New(
+ alloc(), instancePointer_, wasm::Instance::offsetOfBoundsCheckLimit(),
+ type, aliases);
+ curBlock_->add(load);
+ return load;
+ }
+
+ // Return true if the access requires an alignment check. If so, sets
+ // *mustAdd to true if the offset must be added to the pointer before
+ // checking.
+ bool needAlignmentCheck(MemoryAccessDesc* access, MDefinition* base,
+ bool* mustAdd) {
+ MOZ_ASSERT(!*mustAdd);
+
+ // asm.js accesses are always aligned and need no checks.
+ if (moduleEnv_.isAsmJS() || !access->isAtomic()) {
+ return false;
+ }
+
+ // If the EA is known and aligned it will need no checks.
+ if (base->isConstant()) {
+ // We only care about the low bits, so overflow is OK, as is chopping off
+ // the high bits of an i64 pointer.
+ uint32_t ptr = 0;
+ if (isMem64()) {
+ ptr = uint32_t(base->toConstant()->toInt64());
+ } else {
+ ptr = base->toConstant()->toInt32();
+ }
+ if (((ptr + access->offset64()) & (access->byteSize() - 1)) == 0) {
+ return false;
+ }
+ }
+
+ // If the offset is aligned then the EA is just the pointer, for
+ // the purposes of this check.
+ *mustAdd = (access->offset64() & (access->byteSize() - 1)) != 0;
+ return true;
+ }
+
+ // Fold a constant base into the offset and make the base 0, provided the
+ // offset stays below the guard limit. The reason for folding the base into
+ // the offset rather than vice versa is that a small offset can be ignored
+ // by both explicit bounds checking and bounds check elimination.
+ void foldConstantPointer(MemoryAccessDesc* access, MDefinition** base) {
+ uint32_t offsetGuardLimit =
+ GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled());
+
+ if ((*base)->isConstant()) {
+ uint64_t basePtr = 0;
+ if (isMem64()) {
+ basePtr = uint64_t((*base)->toConstant()->toInt64());
+ } else {
+ basePtr = uint64_t(int64_t((*base)->toConstant()->toInt32()));
+ }
+
+ uint64_t offset = access->offset64();
+
+ if (offset < offsetGuardLimit && basePtr < offsetGuardLimit - offset) {
+ offset += uint32_t(basePtr);
+ access->setOffset32(uint32_t(offset));
+ *base = isMem64() ? constantI64(int64_t(0)) : constantI32(0);
+ }
+ }
+ }
+
+ // If the offset must be added because it is large or because the true EA must
+ // be checked, compute the effective address, trapping on overflow.
+ void maybeComputeEffectiveAddress(MemoryAccessDesc* access,
+ MDefinition** base, bool mustAddOffset) {
+ uint32_t offsetGuardLimit =
+ GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled());
+
+ if (access->offset64() >= offsetGuardLimit ||
+ access->offset64() > UINT32_MAX || mustAddOffset ||
+ !JitOptions.wasmFoldOffsets) {
+ *base = computeEffectiveAddress(*base, access);
+ }
+ }
+
+ MWasmLoadInstance* needBoundsCheck() {
+#ifdef JS_64BIT
+ // For 32-bit base pointers:
+ //
+ // If the bounds check uses the full 64 bits of the bounds check limit, then
+ // the base pointer must be zero-extended to 64 bits before checking and
+ // wrapped back to 32-bits after Spectre masking. (And it's important that
+ // the value we end up with has flowed through the Spectre mask.)
+ //
+ // If the memory's max size is known to be smaller than 64K pages exactly,
+ // we can use a 32-bit check and avoid extension and wrapping.
+ static_assert(0x100000000 % PageSize == 0);
+ bool mem32LimitIs64Bits = isMem32() &&
+ !moduleEnv_.memory->boundsCheckLimitIs32Bits() &&
+ MaxMemoryPages(moduleEnv_.memory->indexType()) >=
+ Pages(0x100000000 / PageSize);
+#else
+ // On 32-bit platforms we have no more than 2GB memory and the limit for a
+ // 32-bit base pointer is never a 64-bit value.
+ bool mem32LimitIs64Bits = false;
+#endif
+ return maybeLoadBoundsCheckLimit(
+ mem32LimitIs64Bits || isMem64() ? MIRType::Int64 : MIRType::Int32);
+ }
+
+ void performBoundsCheck(MDefinition** base,
+ MWasmLoadInstance* boundsCheckLimit) {
+ // At the outset, actualBase could be the result of pretty much any integer
+ // operation, or it could be the load of an integer constant. If its type
+ // is i32, we may assume the value has a canonical representation for the
+ // platform, see doc block in MacroAssembler.h.
+ MDefinition* actualBase = *base;
+
+ // Extend an i32 index value to perform a 64-bit bounds check if the memory
+ // can be 4GB or larger.
+ bool extendAndWrapIndex =
+ isMem32() && boundsCheckLimit->type() == MIRType::Int64;
+ if (extendAndWrapIndex) {
+ auto* extended = MWasmExtendU32Index::New(alloc(), actualBase);
+ curBlock_->add(extended);
+ actualBase = extended;
+ }
+
+ auto* ins =
+ MWasmBoundsCheck::New(alloc(), actualBase, boundsCheckLimit,
+ bytecodeOffset(), MWasmBoundsCheck::Memory0);
+ curBlock_->add(ins);
+ actualBase = ins;
+
+ // If we're masking, then we update *base to create a dependency chain
+ // through the masked index. But we will first need to wrap the index
+ // value if it was extended above.
+ if (JitOptions.spectreIndexMasking) {
+ if (extendAndWrapIndex) {
+ auto* wrapped = MWasmWrapU32Index::New(alloc(), actualBase);
+ curBlock_->add(wrapped);
+ actualBase = wrapped;
+ }
+ *base = actualBase;
+ }
+ }
+
+ // Perform all necessary checking before a wasm heap access, based on the
+ // attributes of the access and base pointer.
+ //
+ // For 64-bit indices on platforms that are limited to indices that fit into
+ // 32 bits (all 32-bit platforms and mips64), this returns a bounds-checked
+ // `base` that has type Int32. Lowering code depends on this and will assert
+ // that the base has this type. See the end of this function.
+
+ void checkOffsetAndAlignmentAndBounds(MemoryAccessDesc* access,
+ MDefinition** base) {
+ MOZ_ASSERT(!inDeadCode());
+ MOZ_ASSERT(!moduleEnv_.isAsmJS());
+
+ // Attempt to fold an offset into a constant base pointer so as to simplify
+ // the addressing expression. This may update *base.
+ foldConstantPointer(access, base);
+
+ // Determine whether an alignment check is needed and whether the offset
+ // must be checked too.
+ bool mustAddOffsetForAlignmentCheck = false;
+ bool alignmentCheck =
+ needAlignmentCheck(access, *base, &mustAddOffsetForAlignmentCheck);
+
+ // If bounds checking or alignment checking requires it, compute the
+ // effective address: add the offset into the pointer and trap on overflow.
+ // This may update *base.
+ maybeComputeEffectiveAddress(access, base, mustAddOffsetForAlignmentCheck);
+
+ // Emit the alignment check if necessary; it traps if it fails.
+ if (alignmentCheck) {
+ curBlock_->add(MWasmAlignmentCheck::New(
+ alloc(), *base, access->byteSize(), bytecodeOffset()));
+ }
+
+ // Emit the bounds check if necessary; it traps if it fails. This may
+ // update *base.
+ MWasmLoadInstance* boundsCheckLimit = needBoundsCheck();
+ if (boundsCheckLimit) {
+ performBoundsCheck(base, boundsCheckLimit);
+ }
+
+#ifndef JS_64BIT
+ if (isMem64()) {
+ // We must have had an explicit bounds check (or one was elided if it was
+ // proved redundant), and on 32-bit systems the index will for sure fit in
+ // 32 bits: the max memory is 2GB. So chop the index down to 32-bit to
+ // simplify the back-end.
+ MOZ_ASSERT((*base)->type() == MIRType::Int64);
+ MOZ_ASSERT(!moduleEnv_.hugeMemoryEnabled());
+ auto* chopped = MWasmWrapU32Index::New(alloc(), *base);
+ MOZ_ASSERT(chopped->type() == MIRType::Int32);
+ curBlock_->add(chopped);
+ *base = chopped;
+ }
+#endif
+ }
+
+ bool isSmallerAccessForI64(ValType result, const MemoryAccessDesc* access) {
+ if (result == ValType::I64 && access->byteSize() <= 4) {
+ // These smaller accesses should all be zero-extending.
+ MOZ_ASSERT(!isSignedIntType(access->type()));
+ return true;
+ }
+ return false;
+ }
+
+ public:
+ bool isMem32() { return moduleEnv_.memory->indexType() == IndexType::I32; }
+ bool isMem64() { return moduleEnv_.memory->indexType() == IndexType::I64; }
+
+ // Sometimes, we need to determine the memory type before the opcode reader
+ // that will reject a memory opcode in the presence of no-memory gets a chance
+ // to do so. This predicate is safe.
+ bool isNoMemOrMem32() {
+ return !moduleEnv_.usesMemory() ||
+ moduleEnv_.memory->indexType() == IndexType::I32;
+ }
+
+ // Add the offset into the pointer to yield the EA; trap on overflow.
+ MDefinition* computeEffectiveAddress(MDefinition* base,
+ MemoryAccessDesc* access) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ uint64_t offset = access->offset64();
+ if (offset == 0) {
+ return base;
+ }
+ auto* ins = MWasmAddOffset::New(alloc(), base, offset, bytecodeOffset());
+ curBlock_->add(ins);
+ access->clearOffset();
+ return ins;
+ }
+
+ MDefinition* load(MDefinition* base, MemoryAccessDesc* access,
+ ValType result) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MInstruction* load = nullptr;
+ if (moduleEnv_.isAsmJS()) {
+ MOZ_ASSERT(access->offset64() == 0);
+ MWasmLoadInstance* boundsCheckLimit =
+ maybeLoadBoundsCheckLimit(MIRType::Int32);
+ load = MAsmJSLoadHeap::New(alloc(), memoryBase, base, boundsCheckLimit,
+ access->type());
+ } else {
+ checkOffsetAndAlignmentAndBounds(access, &base);
+#ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+#endif
+ load = MWasmLoad::New(alloc(), memoryBase, base, *access,
+ result.toMIRType());
+ }
+ if (!load) {
+ return nullptr;
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ void store(MDefinition* base, MemoryAccessDesc* access, MDefinition* v) {
+ if (inDeadCode()) {
+ return;
+ }
+
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MInstruction* store = nullptr;
+ if (moduleEnv_.isAsmJS()) {
+ MOZ_ASSERT(access->offset64() == 0);
+ MWasmLoadInstance* boundsCheckLimit =
+ maybeLoadBoundsCheckLimit(MIRType::Int32);
+ store = MAsmJSStoreHeap::New(alloc(), memoryBase, base, boundsCheckLimit,
+ access->type(), v);
+ } else {
+ checkOffsetAndAlignmentAndBounds(access, &base);
+#ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+#endif
+ store = MWasmStore::New(alloc(), memoryBase, base, *access, v);
+ }
+ if (!store) {
+ return;
+ }
+ curBlock_->add(store);
+ }
+
+ MDefinition* atomicCompareExchangeHeap(MDefinition* base,
+ MemoryAccessDesc* access,
+ ValType result, MDefinition* oldv,
+ MDefinition* newv) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ checkOffsetAndAlignmentAndBounds(access, &base);
+#ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+#endif
+
+ if (isSmallerAccessForI64(result, access)) {
+ auto* cvtOldv =
+ MWrapInt64ToInt32::New(alloc(), oldv, /*bottomHalf=*/true);
+ curBlock_->add(cvtOldv);
+ oldv = cvtOldv;
+
+ auto* cvtNewv =
+ MWrapInt64ToInt32::New(alloc(), newv, /*bottomHalf=*/true);
+ curBlock_->add(cvtNewv);
+ newv = cvtNewv;
+ }
+
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MInstruction* cas = MWasmCompareExchangeHeap::New(
+ alloc(), bytecodeOffset(), memoryBase, base, *access, oldv, newv,
+ instancePointer_);
+ if (!cas) {
+ return nullptr;
+ }
+ curBlock_->add(cas);
+
+ if (isSmallerAccessForI64(result, access)) {
+ cas = MExtendInt32ToInt64::New(alloc(), cas, true);
+ curBlock_->add(cas);
+ }
+
+ return cas;
+ }
+
+ MDefinition* atomicExchangeHeap(MDefinition* base, MemoryAccessDesc* access,
+ ValType result, MDefinition* value) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ checkOffsetAndAlignmentAndBounds(access, &base);
+#ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+#endif
+
+ if (isSmallerAccessForI64(result, access)) {
+ auto* cvtValue =
+ MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true);
+ curBlock_->add(cvtValue);
+ value = cvtValue;
+ }
+
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MInstruction* xchg =
+ MWasmAtomicExchangeHeap::New(alloc(), bytecodeOffset(), memoryBase,
+ base, *access, value, instancePointer_);
+ if (!xchg) {
+ return nullptr;
+ }
+ curBlock_->add(xchg);
+
+ if (isSmallerAccessForI64(result, access)) {
+ xchg = MExtendInt32ToInt64::New(alloc(), xchg, true);
+ curBlock_->add(xchg);
+ }
+
+ return xchg;
+ }
+
+ MDefinition* atomicBinopHeap(AtomicOp op, MDefinition* base,
+ MemoryAccessDesc* access, ValType result,
+ MDefinition* value) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ checkOffsetAndAlignmentAndBounds(access, &base);
+#ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+#endif
+
+ if (isSmallerAccessForI64(result, access)) {
+ auto* cvtValue =
+ MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true);
+ curBlock_->add(cvtValue);
+ value = cvtValue;
+ }
+
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MInstruction* binop =
+ MWasmAtomicBinopHeap::New(alloc(), bytecodeOffset(), op, memoryBase,
+ base, *access, value, instancePointer_);
+ if (!binop) {
+ return nullptr;
+ }
+ curBlock_->add(binop);
+
+ if (isSmallerAccessForI64(result, access)) {
+ binop = MExtendInt32ToInt64::New(alloc(), binop, true);
+ curBlock_->add(binop);
+ }
+
+ return binop;
+ }
+
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* loadSplatSimd128(Scalar::Type viewType,
+ const LinearMemoryAddress<MDefinition*>& addr,
+ wasm::SimdOp splatOp) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ bytecodeIfNotAsmJS());
+
+ // Generate better code (on x86)
+ // If AVX2 is enabled, more broadcast operators are available.
+ if (viewType == Scalar::Float64
+# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
+ || (js::jit::CPUInfo::IsAVX2Present() &&
+ (viewType == Scalar::Uint8 || viewType == Scalar::Uint16 ||
+ viewType == Scalar::Float32))
+# endif
+ ) {
+ access.setSplatSimd128Load();
+ return load(addr.base, &access, ValType::V128);
+ }
+
+ ValType resultType = ValType::I32;
+ if (viewType == Scalar::Float32) {
+ resultType = ValType::F32;
+ splatOp = wasm::SimdOp::F32x4Splat;
+ }
+ auto* scalar = load(addr.base, &access, resultType);
+ if (!inDeadCode() && !scalar) {
+ return nullptr;
+ }
+ return scalarToSimd128(scalar, splatOp);
+ }
+
+ MDefinition* loadExtendSimd128(const LinearMemoryAddress<MDefinition*>& addr,
+ wasm::SimdOp op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ // Generate better code (on x86) by loading as a double with an
+ // operation that sign extends directly.
+ MemoryAccessDesc access(Scalar::Float64, addr.align, addr.offset,
+ bytecodeIfNotAsmJS());
+ access.setWidenSimd128Load(op);
+ return load(addr.base, &access, ValType::V128);
+ }
+
+ MDefinition* loadZeroSimd128(Scalar::Type viewType, size_t numBytes,
+ const LinearMemoryAddress<MDefinition*>& addr) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ bytecodeIfNotAsmJS());
+ access.setZeroExtendSimd128Load();
+ return load(addr.base, &access, ValType::V128);
+ }
+
+ MDefinition* loadLaneSimd128(uint32_t laneSize,
+ const LinearMemoryAddress<MDefinition*>& addr,
+ uint32_t laneIndex, MDefinition* src) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MemoryAccessDesc access(Scalar::Simd128, addr.align, addr.offset,
+ bytecodeIfNotAsmJS());
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MDefinition* base = addr.base;
+ MOZ_ASSERT(!moduleEnv_.isAsmJS());
+ checkOffsetAndAlignmentAndBounds(&access, &base);
+# ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+# endif
+ MInstruction* load = MWasmLoadLaneSimd128::New(
+ alloc(), memoryBase, base, access, laneSize, laneIndex, src);
+ if (!load) {
+ return nullptr;
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ void storeLaneSimd128(uint32_t laneSize,
+ const LinearMemoryAddress<MDefinition*>& addr,
+ uint32_t laneIndex, MDefinition* src) {
+ if (inDeadCode()) {
+ return;
+ }
+ MemoryAccessDesc access(Scalar::Simd128, addr.align, addr.offset,
+ bytecodeIfNotAsmJS());
+ MWasmLoadInstance* memoryBase = maybeLoadMemoryBase();
+ MDefinition* base = addr.base;
+ MOZ_ASSERT(!moduleEnv_.isAsmJS());
+ checkOffsetAndAlignmentAndBounds(&access, &base);
+# ifndef JS_64BIT
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+# endif
+ MInstruction* store = MWasmStoreLaneSimd128::New(
+ alloc(), memoryBase, base, access, laneSize, laneIndex, src);
+ if (!store) {
+ return;
+ }
+ curBlock_->add(store);
+ }
+#endif // ENABLE_WASM_SIMD
+
+ /************************************************ Global variable accesses */
+
+ MDefinition* loadGlobalVar(unsigned instanceDataOffset, bool isConst,
+ bool isIndirect, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+
+ MInstruction* load;
+ if (isIndirect) {
+ // Pull a pointer to the value out of Instance::globalArea, then
+ // load from that pointer. Note that the pointer is immutable
+ // even though the value it points at may change, hence the use of
+ // |true| for the first node's |isConst| value, irrespective of
+ // the |isConst| formal parameter to this method. The latter
+ // applies to the denoted value as a whole.
+ auto* cellPtr = MWasmLoadInstanceDataField::New(
+ alloc(), MIRType::Pointer, instanceDataOffset,
+ /*isConst=*/true, instancePointer_);
+ curBlock_->add(cellPtr);
+ load = MWasmLoadGlobalCell::New(alloc(), type, cellPtr);
+ } else {
+ // Pull the value directly out of Instance::globalArea.
+ load = MWasmLoadInstanceDataField::New(alloc(), type, instanceDataOffset,
+ isConst, instancePointer_);
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ [[nodiscard]] bool storeGlobalVar(uint32_t lineOrBytecode,
+ uint32_t instanceDataOffset,
+ bool isIndirect, MDefinition* v) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ if (isIndirect) {
+ // Pull a pointer to the value out of Instance::globalArea, then
+ // store through that pointer.
+ auto* valueAddr = MWasmLoadInstanceDataField::New(
+ alloc(), MIRType::Pointer, instanceDataOffset,
+ /*isConst=*/true, instancePointer_);
+ curBlock_->add(valueAddr);
+
+ // Handle a store to a ref-typed field specially
+ if (v->type() == MIRType::RefOrNull) {
+ // Load the previous value for the post-write barrier
+ auto* prevValue =
+ MWasmLoadGlobalCell::New(alloc(), MIRType::RefOrNull, valueAddr);
+ curBlock_->add(prevValue);
+
+ // Store the new value
+ auto* store =
+ MWasmStoreRef::New(alloc(), instancePointer_, valueAddr,
+ /*valueOffset=*/0, v, AliasSet::WasmGlobalCell,
+ WasmPreBarrierKind::Normal);
+ curBlock_->add(store);
+
+ // Call the post-write barrier
+ return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue);
+ }
+
+ auto* store = MWasmStoreGlobalCell::New(alloc(), v, valueAddr);
+ curBlock_->add(store);
+ return true;
+ }
+ // Or else store the value directly in Instance::globalArea.
+
+ // Handle a store to a ref-typed field specially
+ if (v->type() == MIRType::RefOrNull) {
+ // Compute the address of the ref-typed global
+ auto* valueAddr = MWasmDerivedPointer::New(
+ alloc(), instancePointer_,
+ wasm::Instance::offsetInData(instanceDataOffset));
+ curBlock_->add(valueAddr);
+
+ // Load the previous value for the post-write barrier
+ auto* prevValue =
+ MWasmLoadGlobalCell::New(alloc(), MIRType::RefOrNull, valueAddr);
+ curBlock_->add(prevValue);
+
+ // Store the new value
+ auto* store =
+ MWasmStoreRef::New(alloc(), instancePointer_, valueAddr,
+ /*valueOffset=*/0, v, AliasSet::WasmInstanceData,
+ WasmPreBarrierKind::Normal);
+ curBlock_->add(store);
+
+ // Call the post-write barrier
+ return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue);
+ }
+
+ auto* store = MWasmStoreInstanceDataField::New(alloc(), instanceDataOffset,
+ v, instancePointer_);
+ curBlock_->add(store);
+ return true;
+ }
+
+ MDefinition* loadTableField(uint32_t tableIndex, unsigned fieldOffset,
+ MIRType type) {
+ uint32_t instanceDataOffset = wasm::Instance::offsetInData(
+ moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset);
+ auto* load =
+ MWasmLoadInstance::New(alloc(), instancePointer_, instanceDataOffset,
+ type, AliasSet::Load(AliasSet::WasmTableMeta));
+ curBlock_->add(load);
+ return load;
+ }
+
+ MDefinition* loadTableLength(uint32_t tableIndex) {
+ return loadTableField(tableIndex, offsetof(TableInstanceData, length),
+ MIRType::Int32);
+ }
+
+ MDefinition* loadTableElements(uint32_t tableIndex) {
+ return loadTableField(tableIndex, offsetof(TableInstanceData, elements),
+ MIRType::Pointer);
+ }
+
+ MDefinition* tableGetAnyRef(uint32_t tableIndex, MDefinition* index) {
+ // Load the table length and perform a bounds check with spectre index
+ // masking
+ auto* length = loadTableLength(tableIndex);
+ auto* check = MWasmBoundsCheck::New(
+ alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown);
+ curBlock_->add(check);
+ if (JitOptions.spectreIndexMasking) {
+ index = check;
+ }
+
+ // Load the table elements and load the element
+ auto* elements = loadTableElements(tableIndex);
+ auto* element = MWasmLoadTableElement::New(alloc(), elements, index);
+ curBlock_->add(element);
+ return element;
+ }
+
+ [[nodiscard]] bool tableSetAnyRef(uint32_t tableIndex, MDefinition* index,
+ MDefinition* value,
+ uint32_t lineOrBytecode) {
+ // Load the table length and perform a bounds check with spectre index
+ // masking
+ auto* length = loadTableLength(tableIndex);
+ auto* check = MWasmBoundsCheck::New(
+ alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown);
+ curBlock_->add(check);
+ if (JitOptions.spectreIndexMasking) {
+ index = check;
+ }
+
+ // Load the table elements
+ auto* elements = loadTableElements(tableIndex);
+
+ // Load the previous value
+ auto* prevValue = MWasmLoadTableElement::New(alloc(), elements, index);
+ curBlock_->add(prevValue);
+
+ // Compute the value's location for the post barrier
+ auto* loc =
+ MWasmDerivedIndexPointer::New(alloc(), elements, index, ScalePointer);
+ curBlock_->add(loc);
+
+ // Store the new value
+ auto* store = MWasmStoreRef::New(
+ alloc(), instancePointer_, loc, /*valueOffset=*/0, value,
+ AliasSet::WasmTableElement, WasmPreBarrierKind::Normal);
+ curBlock_->add(store);
+
+ // Perform the post barrier
+ return postBarrierPrecise(lineOrBytecode, loc, prevValue);
+ }
+
+ void addInterruptCheck() {
+ if (inDeadCode()) {
+ return;
+ }
+ curBlock_->add(
+ MWasmInterruptCheck::New(alloc(), instancePointer_, bytecodeOffset()));
+ }
+
+ // Perform a post-write barrier to update the generational store buffer. This
+ // version will remove a previous store buffer entry if it is no longer
+ // needed.
+ [[nodiscard]] bool postBarrierPrecise(uint32_t lineOrBytecode,
+ MDefinition* valueAddr,
+ MDefinition* value) {
+ return emitInstanceCall2(lineOrBytecode, SASigPostBarrierPrecise, valueAddr,
+ value);
+ }
+
+ // Perform a post-write barrier to update the generational store buffer. This
+ // version will remove a previous store buffer entry if it is no longer
+ // needed.
+ [[nodiscard]] bool postBarrierPreciseWithOffset(uint32_t lineOrBytecode,
+ MDefinition* valueBase,
+ uint32_t valueOffset,
+ MDefinition* value) {
+ MDefinition* valueOffsetDef = constantI32(int32_t(valueOffset));
+ if (!valueOffsetDef) {
+ return false;
+ }
+ return emitInstanceCall3(lineOrBytecode, SASigPostBarrierPreciseWithOffset,
+ valueBase, valueOffsetDef, value);
+ }
+
+ // Perform a post-write barrier to update the generational store buffer. This
+ // version is the most efficient and only requires the address to store the
+ // value and the new value. It does not remove a previous store buffer entry
+ // if it is no longer needed, you must use a precise post-write barrier for
+ // that.
+ [[nodiscard]] bool postBarrier(uint32_t lineOrBytecode, MDefinition* object,
+ MDefinition* valueBase, uint32_t valueOffset,
+ MDefinition* newValue) {
+ auto* barrier = MWasmPostWriteBarrier::New(
+ alloc(), instancePointer_, object, valueBase, valueOffset, newValue);
+ if (!barrier) {
+ return false;
+ }
+ curBlock_->add(barrier);
+ return true;
+ }
+
+ /***************************************************************** Calls */
+
+ // The IonMonkey backend maintains a single stack offset (from the stack
+ // pointer to the base of the frame) by adding the total amount of spill
+ // space required plus the maximum stack required for argument passing.
+ // Since we do not use IonMonkey's MPrepareCall/MPassArg/MCall, we must
+ // manually accumulate, for the entire function, the maximum required stack
+ // space for argument passing. (This is passed to the CodeGenerator via
+ // MIRGenerator::maxWasmStackArgBytes.) This is just be the maximum of the
+ // stack space required for each individual call (as determined by the call
+ // ABI).
+
+ // Operations that modify a CallCompileState.
+
+ [[nodiscard]] bool passInstance(MIRType instanceType,
+ CallCompileState* args) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ // Should only pass an instance once. And it must be a non-GC pointer.
+ MOZ_ASSERT(args->instanceArg_ == ABIArg());
+ MOZ_ASSERT(instanceType == MIRType::Pointer);
+ args->instanceArg_ = args->abi_.next(MIRType::Pointer);
+ return true;
+ }
+
+ // Do not call this directly. Call one of the passArg() variants instead.
+ [[nodiscard]] bool passArgWorker(MDefinition* argDef, MIRType type,
+ CallCompileState* call) {
+ ABIArg arg = call->abi_.next(type);
+ switch (arg.kind()) {
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ case ABIArg::GPR_PAIR: {
+ auto mirLow =
+ MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ true);
+ curBlock_->add(mirLow);
+ auto mirHigh =
+ MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ false);
+ curBlock_->add(mirHigh);
+ return call->regArgs_.append(
+ MWasmCallBase::Arg(AnyRegister(arg.gpr64().low), mirLow)) &&
+ call->regArgs_.append(
+ MWasmCallBase::Arg(AnyRegister(arg.gpr64().high), mirHigh));
+ }
+#endif
+ case ABIArg::GPR:
+ case ABIArg::FPU:
+ return call->regArgs_.append(MWasmCallBase::Arg(arg.reg(), argDef));
+ case ABIArg::Stack: {
+ auto* mir =
+ MWasmStackArg::New(alloc(), arg.offsetFromArgBase(), argDef);
+ curBlock_->add(mir);
+ return true;
+ }
+ case ABIArg::Uninitialized:
+ MOZ_ASSERT_UNREACHABLE("Uninitialized ABIArg kind");
+ }
+ MOZ_CRASH("Unknown ABIArg kind.");
+ }
+
+ template <typename SpanT>
+ [[nodiscard]] bool passArgs(const DefVector& argDefs, SpanT types,
+ CallCompileState* call) {
+ MOZ_ASSERT(argDefs.length() == types.size());
+ for (uint32_t i = 0; i < argDefs.length(); i++) {
+ MDefinition* def = argDefs[i];
+ ValType type = types[i];
+ if (!passArg(def, type, call)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool passArg(MDefinition* argDef, MIRType type,
+ CallCompileState* call) {
+ if (inDeadCode()) {
+ return true;
+ }
+ return passArgWorker(argDef, type, call);
+ }
+
+ [[nodiscard]] bool passArg(MDefinition* argDef, ValType type,
+ CallCompileState* call) {
+ if (inDeadCode()) {
+ return true;
+ }
+ return passArgWorker(argDef, type.toMIRType(), call);
+ }
+
+ // If the call returns results on the stack, prepare a stack area to receive
+ // them, and pass the address of the stack area to the callee as an additional
+ // argument.
+ [[nodiscard]] bool passStackResultAreaCallArg(const ResultType& resultType,
+ CallCompileState* call) {
+ if (inDeadCode()) {
+ return true;
+ }
+ ABIResultIter iter(resultType);
+ while (!iter.done() && iter.cur().inRegister()) {
+ iter.next();
+ }
+ if (iter.done()) {
+ // No stack results.
+ return true;
+ }
+
+ auto* stackResultArea = MWasmStackResultArea::New(alloc());
+ if (!stackResultArea) {
+ return false;
+ }
+ if (!stackResultArea->init(alloc(), iter.remaining())) {
+ return false;
+ }
+ for (uint32_t base = iter.index(); !iter.done(); iter.next()) {
+ MWasmStackResultArea::StackResult loc(iter.cur().stackOffset(),
+ iter.cur().type().toMIRType());
+ stackResultArea->initResult(iter.index() - base, loc);
+ }
+ curBlock_->add(stackResultArea);
+ if (!passArg(stackResultArea, MIRType::Pointer, call)) {
+ return false;
+ }
+ call->stackResultArea_ = stackResultArea;
+ return true;
+ }
+
+ [[nodiscard]] bool finishCall(CallCompileState* call) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ if (!call->regArgs_.append(
+ MWasmCallBase::Arg(AnyRegister(InstanceReg), instancePointer_))) {
+ return false;
+ }
+
+ uint32_t stackBytes = call->abi_.stackBytesConsumedSoFar();
+
+ maxStackArgBytes_ = std::max(maxStackArgBytes_, stackBytes);
+ return true;
+ }
+
+ // Wrappers for creating various kinds of calls.
+
+ [[nodiscard]] bool collectUnaryCallResult(MIRType type,
+ MDefinition** result) {
+ MInstruction* def;
+ switch (type) {
+ case MIRType::Int32:
+ def = MWasmRegisterResult::New(alloc(), MIRType::Int32, ReturnReg);
+ break;
+ case MIRType::Int64:
+ def = MWasmRegister64Result::New(alloc(), ReturnReg64);
+ break;
+ case MIRType::Float32:
+ def = MWasmFloatRegisterResult::New(alloc(), type, ReturnFloat32Reg);
+ break;
+ case MIRType::Double:
+ def = MWasmFloatRegisterResult::New(alloc(), type, ReturnDoubleReg);
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ def = MWasmFloatRegisterResult::New(alloc(), type, ReturnSimd128Reg);
+ break;
+#endif
+ case MIRType::RefOrNull:
+ def = MWasmRegisterResult::New(alloc(), MIRType::RefOrNull, ReturnReg);
+ break;
+ default:
+ MOZ_CRASH("unexpected MIRType result for builtin call");
+ }
+
+ if (!def) {
+ return false;
+ }
+
+ curBlock_->add(def);
+ *result = def;
+
+ return true;
+ }
+
+ [[nodiscard]] bool collectCallResults(const ResultType& type,
+ MWasmStackResultArea* stackResultArea,
+ DefVector* results) {
+ if (!results->reserve(type.length())) {
+ return false;
+ }
+
+ // The result iterator goes in the order in which results would be popped
+ // off; we want the order in which they would be pushed.
+ ABIResultIter iter(type);
+ uint32_t stackResultCount = 0;
+ while (!iter.done()) {
+ if (iter.cur().onStack()) {
+ stackResultCount++;
+ }
+ iter.next();
+ }
+
+ for (iter.switchToPrev(); !iter.done(); iter.prev()) {
+ if (!mirGen().ensureBallast()) {
+ return false;
+ }
+ const ABIResult& result = iter.cur();
+ MInstruction* def;
+ if (result.inRegister()) {
+ switch (result.type().kind()) {
+ case wasm::ValType::I32:
+ def =
+ MWasmRegisterResult::New(alloc(), MIRType::Int32, result.gpr());
+ break;
+ case wasm::ValType::I64:
+ def = MWasmRegister64Result::New(alloc(), result.gpr64());
+ break;
+ case wasm::ValType::F32:
+ def = MWasmFloatRegisterResult::New(alloc(), MIRType::Float32,
+ result.fpr());
+ break;
+ case wasm::ValType::F64:
+ def = MWasmFloatRegisterResult::New(alloc(), MIRType::Double,
+ result.fpr());
+ break;
+ case wasm::ValType::Ref:
+ def = MWasmRegisterResult::New(alloc(), MIRType::RefOrNull,
+ result.gpr());
+ break;
+ case wasm::ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ def = MWasmFloatRegisterResult::New(alloc(), MIRType::Simd128,
+ result.fpr());
+#else
+ return this->iter().fail("Ion has no SIMD support yet");
+#endif
+ }
+ } else {
+ MOZ_ASSERT(stackResultArea);
+ MOZ_ASSERT(stackResultCount);
+ uint32_t idx = --stackResultCount;
+ def = MWasmStackResult::New(alloc(), stackResultArea, idx);
+ }
+
+ if (!def) {
+ return false;
+ }
+ curBlock_->add(def);
+ results->infallibleAppend(def);
+ }
+
+ MOZ_ASSERT(results->length() == type.length());
+
+ return true;
+ }
+
+ [[nodiscard]] bool catchableCall(const CallSiteDesc& desc,
+ const CalleeDesc& callee,
+ const MWasmCallBase::Args& args,
+ const ArgTypeVector& argTypes,
+ MDefinition* indexOrRef = nullptr) {
+ MWasmCallTryDesc tryDesc;
+ if (!beginTryCall(&tryDesc)) {
+ return false;
+ }
+
+ MInstruction* ins;
+ if (tryDesc.inTry) {
+ ins = MWasmCallCatchable::New(alloc(), desc, callee, args,
+ StackArgAreaSizeUnaligned(argTypes),
+ tryDesc, indexOrRef);
+ } else {
+ ins = MWasmCallUncatchable::New(alloc(), desc, callee, args,
+ StackArgAreaSizeUnaligned(argTypes),
+ indexOrRef);
+ }
+ if (!ins) {
+ return false;
+ }
+ curBlock_->add(ins);
+
+ return finishTryCall(&tryDesc);
+ }
+
+ [[nodiscard]] bool callDirect(const FuncType& funcType, uint32_t funcIndex,
+ uint32_t lineOrBytecode,
+ const CallCompileState& call,
+ DefVector* results) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Func);
+ ResultType resultType = ResultType::Vector(funcType.results());
+ auto callee = CalleeDesc::function(funcIndex);
+ ArgTypeVector args(funcType);
+
+ if (!catchableCall(desc, callee, call.regArgs_, args)) {
+ return false;
+ }
+ return collectCallResults(resultType, call.stackResultArea_, results);
+ }
+
+ [[nodiscard]] bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
+ MDefinition* index, uint32_t lineOrBytecode,
+ const CallCompileState& call,
+ DefVector* results) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType();
+ CallIndirectId callIndirectId =
+ CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex);
+
+ CalleeDesc callee;
+ if (moduleEnv_.isAsmJS()) {
+ MOZ_ASSERT(tableIndex == 0);
+ MOZ_ASSERT(callIndirectId.kind() == CallIndirectIdKind::AsmJS);
+ uint32_t tableIndex = moduleEnv_.asmJSSigToTableIndex[funcTypeIndex];
+ const TableDesc& table = moduleEnv_.tables[tableIndex];
+ MOZ_ASSERT(IsPowerOfTwo(table.initialLength));
+
+ MDefinition* mask = constantI32(int32_t(table.initialLength - 1));
+ MBitAnd* maskedIndex = MBitAnd::New(alloc(), index, mask, MIRType::Int32);
+ curBlock_->add(maskedIndex);
+
+ index = maskedIndex;
+ callee = CalleeDesc::asmJSTable(moduleEnv_, tableIndex);
+ } else {
+ MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
+ const TableDesc& table = moduleEnv_.tables[tableIndex];
+ callee =
+ CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId);
+ }
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Indirect);
+ ArgTypeVector args(funcType);
+ ResultType resultType = ResultType::Vector(funcType.results());
+
+ if (!catchableCall(desc, callee, call.regArgs_, args, index)) {
+ return false;
+ }
+ return collectCallResults(resultType, call.stackResultArea_, results);
+ }
+
+ [[nodiscard]] bool callImport(unsigned instanceDataOffset,
+ uint32_t lineOrBytecode,
+ const CallCompileState& call,
+ const FuncType& funcType, DefVector* results) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Import);
+ auto callee = CalleeDesc::import(instanceDataOffset);
+ ArgTypeVector args(funcType);
+ ResultType resultType = ResultType::Vector(funcType.results());
+
+ if (!catchableCall(desc, callee, call.regArgs_, args)) {
+ return false;
+ }
+ return collectCallResults(resultType, call.stackResultArea_, results);
+ }
+
+ [[nodiscard]] bool builtinCall(const SymbolicAddressSignature& builtin,
+ uint32_t lineOrBytecode,
+ const CallCompileState& call,
+ MDefinition** def) {
+ if (inDeadCode()) {
+ *def = nullptr;
+ return true;
+ }
+
+ MOZ_ASSERT(builtin.failureMode == FailureMode::Infallible);
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic);
+ auto callee = CalleeDesc::builtin(builtin.identity);
+ auto* ins = MWasmCallUncatchable::New(alloc(), desc, callee, call.regArgs_,
+ StackArgAreaSizeUnaligned(builtin));
+ if (!ins) {
+ return false;
+ }
+
+ curBlock_->add(ins);
+
+ return collectUnaryCallResult(builtin.retType, def);
+ }
+
+ [[nodiscard]] bool builtinInstanceMethodCall(
+ const SymbolicAddressSignature& builtin, uint32_t lineOrBytecode,
+ const CallCompileState& call, MDefinition** def = nullptr) {
+ MOZ_ASSERT_IF(!def, builtin.retType == MIRType::None);
+ if (inDeadCode()) {
+ if (def) {
+ *def = nullptr;
+ }
+ return true;
+ }
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic);
+ auto* ins = MWasmCallUncatchable::NewBuiltinInstanceMethodCall(
+ alloc(), desc, builtin.identity, builtin.failureMode, call.instanceArg_,
+ call.regArgs_, StackArgAreaSizeUnaligned(builtin));
+ if (!ins) {
+ return false;
+ }
+
+ curBlock_->add(ins);
+
+ return def ? collectUnaryCallResult(builtin.retType, def) : true;
+ }
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+ [[nodiscard]] bool callRef(const FuncType& funcType, MDefinition* ref,
+ uint32_t lineOrBytecode,
+ const CallCompileState& call, DefVector* results) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ CalleeDesc callee = CalleeDesc::wasmFuncRef();
+
+ CallSiteDesc desc(lineOrBytecode, CallSiteDesc::FuncRef);
+ ArgTypeVector args(funcType);
+ ResultType resultType = ResultType::Vector(funcType.results());
+
+ if (!catchableCall(desc, callee, call.regArgs_, args, ref)) {
+ return false;
+ }
+ return collectCallResults(resultType, call.stackResultArea_, results);
+ }
+
+#endif // ENABLE_WASM_FUNCTION_REFERENCES
+
+ /*********************************************** Control flow generation */
+
+ inline bool inDeadCode() const { return curBlock_ == nullptr; }
+
+ [[nodiscard]] bool returnValues(const DefVector& values) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ if (values.empty()) {
+ curBlock_->end(MWasmReturnVoid::New(alloc(), instancePointer_));
+ } else {
+ ResultType resultType = ResultType::Vector(funcType().results());
+ ABIResultIter iter(resultType);
+ // Switch to iterate in FIFO order instead of the default LIFO.
+ while (!iter.done()) {
+ iter.next();
+ }
+ iter.switchToPrev();
+ for (uint32_t i = 0; !iter.done(); iter.prev(), i++) {
+ if (!mirGen().ensureBallast()) {
+ return false;
+ }
+ const ABIResult& result = iter.cur();
+ if (result.onStack()) {
+ MOZ_ASSERT(iter.remaining() > 1);
+ if (result.type().isRefRepr()) {
+ auto* store = MWasmStoreRef::New(
+ alloc(), instancePointer_, stackResultPointer_,
+ result.stackOffset(), values[i], AliasSet::WasmStackResult,
+ WasmPreBarrierKind::None);
+ curBlock_->add(store);
+ } else {
+ auto* store = MWasmStoreStackResult::New(
+ alloc(), stackResultPointer_, result.stackOffset(), values[i]);
+ curBlock_->add(store);
+ }
+ } else {
+ MOZ_ASSERT(iter.remaining() == 1);
+ MOZ_ASSERT(i + 1 == values.length());
+ curBlock_->end(
+ MWasmReturn::New(alloc(), values[i], instancePointer_));
+ }
+ }
+ }
+ curBlock_ = nullptr;
+ return true;
+ }
+
+ void unreachableTrap() {
+ if (inDeadCode()) {
+ return;
+ }
+
+ auto* ins =
+ MWasmTrap::New(alloc(), wasm::Trap::Unreachable, bytecodeOffset());
+ curBlock_->end(ins);
+ curBlock_ = nullptr;
+ }
+
+ private:
+ static uint32_t numPushed(MBasicBlock* block) {
+ return block->stackDepth() - block->info().firstStackSlot();
+ }
+
+ public:
+ [[nodiscard]] bool pushDefs(const DefVector& defs) {
+ if (inDeadCode()) {
+ return true;
+ }
+ MOZ_ASSERT(numPushed(curBlock_) == 0);
+ if (!curBlock_->ensureHasSlots(defs.length())) {
+ return false;
+ }
+ for (MDefinition* def : defs) {
+ MOZ_ASSERT(def->type() != MIRType::None);
+ curBlock_->push(def);
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool popPushedDefs(DefVector* defs) {
+ size_t n = numPushed(curBlock_);
+ if (!defs->resizeUninitialized(n)) {
+ return false;
+ }
+ for (; n > 0; n--) {
+ MDefinition* def = curBlock_->pop();
+ MOZ_ASSERT(def->type() != MIRType::Value);
+ (*defs)[n - 1] = def;
+ }
+ return true;
+ }
+
+ private:
+ [[nodiscard]] bool addJoinPredecessor(const DefVector& defs,
+ MBasicBlock** joinPred) {
+ *joinPred = curBlock_;
+ if (inDeadCode()) {
+ return true;
+ }
+ return pushDefs(defs);
+ }
+
+ public:
+ [[nodiscard]] bool branchAndStartThen(MDefinition* cond,
+ MBasicBlock** elseBlock) {
+ if (inDeadCode()) {
+ *elseBlock = nullptr;
+ } else {
+ MBasicBlock* thenBlock;
+ if (!newBlock(curBlock_, &thenBlock)) {
+ return false;
+ }
+ if (!newBlock(curBlock_, elseBlock)) {
+ return false;
+ }
+
+ curBlock_->end(MTest::New(alloc(), cond, thenBlock, *elseBlock));
+
+ curBlock_ = thenBlock;
+ mirGraph().moveBlockToEnd(curBlock_);
+ }
+
+ return startBlock();
+ }
+
+ [[nodiscard]] bool switchToElse(MBasicBlock* elseBlock,
+ MBasicBlock** thenJoinPred) {
+ DefVector values;
+ if (!finishBlock(&values)) {
+ return false;
+ }
+
+ if (!elseBlock) {
+ *thenJoinPred = nullptr;
+ } else {
+ if (!addJoinPredecessor(values, thenJoinPred)) {
+ return false;
+ }
+
+ curBlock_ = elseBlock;
+ mirGraph().moveBlockToEnd(curBlock_);
+ }
+
+ return startBlock();
+ }
+
+ [[nodiscard]] bool joinIfElse(MBasicBlock* thenJoinPred, DefVector* defs) {
+ DefVector values;
+ if (!finishBlock(&values)) {
+ return false;
+ }
+
+ if (!thenJoinPred && inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* elseJoinPred;
+ if (!addJoinPredecessor(values, &elseJoinPred)) {
+ return false;
+ }
+
+ mozilla::Array<MBasicBlock*, 2> blocks;
+ size_t numJoinPreds = 0;
+ if (thenJoinPred) {
+ blocks[numJoinPreds++] = thenJoinPred;
+ }
+ if (elseJoinPred) {
+ blocks[numJoinPreds++] = elseJoinPred;
+ }
+
+ if (numJoinPreds == 0) {
+ return true;
+ }
+
+ MBasicBlock* join;
+ if (!goToNewBlock(blocks[0], &join)) {
+ return false;
+ }
+ for (size_t i = 1; i < numJoinPreds; ++i) {
+ if (!goToExistingBlock(blocks[i], join)) {
+ return false;
+ }
+ }
+
+ curBlock_ = join;
+ return popPushedDefs(defs);
+ }
+
+ [[nodiscard]] bool startBlock() {
+ MOZ_ASSERT_IF(blockDepth_ < blockPatches_.length(),
+ blockPatches_[blockDepth_].empty());
+ blockDepth_++;
+ return true;
+ }
+
+ [[nodiscard]] bool finishBlock(DefVector* defs) {
+ MOZ_ASSERT(blockDepth_);
+ uint32_t topLabel = --blockDepth_;
+ return bindBranches(topLabel, defs);
+ }
+
+ [[nodiscard]] bool startLoop(MBasicBlock** loopHeader, size_t paramCount) {
+ *loopHeader = nullptr;
+
+ blockDepth_++;
+ loopDepth_++;
+
+ if (inDeadCode()) {
+ return true;
+ }
+
+ // Create the loop header.
+ MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_ - 1);
+ *loopHeader = MBasicBlock::New(mirGraph(), info(), curBlock_,
+ MBasicBlock::PENDING_LOOP_HEADER);
+ if (!*loopHeader) {
+ return false;
+ }
+
+ (*loopHeader)->setLoopDepth(loopDepth_);
+ mirGraph().addBlock(*loopHeader);
+ curBlock_->end(MGoto::New(alloc(), *loopHeader));
+
+ DefVector loopParams;
+ if (!iter().getResults(paramCount, &loopParams)) {
+ return false;
+ }
+ for (size_t i = 0; i < paramCount; i++) {
+ MPhi* phi = MPhi::New(alloc(), loopParams[i]->type());
+ if (!phi) {
+ return false;
+ }
+ if (!phi->reserveLength(2)) {
+ return false;
+ }
+ (*loopHeader)->addPhi(phi);
+ phi->addInput(loopParams[i]);
+ loopParams[i] = phi;
+ }
+ iter().setResults(paramCount, loopParams);
+
+ MBasicBlock* body;
+ if (!goToNewBlock(*loopHeader, &body)) {
+ return false;
+ }
+ curBlock_ = body;
+ return true;
+ }
+
+ private:
+ void fixupRedundantPhis(MBasicBlock* b) {
+ for (size_t i = 0, depth = b->stackDepth(); i < depth; i++) {
+ MDefinition* def = b->getSlot(i);
+ if (def->isUnused()) {
+ b->setSlot(i, def->toPhi()->getOperand(0));
+ }
+ }
+ }
+
+ [[nodiscard]] bool setLoopBackedge(MBasicBlock* loopEntry,
+ MBasicBlock* loopBody,
+ MBasicBlock* backedge, size_t paramCount) {
+ if (!loopEntry->setBackedgeWasm(backedge, paramCount)) {
+ return false;
+ }
+
+ // Flag all redundant phis as unused.
+ for (MPhiIterator phi = loopEntry->phisBegin(); phi != loopEntry->phisEnd();
+ phi++) {
+ MOZ_ASSERT(phi->numOperands() == 2);
+ if (phi->getOperand(0) == phi->getOperand(1)) {
+ phi->setUnused();
+ }
+ }
+
+ // Fix up phis stored in the slots Vector of pending blocks.
+ for (ControlFlowPatchVector& patches : blockPatches_) {
+ for (ControlFlowPatch& p : patches) {
+ MBasicBlock* block = p.ins->block();
+ if (block->loopDepth() >= loopEntry->loopDepth()) {
+ fixupRedundantPhis(block);
+ }
+ }
+ }
+
+ // The loop body, if any, might be referencing recycled phis too.
+ if (loopBody) {
+ fixupRedundantPhis(loopBody);
+ }
+
+ // Pending jumps to an enclosing try-catch may reference the recycled phis.
+ // We have to search above all enclosing try blocks, as a delegate may move
+ // patches around.
+ for (uint32_t depth = 0; depth < iter().controlStackDepth(); depth++) {
+ LabelKind kind = iter().controlKind(depth);
+ if (kind != LabelKind::Try && kind != LabelKind::Body) {
+ continue;
+ }
+ Control& control = iter().controlItem(depth);
+ for (MControlInstruction* patch : control.tryPadPatches) {
+ MBasicBlock* block = patch->block();
+ if (block->loopDepth() >= loopEntry->loopDepth()) {
+ fixupRedundantPhis(block);
+ }
+ }
+ }
+
+ // Discard redundant phis and add to the free list.
+ for (MPhiIterator phi = loopEntry->phisBegin();
+ phi != loopEntry->phisEnd();) {
+ MPhi* entryDef = *phi++;
+ if (!entryDef->isUnused()) {
+ continue;
+ }
+
+ entryDef->justReplaceAllUsesWith(entryDef->getOperand(0));
+ loopEntry->discardPhi(entryDef);
+ mirGraph().addPhiToFreeList(entryDef);
+ }
+
+ return true;
+ }
+
+ public:
+ [[nodiscard]] bool closeLoop(MBasicBlock* loopHeader,
+ DefVector* loopResults) {
+ MOZ_ASSERT(blockDepth_ >= 1);
+ MOZ_ASSERT(loopDepth_);
+
+ uint32_t headerLabel = blockDepth_ - 1;
+
+ if (!loopHeader) {
+ MOZ_ASSERT(inDeadCode());
+ MOZ_ASSERT(headerLabel >= blockPatches_.length() ||
+ blockPatches_[headerLabel].empty());
+ blockDepth_--;
+ loopDepth_--;
+ return true;
+ }
+
+ // Op::Loop doesn't have an implicit backedge so temporarily set
+ // aside the end of the loop body to bind backedges.
+ MBasicBlock* loopBody = curBlock_;
+ curBlock_ = nullptr;
+
+ // As explained in bug 1253544, Ion apparently has an invariant that
+ // there is only one backedge to loop headers. To handle wasm's ability
+ // to have multiple backedges to the same loop header, we bind all those
+ // branches as forward jumps to a single backward jump. This is
+ // unfortunate but the optimizer is able to fold these into single jumps
+ // to backedges.
+ DefVector backedgeValues;
+ if (!bindBranches(headerLabel, &backedgeValues)) {
+ return false;
+ }
+
+ MOZ_ASSERT(loopHeader->loopDepth() == loopDepth_);
+
+ if (curBlock_) {
+ // We're on the loop backedge block, created by bindBranches.
+ for (size_t i = 0, n = numPushed(curBlock_); i != n; i++) {
+ curBlock_->pop();
+ }
+
+ if (!pushDefs(backedgeValues)) {
+ return false;
+ }
+
+ MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_);
+ curBlock_->end(MGoto::New(alloc(), loopHeader));
+ if (!setLoopBackedge(loopHeader, loopBody, curBlock_,
+ backedgeValues.length())) {
+ return false;
+ }
+ }
+
+ curBlock_ = loopBody;
+
+ loopDepth_--;
+
+ // If the loop depth still at the inner loop body, correct it.
+ if (curBlock_ && curBlock_->loopDepth() != loopDepth_) {
+ MBasicBlock* out;
+ if (!goToNewBlock(curBlock_, &out)) {
+ return false;
+ }
+ curBlock_ = out;
+ }
+
+ blockDepth_ -= 1;
+ return inDeadCode() || popPushedDefs(loopResults);
+ }
+
+ [[nodiscard]] bool addControlFlowPatch(MControlInstruction* ins,
+ uint32_t relative, uint32_t index) {
+ MOZ_ASSERT(relative < blockDepth_);
+ uint32_t absolute = blockDepth_ - 1 - relative;
+
+ if (absolute >= blockPatches_.length() &&
+ !blockPatches_.resize(absolute + 1)) {
+ return false;
+ }
+
+ return blockPatches_[absolute].append(ControlFlowPatch(ins, index));
+ }
+
+ [[nodiscard]] bool br(uint32_t relativeDepth, const DefVector& values) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MGoto* jump = MGoto::New(alloc());
+ if (!addControlFlowPatch(jump, relativeDepth, MGoto::TargetIndex)) {
+ return false;
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(jump);
+ curBlock_ = nullptr;
+ return true;
+ }
+
+ [[nodiscard]] bool brIf(uint32_t relativeDepth, const DefVector& values,
+ MDefinition* condition) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* joinBlock = nullptr;
+ if (!newBlock(curBlock_, &joinBlock)) {
+ return false;
+ }
+
+ MTest* test = MTest::New(alloc(), condition, nullptr, joinBlock);
+ if (!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) {
+ return false;
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(test);
+ curBlock_ = joinBlock;
+ return true;
+ }
+
+ [[nodiscard]] bool brTable(MDefinition* operand, uint32_t defaultDepth,
+ const Uint32Vector& depths,
+ const DefVector& values) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ size_t numCases = depths.length();
+ MOZ_ASSERT(numCases <= INT32_MAX);
+ MOZ_ASSERT(numCases);
+
+ MTableSwitch* table =
+ MTableSwitch::New(alloc(), operand, 0, int32_t(numCases - 1));
+
+ size_t defaultIndex;
+ if (!table->addDefault(nullptr, &defaultIndex)) {
+ return false;
+ }
+ if (!addControlFlowPatch(table, defaultDepth, defaultIndex)) {
+ return false;
+ }
+
+ using IndexToCaseMap =
+ HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy>;
+
+ IndexToCaseMap indexToCase;
+ if (!indexToCase.put(defaultDepth, defaultIndex)) {
+ return false;
+ }
+
+ for (size_t i = 0; i < numCases; i++) {
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+
+ uint32_t depth = depths[i];
+
+ size_t caseIndex;
+ IndexToCaseMap::AddPtr p = indexToCase.lookupForAdd(depth);
+ if (!p) {
+ if (!table->addSuccessor(nullptr, &caseIndex)) {
+ return false;
+ }
+ if (!addControlFlowPatch(table, depth, caseIndex)) {
+ return false;
+ }
+ if (!indexToCase.add(p, depth, caseIndex)) {
+ return false;
+ }
+ } else {
+ caseIndex = p->value();
+ }
+
+ if (!table->addCase(caseIndex)) {
+ return false;
+ }
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(table);
+ curBlock_ = nullptr;
+
+ return true;
+ }
+
+ /********************************************************** Exceptions ***/
+
+ bool inTryBlock(uint32_t* relativeDepth) {
+ return iter().controlFindInnermost(LabelKind::Try, relativeDepth);
+ }
+
+ bool inTryCode() {
+ uint32_t relativeDepth;
+ return inTryBlock(&relativeDepth);
+ }
+
+ MDefinition* loadTag(uint32_t tagIndex) {
+ MWasmLoadInstanceDataField* tag = MWasmLoadInstanceDataField::New(
+ alloc(), MIRType::RefOrNull,
+ moduleEnv_.offsetOfTagInstanceData(tagIndex), true, instancePointer_);
+ curBlock_->add(tag);
+ return tag;
+ }
+
+ void loadPendingExceptionState(MInstruction** exception, MInstruction** tag) {
+ *exception = MWasmLoadInstance::New(
+ alloc(), instancePointer_, wasm::Instance::offsetOfPendingException(),
+ MIRType::RefOrNull, AliasSet::Load(AliasSet::WasmPendingException));
+ curBlock_->add(*exception);
+
+ *tag = MWasmLoadInstance::New(
+ alloc(), instancePointer_,
+ wasm::Instance::offsetOfPendingExceptionTag(), MIRType::RefOrNull,
+ AliasSet::Load(AliasSet::WasmPendingException));
+ curBlock_->add(*tag);
+ }
+
+ [[nodiscard]] bool setPendingExceptionState(MDefinition* exception,
+ MDefinition* tag) {
+ // Set the pending exception object
+ auto* exceptionAddr = MWasmDerivedPointer::New(
+ alloc(), instancePointer_, Instance::offsetOfPendingException());
+ curBlock_->add(exceptionAddr);
+ auto* setException = MWasmStoreRef::New(
+ alloc(), instancePointer_, exceptionAddr, /*valueOffset=*/0, exception,
+ AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
+ curBlock_->add(setException);
+ if (!postBarrierPrecise(/*lineOrBytecode=*/0, exceptionAddr, exception)) {
+ return false;
+ }
+
+ // Set the pending exception tag object
+ auto* exceptionTagAddr = MWasmDerivedPointer::New(
+ alloc(), instancePointer_, Instance::offsetOfPendingExceptionTag());
+ curBlock_->add(exceptionTagAddr);
+ auto* setExceptionTag = MWasmStoreRef::New(
+ alloc(), instancePointer_, exceptionTagAddr, /*valueOffset=*/0, tag,
+ AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
+ curBlock_->add(setExceptionTag);
+ return postBarrierPrecise(/*lineOrBytecode=*/0, exceptionTagAddr, tag);
+ }
+
+ [[nodiscard]] bool addPadPatch(MControlInstruction* ins,
+ size_t relativeTryDepth) {
+ Control& tryControl = iter().controlItem(relativeTryDepth);
+ ControlInstructionVector& padPatches = tryControl.tryPadPatches;
+ return padPatches.emplaceBack(ins);
+ }
+
+ [[nodiscard]] bool endWithPadPatch(uint32_t relativeTryDepth) {
+ MGoto* jumpToLandingPad = MGoto::New(alloc());
+ curBlock_->end(jumpToLandingPad);
+ return addPadPatch(jumpToLandingPad, relativeTryDepth);
+ }
+
+ [[nodiscard]] bool delegatePadPatches(const ControlInstructionVector& patches,
+ uint32_t relativeDepth) {
+ if (patches.empty()) {
+ return true;
+ }
+
+ // Find where we are delegating the pad patches to.
+ uint32_t targetRelativeDepth;
+ if (!iter().controlFindInnermostFrom(LabelKind::Try, relativeDepth,
+ &targetRelativeDepth)) {
+ MOZ_ASSERT(relativeDepth <= blockDepth_ - 1);
+ targetRelativeDepth = blockDepth_ - 1;
+ }
+ // Append the delegate's pad patches to the target's.
+ for (MControlInstruction* ins : patches) {
+ if (!addPadPatch(ins, targetRelativeDepth)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool beginTryCall(MWasmCallTryDesc* call) {
+ call->inTry = inTryBlock(&call->relativeTryDepth);
+ if (!call->inTry) {
+ return true;
+ }
+ // Allocate a try note
+ if (!tryNotes_.append(wasm::TryNote())) {
+ return false;
+ }
+ call->tryNoteIndex = tryNotes_.length() - 1;
+ // Allocate blocks for fallthrough and exceptions
+ return newBlock(curBlock_, &call->fallthroughBlock) &&
+ newBlock(curBlock_, &call->prePadBlock);
+ }
+
+ [[nodiscard]] bool finishTryCall(MWasmCallTryDesc* call) {
+ if (!call->inTry) {
+ return true;
+ }
+
+ // Switch to the prePadBlock
+ MBasicBlock* callBlock = curBlock_;
+ curBlock_ = call->prePadBlock;
+
+ // Mark this as the landing pad for the call
+ curBlock_->add(
+ MWasmCallLandingPrePad::New(alloc(), callBlock, call->tryNoteIndex));
+
+ // End with a pending jump to the landing pad
+ if (!endWithPadPatch(call->relativeTryDepth)) {
+ return false;
+ }
+
+ // Compilation continues in the fallthroughBlock.
+ curBlock_ = call->fallthroughBlock;
+ return true;
+ }
+
+ // Create a landing pad for a try block if there are any throwing
+ // instructions.
+ [[nodiscard]] bool createTryLandingPadIfNeeded(Control& control,
+ MBasicBlock** landingPad) {
+ // If there are no pad-patches for this try control, it means there are no
+ // instructions in the try code that could throw an exception. In this
+ // case, all the catches are dead code, and the try code ends up equivalent
+ // to a plain wasm block.
+ ControlInstructionVector& patches = control.tryPadPatches;
+ if (patches.empty()) {
+ *landingPad = nullptr;
+ return true;
+ }
+
+ // Otherwise, if there are (pad-) branches from places in the try code that
+ // may throw an exception, bind these branches to a new landing pad
+ // block. This is done similarly to what is done in bindBranches.
+ MControlInstruction* ins = patches[0];
+ MBasicBlock* pred = ins->block();
+ if (!newBlock(pred, landingPad)) {
+ return false;
+ }
+ ins->replaceSuccessor(0, *landingPad);
+ for (size_t i = 1; i < patches.length(); i++) {
+ ins = patches[i];
+ pred = ins->block();
+ if (!(*landingPad)->addPredecessor(alloc(), pred)) {
+ return false;
+ }
+ ins->replaceSuccessor(0, *landingPad);
+ }
+
+ // Set up the slots in the landing pad block.
+ if (!setupLandingPadSlots(*landingPad)) {
+ return false;
+ }
+
+ // Clear the now bound pad patches.
+ patches.clear();
+ return true;
+ }
+
+ // Consume the pending exception state from instance, and set up the slots
+ // of the landing pad with the exception state.
+ [[nodiscard]] bool setupLandingPadSlots(MBasicBlock* landingPad) {
+ MBasicBlock* prevBlock = curBlock_;
+ curBlock_ = landingPad;
+
+ // Load the pending exception and tag
+ MInstruction* exception;
+ MInstruction* tag;
+ loadPendingExceptionState(&exception, &tag);
+
+ // Clear the pending exception and tag
+ auto* null = constantNullRef();
+ if (!setPendingExceptionState(null, null)) {
+ return false;
+ }
+
+ // Push the exception and its tag on the stack to make them available
+ // to the landing pad blocks.
+ if (!landingPad->ensureHasSlots(2)) {
+ return false;
+ }
+ landingPad->push(exception);
+ landingPad->push(tag);
+
+ curBlock_ = prevBlock;
+ return true;
+ }
+
+ [[nodiscard]] bool startTry(MBasicBlock** curBlock) {
+ *curBlock = curBlock_;
+ return startBlock();
+ }
+
+ [[nodiscard]] bool joinTryOrCatchBlock(Control& control) {
+ // If the try or catch block ended with dead code, there is no need to
+ // do any control flow join.
+ if (inDeadCode()) {
+ return true;
+ }
+
+ // This is a split path which we'll need to join later, using a control
+ // flow patch.
+ MOZ_ASSERT(!curBlock_->hasLastIns());
+ MGoto* jump = MGoto::New(alloc());
+ if (!addControlFlowPatch(jump, 0, MGoto::TargetIndex)) {
+ return false;
+ }
+
+ // Finish the current block with the control flow patch instruction.
+ curBlock_->end(jump);
+ return true;
+ }
+
+ // Finish the previous block (either a try or catch block) and then setup a
+ // new catch block.
+ [[nodiscard]] bool switchToCatch(Control& control, const LabelKind& fromKind,
+ uint32_t tagIndex) {
+ // If there is no control block, then either:
+ // - the entry of the try block is dead code, or
+ // - there is no landing pad for the try-catch.
+ // In either case, any catch will be dead code.
+ if (!control.block) {
+ MOZ_ASSERT(inDeadCode());
+ return true;
+ }
+
+ // Join the previous try or catch block with a patch to the future join of
+ // the whole try-catch block.
+ if (!joinTryOrCatchBlock(control)) {
+ return false;
+ }
+
+ // If we are switching from the try block, create the landing pad. This is
+ // guaranteed to happen once and only once before processing catch blocks.
+ if (fromKind == LabelKind::Try) {
+ MBasicBlock* padBlock = nullptr;
+ if (!createTryLandingPadIfNeeded(control, &padBlock)) {
+ return false;
+ }
+ // Set the control block for this try-catch to the landing pad.
+ control.block = padBlock;
+ }
+
+ // If there is no landing pad, then this and following catches are dead
+ // code.
+ if (!control.block) {
+ curBlock_ = nullptr;
+ return true;
+ }
+
+ // Switch to the landing pad.
+ curBlock_ = control.block;
+
+ // Handle a catch_all by immediately jumping to a new block. We require a
+ // new block (as opposed to just emitting the catch_all code in the current
+ // block) because rethrow requires the exception/tag to be present in the
+ // landing pad's slots, while the catch_all block must not have the
+ // exception/tag in slots.
+ if (tagIndex == CatchAllIndex) {
+ MBasicBlock* catchAllBlock = nullptr;
+ if (!goToNewBlock(curBlock_, &catchAllBlock)) {
+ return false;
+ }
+ // Compilation will continue in the catch_all block.
+ curBlock_ = catchAllBlock;
+ // Remove the tag and exception slots from the block, they are no
+ // longer necessary.
+ curBlock_->pop();
+ curBlock_->pop();
+ return true;
+ }
+
+ // Handle a tagged catch by doing a compare and branch on the tag index,
+ // jumping to a catch block if they match, or else to a fallthrough block
+ // to continue the landing pad.
+ MBasicBlock* catchBlock = nullptr;
+ MBasicBlock* fallthroughBlock = nullptr;
+ if (!newBlock(curBlock_, &catchBlock) ||
+ !newBlock(curBlock_, &fallthroughBlock)) {
+ return false;
+ }
+
+ // Get the exception and its tag from the slots we pushed when adding
+ // control flow patches.
+ MDefinition* exceptionTag = curBlock_->pop();
+ MDefinition* exception = curBlock_->pop();
+
+ // Branch to the catch block if the exception's tag matches this catch
+ // block's tag.
+ MDefinition* catchTag = loadTag(tagIndex);
+ MDefinition* matchesCatchTag =
+ compare(exceptionTag, catchTag, JSOp::Eq, MCompare::Compare_RefOrNull);
+ curBlock_->end(
+ MTest::New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock));
+
+ // The landing pad will continue in the fallthrough block
+ control.block = fallthroughBlock;
+
+ // Set up the catch block by extracting the values from the exception
+ // object.
+ curBlock_ = catchBlock;
+
+ // Remove the tag and exception slots from the block, they are no
+ // longer necessary.
+ curBlock_->pop();
+ curBlock_->pop();
+
+ // Extract the exception values for the catch block
+ DefVector values;
+ if (!loadExceptionValues(exception, tagIndex, &values)) {
+ return false;
+ }
+ iter().setResults(values.length(), values);
+ return true;
+ }
+
+ [[nodiscard]] bool loadExceptionValues(MDefinition* exception,
+ uint32_t tagIndex, DefVector* values) {
+ SharedTagType tagType = moduleEnv().tags[tagIndex].type;
+ const ValTypeVector& params = tagType->argTypes_;
+ const TagOffsetVector& offsets = tagType->argOffsets_;
+
+ // Get the data pointer from the exception object
+ auto* data = MWasmLoadField::New(
+ alloc(), exception, WasmExceptionObject::offsetOfData(),
+ MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any));
+ if (!data) {
+ return false;
+ }
+ curBlock_->add(data);
+
+ // Presize the values vector to the number of params
+ if (!values->reserve(params.length())) {
+ return false;
+ }
+
+ // Load each value from the data pointer
+ for (size_t i = 0; i < params.length(); i++) {
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+ auto* load = MWasmLoadFieldKA::New(
+ alloc(), exception, data, offsets[i], params[i].toMIRType(),
+ MWideningOp::None, AliasSet::Load(AliasSet::Any));
+ if (!load || !values->append(load)) {
+ return false;
+ }
+ curBlock_->add(load);
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool finishTryCatch(LabelKind kind, Control& control,
+ DefVector* defs) {
+ switch (kind) {
+ case LabelKind::Try: {
+ // This is a catchless try, we must delegate all throwing instructions
+ // to the nearest enclosing try block if one exists, or else to the
+ // body block which will handle it in emitBodyDelegateThrowPad. We
+ // specify a relativeDepth of '1' to delegate outside of the still
+ // active try block.
+ uint32_t relativeDepth = 1;
+ if (!delegatePadPatches(control.tryPadPatches, relativeDepth)) {
+ return false;
+ }
+ break;
+ }
+ case LabelKind::Catch: {
+ // This is a try without a catch_all, we must have a rethrow at the end
+ // of the landing pad (if any).
+ MBasicBlock* padBlock = control.block;
+ if (padBlock) {
+ MBasicBlock* prevBlock = curBlock_;
+ curBlock_ = padBlock;
+ MDefinition* tag = curBlock_->pop();
+ MDefinition* exception = curBlock_->pop();
+ if (!throwFrom(exception, tag)) {
+ return false;
+ }
+ curBlock_ = prevBlock;
+ }
+ break;
+ }
+ case LabelKind::CatchAll:
+ // This is a try with a catch_all, and requires no special handling.
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ // Finish the block, joining the try and catch blocks
+ return finishBlock(defs);
+ }
+
+ [[nodiscard]] bool emitBodyDelegateThrowPad(Control& control) {
+ // Create a landing pad for any throwing instructions
+ MBasicBlock* padBlock;
+ if (!createTryLandingPadIfNeeded(control, &padBlock)) {
+ return false;
+ }
+
+ // If no landing pad was necessary, then we don't need to do anything here
+ if (!padBlock) {
+ return true;
+ }
+
+ // Switch to the landing pad and rethrow the exception
+ MBasicBlock* prevBlock = curBlock_;
+ curBlock_ = padBlock;
+ MDefinition* tag = curBlock_->pop();
+ MDefinition* exception = curBlock_->pop();
+ if (!throwFrom(exception, tag)) {
+ return false;
+ }
+ curBlock_ = prevBlock;
+ return true;
+ }
+
+ [[nodiscard]] bool emitNewException(MDefinition* tag,
+ MDefinition** exception) {
+ return emitInstanceCall1(readBytecodeOffset(), SASigExceptionNew, tag,
+ exception);
+ }
+
+ [[nodiscard]] bool emitThrow(uint32_t tagIndex, const DefVector& argValues) {
+ if (inDeadCode()) {
+ return true;
+ }
+ uint32_t bytecodeOffset = readBytecodeOffset();
+
+ // Load the tag
+ MDefinition* tag = loadTag(tagIndex);
+ if (!tag) {
+ return false;
+ }
+
+ // Allocate an exception object
+ MDefinition* exception;
+ if (!emitNewException(tag, &exception)) {
+ return false;
+ }
+
+ // Load the data pointer from the object
+ auto* data = MWasmLoadField::New(
+ alloc(), exception, WasmExceptionObject::offsetOfData(),
+ MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any));
+ if (!data) {
+ return false;
+ }
+ curBlock_->add(data);
+
+ // Store the params into the data pointer
+ SharedTagType tagType = moduleEnv_.tags[tagIndex].type;
+ for (size_t i = 0; i < tagType->argOffsets_.length(); i++) {
+ if (!mirGen_.ensureBallast()) {
+ return false;
+ }
+ ValType type = tagType->argTypes_[i];
+ uint32_t offset = tagType->argOffsets_[i];
+
+ if (!type.isRefRepr()) {
+ auto* store = MWasmStoreFieldKA::New(alloc(), exception, data, offset,
+ argValues[i], MNarrowingOp::None,
+ AliasSet::Store(AliasSet::Any));
+ if (!store) {
+ return false;
+ }
+ curBlock_->add(store);
+ continue;
+ }
+
+ // Store the new value
+ auto* store = MWasmStoreFieldRefKA::New(
+ alloc(), instancePointer_, exception, data, offset, argValues[i],
+ AliasSet::Store(AliasSet::Any), Nothing(), WasmPreBarrierKind::None);
+ if (!store) {
+ return false;
+ }
+ curBlock_->add(store);
+
+ // Call the post-write barrier
+ if (!postBarrier(bytecodeOffset, exception, data, offset, argValues[i])) {
+ return false;
+ }
+ }
+
+ // Throw the exception
+ return throwFrom(exception, tag);
+ }
+
+ [[nodiscard]] bool throwFrom(MDefinition* exn, MDefinition* tag) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ // Check if there is a local catching try control, and if so, then add a
+ // pad-patch to its tryPadPatches.
+ uint32_t relativeTryDepth;
+ if (inTryBlock(&relativeTryDepth)) {
+ // Set the pending exception state, the landing pad will read from this
+ if (!setPendingExceptionState(exn, tag)) {
+ return false;
+ }
+
+ // End with a pending jump to the landing pad
+ if (!endWithPadPatch(relativeTryDepth)) {
+ return false;
+ }
+ curBlock_ = nullptr;
+ return true;
+ }
+
+ // If there is no surrounding catching block, call an instance method to
+ // throw the exception.
+ if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exn)) {
+ return false;
+ }
+ unreachableTrap();
+
+ curBlock_ = nullptr;
+ return true;
+ }
+
+ [[nodiscard]] bool emitRethrow(uint32_t relativeDepth) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ Control& control = iter().controlItem(relativeDepth);
+ MBasicBlock* pad = control.block;
+ MOZ_ASSERT(pad);
+ MOZ_ASSERT(pad->nslots() > 1);
+ MOZ_ASSERT(iter().controlKind(relativeDepth) == LabelKind::Catch ||
+ iter().controlKind(relativeDepth) == LabelKind::CatchAll);
+
+ // The exception will always be the last slot in the landing pad.
+ size_t exnSlotPosition = pad->nslots() - 2;
+ MDefinition* tag = pad->getSlot(exnSlotPosition + 1);
+ MDefinition* exception = pad->getSlot(exnSlotPosition);
+ MOZ_ASSERT(exception->type() == MIRType::RefOrNull &&
+ tag->type() == MIRType::RefOrNull);
+ return throwFrom(exception, tag);
+ }
+
+ /*********************************************** Instance call helpers ***/
+
+ // Do not call this function directly -- it offers no protection against
+ // mis-counting of arguments. Instead call one of
+ // ::emitInstanceCall{0,1,2,3,4,5,6}.
+ //
+ // Emits a call to the Instance function indicated by `callee`. This is
+ // assumed to take an Instance pointer as its first argument. The remaining
+ // args are taken from `args`, which is assumed to hold `numArgs` entries.
+ // If `result` is non-null, the MDefinition* holding the return value is
+ // written to `*result`.
+ [[nodiscard]] bool emitInstanceCallN(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition** args, size_t numArgs,
+ MDefinition** result = nullptr) {
+ // Check that the first formal parameter is plausibly an Instance pointer.
+ MOZ_ASSERT(callee.numArgs > 0);
+ MOZ_ASSERT(callee.argTypes[0] == MIRType::Pointer);
+ // Check we agree on the number of args.
+ MOZ_ASSERT(numArgs + 1 /* the instance pointer */ == callee.numArgs);
+ // Check we agree on whether a value is returned.
+ MOZ_ASSERT((result == nullptr) == (callee.retType == MIRType::None));
+
+ // If we are in dead code, it can happen that some of the `args` entries
+ // are nullptr, which will look like an OOM to the logic below. So exit
+ // at this point. `passInstance`, `passArg`, `finishCall` and
+ // `builtinInstanceMethodCall` all do nothing in dead code, so it's valid
+ // to exit here.
+ if (inDeadCode()) {
+ if (result) {
+ *result = nullptr;
+ }
+ return true;
+ }
+
+ // Check all args for signs of OOMness before attempting to allocating any
+ // more memory.
+ for (size_t i = 0; i < numArgs; i++) {
+ if (!args[i]) {
+ if (result) {
+ *result = nullptr;
+ }
+ return false;
+ }
+ }
+
+ // Finally, construct the call.
+ CallCompileState ccsArgs;
+ if (!passInstance(callee.argTypes[0], &ccsArgs)) {
+ return false;
+ }
+ for (size_t i = 0; i < numArgs; i++) {
+ if (!passArg(args[i], callee.argTypes[i + 1], &ccsArgs)) {
+ return false;
+ }
+ }
+ if (!finishCall(&ccsArgs)) {
+ return false;
+ }
+ return builtinInstanceMethodCall(callee, lineOrBytecode, ccsArgs, result);
+ }
+
+ [[nodiscard]] bool emitInstanceCall0(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition** result = nullptr) {
+ MDefinition* args[0] = {};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 0, result);
+ }
+ [[nodiscard]] bool emitInstanceCall1(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1,
+ MDefinition** result = nullptr) {
+ MDefinition* args[1] = {arg1};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 1, result);
+ }
+ [[nodiscard]] bool emitInstanceCall2(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1, MDefinition* arg2,
+ MDefinition** result = nullptr) {
+ MDefinition* args[2] = {arg1, arg2};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 2, result);
+ }
+ [[nodiscard]] bool emitInstanceCall3(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1, MDefinition* arg2,
+ MDefinition* arg3,
+ MDefinition** result = nullptr) {
+ MDefinition* args[3] = {arg1, arg2, arg3};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 3, result);
+ }
+ [[nodiscard]] bool emitInstanceCall4(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1, MDefinition* arg2,
+ MDefinition* arg3, MDefinition* arg4,
+ MDefinition** result = nullptr) {
+ MDefinition* args[4] = {arg1, arg2, arg3, arg4};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 4, result);
+ }
+ [[nodiscard]] bool emitInstanceCall5(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1, MDefinition* arg2,
+ MDefinition* arg3, MDefinition* arg4,
+ MDefinition* arg5,
+ MDefinition** result = nullptr) {
+ MDefinition* args[5] = {arg1, arg2, arg3, arg4, arg5};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 5, result);
+ }
+ [[nodiscard]] bool emitInstanceCall6(uint32_t lineOrBytecode,
+ const SymbolicAddressSignature& callee,
+ MDefinition* arg1, MDefinition* arg2,
+ MDefinition* arg3, MDefinition* arg4,
+ MDefinition* arg5, MDefinition* arg6,
+ MDefinition** result = nullptr) {
+ MDefinition* args[6] = {arg1, arg2, arg3, arg4, arg5, arg6};
+ return emitInstanceCallN(lineOrBytecode, callee, args, 6, result);
+ }
+
+ /******************************** WasmGC: low level load/store helpers ***/
+
+ // Given a (FieldType, FieldExtension) pair, produce the (MIRType,
+ // MWideningOp) pair that will give the correct operation for reading the
+ // value from memory.
+ static void fieldLoadInfoToMIR(FieldType type, FieldWideningOp wideningOp,
+ MIRType* mirType, MWideningOp* mirWideningOp) {
+ switch (type.kind()) {
+ case FieldType::I8: {
+ switch (wideningOp) {
+ case FieldWideningOp::Signed:
+ *mirType = MIRType::Int32;
+ *mirWideningOp = MWideningOp::FromS8;
+ return;
+ case FieldWideningOp::Unsigned:
+ *mirType = MIRType::Int32;
+ *mirWideningOp = MWideningOp::FromU8;
+ return;
+ default:
+ MOZ_CRASH();
+ }
+ }
+ case FieldType::I16: {
+ switch (wideningOp) {
+ case FieldWideningOp::Signed:
+ *mirType = MIRType::Int32;
+ *mirWideningOp = MWideningOp::FromS16;
+ return;
+ case FieldWideningOp::Unsigned:
+ *mirType = MIRType::Int32;
+ *mirWideningOp = MWideningOp::FromU16;
+ return;
+ default:
+ MOZ_CRASH();
+ }
+ }
+ default: {
+ switch (wideningOp) {
+ case FieldWideningOp::None:
+ *mirType = type.toMIRType();
+ *mirWideningOp = MWideningOp::None;
+ return;
+ default:
+ MOZ_CRASH();
+ }
+ }
+ }
+ }
+
+ // Given a FieldType, produce the MNarrowingOp required for writing the
+ // value to memory.
+ static MNarrowingOp fieldStoreInfoToMIR(FieldType type) {
+ switch (type.kind()) {
+ case FieldType::I8:
+ return MNarrowingOp::To8;
+ case FieldType::I16:
+ return MNarrowingOp::To16;
+ default:
+ return MNarrowingOp::None;
+ }
+ }
+
+ // Generate a write of `value` at address `base + offset`, where `offset` is
+ // known at JIT time. If the written value is a reftype, the previous value
+ // at `base + offset` will be retrieved and handed off to the post-write
+ // barrier. `keepAlive` will be referenced by the instruction so as to hold
+ // it live (from the GC's point of view).
+ [[nodiscard]] bool writeGcValueAtBasePlusOffset(
+ uint32_t lineOrBytecode, FieldType fieldType, MDefinition* keepAlive,
+ AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
+ uint32_t offset, bool needsTrapInfo, WasmPreBarrierKind preBarrierKind) {
+ MOZ_ASSERT(aliasBitset != 0);
+ MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull);
+ MOZ_ASSERT(fieldType.widenToValType().toMIRType() == value->type());
+ MNarrowingOp narrowingOp = fieldStoreInfoToMIR(fieldType);
+
+ if (!fieldType.isRefRepr()) {
+ MaybeTrapSiteInfo maybeTrap;
+ if (needsTrapInfo) {
+ maybeTrap.emplace(getTrapSiteInfo());
+ }
+ auto* store = MWasmStoreFieldKA::New(
+ alloc(), keepAlive, base, offset, value, narrowingOp,
+ AliasSet::Store(aliasBitset), maybeTrap);
+ if (!store) {
+ return false;
+ }
+ curBlock_->add(store);
+ return true;
+ }
+
+ // Otherwise it's a ref store. Load the previous value so we can show it
+ // to the post-write barrier.
+ //
+ // Optimisation opportunity: for the case where this field write results
+ // from struct.new, the old value is always zero. So we should synthesise
+ // a suitable zero constant rather than reading it from the object. See
+ // also bug 1799999.
+ MOZ_ASSERT(narrowingOp == MNarrowingOp::None);
+ MOZ_ASSERT(fieldType.widenToValType() == fieldType.valType());
+
+ // Store the new value
+ auto* store = MWasmStoreFieldRefKA::New(
+ alloc(), instancePointer_, keepAlive, base, offset, value,
+ AliasSet::Store(aliasBitset), mozilla::Some(getTrapSiteInfo()),
+ preBarrierKind);
+ if (!store) {
+ return false;
+ }
+ curBlock_->add(store);
+
+ // Call the post-write barrier
+ return postBarrier(lineOrBytecode, keepAlive, base, offset, value);
+ }
+
+ // Generate a write of `value` at address `base + index * scale`, where
+ // `scale` is known at JIT-time. If the written value is a reftype, the
+ // previous value at `base + index * scale` will be retrieved and handed off
+ // to the post-write barrier. `keepAlive` will be referenced by the
+ // instruction so as to hold it live (from the GC's point of view).
+ [[nodiscard]] bool writeGcValueAtBasePlusScaledIndex(
+ uint32_t lineOrBytecode, FieldType fieldType, MDefinition* keepAlive,
+ AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
+ uint32_t scale, MDefinition* index, WasmPreBarrierKind preBarrierKind) {
+ MOZ_ASSERT(aliasBitset != 0);
+ MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull);
+ MOZ_ASSERT(fieldType.widenToValType().toMIRType() == value->type());
+ MOZ_ASSERT(scale == 1 || scale == 2 || scale == 4 || scale == 8 ||
+ scale == 16);
+
+ // Currently there's no single MIR node that this can be translated into.
+ // So compute the final address "manually", then store directly to that
+ // address. See bug 1802287.
+ MDefinition* scaleDef = constantTargetWord(intptr_t(scale));
+ if (!scaleDef) {
+ return false;
+ }
+ MDefinition* finalAddr = computeBasePlusScaledIndex(base, scaleDef, index);
+ if (!finalAddr) {
+ return false;
+ }
+
+ return writeGcValueAtBasePlusOffset(
+ lineOrBytecode, fieldType, keepAlive, aliasBitset, value, finalAddr,
+ /*offset=*/0,
+ /*needsTrapInfo=*/false, preBarrierKind);
+ }
+
+ // Generate a read from address `base + offset`, where `offset` is known at
+ // JIT time. The loaded value will be widened as described by `fieldType`
+ // and `fieldWideningOp`. `keepAlive` will be referenced by the instruction
+ // so as to hold it live (from the GC's point of view).
+ [[nodiscard]] MDefinition* readGcValueAtBasePlusOffset(
+ FieldType fieldType, FieldWideningOp fieldWideningOp,
+ MDefinition* keepAlive, AliasSet::Flag aliasBitset, MDefinition* base,
+ uint32_t offset, bool needsTrapInfo) {
+ MOZ_ASSERT(aliasBitset != 0);
+ MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull);
+ MIRType mirType;
+ MWideningOp mirWideningOp;
+ fieldLoadInfoToMIR(fieldType, fieldWideningOp, &mirType, &mirWideningOp);
+ MaybeTrapSiteInfo maybeTrap;
+ if (needsTrapInfo) {
+ maybeTrap.emplace(getTrapSiteInfo());
+ }
+ auto* load = MWasmLoadFieldKA::New(alloc(), keepAlive, base, offset,
+ mirType, mirWideningOp,
+ AliasSet::Load(aliasBitset), maybeTrap);
+ if (!load) {
+ return nullptr;
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ // Generate a read from address `base + index * scale`, where `scale` is
+ // known at JIT-time. The loaded value will be widened as described by
+ // `fieldType` and `fieldWideningOp`. `keepAlive` will be referenced by the
+ // instruction so as to hold it live (from the GC's point of view).
+ [[nodiscard]] MDefinition* readGcValueAtBasePlusScaledIndex(
+ FieldType fieldType, FieldWideningOp fieldWideningOp,
+ MDefinition* keepAlive, AliasSet::Flag aliasBitset, MDefinition* base,
+ uint32_t scale, MDefinition* index) {
+ MOZ_ASSERT(aliasBitset != 0);
+ MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull);
+ MOZ_ASSERT(scale == 1 || scale == 2 || scale == 4 || scale == 8 ||
+ scale == 16);
+
+ // Currently there's no single MIR node that this can be translated into.
+ // So compute the final address "manually", then store directly to that
+ // address. See bug 1802287.
+ MDefinition* scaleDef = constantTargetWord(intptr_t(scale));
+ if (!scaleDef) {
+ return nullptr;
+ }
+ MDefinition* finalAddr = computeBasePlusScaledIndex(base, scaleDef, index);
+ if (!finalAddr) {
+ return nullptr;
+ }
+
+ MIRType mirType;
+ MWideningOp mirWideningOp;
+ fieldLoadInfoToMIR(fieldType, fieldWideningOp, &mirType, &mirWideningOp);
+ auto* load = MWasmLoadFieldKA::New(alloc(), keepAlive, finalAddr,
+ /*offset=*/0, mirType, mirWideningOp,
+ AliasSet::Load(aliasBitset),
+ mozilla::Some(getTrapSiteInfo()));
+ if (!load) {
+ return nullptr;
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ /************************************************ WasmGC: type helpers ***/
+
+ // Returns an MDefinition holding the supertype vector for `typeIndex`.
+ [[nodiscard]] MDefinition* loadSuperTypeVector(uint32_t typeIndex) {
+ uint32_t superTypeVectorOffset =
+ moduleEnv().offsetOfSuperTypeVector(typeIndex);
+
+ auto* load = MWasmLoadInstanceDataField::New(
+ alloc(), MIRType::Pointer, superTypeVectorOffset,
+ /*isConst=*/true, instancePointer_);
+ if (!load) {
+ return nullptr;
+ }
+ curBlock_->add(load);
+ return load;
+ }
+
+ [[nodiscard]] MDefinition* loadTypeDefInstanceData(uint32_t typeIndex) {
+ size_t offset = Instance::offsetInData(
+ moduleEnv_.offsetOfTypeDefInstanceData(typeIndex));
+ auto* result = MWasmDerivedPointer::New(alloc(), instancePointer_, offset);
+ if (!result) {
+ return nullptr;
+ }
+ curBlock_->add(result);
+ return result;
+ }
+
+ /********************************************** WasmGC: struct helpers ***/
+
+ // Helper function for EmitStruct{New,Set}: given a MIR pointer to a
+ // WasmStructObject, a MIR pointer to a value, and a field descriptor,
+ // generate MIR to write the value to the relevant field in the object.
+ [[nodiscard]] bool writeValueToStructField(
+ uint32_t lineOrBytecode, const StructField& field,
+ MDefinition* structObject, MDefinition* value,
+ WasmPreBarrierKind preBarrierKind) {
+ FieldType fieldType = field.type;
+ uint32_t fieldOffset = field.offset;
+
+ bool areaIsOutline;
+ uint32_t areaOffset;
+ WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
+ &areaIsOutline, &areaOffset);
+
+ // Make `base` point at the first byte of either the struct object as a
+ // whole or of the out-of-line data area. And adjust `areaOffset`
+ // accordingly.
+ MDefinition* base;
+ bool needsTrapInfo;
+ if (areaIsOutline) {
+ auto* load = MWasmLoadField::New(
+ alloc(), structObject, WasmStructObject::offsetOfOutlineData(),
+ MIRType::Pointer, MWideningOp::None,
+ AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
+ mozilla::Some(getTrapSiteInfo()));
+ if (!load) {
+ return false;
+ }
+ curBlock_->add(load);
+ base = load;
+ needsTrapInfo = false;
+ } else {
+ base = structObject;
+ needsTrapInfo = true;
+ areaOffset += WasmStructObject::offsetOfInlineData();
+ }
+ // The transaction is to happen at `base + areaOffset`, so to speak.
+ // After this point we must ignore `fieldOffset`.
+
+ // The alias set denoting the field's location, although lacking a
+ // Load-vs-Store indication at this point.
+ AliasSet::Flag fieldAliasSet = areaIsOutline
+ ? AliasSet::WasmStructOutlineDataArea
+ : AliasSet::WasmStructInlineDataArea;
+
+ return writeGcValueAtBasePlusOffset(lineOrBytecode, fieldType, structObject,
+ fieldAliasSet, value, base, areaOffset,
+ needsTrapInfo, preBarrierKind);
+ }
+
+ // Helper function for EmitStructGet: given a MIR pointer to a
+ // WasmStructObject, a field descriptor and a field widening operation,
+ // generate MIR to read the value from the relevant field in the object.
+ [[nodiscard]] MDefinition* readValueFromStructField(
+ const StructField& field, FieldWideningOp wideningOp,
+ MDefinition* structObject) {
+ FieldType fieldType = field.type;
+ uint32_t fieldOffset = field.offset;
+
+ bool areaIsOutline;
+ uint32_t areaOffset;
+ WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
+ &areaIsOutline, &areaOffset);
+
+ // Make `base` point at the first byte of either the struct object as a
+ // whole or of the out-of-line data area. And adjust `areaOffset`
+ // accordingly.
+ MDefinition* base;
+ bool needsTrapInfo;
+ if (areaIsOutline) {
+ auto* loadOOLptr = MWasmLoadField::New(
+ alloc(), structObject, WasmStructObject::offsetOfOutlineData(),
+ MIRType::Pointer, MWideningOp::None,
+ AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
+ mozilla::Some(getTrapSiteInfo()));
+ if (!loadOOLptr) {
+ return nullptr;
+ }
+ curBlock_->add(loadOOLptr);
+ base = loadOOLptr;
+ needsTrapInfo = false;
+ } else {
+ base = structObject;
+ needsTrapInfo = true;
+ areaOffset += WasmStructObject::offsetOfInlineData();
+ }
+ // The transaction is to happen at `base + areaOffset`, so to speak.
+ // After this point we must ignore `fieldOffset`.
+
+ // The alias set denoting the field's location, although lacking a
+ // Load-vs-Store indication at this point.
+ AliasSet::Flag fieldAliasSet = areaIsOutline
+ ? AliasSet::WasmStructOutlineDataArea
+ : AliasSet::WasmStructInlineDataArea;
+
+ return readGcValueAtBasePlusOffset(fieldType, wideningOp, structObject,
+ fieldAliasSet, base, areaOffset,
+ needsTrapInfo);
+ }
+
+ /********************************* WasmGC: address-arithmetic helpers ***/
+
+ inline bool targetIs64Bit() const {
+#ifdef JS_64BIT
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Generate MIR to unsigned widen `val` out to the target word size. If
+ // `val` is already at the target word size, this is a no-op. The only
+ // other allowed case is where `val` is Int32 and we're compiling for a
+ // 64-bit target, in which case a widen is generated.
+ [[nodiscard]] MDefinition* unsignedWidenToTargetWord(MDefinition* val) {
+ if (targetIs64Bit()) {
+ if (val->type() == MIRType::Int32) {
+ auto* ext = MExtendInt32ToInt64::New(alloc(), val, /*isUnsigned=*/true);
+ if (!ext) {
+ return nullptr;
+ }
+ curBlock_->add(ext);
+ return ext;
+ }
+ MOZ_ASSERT(val->type() == MIRType::Int64);
+ return val;
+ }
+ MOZ_ASSERT(val->type() == MIRType::Int32);
+ return val;
+ }
+
+ // Compute `base + index * scale`, for both 32- and 64-bit targets. For the
+ // convenience of callers, on a 64-bit target, `index` and `scale` can
+ // (independently) be either Int32 or Int64; in the former case they will be
+ // zero-extended before the multiplication, so that both the multiplication
+ // and addition are done at the target word size.
+ [[nodiscard]] MDefinition* computeBasePlusScaledIndex(MDefinition* base,
+ MDefinition* scale,
+ MDefinition* index) {
+ // On a 32-bit target, require:
+ // base : Int32 (== TargetWordMIRType())
+ // index, scale : Int32
+ // Calculate base +32 (index *32 scale)
+ //
+ // On a 64-bit target, require:
+ // base : Int64 (== TargetWordMIRType())
+ // index, scale: either Int32 or Int64 (any combination is OK)
+ // Calculate base +64 (u-widen to 64(index)) *64 (u-widen to 64(scale))
+ //
+ // Final result type is the same as that of `base`.
+
+ MOZ_ASSERT(base->type() == TargetWordMIRType());
+
+ // Widen `index` if necessary, producing `indexW`.
+ MDefinition* indexW = unsignedWidenToTargetWord(index);
+ if (!indexW) {
+ return nullptr;
+ }
+ // Widen `scale` if necessary, producing `scaleW`.
+ MDefinition* scaleW = unsignedWidenToTargetWord(scale);
+ if (!scaleW) {
+ return nullptr;
+ }
+ // Compute `scaledIndex = indexW * scaleW`.
+ MIRType targetWordType = TargetWordMIRType();
+ bool targetIs64 = targetWordType == MIRType::Int64;
+ MMul* scaledIndex =
+ MMul::NewWasm(alloc(), indexW, scaleW, targetWordType,
+ targetIs64 ? MMul::Mode::Normal : MMul::Mode::Integer,
+ /*mustPreserveNan=*/false);
+ if (!scaledIndex) {
+ return nullptr;
+ }
+ // Compute `result = base + scaledIndex`.
+ curBlock_->add(scaledIndex);
+ MAdd* result = MAdd::NewWasm(alloc(), base, scaledIndex, targetWordType);
+ if (!result) {
+ return nullptr;
+ }
+ curBlock_->add(result);
+ return result;
+ }
+
+ /********************************************** WasmGC: array helpers ***/
+
+ // Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
+ // return the contents of the WasmArrayObject::numElements_ field.
+ // Adds trap site info for the null check.
+ [[nodiscard]] MDefinition* getWasmArrayObjectNumElements(
+ MDefinition* arrayObject) {
+ MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull);
+
+ auto* numElements = MWasmLoadField::New(
+ alloc(), arrayObject, WasmArrayObject::offsetOfNumElements(),
+ MIRType::Int32, MWideningOp::None,
+ AliasSet::Load(AliasSet::WasmArrayNumElements),
+ mozilla::Some(getTrapSiteInfo()));
+ if (!numElements) {
+ return nullptr;
+ }
+ curBlock_->add(numElements);
+
+ return numElements;
+ }
+
+ // Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
+ // return the contents of the WasmArrayObject::data_ field.
+ [[nodiscard]] MDefinition* getWasmArrayObjectData(MDefinition* arrayObject) {
+ MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull);
+
+ auto* data = MWasmLoadField::New(
+ alloc(), arrayObject, WasmArrayObject::offsetOfData(),
+ TargetWordMIRType(), MWideningOp::None,
+ AliasSet::Load(AliasSet::WasmArrayDataPointer),
+ mozilla::Some(getTrapSiteInfo()));
+ if (!data) {
+ return nullptr;
+ }
+ curBlock_->add(data);
+
+ return data;
+ }
+
+ // Given a JIT-time-known type index `typeIndex` and a run-time known number
+ // of elements `numElements`, create MIR to call `Instance::arrayNew`,
+ // producing an array with the relevant type and size and initialized with
+ // `typeIndex`s default value.
+ [[nodiscard]] MDefinition* createDefaultInitializedArrayObject(
+ uint32_t lineOrBytecode, uint32_t typeIndex, MDefinition* numElements) {
+ // Get the type definition for the array as a whole.
+ MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return nullptr;
+ }
+
+ // Create call: arrayObject = Instance::arrayNew(numElements, typeDefData)
+ // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated
+ // by this call will trap.
+ MDefinition* arrayObject;
+ if (!emitInstanceCall2(lineOrBytecode, SASigArrayNew, numElements,
+ typeDefData, &arrayObject)) {
+ return nullptr;
+ }
+
+ return arrayObject;
+ }
+
+ [[nodiscard]] MDefinition* createUninitializedArrayObject(
+ uint32_t lineOrBytecode, uint32_t typeIndex, MDefinition* numElements) {
+ // Get the type definition for the array as a whole.
+ MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return nullptr;
+ }
+
+ // Create call: arrayObject = Instance::arrayNewUninit(numElements,
+ // typeDefData) If the requested size exceeds MaxArrayPayloadBytes, the MIR
+ // generated by this call will trap.
+ MDefinition* arrayObject;
+ if (!emitInstanceCall2(lineOrBytecode, SASigArrayNewUninit, numElements,
+ typeDefData, &arrayObject)) {
+ return nullptr;
+ }
+
+ return arrayObject;
+ }
+
+ // This emits MIR to perform several actions common to array loads and
+ // stores. Given `arrayObject`, that points to a WasmArrayObject, and an
+ // index value `index`, it:
+ //
+ // * Generates a trap if the array pointer is null
+ // * Gets the size of the array
+ // * Emits a bounds check of `index` against the array size
+ // * Retrieves the OOL object pointer from the array
+ // * Includes check for null via signal handler.
+ //
+ // The returned value is for the OOL object pointer.
+ [[nodiscard]] MDefinition* setupForArrayAccess(MDefinition* arrayObject,
+ MDefinition* index) {
+ MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ // Check for null is done in getWasmArrayObjectNumElements.
+
+ // Get the size value for the array.
+ MDefinition* numElements = getWasmArrayObjectNumElements(arrayObject);
+ if (!numElements) {
+ return nullptr;
+ }
+
+ // Create a bounds check.
+ auto* boundsCheck =
+ MWasmBoundsCheck::New(alloc(), index, numElements, bytecodeOffset(),
+ MWasmBoundsCheck::Target::Unknown);
+ if (!boundsCheck) {
+ return nullptr;
+ }
+ curBlock_->add(boundsCheck);
+
+ // Get the address of the first byte of the (OOL) data area.
+ return getWasmArrayObjectData(arrayObject);
+ }
+
+ // This routine generates all MIR required for `array.new`. The returned
+ // value is for the newly created array.
+ [[nodiscard]] MDefinition* createArrayNewCallAndLoop(uint32_t lineOrBytecode,
+ uint32_t typeIndex,
+ MDefinition* numElements,
+ MDefinition* fillValue) {
+ const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
+
+ // Create the array object, default-initialized.
+ MDefinition* arrayObject =
+ createUninitializedArrayObject(lineOrBytecode, typeIndex, numElements);
+ if (!arrayObject) {
+ return nullptr;
+ }
+
+ mozilla::DebugOnly<MIRType> fillValueMIRType = fillValue->type();
+ FieldType fillValueFieldType = arrayType.elementType_;
+ MOZ_ASSERT(fillValueFieldType.widenToValType().toMIRType() ==
+ fillValueMIRType);
+
+ uint32_t elemSize = fillValueFieldType.size();
+ MOZ_ASSERT(elemSize >= 1 && elemSize <= 16);
+
+ // Make `base` point at the first byte of the (OOL) data area.
+ MDefinition* base = getWasmArrayObjectData(arrayObject);
+ if (!base) {
+ return nullptr;
+ }
+
+ // We have:
+ // base : TargetWord
+ // numElements : Int32
+ // fillValue : <any FieldType>
+ // $elemSize = arrayType.elementType_.size(); 1, 2, 4, 8 or 16
+ //
+ // Generate MIR:
+ // <in current block>
+ // limit : TargetWord = base + nElems * elemSize
+ // if (limit == base) goto after; // skip loop if trip count == 0
+ // // optimisation (not done): skip loop if fill value == 0
+ // loop:
+ // ptrPhi = phi(base, ptrNext)
+ // *ptrPhi = fillValue
+ // ptrNext = ptrPhi + $elemSize
+ // if (ptrNext <u limit) goto loop;
+ // after:
+ //
+ // We construct the loop "manually" rather than using
+ // FunctionCompiler::{startLoop,closeLoop} as the latter have awareness of
+ // the wasm view of loops, whereas the loop we're building here is not a
+ // wasm-level loop.
+ // ==== Create the "loop" and "after" blocks ====
+ MBasicBlock* loopBlock;
+ if (!newBlock(curBlock_, &loopBlock, MBasicBlock::LOOP_HEADER)) {
+ return nullptr;
+ }
+ MBasicBlock* afterBlock;
+ if (!newBlock(loopBlock, &afterBlock)) {
+ return nullptr;
+ }
+
+ // ==== Fill in the remainder of the block preceding the loop ====
+ MDefinition* elemSizeDef = constantTargetWord(intptr_t(elemSize));
+ if (!elemSizeDef) {
+ return nullptr;
+ }
+
+ MDefinition* limit =
+ computeBasePlusScaledIndex(base, elemSizeDef, numElements);
+ if (!limit) {
+ return nullptr;
+ }
+
+ // Use JSOp::StrictEq, not ::Eq, so that the comparison (and eventually
+ // the entire initialisation loop) will be folded out in the case where
+ // the number of elements is zero. See MCompare::tryFoldEqualOperands.
+ MDefinition* limitEqualsBase = compare(
+ limit, base, JSOp::StrictEq,
+ targetIs64Bit() ? MCompare::Compare_UInt64 : MCompare::Compare_UInt32);
+ if (!limitEqualsBase) {
+ return nullptr;
+ }
+ MTest* skipIfLimitEqualsBase =
+ MTest::New(alloc(), limitEqualsBase, afterBlock, loopBlock);
+ if (!skipIfLimitEqualsBase) {
+ return nullptr;
+ }
+ curBlock_->end(skipIfLimitEqualsBase);
+ if (!afterBlock->addPredecessor(alloc(), curBlock_)) {
+ return nullptr;
+ }
+ // Optimisation opportunity: if the fill value is zero, maybe we should
+ // likewise skip over the initialisation loop entirely (and, if the zero
+ // value is visible at JIT time, the loop will be removed). For the
+ // reftyped case, that would be a big win since each iteration requires a
+ // call to the post-write barrier routine.
+
+ // ==== Fill in the loop block as best we can ====
+ curBlock_ = loopBlock;
+ MPhi* ptrPhi = MPhi::New(alloc(), TargetWordMIRType());
+ if (!ptrPhi) {
+ return nullptr;
+ }
+ if (!ptrPhi->reserveLength(2)) {
+ return nullptr;
+ }
+ ptrPhi->addInput(base);
+ curBlock_->addPhi(ptrPhi);
+ curBlock_->setLoopDepth(loopDepth_ + 1);
+
+ // Because we have the exact address to hand, use
+ // `writeGcValueAtBasePlusOffset` rather than
+ // `writeGcValueAtBasePlusScaledIndex` to do the store.
+ if (!writeGcValueAtBasePlusOffset(
+ lineOrBytecode, fillValueFieldType, arrayObject,
+ AliasSet::WasmArrayDataArea, fillValue, ptrPhi, /*offset=*/0,
+ /*needsTrapInfo=*/false, WasmPreBarrierKind::None)) {
+ return nullptr;
+ }
+
+ auto* ptrNext =
+ MAdd::NewWasm(alloc(), ptrPhi, elemSizeDef, TargetWordMIRType());
+ if (!ptrNext) {
+ return nullptr;
+ }
+ curBlock_->add(ptrNext);
+ ptrPhi->addInput(ptrNext);
+
+ MDefinition* ptrNextLtuLimit = compare(
+ ptrNext, limit, JSOp::Lt,
+ targetIs64Bit() ? MCompare::Compare_UInt64 : MCompare::Compare_UInt32);
+ if (!ptrNextLtuLimit) {
+ return nullptr;
+ }
+ auto* continueIfPtrNextLtuLimit =
+ MTest::New(alloc(), ptrNextLtuLimit, loopBlock, afterBlock);
+ if (!continueIfPtrNextLtuLimit) {
+ return nullptr;
+ }
+ curBlock_->end(continueIfPtrNextLtuLimit);
+ if (!loopBlock->addPredecessor(alloc(), loopBlock)) {
+ return nullptr;
+ }
+ // ==== Loop block completed ====
+
+ curBlock_ = afterBlock;
+ return arrayObject;
+ }
+
+ /*********************************************** WasmGC: other helpers ***/
+
+ // Generate MIR that causes a trap of kind `trapKind` if `arg` is zero.
+ // Currently `arg` may only be a MIRType::Int32, but that requirement could
+ // be relaxed if needed in future.
+ [[nodiscard]] bool trapIfZero(wasm::Trap trapKind, MDefinition* arg) {
+ MOZ_ASSERT(arg->type() == MIRType::Int32);
+
+ MBasicBlock* trapBlock = nullptr;
+ if (!newBlock(curBlock_, &trapBlock)) {
+ return false;
+ }
+
+ auto* trap = MWasmTrap::New(alloc(), trapKind, bytecodeOffset());
+ if (!trap) {
+ return false;
+ }
+ trapBlock->end(trap);
+
+ MBasicBlock* joinBlock = nullptr;
+ if (!newBlock(curBlock_, &joinBlock)) {
+ return false;
+ }
+
+ auto* test = MTest::New(alloc(), arg, joinBlock, trapBlock);
+ if (!test) {
+ return false;
+ }
+ curBlock_->end(test);
+ curBlock_ = joinBlock;
+ return true;
+ }
+
+ [[nodiscard]] MDefinition* isGcObjectSubtypeOf(MDefinition* object,
+ RefType sourceType,
+ RefType destType) {
+ MInstruction* isSubTypeOf = nullptr;
+ if (destType.isTypeRef()) {
+ uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef());
+ MDefinition* superSuperTypeVector = loadSuperTypeVector(typeIndex);
+ isSubTypeOf = MWasmGcObjectIsSubtypeOfConcrete::New(
+ alloc(), object, superSuperTypeVector, sourceType, destType);
+ } else {
+ isSubTypeOf = MWasmGcObjectIsSubtypeOfAbstract::New(alloc(), object,
+ sourceType, destType);
+ }
+ MOZ_ASSERT(isSubTypeOf);
+
+ curBlock_->add(isSubTypeOf);
+ return isSubTypeOf;
+ }
+
+ // Generate MIR that attempts to downcast `ref` to `castToTypeDef`. If the
+ // downcast fails, we trap. If it succeeds, then `ref` can be assumed to
+ // have a type that is a subtype of (or the same as) `castToTypeDef` after
+ // this point.
+ [[nodiscard]] bool refCast(MDefinition* ref, RefType sourceType,
+ RefType destType) {
+ MDefinition* success = isGcObjectSubtypeOf(ref, sourceType, destType);
+ if (!success) {
+ return false;
+ }
+
+ // Trap if `success` is zero. If it's nonzero, we have established that
+ // `ref <: castToTypeDef`.
+ return trapIfZero(wasm::Trap::BadCast, success);
+ }
+
+ // Generate MIR that computes a boolean value indicating whether or not it
+ // is possible to downcast `ref` to `destType`.
+ [[nodiscard]] MDefinition* refTest(MDefinition* ref, RefType sourceType,
+ RefType destType) {
+ return isGcObjectSubtypeOf(ref, sourceType, destType);
+ }
+
+ // Generates MIR for br_on_cast and br_on_cast_fail.
+ [[nodiscard]] bool brOnCastCommon(bool onSuccess, uint32_t labelRelativeDepth,
+ RefType sourceType, RefType destType,
+ const ResultType& labelType,
+ const DefVector& values) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* fallthroughBlock = nullptr;
+ if (!newBlock(curBlock_, &fallthroughBlock)) {
+ return false;
+ }
+
+ // `values` are the values in the top block-value on the stack. Since the
+ // argument to `br_on_cast{_fail}` is at the top of the stack, it is the
+ // last element in `values`.
+ //
+ // For both br_on_cast and br_on_cast_fail, the OpIter validation routines
+ // ensure that `values` is non-empty (by rejecting the case
+ // `labelType->length() < 1`) and that the last value in `values` is
+ // reftyped.
+ MOZ_RELEASE_ASSERT(values.length() > 0);
+ MDefinition* ref = values.back();
+ MOZ_ASSERT(ref->type() == MIRType::RefOrNull);
+
+ MDefinition* success = isGcObjectSubtypeOf(ref, sourceType, destType);
+ if (!success) {
+ return false;
+ }
+
+ MTest* test;
+ if (onSuccess) {
+ test = MTest::New(alloc(), success, nullptr, fallthroughBlock);
+ if (!test || !addControlFlowPatch(test, labelRelativeDepth,
+ MTest::TrueBranchIndex)) {
+ return false;
+ }
+ } else {
+ test = MTest::New(alloc(), success, fallthroughBlock, nullptr);
+ if (!test || !addControlFlowPatch(test, labelRelativeDepth,
+ MTest::FalseBranchIndex)) {
+ return false;
+ }
+ }
+
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(test);
+ curBlock_ = fallthroughBlock;
+ return true;
+ }
+
+ [[nodiscard]] bool brOnNonStruct(const DefVector& values) {
+ if (inDeadCode()) {
+ return true;
+ }
+
+ MBasicBlock* fallthroughBlock = nullptr;
+ if (!newBlock(curBlock_, &fallthroughBlock)) {
+ return false;
+ }
+
+ MOZ_ASSERT(values.length() > 0);
+ MOZ_ASSERT(values.back()->type() == MIRType::RefOrNull);
+
+ MGoto* jump = MGoto::New(alloc(), fallthroughBlock);
+ if (!jump) {
+ return false;
+ }
+ if (!pushDefs(values)) {
+ return false;
+ }
+
+ curBlock_->end(jump);
+ curBlock_ = fallthroughBlock;
+ return true;
+ }
+
+ /************************************************************ DECODING ***/
+
+ // AsmJS adds a line number to `callSiteLineNums` for certain operations that
+ // are represented by a JS call, such as math builtins. We use these line
+ // numbers when calling builtins. This method will read from
+ // `callSiteLineNums` when we are using AsmJS, or else return the current
+ // bytecode offset.
+ //
+ // This method MUST be called from opcodes that AsmJS will emit a call site
+ // line number for, or else the arrays will get out of sync. Other opcodes
+ // must use `readBytecodeOffset` below.
+ uint32_t readCallSiteLineOrBytecode() {
+ if (!func_.callSiteLineNums.empty()) {
+ return func_.callSiteLineNums[lastReadCallSite_++];
+ }
+ return iter_.lastOpcodeOffset();
+ }
+
+ // Return the current bytecode offset.
+ uint32_t readBytecodeOffset() { return iter_.lastOpcodeOffset(); }
+
+ TrapSiteInfo getTrapSiteInfo() {
+ return TrapSiteInfo(wasm::BytecodeOffset(readBytecodeOffset()));
+ }
+
+#if DEBUG
+ bool done() const { return iter_.done(); }
+#endif
+
+ /*************************************************************************/
+ private:
+ [[nodiscard]] bool newBlock(MBasicBlock* pred, MBasicBlock** block,
+ MBasicBlock::Kind kind = MBasicBlock::NORMAL) {
+ *block = MBasicBlock::New(mirGraph(), info(), pred, kind);
+ if (!*block) {
+ return false;
+ }
+ mirGraph().addBlock(*block);
+ (*block)->setLoopDepth(loopDepth_);
+ return true;
+ }
+
+ [[nodiscard]] bool goToNewBlock(MBasicBlock* pred, MBasicBlock** block) {
+ if (!newBlock(pred, block)) {
+ return false;
+ }
+ pred->end(MGoto::New(alloc(), *block));
+ return true;
+ }
+
+ [[nodiscard]] bool goToExistingBlock(MBasicBlock* prev, MBasicBlock* next) {
+ MOZ_ASSERT(prev);
+ MOZ_ASSERT(next);
+ prev->end(MGoto::New(alloc(), next));
+ return next->addPredecessor(alloc(), prev);
+ }
+
+ [[nodiscard]] bool bindBranches(uint32_t absolute, DefVector* defs) {
+ if (absolute >= blockPatches_.length() || blockPatches_[absolute].empty()) {
+ return inDeadCode() || popPushedDefs(defs);
+ }
+
+ ControlFlowPatchVector& patches = blockPatches_[absolute];
+ MControlInstruction* ins = patches[0].ins;
+ MBasicBlock* pred = ins->block();
+
+ MBasicBlock* join = nullptr;
+ if (!newBlock(pred, &join)) {
+ return false;
+ }
+
+ pred->mark();
+ ins->replaceSuccessor(patches[0].index, join);
+
+ for (size_t i = 1; i < patches.length(); i++) {
+ ins = patches[i].ins;
+
+ pred = ins->block();
+ if (!pred->isMarked()) {
+ if (!join->addPredecessor(alloc(), pred)) {
+ return false;
+ }
+ pred->mark();
+ }
+
+ ins->replaceSuccessor(patches[i].index, join);
+ }
+
+ MOZ_ASSERT_IF(curBlock_, !curBlock_->isMarked());
+ for (uint32_t i = 0; i < join->numPredecessors(); i++) {
+ join->getPredecessor(i)->unmark();
+ }
+
+ if (curBlock_ && !goToExistingBlock(curBlock_, join)) {
+ return false;
+ }
+
+ curBlock_ = join;
+
+ if (!popPushedDefs(defs)) {
+ return false;
+ }
+
+ patches.clear();
+ return true;
+ }
+};
+
+template <>
+MDefinition* FunctionCompiler::unary<MToFloat32>(MDefinition* op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MToFloat32::New(alloc(), op, mustPreserveNaN(op->type()));
+ curBlock_->add(ins);
+ return ins;
+}
+
+template <>
+MDefinition* FunctionCompiler::unary<MWasmBuiltinTruncateToInt32>(
+ MDefinition* op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_,
+ bytecodeOffset());
+ curBlock_->add(ins);
+ return ins;
+}
+
+template <>
+MDefinition* FunctionCompiler::unary<MNot>(MDefinition* op) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MNot::NewInt32(alloc(), op);
+ curBlock_->add(ins);
+ return ins;
+}
+
+template <>
+MDefinition* FunctionCompiler::unary<MAbs>(MDefinition* op, MIRType type) {
+ if (inDeadCode()) {
+ return nullptr;
+ }
+ auto* ins = MAbs::NewWasm(alloc(), op, type);
+ curBlock_->add(ins);
+ return ins;
+}
+
+} // end anonymous namespace
+
+static bool EmitI32Const(FunctionCompiler& f) {
+ int32_t i32;
+ if (!f.iter().readI32Const(&i32)) {
+ return false;
+ }
+
+ f.iter().setResult(f.constantI32(i32));
+ return true;
+}
+
+static bool EmitI64Const(FunctionCompiler& f) {
+ int64_t i64;
+ if (!f.iter().readI64Const(&i64)) {
+ return false;
+ }
+
+ f.iter().setResult(f.constantI64(i64));
+ return true;
+}
+
+static bool EmitF32Const(FunctionCompiler& f) {
+ float f32;
+ if (!f.iter().readF32Const(&f32)) {
+ return false;
+ }
+
+ f.iter().setResult(f.constantF32(f32));
+ return true;
+}
+
+static bool EmitF64Const(FunctionCompiler& f) {
+ double f64;
+ if (!f.iter().readF64Const(&f64)) {
+ return false;
+ }
+
+ f.iter().setResult(f.constantF64(f64));
+ return true;
+}
+
+static bool EmitBlock(FunctionCompiler& f) {
+ ResultType params;
+ return f.iter().readBlock(&params) && f.startBlock();
+}
+
+static bool EmitLoop(FunctionCompiler& f) {
+ ResultType params;
+ if (!f.iter().readLoop(&params)) {
+ return false;
+ }
+
+ MBasicBlock* loopHeader;
+ if (!f.startLoop(&loopHeader, params.length())) {
+ return false;
+ }
+
+ f.addInterruptCheck();
+
+ f.iter().controlItem().setBlock(loopHeader);
+ return true;
+}
+
+static bool EmitIf(FunctionCompiler& f) {
+ ResultType params;
+ MDefinition* condition = nullptr;
+ if (!f.iter().readIf(&params, &condition)) {
+ return false;
+ }
+
+ MBasicBlock* elseBlock;
+ if (!f.branchAndStartThen(condition, &elseBlock)) {
+ return false;
+ }
+
+ f.iter().controlItem().setBlock(elseBlock);
+ return true;
+}
+
+static bool EmitElse(FunctionCompiler& f) {
+ ResultType paramType;
+ ResultType resultType;
+ DefVector thenValues;
+ if (!f.iter().readElse(&paramType, &resultType, &thenValues)) {
+ return false;
+ }
+
+ if (!f.pushDefs(thenValues)) {
+ return false;
+ }
+
+ Control& control = f.iter().controlItem();
+ if (!f.switchToElse(control.block, &control.block)) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool EmitEnd(FunctionCompiler& f) {
+ LabelKind kind;
+ ResultType type;
+ DefVector preJoinDefs;
+ DefVector resultsForEmptyElse;
+ if (!f.iter().readEnd(&kind, &type, &preJoinDefs, &resultsForEmptyElse)) {
+ return false;
+ }
+
+ Control& control = f.iter().controlItem();
+ MBasicBlock* block = control.block;
+
+ if (!f.pushDefs(preJoinDefs)) {
+ return false;
+ }
+
+ // Every label case is responsible to pop the control item at the appropriate
+ // time for the label case
+ DefVector postJoinDefs;
+ switch (kind) {
+ case LabelKind::Body:
+ if (!f.emitBodyDelegateThrowPad(control)) {
+ return false;
+ }
+ if (!f.finishBlock(&postJoinDefs)) {
+ return false;
+ }
+ if (!f.returnValues(postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ MOZ_ASSERT(f.iter().controlStackEmpty());
+ return f.iter().endFunction(f.iter().end());
+ case LabelKind::Block:
+ if (!f.finishBlock(&postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ break;
+ case LabelKind::Loop:
+ if (!f.closeLoop(block, &postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ break;
+ case LabelKind::Then: {
+ // If we didn't see an Else, create a trivial else block so that we create
+ // a diamond anyway, to preserve Ion invariants.
+ if (!f.switchToElse(block, &block)) {
+ return false;
+ }
+
+ if (!f.pushDefs(resultsForEmptyElse)) {
+ return false;
+ }
+
+ if (!f.joinIfElse(block, &postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ break;
+ }
+ case LabelKind::Else:
+ if (!f.joinIfElse(block, &postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ break;
+ case LabelKind::Try:
+ case LabelKind::Catch:
+ case LabelKind::CatchAll:
+ if (!f.finishTryCatch(kind, control, &postJoinDefs)) {
+ return false;
+ }
+ f.iter().popEnd();
+ break;
+ }
+
+ MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == type.length());
+ f.iter().setResults(postJoinDefs.length(), postJoinDefs);
+
+ return true;
+}
+
+static bool EmitBr(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ ResultType type;
+ DefVector values;
+ if (!f.iter().readBr(&relativeDepth, &type, &values)) {
+ return false;
+ }
+
+ return f.br(relativeDepth, values);
+}
+
+static bool EmitBrIf(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ ResultType type;
+ DefVector values;
+ MDefinition* condition;
+ if (!f.iter().readBrIf(&relativeDepth, &type, &values, &condition)) {
+ return false;
+ }
+
+ return f.brIf(relativeDepth, values, condition);
+}
+
+static bool EmitBrTable(FunctionCompiler& f) {
+ Uint32Vector depths;
+ uint32_t defaultDepth;
+ ResultType branchValueType;
+ DefVector branchValues;
+ MDefinition* index;
+ if (!f.iter().readBrTable(&depths, &defaultDepth, &branchValueType,
+ &branchValues, &index)) {
+ return false;
+ }
+
+ // If all the targets are the same, or there are no targets, we can just
+ // use a goto. This is not just an optimization: MaybeFoldConditionBlock
+ // assumes that tables have more than one successor.
+ bool allSameDepth = true;
+ for (uint32_t depth : depths) {
+ if (depth != defaultDepth) {
+ allSameDepth = false;
+ break;
+ }
+ }
+
+ if (allSameDepth) {
+ return f.br(defaultDepth, branchValues);
+ }
+
+ return f.brTable(index, defaultDepth, depths, branchValues);
+}
+
+static bool EmitReturn(FunctionCompiler& f) {
+ DefVector values;
+ if (!f.iter().readReturn(&values)) {
+ return false;
+ }
+
+ return f.returnValues(values);
+}
+
+static bool EmitUnreachable(FunctionCompiler& f) {
+ if (!f.iter().readUnreachable()) {
+ return false;
+ }
+
+ f.unreachableTrap();
+ return true;
+}
+
+static bool EmitTry(FunctionCompiler& f) {
+ ResultType params;
+ if (!f.iter().readTry(&params)) {
+ return false;
+ }
+
+ MBasicBlock* curBlock = nullptr;
+ if (!f.startTry(&curBlock)) {
+ return false;
+ }
+
+ f.iter().controlItem().setBlock(curBlock);
+ return true;
+}
+
+static bool EmitCatch(FunctionCompiler& f) {
+ LabelKind kind;
+ uint32_t tagIndex;
+ ResultType paramType, resultType;
+ DefVector tryValues;
+ if (!f.iter().readCatch(&kind, &tagIndex, &paramType, &resultType,
+ &tryValues)) {
+ return false;
+ }
+
+ // Pushing the results of the previous block, to properly join control flow
+ // after the try and after each handler, as well as potential control flow
+ // patches from other instrunctions. This is similar to what is done for
+ // if-then-else control flow and for most other control control flow joins.
+ if (!f.pushDefs(tryValues)) {
+ return false;
+ }
+
+ return f.switchToCatch(f.iter().controlItem(), kind, tagIndex);
+}
+
+static bool EmitCatchAll(FunctionCompiler& f) {
+ LabelKind kind;
+ ResultType paramType, resultType;
+ DefVector tryValues;
+ if (!f.iter().readCatchAll(&kind, &paramType, &resultType, &tryValues)) {
+ return false;
+ }
+
+ // Pushing the results of the previous block, to properly join control flow
+ // after the try and after each handler, as well as potential control flow
+ // patches from other instrunctions.
+ if (!f.pushDefs(tryValues)) {
+ return false;
+ }
+
+ return f.switchToCatch(f.iter().controlItem(), kind, CatchAllIndex);
+}
+
+static bool EmitDelegate(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ ResultType resultType;
+ DefVector tryValues;
+ if (!f.iter().readDelegate(&relativeDepth, &resultType, &tryValues)) {
+ return false;
+ }
+
+ Control& control = f.iter().controlItem();
+ MBasicBlock* block = control.block;
+
+ // Unless the entire try-delegate is dead code, delegate any pad-patches from
+ // this try to the next try-block above relativeDepth.
+ if (block) {
+ ControlInstructionVector& delegatePadPatches = control.tryPadPatches;
+ if (!f.delegatePadPatches(delegatePadPatches, relativeDepth)) {
+ return false;
+ }
+ }
+ f.iter().popDelegate();
+
+ // Push the results of the previous block, and join control flow with
+ // potential control flow patches from other instrunctions in the try code.
+ // This is similar to what is done for EmitEnd.
+ if (!f.pushDefs(tryValues)) {
+ return false;
+ }
+ DefVector postJoinDefs;
+ if (!f.finishBlock(&postJoinDefs)) {
+ return false;
+ }
+ MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == resultType.length());
+ f.iter().setResults(postJoinDefs.length(), postJoinDefs);
+
+ return true;
+}
+
+static bool EmitThrow(FunctionCompiler& f) {
+ uint32_t tagIndex;
+ DefVector argValues;
+ if (!f.iter().readThrow(&tagIndex, &argValues)) {
+ return false;
+ }
+
+ return f.emitThrow(tagIndex, argValues);
+}
+
+static bool EmitRethrow(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ if (!f.iter().readRethrow(&relativeDepth)) {
+ return false;
+ }
+
+ return f.emitRethrow(relativeDepth);
+}
+
+static bool EmitCallArgs(FunctionCompiler& f, const FuncType& funcType,
+ const DefVector& args, CallCompileState* call) {
+ for (size_t i = 0, n = funcType.args().length(); i < n; ++i) {
+ if (!f.mirGen().ensureBallast()) {
+ return false;
+ }
+ if (!f.passArg(args[i], funcType.args()[i], call)) {
+ return false;
+ }
+ }
+
+ ResultType resultType = ResultType::Vector(funcType.results());
+ if (!f.passStackResultAreaCallArg(resultType, call)) {
+ return false;
+ }
+
+ return f.finishCall(call);
+}
+
+static bool EmitCall(FunctionCompiler& f, bool asmJSFuncDef) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t funcIndex;
+ DefVector args;
+ if (asmJSFuncDef) {
+ if (!f.iter().readOldCallDirect(f.moduleEnv().numFuncImports, &funcIndex,
+ &args)) {
+ return false;
+ }
+ } else {
+ if (!f.iter().readCall(&funcIndex, &args)) {
+ return false;
+ }
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const FuncType& funcType = *f.moduleEnv().funcs[funcIndex].type;
+
+ CallCompileState call;
+ if (!EmitCallArgs(f, funcType, args, &call)) {
+ return false;
+ }
+
+ DefVector results;
+ if (f.moduleEnv().funcIsImport(funcIndex)) {
+ uint32_t instanceDataOffset =
+ f.moduleEnv().offsetOfFuncImportInstanceData(funcIndex);
+ if (!f.callImport(instanceDataOffset, lineOrBytecode, call, funcType,
+ &results)) {
+ return false;
+ }
+ } else {
+ if (!f.callDirect(funcType, funcIndex, lineOrBytecode, call, &results)) {
+ return false;
+ }
+ }
+
+ f.iter().setResults(results.length(), results);
+ return true;
+}
+
+static bool EmitCallIndirect(FunctionCompiler& f, bool oldStyle) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t funcTypeIndex;
+ uint32_t tableIndex;
+ MDefinition* callee;
+ DefVector args;
+ if (oldStyle) {
+ tableIndex = 0;
+ if (!f.iter().readOldCallIndirect(&funcTypeIndex, &callee, &args)) {
+ return false;
+ }
+ } else {
+ if (!f.iter().readCallIndirect(&funcTypeIndex, &tableIndex, &callee,
+ &args)) {
+ return false;
+ }
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const FuncType& funcType = (*f.moduleEnv().types)[funcTypeIndex].funcType();
+
+ CallCompileState call;
+ if (!EmitCallArgs(f, funcType, args, &call)) {
+ return false;
+ }
+
+ DefVector results;
+ if (!f.callIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode, call,
+ &results)) {
+ return false;
+ }
+
+ f.iter().setResults(results.length(), results);
+ return true;
+}
+
+static bool EmitGetLocal(FunctionCompiler& f) {
+ uint32_t id;
+ if (!f.iter().readGetLocal(f.locals(), &id)) {
+ return false;
+ }
+
+ f.iter().setResult(f.getLocalDef(id));
+ return true;
+}
+
+static bool EmitSetLocal(FunctionCompiler& f) {
+ uint32_t id;
+ MDefinition* value;
+ if (!f.iter().readSetLocal(f.locals(), &id, &value)) {
+ return false;
+ }
+
+ f.assign(id, value);
+ return true;
+}
+
+static bool EmitTeeLocal(FunctionCompiler& f) {
+ uint32_t id;
+ MDefinition* value;
+ if (!f.iter().readTeeLocal(f.locals(), &id, &value)) {
+ return false;
+ }
+
+ f.assign(id, value);
+ return true;
+}
+
+static bool EmitGetGlobal(FunctionCompiler& f) {
+ uint32_t id;
+ if (!f.iter().readGetGlobal(&id)) {
+ return false;
+ }
+
+ const GlobalDesc& global = f.moduleEnv().globals[id];
+ if (!global.isConstant()) {
+ f.iter().setResult(f.loadGlobalVar(global.offset(), !global.isMutable(),
+ global.isIndirect(),
+ global.type().toMIRType()));
+ return true;
+ }
+
+ LitVal value = global.constantValue();
+
+ MDefinition* result;
+ switch (value.type().kind()) {
+ case ValType::I32:
+ result = f.constantI32(int32_t(value.i32()));
+ break;
+ case ValType::I64:
+ result = f.constantI64(int64_t(value.i64()));
+ break;
+ case ValType::F32:
+ result = f.constantF32(value.f32());
+ break;
+ case ValType::F64:
+ result = f.constantF64(value.f64());
+ break;
+ case ValType::V128:
+#ifdef ENABLE_WASM_SIMD
+ result = f.constantV128(value.v128());
+ break;
+#else
+ return f.iter().fail("Ion has no SIMD support yet");
+#endif
+ case ValType::Ref:
+ MOZ_ASSERT(value.ref().isNull());
+ result = f.constantNullRef();
+ break;
+ default:
+ MOZ_CRASH("unexpected type in EmitGetGlobal");
+ }
+
+ f.iter().setResult(result);
+ return true;
+}
+
+static bool EmitSetGlobal(FunctionCompiler& f) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ uint32_t id;
+ MDefinition* value;
+ if (!f.iter().readSetGlobal(&id, &value)) {
+ return false;
+ }
+
+ const GlobalDesc& global = f.moduleEnv().globals[id];
+ MOZ_ASSERT(global.isMutable());
+ return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(),
+ value);
+}
+
+static bool EmitTeeGlobal(FunctionCompiler& f) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ uint32_t id;
+ MDefinition* value;
+ if (!f.iter().readTeeGlobal(&id, &value)) {
+ return false;
+ }
+
+ const GlobalDesc& global = f.moduleEnv().globals[id];
+ MOZ_ASSERT(global.isMutable());
+
+ return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(),
+ value);
+}
+
+template <typename MIRClass>
+static bool EmitUnary(FunctionCompiler& f, ValType operandType) {
+ MDefinition* input;
+ if (!f.iter().readUnary(operandType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unary<MIRClass>(input));
+ return true;
+}
+
+template <typename MIRClass>
+static bool EmitConversion(FunctionCompiler& f, ValType operandType,
+ ValType resultType) {
+ MDefinition* input;
+ if (!f.iter().readConversion(operandType, resultType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unary<MIRClass>(input));
+ return true;
+}
+
+template <typename MIRClass>
+static bool EmitUnaryWithType(FunctionCompiler& f, ValType operandType,
+ MIRType mirType) {
+ MDefinition* input;
+ if (!f.iter().readUnary(operandType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unary<MIRClass>(input, mirType));
+ return true;
+}
+
+template <typename MIRClass>
+static bool EmitConversionWithType(FunctionCompiler& f, ValType operandType,
+ ValType resultType, MIRType mirType) {
+ MDefinition* input;
+ if (!f.iter().readConversion(operandType, resultType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unary<MIRClass>(input, mirType));
+ return true;
+}
+
+static bool EmitTruncate(FunctionCompiler& f, ValType operandType,
+ ValType resultType, bool isUnsigned,
+ bool isSaturating) {
+ MDefinition* input = nullptr;
+ if (!f.iter().readConversion(operandType, resultType, &input)) {
+ return false;
+ }
+
+ TruncFlags flags = 0;
+ if (isUnsigned) {
+ flags |= TRUNC_UNSIGNED;
+ }
+ if (isSaturating) {
+ flags |= TRUNC_SATURATING;
+ }
+ if (resultType == ValType::I32) {
+ if (f.moduleEnv().isAsmJS()) {
+ if (input && (input->type() == MIRType::Double ||
+ input->type() == MIRType::Float32)) {
+ f.iter().setResult(f.unary<MWasmBuiltinTruncateToInt32>(input));
+ } else {
+ f.iter().setResult(f.unary<MTruncateToInt32>(input));
+ }
+ } else {
+ f.iter().setResult(f.truncate<MWasmTruncateToInt32>(input, flags));
+ }
+ } else {
+ MOZ_ASSERT(resultType == ValType::I64);
+ MOZ_ASSERT(!f.moduleEnv().isAsmJS());
+#if defined(JS_CODEGEN_ARM)
+ f.iter().setResult(f.truncateWithInstance(input, flags));
+#else
+ f.iter().setResult(f.truncate<MWasmTruncateToInt64>(input, flags));
+#endif
+ }
+ return true;
+}
+
+static bool EmitSignExtend(FunctionCompiler& f, uint32_t srcSize,
+ uint32_t targetSize) {
+ MDefinition* input;
+ ValType type = targetSize == 4 ? ValType::I32 : ValType::I64;
+ if (!f.iter().readConversion(type, type, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.signExtend(input, srcSize, targetSize));
+ return true;
+}
+
+static bool EmitExtendI32(FunctionCompiler& f, bool isUnsigned) {
+ MDefinition* input;
+ if (!f.iter().readConversion(ValType::I32, ValType::I64, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.extendI32(input, isUnsigned));
+ return true;
+}
+
+static bool EmitConvertI64ToFloatingPoint(FunctionCompiler& f,
+ ValType resultType, MIRType mirType,
+ bool isUnsigned) {
+ MDefinition* input;
+ if (!f.iter().readConversion(ValType::I64, resultType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.convertI64ToFloatingPoint(input, mirType, isUnsigned));
+ return true;
+}
+
+static bool EmitReinterpret(FunctionCompiler& f, ValType resultType,
+ ValType operandType, MIRType mirType) {
+ MDefinition* input;
+ if (!f.iter().readConversion(operandType, resultType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unary<MWasmReinterpret>(input, mirType));
+ return true;
+}
+
+static bool EmitAdd(FunctionCompiler& f, ValType type, MIRType mirType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(type, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.add(lhs, rhs, mirType));
+ return true;
+}
+
+static bool EmitSub(FunctionCompiler& f, ValType type, MIRType mirType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(type, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.sub(lhs, rhs, mirType));
+ return true;
+}
+
+static bool EmitRotate(FunctionCompiler& f, ValType type, bool isLeftRotation) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(type, &lhs, &rhs)) {
+ return false;
+ }
+
+ MDefinition* result = f.rotate(lhs, rhs, type.toMIRType(), isLeftRotation);
+ f.iter().setResult(result);
+ return true;
+}
+
+static bool EmitBitNot(FunctionCompiler& f, ValType operandType) {
+ MDefinition* input;
+ if (!f.iter().readUnary(operandType, &input)) {
+ return false;
+ }
+
+ f.iter().setResult(f.bitnot(input));
+ return true;
+}
+
+static bool EmitBitwiseAndOrXor(FunctionCompiler& f, ValType operandType,
+ MIRType mirType,
+ MWasmBinaryBitwise::SubOpcode subOpc) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.binary<MWasmBinaryBitwise>(lhs, rhs, mirType, subOpc));
+ return true;
+}
+
+template <typename MIRClass>
+static bool EmitShift(FunctionCompiler& f, ValType operandType,
+ MIRType mirType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.binary<MIRClass>(lhs, rhs, mirType));
+ return true;
+}
+
+static bool EmitUrsh(FunctionCompiler& f, ValType operandType,
+ MIRType mirType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.ursh(lhs, rhs, mirType));
+ return true;
+}
+
+static bool EmitMul(FunctionCompiler& f, ValType operandType, MIRType mirType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(
+ f.mul(lhs, rhs, mirType,
+ mirType == MIRType::Int32 ? MMul::Integer : MMul::Normal));
+ return true;
+}
+
+static bool EmitDiv(FunctionCompiler& f, ValType operandType, MIRType mirType,
+ bool isUnsigned) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.div(lhs, rhs, mirType, isUnsigned));
+ return true;
+}
+
+static bool EmitRem(FunctionCompiler& f, ValType operandType, MIRType mirType,
+ bool isUnsigned) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.mod(lhs, rhs, mirType, isUnsigned));
+ return true;
+}
+
+static bool EmitMinMax(FunctionCompiler& f, ValType operandType,
+ MIRType mirType, bool isMax) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.minMax(lhs, rhs, mirType, isMax));
+ return true;
+}
+
+static bool EmitCopySign(FunctionCompiler& f, ValType operandType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.binary<MCopySign>(lhs, rhs, operandType.toMIRType()));
+ return true;
+}
+
+static bool EmitComparison(FunctionCompiler& f, ValType operandType,
+ JSOp compareOp, MCompare::CompareType compareType) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readComparison(operandType, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.compare(lhs, rhs, compareOp, compareType));
+ return true;
+}
+
+static bool EmitSelect(FunctionCompiler& f, bool typed) {
+ StackType type;
+ MDefinition* trueValue;
+ MDefinition* falseValue;
+ MDefinition* condition;
+ if (!f.iter().readSelect(typed, &type, &trueValue, &falseValue, &condition)) {
+ return false;
+ }
+
+ f.iter().setResult(f.select(trueValue, falseValue, condition));
+ return true;
+}
+
+static bool EmitLoad(FunctionCompiler& f, ValType type, Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readLoad(type, Scalar::byteSize(viewType), &addr)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ f.bytecodeIfNotAsmJS());
+ auto* ins = f.load(addr.base, &access, type);
+ if (!f.inDeadCode() && !ins) {
+ return false;
+ }
+
+ f.iter().setResult(ins);
+ return true;
+}
+
+static bool EmitStore(FunctionCompiler& f, ValType resultType,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readStore(resultType, Scalar::byteSize(viewType), &addr,
+ &value)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ f.bytecodeIfNotAsmJS());
+
+ f.store(addr.base, &access, value);
+ return true;
+}
+
+static bool EmitTeeStore(FunctionCompiler& f, ValType resultType,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
+ &value)) {
+ return false;
+ }
+
+ MOZ_ASSERT(f.isMem32()); // asm.js opcode
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ f.bytecodeIfNotAsmJS());
+
+ f.store(addr.base, &access, value);
+ return true;
+}
+
+static bool EmitTeeStoreWithCoercion(FunctionCompiler& f, ValType resultType,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
+ &value)) {
+ return false;
+ }
+
+ if (resultType == ValType::F32 && viewType == Scalar::Float64) {
+ value = f.unary<MToDouble>(value);
+ } else if (resultType == ValType::F64 && viewType == Scalar::Float32) {
+ value = f.unary<MToFloat32>(value);
+ } else {
+ MOZ_CRASH("unexpected coerced store");
+ }
+
+ MOZ_ASSERT(f.isMem32()); // asm.js opcode
+ MemoryAccessDesc access(viewType, addr.align, addr.offset,
+ f.bytecodeIfNotAsmJS());
+
+ f.store(addr.base, &access, value);
+ return true;
+}
+
+static bool TryInlineUnaryBuiltin(FunctionCompiler& f, SymbolicAddress callee,
+ MDefinition* input) {
+ if (!input) {
+ return false;
+ }
+
+ MOZ_ASSERT(IsFloatingPointType(input->type()));
+
+ RoundingMode mode;
+ if (!IsRoundingFunction(callee, &mode)) {
+ return false;
+ }
+
+ if (!MNearbyInt::HasAssemblerSupport(mode)) {
+ return false;
+ }
+
+ f.iter().setResult(f.nearbyInt(input, mode));
+ return true;
+}
+
+static bool EmitUnaryMathBuiltinCall(FunctionCompiler& f,
+ const SymbolicAddressSignature& callee) {
+ MOZ_ASSERT(callee.numArgs == 1);
+
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ MDefinition* input;
+ if (!f.iter().readUnary(ValType::fromMIRType(callee.argTypes[0]), &input)) {
+ return false;
+ }
+
+ if (TryInlineUnaryBuiltin(f, callee.identity, input)) {
+ return true;
+ }
+
+ CallCompileState call;
+ if (!f.passArg(input, callee.argTypes[0], &call)) {
+ return false;
+ }
+
+ if (!f.finishCall(&call)) {
+ return false;
+ }
+
+ MDefinition* def;
+ if (!f.builtinCall(callee, lineOrBytecode, call, &def)) {
+ return false;
+ }
+
+ f.iter().setResult(def);
+ return true;
+}
+
+static bool EmitBinaryMathBuiltinCall(FunctionCompiler& f,
+ const SymbolicAddressSignature& callee) {
+ MOZ_ASSERT(callee.numArgs == 2);
+ MOZ_ASSERT(callee.argTypes[0] == callee.argTypes[1]);
+
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ CallCompileState call;
+ MDefinition* lhs;
+ MDefinition* rhs;
+ // This call to readBinary assumes both operands have the same type.
+ if (!f.iter().readBinary(ValType::fromMIRType(callee.argTypes[0]), &lhs,
+ &rhs)) {
+ return false;
+ }
+
+ if (!f.passArg(lhs, callee.argTypes[0], &call)) {
+ return false;
+ }
+
+ if (!f.passArg(rhs, callee.argTypes[1], &call)) {
+ return false;
+ }
+
+ if (!f.finishCall(&call)) {
+ return false;
+ }
+
+ MDefinition* def;
+ if (!f.builtinCall(callee, lineOrBytecode, call, &def)) {
+ return false;
+ }
+
+ f.iter().setResult(def);
+ return true;
+}
+
+static bool EmitMemoryGrow(FunctionCompiler& f) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* delta;
+ if (!f.iter().readMemoryGrow(&delta)) {
+ return false;
+ }
+
+ const SymbolicAddressSignature& callee =
+ f.isNoMemOrMem32() ? SASigMemoryGrowM32 : SASigMemoryGrowM64;
+
+ MDefinition* ret;
+ if (!f.emitInstanceCall1(bytecodeOffset, callee, delta, &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitMemorySize(FunctionCompiler& f) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ if (!f.iter().readMemorySize()) {
+ return false;
+ }
+
+ const SymbolicAddressSignature& callee =
+ f.isNoMemOrMem32() ? SASigMemorySizeM32 : SASigMemorySizeM64;
+
+ MDefinition* ret;
+ if (!f.emitInstanceCall0(bytecodeOffset, callee, &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitAtomicCmpXchg(FunctionCompiler& f, ValType type,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* oldValue;
+ MDefinition* newValue;
+ if (!f.iter().readAtomicCmpXchg(&addr, type, byteSize(viewType), &oldValue,
+ &newValue)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(),
+ Synchronization::Full());
+ auto* ins =
+ f.atomicCompareExchangeHeap(addr.base, &access, type, oldValue, newValue);
+ if (!f.inDeadCode() && !ins) {
+ return false;
+ }
+
+ f.iter().setResult(ins);
+ return true;
+}
+
+static bool EmitAtomicLoad(FunctionCompiler& f, ValType type,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readAtomicLoad(&addr, type, byteSize(viewType))) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(),
+ Synchronization::Load());
+ auto* ins = f.load(addr.base, &access, type);
+ if (!f.inDeadCode() && !ins) {
+ return false;
+ }
+
+ f.iter().setResult(ins);
+ return true;
+}
+
+static bool EmitAtomicRMW(FunctionCompiler& f, ValType type,
+ Scalar::Type viewType, jit::AtomicOp op) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(),
+ Synchronization::Full());
+ auto* ins = f.atomicBinopHeap(op, addr.base, &access, type, value);
+ if (!f.inDeadCode() && !ins) {
+ return false;
+ }
+
+ f.iter().setResult(ins);
+ return true;
+}
+
+static bool EmitAtomicStore(FunctionCompiler& f, ValType type,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readAtomicStore(&addr, type, byteSize(viewType), &value)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(),
+ Synchronization::Store());
+ f.store(addr.base, &access, value);
+ return true;
+}
+
+static bool EmitWait(FunctionCompiler& f, ValType type, uint32_t byteSize) {
+ MOZ_ASSERT(type == ValType::I32 || type == ValType::I64);
+ MOZ_ASSERT(type.size() == byteSize);
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* expected;
+ MDefinition* timeout;
+ if (!f.iter().readWait(&addr, type, byteSize, &expected, &timeout)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(type == ValType::I32 ? Scalar::Int32 : Scalar::Int64,
+ addr.align, addr.offset, f.bytecodeOffset());
+ MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access);
+ if (!f.inDeadCode() && !ptr) {
+ return false;
+ }
+
+ const SymbolicAddressSignature& callee =
+ f.isNoMemOrMem32()
+ ? (type == ValType::I32 ? SASigWaitI32M32 : SASigWaitI64M32)
+ : (type == ValType::I32 ? SASigWaitI32M64 : SASigWaitI64M64);
+ MOZ_ASSERT(type.toMIRType() == callee.argTypes[2]);
+
+ MDefinition* ret;
+ if (!f.emitInstanceCall3(bytecodeOffset, callee, ptr, expected, timeout,
+ &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitFence(FunctionCompiler& f) {
+ if (!f.iter().readFence()) {
+ return false;
+ }
+
+ f.fence();
+ return true;
+}
+
+static bool EmitWake(FunctionCompiler& f) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* count;
+ if (!f.iter().readWake(&addr, &count)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(Scalar::Int32, addr.align, addr.offset,
+ f.bytecodeOffset());
+ MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access);
+ if (!f.inDeadCode() && !ptr) {
+ return false;
+ }
+
+ const SymbolicAddressSignature& callee =
+ f.isNoMemOrMem32() ? SASigWakeM32 : SASigWakeM64;
+
+ MDefinition* ret;
+ if (!f.emitInstanceCall2(bytecodeOffset, callee, ptr, count, &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitAtomicXchg(FunctionCompiler& f, ValType type,
+ Scalar::Type viewType) {
+ LinearMemoryAddress<MDefinition*> addr;
+ MDefinition* value;
+ if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
+ return false;
+ }
+
+ MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(),
+ Synchronization::Full());
+ MDefinition* ins = f.atomicExchangeHeap(addr.base, &access, type, value);
+ if (!f.inDeadCode() && !ins) {
+ return false;
+ }
+
+ f.iter().setResult(ins);
+ return true;
+}
+
+static bool EmitMemCopyCall(FunctionCompiler& f, MDefinition* dst,
+ MDefinition* src, MDefinition* len) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* memoryBase = f.memoryBase();
+ const SymbolicAddressSignature& callee =
+ (f.moduleEnv().usesSharedMemory()
+ ? (f.isMem32() ? SASigMemCopySharedM32 : SASigMemCopySharedM64)
+ : (f.isMem32() ? SASigMemCopyM32 : SASigMemCopyM64));
+
+ return f.emitInstanceCall4(bytecodeOffset, callee, dst, src, len, memoryBase);
+}
+
+static bool EmitMemCopyInline(FunctionCompiler& f, MDefinition* dst,
+ MDefinition* src, uint32_t length) {
+ MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryCopyLength);
+
+ // Compute the number of copies of each width we will need to do
+ size_t remainder = length;
+#ifdef ENABLE_WASM_SIMD
+ size_t numCopies16 = 0;
+ if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
+ numCopies16 = remainder / sizeof(V128);
+ remainder %= sizeof(V128);
+ }
+#endif
+#ifdef JS_64BIT
+ size_t numCopies8 = remainder / sizeof(uint64_t);
+ remainder %= sizeof(uint64_t);
+#endif
+ size_t numCopies4 = remainder / sizeof(uint32_t);
+ remainder %= sizeof(uint32_t);
+ size_t numCopies2 = remainder / sizeof(uint16_t);
+ remainder %= sizeof(uint16_t);
+ size_t numCopies1 = remainder;
+
+ // Load all source bytes from low to high using the widest transfer width we
+ // can for the system. We will trap without writing anything if any source
+ // byte is out-of-bounds.
+ size_t offset = 0;
+ DefVector loadedValues;
+
+#ifdef ENABLE_WASM_SIMD
+ for (uint32_t i = 0; i < numCopies16; i++) {
+ MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset());
+ auto* load = f.load(src, &access, ValType::V128);
+ if (!load || !loadedValues.append(load)) {
+ return false;
+ }
+
+ offset += sizeof(V128);
+ }
+#endif
+
+#ifdef JS_64BIT
+ for (uint32_t i = 0; i < numCopies8; i++) {
+ MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset());
+ auto* load = f.load(src, &access, ValType::I64);
+ if (!load || !loadedValues.append(load)) {
+ return false;
+ }
+
+ offset += sizeof(uint64_t);
+ }
+#endif
+
+ for (uint32_t i = 0; i < numCopies4; i++) {
+ MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset());
+ auto* load = f.load(src, &access, ValType::I32);
+ if (!load || !loadedValues.append(load)) {
+ return false;
+ }
+
+ offset += sizeof(uint32_t);
+ }
+
+ if (numCopies2) {
+ MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset());
+ auto* load = f.load(src, &access, ValType::I32);
+ if (!load || !loadedValues.append(load)) {
+ return false;
+ }
+
+ offset += sizeof(uint16_t);
+ }
+
+ if (numCopies1) {
+ MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset());
+ auto* load = f.load(src, &access, ValType::I32);
+ if (!load || !loadedValues.append(load)) {
+ return false;
+ }
+ }
+
+ // Store all source bytes to the destination from high to low. We will trap
+ // without writing anything on the first store if any dest byte is
+ // out-of-bounds.
+ offset = length;
+
+ if (numCopies1) {
+ offset -= sizeof(uint8_t);
+
+ MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset());
+ auto* value = loadedValues.popCopy();
+ f.store(dst, &access, value);
+ }
+
+ if (numCopies2) {
+ offset -= sizeof(uint16_t);
+
+ MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset());
+ auto* value = loadedValues.popCopy();
+ f.store(dst, &access, value);
+ }
+
+ for (uint32_t i = 0; i < numCopies4; i++) {
+ offset -= sizeof(uint32_t);
+
+ MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset());
+ auto* value = loadedValues.popCopy();
+ f.store(dst, &access, value);
+ }
+
+#ifdef JS_64BIT
+ for (uint32_t i = 0; i < numCopies8; i++) {
+ offset -= sizeof(uint64_t);
+
+ MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset());
+ auto* value = loadedValues.popCopy();
+ f.store(dst, &access, value);
+ }
+#endif
+
+#ifdef ENABLE_WASM_SIMD
+ for (uint32_t i = 0; i < numCopies16; i++) {
+ offset -= sizeof(V128);
+
+ MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset());
+ auto* value = loadedValues.popCopy();
+ f.store(dst, &access, value);
+ }
+#endif
+
+ return true;
+}
+
+static bool EmitMemCopy(FunctionCompiler& f) {
+ MDefinition *dst, *src, *len;
+ uint32_t dstMemIndex;
+ uint32_t srcMemIndex;
+ if (!f.iter().readMemOrTableCopy(true, &dstMemIndex, &dst, &srcMemIndex, &src,
+ &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ if (len->isConstant()) {
+ uint64_t length = f.isMem32() ? len->toConstant()->toInt32()
+ : len->toConstant()->toInt64();
+ static_assert(MaxInlineMemoryCopyLength <= UINT32_MAX);
+ if (length != 0 && length <= MaxInlineMemoryCopyLength) {
+ return EmitMemCopyInline(f, dst, src, uint32_t(length));
+ }
+ }
+
+ return EmitMemCopyCall(f, dst, src, len);
+}
+
+static bool EmitTableCopy(FunctionCompiler& f) {
+ MDefinition *dst, *src, *len;
+ uint32_t dstTableIndex;
+ uint32_t srcTableIndex;
+ if (!f.iter().readMemOrTableCopy(false, &dstTableIndex, &dst, &srcTableIndex,
+ &src, &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+ MDefinition* dti = f.constantI32(int32_t(dstTableIndex));
+ MDefinition* sti = f.constantI32(int32_t(srcTableIndex));
+
+ return f.emitInstanceCall5(bytecodeOffset, SASigTableCopy, dst, src, len, dti,
+ sti);
+}
+
+static bool EmitDataOrElemDrop(FunctionCompiler& f, bool isData) {
+ uint32_t segIndexVal = 0;
+ if (!f.iter().readDataOrElemDrop(isData, &segIndexVal)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* segIndex = f.constantI32(int32_t(segIndexVal));
+
+ const SymbolicAddressSignature& callee =
+ isData ? SASigDataDrop : SASigElemDrop;
+ return f.emitInstanceCall1(bytecodeOffset, callee, segIndex);
+}
+
+static bool EmitMemFillCall(FunctionCompiler& f, MDefinition* start,
+ MDefinition* val, MDefinition* len) {
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* memoryBase = f.memoryBase();
+
+ const SymbolicAddressSignature& callee =
+ (f.moduleEnv().usesSharedMemory()
+ ? (f.isMem32() ? SASigMemFillSharedM32 : SASigMemFillSharedM64)
+ : (f.isMem32() ? SASigMemFillM32 : SASigMemFillM64));
+ return f.emitInstanceCall4(bytecodeOffset, callee, start, val, len,
+ memoryBase);
+}
+
+static bool EmitMemFillInline(FunctionCompiler& f, MDefinition* start,
+ MDefinition* val, uint32_t length) {
+ MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryFillLength);
+ uint32_t value = val->toConstant()->toInt32();
+
+ // Compute the number of copies of each width we will need to do
+ size_t remainder = length;
+#ifdef ENABLE_WASM_SIMD
+ size_t numCopies16 = 0;
+ if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
+ numCopies16 = remainder / sizeof(V128);
+ remainder %= sizeof(V128);
+ }
+#endif
+#ifdef JS_64BIT
+ size_t numCopies8 = remainder / sizeof(uint64_t);
+ remainder %= sizeof(uint64_t);
+#endif
+ size_t numCopies4 = remainder / sizeof(uint32_t);
+ remainder %= sizeof(uint32_t);
+ size_t numCopies2 = remainder / sizeof(uint16_t);
+ remainder %= sizeof(uint16_t);
+ size_t numCopies1 = remainder;
+
+ // Generate splatted definitions for wider fills as needed
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* val16 = numCopies16 ? f.constantV128(V128(value)) : nullptr;
+#endif
+#ifdef JS_64BIT
+ MDefinition* val8 =
+ numCopies8 ? f.constantI64(int64_t(SplatByteToUInt<uint64_t>(value, 8)))
+ : nullptr;
+#endif
+ MDefinition* val4 =
+ numCopies4 ? f.constantI32(int32_t(SplatByteToUInt<uint32_t>(value, 4)))
+ : nullptr;
+ MDefinition* val2 =
+ numCopies2 ? f.constantI32(int32_t(SplatByteToUInt<uint32_t>(value, 2)))
+ : nullptr;
+
+ // Store the fill value to the destination from high to low. We will trap
+ // without writing anything on the first store if any dest byte is
+ // out-of-bounds.
+ size_t offset = length;
+
+ if (numCopies1) {
+ offset -= sizeof(uint8_t);
+
+ MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset());
+ f.store(start, &access, val);
+ }
+
+ if (numCopies2) {
+ offset -= sizeof(uint16_t);
+
+ MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset());
+ f.store(start, &access, val2);
+ }
+
+ for (uint32_t i = 0; i < numCopies4; i++) {
+ offset -= sizeof(uint32_t);
+
+ MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset());
+ f.store(start, &access, val4);
+ }
+
+#ifdef JS_64BIT
+ for (uint32_t i = 0; i < numCopies8; i++) {
+ offset -= sizeof(uint64_t);
+
+ MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset());
+ f.store(start, &access, val8);
+ }
+#endif
+
+#ifdef ENABLE_WASM_SIMD
+ for (uint32_t i = 0; i < numCopies16; i++) {
+ offset -= sizeof(V128);
+
+ MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset());
+ f.store(start, &access, val16);
+ }
+#endif
+
+ return true;
+}
+
+static bool EmitMemFill(FunctionCompiler& f) {
+ MDefinition *start, *val, *len;
+ if (!f.iter().readMemFill(&start, &val, &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ if (len->isConstant() && val->isConstant()) {
+ uint64_t length = f.isMem32() ? len->toConstant()->toInt32()
+ : len->toConstant()->toInt64();
+ static_assert(MaxInlineMemoryFillLength <= UINT32_MAX);
+ if (length != 0 && length <= MaxInlineMemoryFillLength) {
+ return EmitMemFillInline(f, start, val, uint32_t(length));
+ }
+ }
+
+ return EmitMemFillCall(f, start, val, len);
+}
+
+static bool EmitMemOrTableInit(FunctionCompiler& f, bool isMem) {
+ uint32_t segIndexVal = 0, dstTableIndex = 0;
+ MDefinition *dstOff, *srcOff, *len;
+ if (!f.iter().readMemOrTableInit(isMem, &segIndexVal, &dstTableIndex, &dstOff,
+ &srcOff, &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* segIndex = f.constantI32(int32_t(segIndexVal));
+
+ if (isMem) {
+ const SymbolicAddressSignature& callee =
+ f.isMem32() ? SASigMemInitM32 : SASigMemInitM64;
+ return f.emitInstanceCall4(bytecodeOffset, callee, dstOff, srcOff, len,
+ segIndex);
+ }
+
+ MDefinition* dti = f.constantI32(int32_t(dstTableIndex));
+ return f.emitInstanceCall5(bytecodeOffset, SASigTableInit, dstOff, srcOff,
+ len, segIndex, dti);
+}
+
+// Note, table.{get,grow,set} on table(funcref) are currently rejected by the
+// verifier.
+
+static bool EmitTableFill(FunctionCompiler& f) {
+ uint32_t tableIndex;
+ MDefinition *start, *val, *len;
+ if (!f.iter().readTableFill(&tableIndex, &start, &val, &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
+ if (!tableIndexArg) {
+ return false;
+ }
+
+ return f.emitInstanceCall4(bytecodeOffset, SASigTableFill, start, val, len,
+ tableIndexArg);
+}
+
+#if ENABLE_WASM_MEMORY_CONTROL
+static bool EmitMemDiscard(FunctionCompiler& f) {
+ MDefinition *start, *len;
+ if (!f.iter().readMemDiscard(&start, &len)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* memoryBase = f.memoryBase();
+
+ const SymbolicAddressSignature& callee =
+ (f.moduleEnv().usesSharedMemory()
+ ? (f.isMem32() ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64)
+ : (f.isMem32() ? SASigMemDiscardM32 : SASigMemDiscardM64));
+ return f.emitInstanceCall3(bytecodeOffset, callee, start, len, memoryBase);
+}
+#endif
+
+static bool EmitTableGet(FunctionCompiler& f) {
+ uint32_t tableIndex;
+ MDefinition* index;
+ if (!f.iter().readTableGet(&tableIndex, &index)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const TableDesc& table = f.moduleEnv().tables[tableIndex];
+ if (table.elemType.tableRepr() == TableRepr::Ref) {
+ MDefinition* ret = f.tableGetAnyRef(tableIndex, index);
+ if (!ret) {
+ return false;
+ }
+ f.iter().setResult(ret);
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
+ if (!tableIndexArg) {
+ return false;
+ }
+
+ // The return value here is either null, denoting an error, or a short-lived
+ // pointer to a location containing a possibly-null ref.
+ MDefinition* ret;
+ if (!f.emitInstanceCall2(bytecodeOffset, SASigTableGet, index, tableIndexArg,
+ &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitTableGrow(FunctionCompiler& f) {
+ uint32_t tableIndex;
+ MDefinition* initValue;
+ MDefinition* delta;
+ if (!f.iter().readTableGrow(&tableIndex, &initValue, &delta)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
+ if (!tableIndexArg) {
+ return false;
+ }
+
+ MDefinition* ret;
+ if (!f.emitInstanceCall3(bytecodeOffset, SASigTableGrow, initValue, delta,
+ tableIndexArg, &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitTableSet(FunctionCompiler& f) {
+ uint32_t tableIndex;
+ MDefinition* index;
+ MDefinition* value;
+ if (!f.iter().readTableSet(&tableIndex, &index, &value)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ const TableDesc& table = f.moduleEnv().tables[tableIndex];
+ if (table.elemType.tableRepr() == TableRepr::Ref) {
+ return f.tableSetAnyRef(tableIndex, index, value, bytecodeOffset);
+ }
+
+ MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
+ if (!tableIndexArg) {
+ return false;
+ }
+
+ return f.emitInstanceCall3(bytecodeOffset, SASigTableSet, index, value,
+ tableIndexArg);
+}
+
+static bool EmitTableSize(FunctionCompiler& f) {
+ uint32_t tableIndex;
+ if (!f.iter().readTableSize(&tableIndex)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MDefinition* length = f.loadTableLength(tableIndex);
+ if (!length) {
+ return false;
+ }
+
+ f.iter().setResult(length);
+ return true;
+}
+
+static bool EmitRefFunc(FunctionCompiler& f) {
+ uint32_t funcIndex;
+ if (!f.iter().readRefFunc(&funcIndex)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+
+ MDefinition* funcIndexArg = f.constantI32(int32_t(funcIndex));
+ if (!funcIndexArg) {
+ return false;
+ }
+
+ // The return value here is either null, denoting an error, or a short-lived
+ // pointer to a location containing a possibly-null ref.
+ MDefinition* ret;
+ if (!f.emitInstanceCall1(bytecodeOffset, SASigRefFunc, funcIndexArg, &ret)) {
+ return false;
+ }
+
+ f.iter().setResult(ret);
+ return true;
+}
+
+static bool EmitRefNull(FunctionCompiler& f) {
+ RefType type;
+ if (!f.iter().readRefNull(&type)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MDefinition* nullVal = f.constantNullRef();
+ if (!nullVal) {
+ return false;
+ }
+ f.iter().setResult(nullVal);
+ return true;
+}
+
+static bool EmitRefIsNull(FunctionCompiler& f) {
+ MDefinition* input;
+ if (!f.iter().readRefIsNull(&input)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MDefinition* nullVal = f.constantNullRef();
+ if (!nullVal) {
+ return false;
+ }
+ f.iter().setResult(
+ f.compare(input, nullVal, JSOp::Eq, MCompare::Compare_RefOrNull));
+ return true;
+}
+
+#ifdef ENABLE_WASM_SIMD
+static bool EmitConstSimd128(FunctionCompiler& f) {
+ V128 v128;
+ if (!f.iter().readV128Const(&v128)) {
+ return false;
+ }
+
+ f.iter().setResult(f.constantV128(v128));
+ return true;
+}
+
+static bool EmitBinarySimd128(FunctionCompiler& f, bool commutative,
+ SimdOp op) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readBinary(ValType::V128, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.binarySimd128(lhs, rhs, commutative, op));
+ return true;
+}
+
+static bool EmitTernarySimd128(FunctionCompiler& f, wasm::SimdOp op) {
+ MDefinition* v0;
+ MDefinition* v1;
+ MDefinition* v2;
+ if (!f.iter().readTernary(ValType::V128, &v0, &v1, &v2)) {
+ return false;
+ }
+
+ f.iter().setResult(f.ternarySimd128(v0, v1, v2, op));
+ return true;
+}
+
+static bool EmitShiftSimd128(FunctionCompiler& f, SimdOp op) {
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readVectorShift(&lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.shiftSimd128(lhs, rhs, op));
+ return true;
+}
+
+static bool EmitSplatSimd128(FunctionCompiler& f, ValType inType, SimdOp op) {
+ MDefinition* src;
+ if (!f.iter().readConversion(inType, ValType::V128, &src)) {
+ return false;
+ }
+
+ f.iter().setResult(f.scalarToSimd128(src, op));
+ return true;
+}
+
+static bool EmitUnarySimd128(FunctionCompiler& f, SimdOp op) {
+ MDefinition* src;
+ if (!f.iter().readUnary(ValType::V128, &src)) {
+ return false;
+ }
+
+ f.iter().setResult(f.unarySimd128(src, op));
+ return true;
+}
+
+static bool EmitReduceSimd128(FunctionCompiler& f, SimdOp op) {
+ MDefinition* src;
+ if (!f.iter().readConversion(ValType::V128, ValType::I32, &src)) {
+ return false;
+ }
+
+ f.iter().setResult(f.reduceSimd128(src, op, ValType::I32));
+ return true;
+}
+
+static bool EmitExtractLaneSimd128(FunctionCompiler& f, ValType outType,
+ uint32_t laneLimit, SimdOp op) {
+ uint32_t laneIndex;
+ MDefinition* src;
+ if (!f.iter().readExtractLane(outType, laneLimit, &laneIndex, &src)) {
+ return false;
+ }
+
+ f.iter().setResult(f.reduceSimd128(src, op, outType, laneIndex));
+ return true;
+}
+
+static bool EmitReplaceLaneSimd128(FunctionCompiler& f, ValType laneType,
+ uint32_t laneLimit, SimdOp op) {
+ uint32_t laneIndex;
+ MDefinition* lhs;
+ MDefinition* rhs;
+ if (!f.iter().readReplaceLane(laneType, laneLimit, &laneIndex, &lhs, &rhs)) {
+ return false;
+ }
+
+ f.iter().setResult(f.replaceLaneSimd128(lhs, rhs, laneIndex, op));
+ return true;
+}
+
+static bool EmitShuffleSimd128(FunctionCompiler& f) {
+ MDefinition* v1;
+ MDefinition* v2;
+ V128 control;
+ if (!f.iter().readVectorShuffle(&v1, &v2, &control)) {
+ return false;
+ }
+
+ f.iter().setResult(f.shuffleSimd128(v1, v2, control));
+ return true;
+}
+
+static bool EmitLoadSplatSimd128(FunctionCompiler& f, Scalar::Type viewType,
+ wasm::SimdOp splatOp) {
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readLoadSplat(Scalar::byteSize(viewType), &addr)) {
+ return false;
+ }
+
+ f.iter().setResult(f.loadSplatSimd128(viewType, addr, splatOp));
+ return true;
+}
+
+static bool EmitLoadExtendSimd128(FunctionCompiler& f, wasm::SimdOp op) {
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readLoadExtend(&addr)) {
+ return false;
+ }
+
+ f.iter().setResult(f.loadExtendSimd128(addr, op));
+ return true;
+}
+
+static bool EmitLoadZeroSimd128(FunctionCompiler& f, Scalar::Type viewType,
+ size_t numBytes) {
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readLoadSplat(numBytes, &addr)) {
+ return false;
+ }
+
+ f.iter().setResult(f.loadZeroSimd128(viewType, numBytes, addr));
+ return true;
+}
+
+static bool EmitLoadLaneSimd128(FunctionCompiler& f, uint32_t laneSize) {
+ uint32_t laneIndex;
+ MDefinition* src;
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readLoadLane(laneSize, &addr, &laneIndex, &src)) {
+ return false;
+ }
+
+ f.iter().setResult(f.loadLaneSimd128(laneSize, addr, laneIndex, src));
+ return true;
+}
+
+static bool EmitStoreLaneSimd128(FunctionCompiler& f, uint32_t laneSize) {
+ uint32_t laneIndex;
+ MDefinition* src;
+ LinearMemoryAddress<MDefinition*> addr;
+ if (!f.iter().readStoreLane(laneSize, &addr, &laneIndex, &src)) {
+ return false;
+ }
+
+ f.storeLaneSimd128(laneSize, addr, laneIndex, src);
+ return true;
+}
+
+#endif // ENABLE_WASM_SIMD
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+static bool EmitRefAsNonNull(FunctionCompiler& f) {
+ MDefinition* value;
+ if (!f.iter().readRefAsNonNull(&value)) {
+ return false;
+ }
+
+ return f.refAsNonNull(value);
+}
+
+static bool EmitBrOnNull(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ ResultType type;
+ DefVector values;
+ MDefinition* condition;
+ if (!f.iter().readBrOnNull(&relativeDepth, &type, &values, &condition)) {
+ return false;
+ }
+
+ return f.brOnNull(relativeDepth, values, type, condition);
+}
+
+static bool EmitBrOnNonNull(FunctionCompiler& f) {
+ uint32_t relativeDepth;
+ ResultType type;
+ DefVector values;
+ MDefinition* condition;
+ if (!f.iter().readBrOnNonNull(&relativeDepth, &type, &values, &condition)) {
+ return false;
+ }
+
+ return f.brOnNonNull(relativeDepth, values, type, condition);
+}
+
+static bool EmitCallRef(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ const FuncType* funcType;
+ MDefinition* callee;
+ DefVector args;
+
+ if (!f.iter().readCallRef(&funcType, &callee, &args)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ CallCompileState call;
+ if (!EmitCallArgs(f, *funcType, args, &call)) {
+ return false;
+ }
+
+ DefVector results;
+ if (!f.callRef(*funcType, callee, lineOrBytecode, call, &results)) {
+ return false;
+ }
+
+ f.iter().setResults(results.length(), results);
+ return true;
+}
+
+#endif // ENABLE_WASM_FUNCTION_REFERENCES
+
+#ifdef ENABLE_WASM_GC
+
+static bool EmitStructNew(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ DefVector args;
+ if (!f.iter().readStructNew(&typeIndex, &args)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
+ MOZ_ASSERT(args.length() == structType.fields_.length());
+
+ // Allocate a default initialized struct. This requires the type definition
+ // for the struct.
+ MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return false;
+ }
+
+ // Create call: structObject = Instance::structNewUninit(typeDefData)
+ MDefinition* structObject;
+ if (!f.emitInstanceCall1(lineOrBytecode, SASigStructNewUninit, typeDefData,
+ &structObject)) {
+ return false;
+ }
+
+ // And fill in the fields.
+ for (uint32_t fieldIndex = 0; fieldIndex < structType.fields_.length();
+ fieldIndex++) {
+ if (!f.mirGen().ensureBallast()) {
+ return false;
+ }
+ const StructField& field = structType.fields_[fieldIndex];
+ if (!f.writeValueToStructField(lineOrBytecode, field, structObject,
+ args[fieldIndex],
+ WasmPreBarrierKind::None)) {
+ return false;
+ }
+ }
+
+ f.iter().setResult(structObject);
+ return true;
+}
+
+static bool EmitStructNewDefault(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ if (!f.iter().readStructNewDefault(&typeIndex)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Allocate a default initialized struct. This requires the type definition
+ // for the struct.
+ MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return false;
+ }
+
+ // Create call: structObject = Instance::structNew(typeDefData)
+ MDefinition* structObject;
+ if (!f.emitInstanceCall1(lineOrBytecode, SASigStructNew, typeDefData,
+ &structObject)) {
+ return false;
+ }
+
+ f.iter().setResult(structObject);
+ return true;
+}
+
+static bool EmitStructSet(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ uint32_t fieldIndex;
+ MDefinition* structObject;
+ MDefinition* value;
+ if (!f.iter().readStructSet(&typeIndex, &fieldIndex, &structObject, &value)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Check for null is done at writeValueToStructField.
+
+ // And fill in the field.
+ const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
+ const StructField& field = structType.fields_[fieldIndex];
+ return f.writeValueToStructField(lineOrBytecode, field, structObject, value,
+ WasmPreBarrierKind::Normal);
+}
+
+static bool EmitStructGet(FunctionCompiler& f, FieldWideningOp wideningOp) {
+ uint32_t typeIndex;
+ uint32_t fieldIndex;
+ MDefinition* structObject;
+ if (!f.iter().readStructGet(&typeIndex, &fieldIndex, wideningOp,
+ &structObject)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Check for null is done at readValueFromStructField.
+
+ // And fetch the data.
+ const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
+ const StructField& field = structType.fields_[fieldIndex];
+ MDefinition* load =
+ f.readValueFromStructField(field, wideningOp, structObject);
+ if (!load) {
+ return false;
+ }
+
+ f.iter().setResult(load);
+ return true;
+}
+
+static bool EmitArrayNew(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ MDefinition* numElements;
+ MDefinition* fillValue;
+ if (!f.iter().readArrayNew(&typeIndex, &numElements, &fillValue)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
+ // this helper will trap.
+ MDefinition* arrayObject = f.createArrayNewCallAndLoop(
+ lineOrBytecode, typeIndex, numElements, fillValue);
+ if (!arrayObject) {
+ return false;
+ }
+
+ f.iter().setResult(arrayObject);
+ return true;
+}
+
+static bool EmitArrayNewDefault(FunctionCompiler& f) {
+ // This is almost identical to EmitArrayNew, except we skip the
+ // initialisation loop.
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ MDefinition* numElements;
+ if (!f.iter().readArrayNewDefault(&typeIndex, &numElements)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Create the array object, default-initialized.
+ MDefinition* arrayObject = f.createDefaultInitializedArrayObject(
+ lineOrBytecode, typeIndex, numElements);
+ if (!arrayObject) {
+ return false;
+ }
+
+ f.iter().setResult(arrayObject);
+ return true;
+}
+
+static bool EmitArrayNewFixed(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex, numElements;
+ DefVector values;
+
+ if (!f.iter().readArrayNewFixed(&typeIndex, &numElements, &values)) {
+ return false;
+ }
+ MOZ_ASSERT(values.length() == numElements);
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MDefinition* numElementsDef = f.constantI32(int32_t(numElements));
+ if (!numElementsDef) {
+ return false;
+ }
+
+ // Create the array object, default-initialized.
+ MDefinition* arrayObject = f.createDefaultInitializedArrayObject(
+ lineOrBytecode, typeIndex, numElementsDef);
+ if (!arrayObject) {
+ return false;
+ }
+
+ // Make `base` point at the first byte of the (OOL) data area.
+ MDefinition* base = f.getWasmArrayObjectData(arrayObject);
+ if (!base) {
+ return false;
+ }
+
+ // Write each element in turn.
+ const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
+ FieldType elemFieldType = arrayType.elementType_;
+ uint32_t elemSize = elemFieldType.size();
+
+ // How do we know that the offset expression `i * elemSize` below remains
+ // within 2^31 (signed-i32) range? In the worst case we will have 16-byte
+ // values, and there can be at most MaxFunctionBytes expressions, if it were
+ // theoretically possible to generate one expression per instruction byte.
+ // Hence the max offset we can be expected to generate is
+ // `16 * MaxFunctionBytes`.
+ static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <=
+ MaxArrayPayloadBytes);
+ MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes);
+
+ for (uint32_t i = 0; i < numElements; i++) {
+ if (!f.mirGen().ensureBallast()) {
+ return false;
+ }
+ // `i * elemSize` is made safe by the assertions above.
+ if (!f.writeGcValueAtBasePlusOffset(
+ lineOrBytecode, elemFieldType, arrayObject,
+ AliasSet::WasmArrayDataArea, values[numElements - 1 - i], base,
+ i * elemSize, false, WasmPreBarrierKind::None)) {
+ return false;
+ }
+ }
+
+ f.iter().setResult(arrayObject);
+ return true;
+}
+
+static bool EmitArrayNewData(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex, segIndex;
+ MDefinition* segByteOffset;
+ MDefinition* numElements;
+ if (!f.iter().readArrayNewData(&typeIndex, &segIndex, &segByteOffset,
+ &numElements)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Get the type definition data for the array as a whole.
+ MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return false;
+ }
+
+ // Other values we need to pass to the instance call:
+ MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
+ if (!segIndexM) {
+ return false;
+ }
+
+ // Create call:
+ // arrayObject = Instance::arrayNewData(segByteOffset:u32, numElements:u32,
+ // typeDefData:word, segIndex:u32)
+ // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
+ // this call will trap.
+ MDefinition* arrayObject;
+ if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewData, segByteOffset,
+ numElements, typeDefData, segIndexM, &arrayObject)) {
+ return false;
+ }
+
+ f.iter().setResult(arrayObject);
+ return true;
+}
+
+static bool EmitArrayNewElem(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex, segIndex;
+ MDefinition* segElemIndex;
+ MDefinition* numElements;
+ if (!f.iter().readArrayNewElem(&typeIndex, &segIndex, &segElemIndex,
+ &numElements)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Get the type definition for the array as a whole.
+ // Get the type definition data for the array as a whole.
+ MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
+ if (!typeDefData) {
+ return false;
+ }
+
+ // Other values we need to pass to the instance call:
+ MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
+ if (!segIndexM) {
+ return false;
+ }
+
+ // Create call:
+ // arrayObject = Instance::arrayNewElem(segElemIndex:u32, numElements:u32,
+ // typeDefData:word, segIndex:u32)
+ // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
+ // this call will trap.
+ MDefinition* arrayObject;
+ if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewElem, segElemIndex,
+ numElements, typeDefData, segIndexM, &arrayObject)) {
+ return false;
+ }
+
+ f.iter().setResult(arrayObject);
+ return true;
+}
+
+static bool EmitArraySet(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ uint32_t typeIndex;
+ MDefinition* value;
+ MDefinition* index;
+ MDefinition* arrayObject;
+ if (!f.iter().readArraySet(&typeIndex, &value, &index, &arrayObject)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Check for null is done at setupForArrayAccess.
+
+ // Create the object null check and the array bounds check and get the OOL
+ // data pointer.
+ MDefinition* base = f.setupForArrayAccess(arrayObject, index);
+ if (!base) {
+ return false;
+ }
+
+ // And do the store.
+ const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
+ FieldType elemFieldType = arrayType.elementType_;
+ uint32_t elemSize = elemFieldType.size();
+ MOZ_ASSERT(elemSize >= 1 && elemSize <= 16);
+
+ return f.writeGcValueAtBasePlusScaledIndex(
+ lineOrBytecode, elemFieldType, arrayObject, AliasSet::WasmArrayDataArea,
+ value, base, elemSize, index, WasmPreBarrierKind::Normal);
+}
+
+static bool EmitArrayGet(FunctionCompiler& f, FieldWideningOp wideningOp) {
+ uint32_t typeIndex;
+ MDefinition* index;
+ MDefinition* arrayObject;
+ if (!f.iter().readArrayGet(&typeIndex, wideningOp, &index, &arrayObject)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Check for null is done at setupForArrayAccess.
+
+ // Create the object null check and the array bounds check and get the OOL
+ // data pointer.
+ MDefinition* base = f.setupForArrayAccess(arrayObject, index);
+ if (!base) {
+ return false;
+ }
+
+ // And do the load.
+ const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
+ FieldType elemFieldType = arrayType.elementType_;
+ uint32_t elemSize = elemFieldType.size();
+ MOZ_ASSERT(elemSize >= 1 && elemSize <= 16);
+
+ MDefinition* load = f.readGcValueAtBasePlusScaledIndex(
+ elemFieldType, wideningOp, arrayObject, AliasSet::WasmArrayDataArea, base,
+ elemSize, index);
+ if (!load) {
+ return false;
+ }
+
+ f.iter().setResult(load);
+ return true;
+}
+
+static bool EmitArrayLen(FunctionCompiler& f, bool decodeIgnoredTypeIndex) {
+ MDefinition* arrayObject;
+ if (!f.iter().readArrayLen(decodeIgnoredTypeIndex, &arrayObject)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ // Check for null is done at getWasmArrayObjectNumElements.
+
+ // Get the size value for the array
+ MDefinition* numElements = f.getWasmArrayObjectNumElements(arrayObject);
+ if (!numElements) {
+ return false;
+ }
+
+ f.iter().setResult(numElements);
+ return true;
+}
+
+static bool EmitArrayCopy(FunctionCompiler& f) {
+ uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
+
+ int32_t elemSize;
+ bool elemsAreRefTyped;
+ MDefinition* dstArrayObject;
+ MDefinition* dstArrayIndex;
+ MDefinition* srcArrayObject;
+ MDefinition* srcArrayIndex;
+ MDefinition* numElements;
+ if (!f.iter().readArrayCopy(&elemSize, &elemsAreRefTyped, &dstArrayObject,
+ &dstArrayIndex, &srcArrayObject, &srcArrayIndex,
+ &numElements)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MOZ_ASSERT_IF(elemsAreRefTyped,
+ size_t(elemSize) == MIRTypeToSize(TargetWordMIRType()));
+ MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 ||
+ elemSize == 4 || elemSize == 8 ||
+ elemSize == 16);
+
+ // A negative element size is used to inform Instance::arrayCopy that the
+ // values are reftyped. This avoids having to pass it an extra boolean
+ // argument.
+ MDefinition* elemSizeDef =
+ f.constantI32(elemsAreRefTyped ? -elemSize : elemSize);
+ if (!elemSizeDef) {
+ return false;
+ }
+
+ // Create call:
+ // Instance::arrayCopy(dstArrayObject:word, dstArrayIndex:u32,
+ // srcArrayObject:word, srcArrayIndex:u32,
+ // numElements:u32,
+ // (elemsAreRefTyped ? -elemSize : elemSize):u32))
+ return f.emitInstanceCall6(lineOrBytecode, SASigArrayCopy, dstArrayObject,
+ dstArrayIndex, srcArrayObject, srcArrayIndex,
+ numElements, elemSizeDef);
+}
+
+static bool EmitRefTestV5(FunctionCompiler& f) {
+ MDefinition* ref;
+ RefType sourceType;
+ uint32_t typeIndex;
+ if (!f.iter().readRefTestV5(&sourceType, &typeIndex, &ref)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const TypeDef& typeDef = f.moduleEnv().types->type(typeIndex);
+ RefType destType = RefType::fromTypeDef(&typeDef, false);
+ MDefinition* success = f.refTest(ref, sourceType, destType);
+ if (!success) {
+ return false;
+ }
+
+ f.iter().setResult(success);
+ return true;
+}
+
+static bool EmitRefCastV5(FunctionCompiler& f) {
+ MDefinition* ref;
+ RefType sourceType;
+ uint32_t typeIndex;
+ if (!f.iter().readRefCastV5(&sourceType, &typeIndex, &ref)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ const TypeDef& typeDef = f.moduleEnv().types->type(typeIndex);
+ RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/true);
+ if (!f.refCast(ref, sourceType, destType)) {
+ return false;
+ }
+
+ f.iter().setResult(ref);
+ return true;
+}
+
+static bool EmitRefTest(FunctionCompiler& f, bool nullable) {
+ MDefinition* ref;
+ RefType sourceType;
+ RefType destType;
+ if (!f.iter().readRefTest(nullable, &sourceType, &destType, &ref)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ MDefinition* success = f.refTest(ref, sourceType, destType);
+ if (!success) {
+ return false;
+ }
+
+ f.iter().setResult(success);
+ return true;
+}
+
+static bool EmitRefCast(FunctionCompiler& f, bool nullable) {
+ MDefinition* ref;
+ RefType sourceType;
+ RefType destType;
+ if (!f.iter().readRefCast(nullable, &sourceType, &destType, &ref)) {
+ return false;
+ }
+
+ if (f.inDeadCode()) {
+ return true;
+ }
+
+ if (!f.refCast(ref, sourceType, destType)) {
+ return false;
+ }
+
+ f.iter().setResult(ref);
+ return true;
+}
+
+static bool EmitBrOnCast(FunctionCompiler& f) {
+ bool onSuccess;
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ RefType destType;
+ ResultType labelType;
+ DefVector values;
+ if (!f.iter().readBrOnCast(&onSuccess, &labelRelativeDepth, &sourceType,
+ &destType, &labelType, &values)) {
+ return false;
+ }
+
+ return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType,
+ labelType, values);
+}
+
+static bool EmitBrOnCastCommonV5(FunctionCompiler& f, bool onSuccess) {
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ uint32_t castTypeIndex;
+ ResultType labelType;
+ DefVector values;
+ if (onSuccess
+ ? !f.iter().readBrOnCastV5(&labelRelativeDepth, &sourceType,
+ &castTypeIndex, &labelType, &values)
+ : !f.iter().readBrOnCastFailV5(&labelRelativeDepth, &sourceType,
+ &castTypeIndex, &labelType, &values)) {
+ return false;
+ }
+
+ const TypeDef& typeDef = f.moduleEnv().types->type(castTypeIndex);
+ RefType type = RefType::fromTypeDef(&typeDef, false);
+ return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, type,
+ labelType, values);
+}
+
+static bool EmitBrOnCastHeapV5(FunctionCompiler& f, bool onSuccess,
+ bool nullable) {
+ uint32_t labelRelativeDepth;
+ RefType sourceType;
+ RefType destType;
+ ResultType labelType;
+ DefVector values;
+ if (onSuccess ? !f.iter().readBrOnCastHeapV5(nullable, &labelRelativeDepth,
+ &sourceType, &destType,
+ &labelType, &values)
+ : !f.iter().readBrOnCastFailHeapV5(
+ nullable, &labelRelativeDepth, &sourceType, &destType,
+ &labelType, &values)) {
+ return false;
+ }
+
+ return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType,
+ labelType, values);
+}
+
+static bool EmitRefAsStructV5(FunctionCompiler& f) {
+ MDefinition* value;
+ if (!f.iter().readConversion(ValType(RefType::any()),
+ ValType(RefType::struct_().asNonNullable()),
+ &value)) {
+ return false;
+ }
+ f.iter().setResult(value);
+ return true;
+}
+
+static bool EmitBrOnNonStructV5(FunctionCompiler& f) {
+ uint32_t labelRelativeDepth;
+ ResultType labelType;
+ DefVector values;
+ if (!f.iter().readBrOnNonStructV5(&labelRelativeDepth, &labelType, &values)) {
+ return false;
+ }
+ return f.brOnNonStruct(values);
+}
+
+static bool EmitExternInternalize(FunctionCompiler& f) {
+ // extern.internalize is a no-op because anyref and extern share the same
+ // representation
+ MDefinition* ref;
+ if (!f.iter().readRefConversion(RefType::extern_(), RefType::any(), &ref)) {
+ return false;
+ }
+
+ f.iter().setResult(ref);
+ return true;
+}
+
+static bool EmitExternExternalize(FunctionCompiler& f) {
+ // extern.externalize is a no-op because anyref and extern share the same
+ // representation
+ MDefinition* ref;
+ if (!f.iter().readRefConversion(RefType::any(), RefType::extern_(), &ref)) {
+ return false;
+ }
+
+ f.iter().setResult(ref);
+ return true;
+}
+
+#endif // ENABLE_WASM_GC
+
+static bool EmitIntrinsic(FunctionCompiler& f) {
+ // It's almost possible to use FunctionCompiler::emitInstanceCallN here.
+ // Unfortunately not currently possible though, since ::emitInstanceCallN
+ // expects an array of arguments along with a size, and that's not what is
+ // available here. It would be possible if we were prepared to copy
+ // `intrinsic->params` into a fixed-sized (16 element?) array, add
+ // `memoryBase`, and make the call.
+ const Intrinsic* intrinsic;
+
+ DefVector params;
+ if (!f.iter().readIntrinsic(&intrinsic, &params)) {
+ return false;
+ }
+
+ uint32_t bytecodeOffset = f.readBytecodeOffset();
+ const SymbolicAddressSignature& callee = intrinsic->signature;
+
+ CallCompileState args;
+ if (!f.passInstance(callee.argTypes[0], &args)) {
+ return false;
+ }
+
+ if (!f.passArgs(params, intrinsic->params, &args)) {
+ return false;
+ }
+
+ MDefinition* memoryBase = f.memoryBase();
+ if (!f.passArg(memoryBase, MIRType::Pointer, &args)) {
+ return false;
+ }
+
+ if (!f.finishCall(&args)) {
+ return false;
+ }
+
+ return f.builtinInstanceMethodCall(callee, bytecodeOffset, args);
+}
+
+static bool EmitBodyExprs(FunctionCompiler& f) {
+ if (!f.iter().startFunction(f.funcIndex(), f.locals())) {
+ return false;
+ }
+
+#define CHECK(c) \
+ if (!(c)) return false; \
+ break
+
+ while (true) {
+ if (!f.mirGen().ensureBallast()) {
+ return false;
+ }
+
+ OpBytes op;
+ if (!f.iter().readOp(&op)) {
+ return false;
+ }
+
+ switch (op.b0) {
+ case uint16_t(Op::End):
+ if (!EmitEnd(f)) {
+ return false;
+ }
+ if (f.iter().controlStackEmpty()) {
+ return true;
+ }
+ break;
+
+ // Control opcodes
+ case uint16_t(Op::Unreachable):
+ CHECK(EmitUnreachable(f));
+ case uint16_t(Op::Nop):
+ CHECK(f.iter().readNop());
+ case uint16_t(Op::Block):
+ CHECK(EmitBlock(f));
+ case uint16_t(Op::Loop):
+ CHECK(EmitLoop(f));
+ case uint16_t(Op::If):
+ CHECK(EmitIf(f));
+ case uint16_t(Op::Else):
+ CHECK(EmitElse(f));
+ case uint16_t(Op::Try):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitTry(f));
+ case uint16_t(Op::Catch):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitCatch(f));
+ case uint16_t(Op::CatchAll):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitCatchAll(f));
+ case uint16_t(Op::Delegate):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ if (!EmitDelegate(f)) {
+ return false;
+ }
+ break;
+ case uint16_t(Op::Throw):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitThrow(f));
+ case uint16_t(Op::Rethrow):
+ if (!f.moduleEnv().exceptionsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitRethrow(f));
+ case uint16_t(Op::Br):
+ CHECK(EmitBr(f));
+ case uint16_t(Op::BrIf):
+ CHECK(EmitBrIf(f));
+ case uint16_t(Op::BrTable):
+ CHECK(EmitBrTable(f));
+ case uint16_t(Op::Return):
+ CHECK(EmitReturn(f));
+
+ // Calls
+ case uint16_t(Op::Call):
+ CHECK(EmitCall(f, /* asmJSFuncDef = */ false));
+ case uint16_t(Op::CallIndirect):
+ CHECK(EmitCallIndirect(f, /* oldStyle = */ false));
+
+ // Parametric operators
+ case uint16_t(Op::Drop):
+ CHECK(f.iter().readDrop());
+ case uint16_t(Op::SelectNumeric):
+ CHECK(EmitSelect(f, /*typed*/ false));
+ case uint16_t(Op::SelectTyped):
+ CHECK(EmitSelect(f, /*typed*/ true));
+
+ // Locals and globals
+ case uint16_t(Op::LocalGet):
+ CHECK(EmitGetLocal(f));
+ case uint16_t(Op::LocalSet):
+ CHECK(EmitSetLocal(f));
+ case uint16_t(Op::LocalTee):
+ CHECK(EmitTeeLocal(f));
+ case uint16_t(Op::GlobalGet):
+ CHECK(EmitGetGlobal(f));
+ case uint16_t(Op::GlobalSet):
+ CHECK(EmitSetGlobal(f));
+ case uint16_t(Op::TableGet):
+ CHECK(EmitTableGet(f));
+ case uint16_t(Op::TableSet):
+ CHECK(EmitTableSet(f));
+
+ // Memory-related operators
+ case uint16_t(Op::I32Load):
+ CHECK(EmitLoad(f, ValType::I32, Scalar::Int32));
+ case uint16_t(Op::I64Load):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Int64));
+ case uint16_t(Op::F32Load):
+ CHECK(EmitLoad(f, ValType::F32, Scalar::Float32));
+ case uint16_t(Op::F64Load):
+ CHECK(EmitLoad(f, ValType::F64, Scalar::Float64));
+ case uint16_t(Op::I32Load8S):
+ CHECK(EmitLoad(f, ValType::I32, Scalar::Int8));
+ case uint16_t(Op::I32Load8U):
+ CHECK(EmitLoad(f, ValType::I32, Scalar::Uint8));
+ case uint16_t(Op::I32Load16S):
+ CHECK(EmitLoad(f, ValType::I32, Scalar::Int16));
+ case uint16_t(Op::I32Load16U):
+ CHECK(EmitLoad(f, ValType::I32, Scalar::Uint16));
+ case uint16_t(Op::I64Load8S):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Int8));
+ case uint16_t(Op::I64Load8U):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Uint8));
+ case uint16_t(Op::I64Load16S):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Int16));
+ case uint16_t(Op::I64Load16U):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Uint16));
+ case uint16_t(Op::I64Load32S):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Int32));
+ case uint16_t(Op::I64Load32U):
+ CHECK(EmitLoad(f, ValType::I64, Scalar::Uint32));
+ case uint16_t(Op::I32Store):
+ CHECK(EmitStore(f, ValType::I32, Scalar::Int32));
+ case uint16_t(Op::I64Store):
+ CHECK(EmitStore(f, ValType::I64, Scalar::Int64));
+ case uint16_t(Op::F32Store):
+ CHECK(EmitStore(f, ValType::F32, Scalar::Float32));
+ case uint16_t(Op::F64Store):
+ CHECK(EmitStore(f, ValType::F64, Scalar::Float64));
+ case uint16_t(Op::I32Store8):
+ CHECK(EmitStore(f, ValType::I32, Scalar::Int8));
+ case uint16_t(Op::I32Store16):
+ CHECK(EmitStore(f, ValType::I32, Scalar::Int16));
+ case uint16_t(Op::I64Store8):
+ CHECK(EmitStore(f, ValType::I64, Scalar::Int8));
+ case uint16_t(Op::I64Store16):
+ CHECK(EmitStore(f, ValType::I64, Scalar::Int16));
+ case uint16_t(Op::I64Store32):
+ CHECK(EmitStore(f, ValType::I64, Scalar::Int32));
+ case uint16_t(Op::MemorySize):
+ CHECK(EmitMemorySize(f));
+ case uint16_t(Op::MemoryGrow):
+ CHECK(EmitMemoryGrow(f));
+
+ // Constants
+ case uint16_t(Op::I32Const):
+ CHECK(EmitI32Const(f));
+ case uint16_t(Op::I64Const):
+ CHECK(EmitI64Const(f));
+ case uint16_t(Op::F32Const):
+ CHECK(EmitF32Const(f));
+ case uint16_t(Op::F64Const):
+ CHECK(EmitF64Const(f));
+
+ // Comparison operators
+ case uint16_t(Op::I32Eqz):
+ CHECK(EmitConversion<MNot>(f, ValType::I32, ValType::I32));
+ case uint16_t(Op::I32Eq):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Eq, MCompare::Compare_Int32));
+ case uint16_t(Op::I32Ne):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Ne, MCompare::Compare_Int32));
+ case uint16_t(Op::I32LtS):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Lt, MCompare::Compare_Int32));
+ case uint16_t(Op::I32LtU):
+ CHECK(EmitComparison(f, ValType::I32, JSOp::Lt,
+ MCompare::Compare_UInt32));
+ case uint16_t(Op::I32GtS):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Gt, MCompare::Compare_Int32));
+ case uint16_t(Op::I32GtU):
+ CHECK(EmitComparison(f, ValType::I32, JSOp::Gt,
+ MCompare::Compare_UInt32));
+ case uint16_t(Op::I32LeS):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Le, MCompare::Compare_Int32));
+ case uint16_t(Op::I32LeU):
+ CHECK(EmitComparison(f, ValType::I32, JSOp::Le,
+ MCompare::Compare_UInt32));
+ case uint16_t(Op::I32GeS):
+ CHECK(
+ EmitComparison(f, ValType::I32, JSOp::Ge, MCompare::Compare_Int32));
+ case uint16_t(Op::I32GeU):
+ CHECK(EmitComparison(f, ValType::I32, JSOp::Ge,
+ MCompare::Compare_UInt32));
+ case uint16_t(Op::I64Eqz):
+ CHECK(EmitConversion<MNot>(f, ValType::I64, ValType::I32));
+ case uint16_t(Op::I64Eq):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Eq, MCompare::Compare_Int64));
+ case uint16_t(Op::I64Ne):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Ne, MCompare::Compare_Int64));
+ case uint16_t(Op::I64LtS):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Lt, MCompare::Compare_Int64));
+ case uint16_t(Op::I64LtU):
+ CHECK(EmitComparison(f, ValType::I64, JSOp::Lt,
+ MCompare::Compare_UInt64));
+ case uint16_t(Op::I64GtS):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Gt, MCompare::Compare_Int64));
+ case uint16_t(Op::I64GtU):
+ CHECK(EmitComparison(f, ValType::I64, JSOp::Gt,
+ MCompare::Compare_UInt64));
+ case uint16_t(Op::I64LeS):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Le, MCompare::Compare_Int64));
+ case uint16_t(Op::I64LeU):
+ CHECK(EmitComparison(f, ValType::I64, JSOp::Le,
+ MCompare::Compare_UInt64));
+ case uint16_t(Op::I64GeS):
+ CHECK(
+ EmitComparison(f, ValType::I64, JSOp::Ge, MCompare::Compare_Int64));
+ case uint16_t(Op::I64GeU):
+ CHECK(EmitComparison(f, ValType::I64, JSOp::Ge,
+ MCompare::Compare_UInt64));
+ case uint16_t(Op::F32Eq):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Eq,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F32Ne):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Ne,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F32Lt):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Lt,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F32Gt):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Gt,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F32Le):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Le,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F32Ge):
+ CHECK(EmitComparison(f, ValType::F32, JSOp::Ge,
+ MCompare::Compare_Float32));
+ case uint16_t(Op::F64Eq):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Eq,
+ MCompare::Compare_Double));
+ case uint16_t(Op::F64Ne):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Ne,
+ MCompare::Compare_Double));
+ case uint16_t(Op::F64Lt):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Lt,
+ MCompare::Compare_Double));
+ case uint16_t(Op::F64Gt):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Gt,
+ MCompare::Compare_Double));
+ case uint16_t(Op::F64Le):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Le,
+ MCompare::Compare_Double));
+ case uint16_t(Op::F64Ge):
+ CHECK(EmitComparison(f, ValType::F64, JSOp::Ge,
+ MCompare::Compare_Double));
+
+ // Numeric operators
+ case uint16_t(Op::I32Clz):
+ CHECK(EmitUnaryWithType<MClz>(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Ctz):
+ CHECK(EmitUnaryWithType<MCtz>(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Popcnt):
+ CHECK(EmitUnaryWithType<MPopcnt>(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Add):
+ CHECK(EmitAdd(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Sub):
+ CHECK(EmitSub(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Mul):
+ CHECK(EmitMul(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32DivS):
+ case uint16_t(Op::I32DivU):
+ CHECK(
+ EmitDiv(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32DivU));
+ case uint16_t(Op::I32RemS):
+ case uint16_t(Op::I32RemU):
+ CHECK(
+ EmitRem(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32RemU));
+ case uint16_t(Op::I32And):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
+ MWasmBinaryBitwise::SubOpcode::And));
+ case uint16_t(Op::I32Or):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
+ MWasmBinaryBitwise::SubOpcode::Or));
+ case uint16_t(Op::I32Xor):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
+ MWasmBinaryBitwise::SubOpcode::Xor));
+ case uint16_t(Op::I32Shl):
+ CHECK(EmitShift<MLsh>(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32ShrS):
+ CHECK(EmitShift<MRsh>(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32ShrU):
+ CHECK(EmitUrsh(f, ValType::I32, MIRType::Int32));
+ case uint16_t(Op::I32Rotl):
+ case uint16_t(Op::I32Rotr):
+ CHECK(EmitRotate(f, ValType::I32, Op(op.b0) == Op::I32Rotl));
+ case uint16_t(Op::I64Clz):
+ CHECK(EmitUnaryWithType<MClz>(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Ctz):
+ CHECK(EmitUnaryWithType<MCtz>(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Popcnt):
+ CHECK(EmitUnaryWithType<MPopcnt>(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Add):
+ CHECK(EmitAdd(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Sub):
+ CHECK(EmitSub(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Mul):
+ CHECK(EmitMul(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64DivS):
+ case uint16_t(Op::I64DivU):
+ CHECK(
+ EmitDiv(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64DivU));
+ case uint16_t(Op::I64RemS):
+ case uint16_t(Op::I64RemU):
+ CHECK(
+ EmitRem(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64RemU));
+ case uint16_t(Op::I64And):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
+ MWasmBinaryBitwise::SubOpcode::And));
+ case uint16_t(Op::I64Or):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
+ MWasmBinaryBitwise::SubOpcode::Or));
+ case uint16_t(Op::I64Xor):
+ CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
+ MWasmBinaryBitwise::SubOpcode::Xor));
+ case uint16_t(Op::I64Shl):
+ CHECK(EmitShift<MLsh>(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64ShrS):
+ CHECK(EmitShift<MRsh>(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64ShrU):
+ CHECK(EmitUrsh(f, ValType::I64, MIRType::Int64));
+ case uint16_t(Op::I64Rotl):
+ case uint16_t(Op::I64Rotr):
+ CHECK(EmitRotate(f, ValType::I64, Op(op.b0) == Op::I64Rotl));
+ case uint16_t(Op::F32Abs):
+ CHECK(EmitUnaryWithType<MAbs>(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Neg):
+ CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Ceil):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilF));
+ case uint16_t(Op::F32Floor):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorF));
+ case uint16_t(Op::F32Trunc):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncF));
+ case uint16_t(Op::F32Nearest):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntF));
+ case uint16_t(Op::F32Sqrt):
+ CHECK(EmitUnaryWithType<MSqrt>(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Add):
+ CHECK(EmitAdd(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Sub):
+ CHECK(EmitSub(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Mul):
+ CHECK(EmitMul(f, ValType::F32, MIRType::Float32));
+ case uint16_t(Op::F32Div):
+ CHECK(EmitDiv(f, ValType::F32, MIRType::Float32,
+ /* isUnsigned = */ false));
+ case uint16_t(Op::F32Min):
+ case uint16_t(Op::F32Max):
+ CHECK(EmitMinMax(f, ValType::F32, MIRType::Float32,
+ Op(op.b0) == Op::F32Max));
+ case uint16_t(Op::F32CopySign):
+ CHECK(EmitCopySign(f, ValType::F32));
+ case uint16_t(Op::F64Abs):
+ CHECK(EmitUnaryWithType<MAbs>(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Neg):
+ CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Ceil):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilD));
+ case uint16_t(Op::F64Floor):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorD));
+ case uint16_t(Op::F64Trunc):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncD));
+ case uint16_t(Op::F64Nearest):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntD));
+ case uint16_t(Op::F64Sqrt):
+ CHECK(EmitUnaryWithType<MSqrt>(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Add):
+ CHECK(EmitAdd(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Sub):
+ CHECK(EmitSub(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Mul):
+ CHECK(EmitMul(f, ValType::F64, MIRType::Double));
+ case uint16_t(Op::F64Div):
+ CHECK(EmitDiv(f, ValType::F64, MIRType::Double,
+ /* isUnsigned = */ false));
+ case uint16_t(Op::F64Min):
+ case uint16_t(Op::F64Max):
+ CHECK(EmitMinMax(f, ValType::F64, MIRType::Double,
+ Op(op.b0) == Op::F64Max));
+ case uint16_t(Op::F64CopySign):
+ CHECK(EmitCopySign(f, ValType::F64));
+
+ // Conversions
+ case uint16_t(Op::I32WrapI64):
+ CHECK(EmitConversion<MWrapInt64ToInt32>(f, ValType::I64, ValType::I32));
+ case uint16_t(Op::I32TruncF32S):
+ case uint16_t(Op::I32TruncF32U):
+ CHECK(EmitTruncate(f, ValType::F32, ValType::I32,
+ Op(op.b0) == Op::I32TruncF32U, false));
+ case uint16_t(Op::I32TruncF64S):
+ case uint16_t(Op::I32TruncF64U):
+ CHECK(EmitTruncate(f, ValType::F64, ValType::I32,
+ Op(op.b0) == Op::I32TruncF64U, false));
+ case uint16_t(Op::I64ExtendI32S):
+ case uint16_t(Op::I64ExtendI32U):
+ CHECK(EmitExtendI32(f, Op(op.b0) == Op::I64ExtendI32U));
+ case uint16_t(Op::I64TruncF32S):
+ case uint16_t(Op::I64TruncF32U):
+ CHECK(EmitTruncate(f, ValType::F32, ValType::I64,
+ Op(op.b0) == Op::I64TruncF32U, false));
+ case uint16_t(Op::I64TruncF64S):
+ case uint16_t(Op::I64TruncF64U):
+ CHECK(EmitTruncate(f, ValType::F64, ValType::I64,
+ Op(op.b0) == Op::I64TruncF64U, false));
+ case uint16_t(Op::F32ConvertI32S):
+ CHECK(EmitConversion<MToFloat32>(f, ValType::I32, ValType::F32));
+ case uint16_t(Op::F32ConvertI32U):
+ CHECK(EmitConversion<MWasmUnsignedToFloat32>(f, ValType::I32,
+ ValType::F32));
+ case uint16_t(Op::F32ConvertI64S):
+ case uint16_t(Op::F32ConvertI64U):
+ CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F32, MIRType::Float32,
+ Op(op.b0) == Op::F32ConvertI64U));
+ case uint16_t(Op::F32DemoteF64):
+ CHECK(EmitConversion<MToFloat32>(f, ValType::F64, ValType::F32));
+ case uint16_t(Op::F64ConvertI32S):
+ CHECK(EmitConversion<MToDouble>(f, ValType::I32, ValType::F64));
+ case uint16_t(Op::F64ConvertI32U):
+ CHECK(EmitConversion<MWasmUnsignedToDouble>(f, ValType::I32,
+ ValType::F64));
+ case uint16_t(Op::F64ConvertI64S):
+ case uint16_t(Op::F64ConvertI64U):
+ CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F64, MIRType::Double,
+ Op(op.b0) == Op::F64ConvertI64U));
+ case uint16_t(Op::F64PromoteF32):
+ CHECK(EmitConversion<MToDouble>(f, ValType::F32, ValType::F64));
+
+ // Reinterpretations
+ case uint16_t(Op::I32ReinterpretF32):
+ CHECK(EmitReinterpret(f, ValType::I32, ValType::F32, MIRType::Int32));
+ case uint16_t(Op::I64ReinterpretF64):
+ CHECK(EmitReinterpret(f, ValType::I64, ValType::F64, MIRType::Int64));
+ case uint16_t(Op::F32ReinterpretI32):
+ CHECK(EmitReinterpret(f, ValType::F32, ValType::I32, MIRType::Float32));
+ case uint16_t(Op::F64ReinterpretI64):
+ CHECK(EmitReinterpret(f, ValType::F64, ValType::I64, MIRType::Double));
+
+#ifdef ENABLE_WASM_GC
+ case uint16_t(Op::RefEq):
+ if (!f.moduleEnv().gcEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitComparison(f, RefType::eq(), JSOp::Eq,
+ MCompare::Compare_RefOrNull));
+#endif
+ case uint16_t(Op::RefFunc):
+ CHECK(EmitRefFunc(f));
+ case uint16_t(Op::RefNull):
+ CHECK(EmitRefNull(f));
+ case uint16_t(Op::RefIsNull):
+ CHECK(EmitRefIsNull(f));
+
+ // Sign extensions
+ case uint16_t(Op::I32Extend8S):
+ CHECK(EmitSignExtend(f, 1, 4));
+ case uint16_t(Op::I32Extend16S):
+ CHECK(EmitSignExtend(f, 2, 4));
+ case uint16_t(Op::I64Extend8S):
+ CHECK(EmitSignExtend(f, 1, 8));
+ case uint16_t(Op::I64Extend16S):
+ CHECK(EmitSignExtend(f, 2, 8));
+ case uint16_t(Op::I64Extend32S):
+ CHECK(EmitSignExtend(f, 4, 8));
+
+#ifdef ENABLE_WASM_FUNCTION_REFERENCES
+ case uint16_t(Op::RefAsNonNull):
+ if (!f.moduleEnv().functionReferencesEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitRefAsNonNull(f));
+ case uint16_t(Op::BrOnNull): {
+ if (!f.moduleEnv().functionReferencesEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitBrOnNull(f));
+ }
+ case uint16_t(Op::BrOnNonNull): {
+ if (!f.moduleEnv().functionReferencesEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitBrOnNonNull(f));
+ }
+ case uint16_t(Op::CallRef): {
+ if (!f.moduleEnv().functionReferencesEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitCallRef(f));
+ }
+#endif
+
+ // Gc operations
+#ifdef ENABLE_WASM_GC
+ case uint16_t(Op::GcPrefix): {
+ if (!f.moduleEnv().gcEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(GcOp::StructNew):
+ CHECK(EmitStructNew(f));
+ case uint32_t(GcOp::StructNewDefault):
+ CHECK(EmitStructNewDefault(f));
+ case uint32_t(GcOp::StructSet):
+ CHECK(EmitStructSet(f));
+ case uint32_t(GcOp::StructGet):
+ CHECK(EmitStructGet(f, FieldWideningOp::None));
+ case uint32_t(GcOp::StructGetS):
+ CHECK(EmitStructGet(f, FieldWideningOp::Signed));
+ case uint32_t(GcOp::StructGetU):
+ CHECK(EmitStructGet(f, FieldWideningOp::Unsigned));
+ case uint32_t(GcOp::ArrayNew):
+ CHECK(EmitArrayNew(f));
+ case uint32_t(GcOp::ArrayNewDefault):
+ CHECK(EmitArrayNewDefault(f));
+ case uint32_t(GcOp::ArrayNewFixed):
+ CHECK(EmitArrayNewFixed(f));
+ case uint32_t(GcOp::ArrayNewData):
+ CHECK(EmitArrayNewData(f));
+ case uint32_t(GcOp::ArrayInitFromElemStaticV5):
+ case uint32_t(GcOp::ArrayNewElem):
+ CHECK(EmitArrayNewElem(f));
+ case uint32_t(GcOp::ArraySet):
+ CHECK(EmitArraySet(f));
+ case uint32_t(GcOp::ArrayGet):
+ CHECK(EmitArrayGet(f, FieldWideningOp::None));
+ case uint32_t(GcOp::ArrayGetS):
+ CHECK(EmitArrayGet(f, FieldWideningOp::Signed));
+ case uint32_t(GcOp::ArrayGetU):
+ CHECK(EmitArrayGet(f, FieldWideningOp::Unsigned));
+ case uint32_t(GcOp::ArrayLenWithTypeIndex):
+ CHECK(EmitArrayLen(f, /*decodeIgnoredTypeIndex=*/true));
+ case uint32_t(GcOp::ArrayLen):
+ CHECK(EmitArrayLen(f, /*decodeIgnoredTypeIndex=*/false));
+ case uint32_t(GcOp::ArrayCopy):
+ CHECK(EmitArrayCopy(f));
+ case uint32_t(GcOp::RefTestV5):
+ CHECK(EmitRefTestV5(f));
+ case uint32_t(GcOp::RefCastV5):
+ CHECK(EmitRefCastV5(f));
+ case uint32_t(GcOp::BrOnCast):
+ CHECK(EmitBrOnCast(f));
+ case uint32_t(GcOp::BrOnCastV5):
+ CHECK(EmitBrOnCastCommonV5(f, /*onSuccess=*/true));
+ case uint32_t(GcOp::BrOnCastFailV5):
+ CHECK(EmitBrOnCastCommonV5(f, /*onSuccess=*/false));
+ case uint32_t(GcOp::BrOnCastHeapV5):
+ CHECK(
+ EmitBrOnCastHeapV5(f, /*onSuccess=*/true, /*nullable=*/false));
+ case uint32_t(GcOp::BrOnCastHeapNullV5):
+ CHECK(EmitBrOnCastHeapV5(f, /*onSuccess=*/true, /*nullable=*/true));
+ case uint32_t(GcOp::BrOnCastFailHeapV5):
+ CHECK(
+ EmitBrOnCastHeapV5(f, /*onSuccess=*/false, /*nullable=*/false));
+ case uint32_t(GcOp::BrOnCastFailHeapNullV5):
+ CHECK(
+ EmitBrOnCastHeapV5(f, /*onSuccess=*/false, /*nullable=*/true));
+ case uint32_t(GcOp::RefAsStructV5):
+ CHECK(EmitRefAsStructV5(f));
+ case uint32_t(GcOp::BrOnNonStructV5):
+ CHECK(EmitBrOnNonStructV5(f));
+ case uint32_t(GcOp::RefTest):
+ CHECK(EmitRefTest(f, /*nullable=*/false));
+ case uint32_t(GcOp::RefTestNull):
+ CHECK(EmitRefTest(f, /*nullable=*/true));
+ case uint32_t(GcOp::RefCast):
+ CHECK(EmitRefCast(f, /*nullable=*/false));
+ case uint32_t(GcOp::RefCastNull):
+ CHECK(EmitRefCast(f, /*nullable=*/true));
+ case uint16_t(GcOp::ExternInternalize):
+ CHECK(EmitExternInternalize(f));
+ case uint16_t(GcOp::ExternExternalize):
+ CHECK(EmitExternExternalize(f));
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ } // switch (op.b1)
+ break;
+ }
+#endif
+
+ // SIMD operations
+#ifdef ENABLE_WASM_SIMD
+ case uint16_t(Op::SimdPrefix): {
+ if (!f.moduleEnv().simdAvailable()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(SimdOp::V128Const):
+ CHECK(EmitConstSimd128(f));
+ case uint32_t(SimdOp::V128Load):
+ CHECK(EmitLoad(f, ValType::V128, Scalar::Simd128));
+ case uint32_t(SimdOp::V128Store):
+ CHECK(EmitStore(f, ValType::V128, Scalar::Simd128));
+ case uint32_t(SimdOp::V128And):
+ case uint32_t(SimdOp::V128Or):
+ case uint32_t(SimdOp::V128Xor):
+ case uint32_t(SimdOp::I8x16AvgrU):
+ case uint32_t(SimdOp::I16x8AvgrU):
+ case uint32_t(SimdOp::I8x16Add):
+ case uint32_t(SimdOp::I8x16AddSatS):
+ case uint32_t(SimdOp::I8x16AddSatU):
+ case uint32_t(SimdOp::I8x16MinS):
+ case uint32_t(SimdOp::I8x16MinU):
+ case uint32_t(SimdOp::I8x16MaxS):
+ case uint32_t(SimdOp::I8x16MaxU):
+ case uint32_t(SimdOp::I16x8Add):
+ case uint32_t(SimdOp::I16x8AddSatS):
+ case uint32_t(SimdOp::I16x8AddSatU):
+ case uint32_t(SimdOp::I16x8Mul):
+ case uint32_t(SimdOp::I16x8MinS):
+ case uint32_t(SimdOp::I16x8MinU):
+ case uint32_t(SimdOp::I16x8MaxS):
+ case uint32_t(SimdOp::I16x8MaxU):
+ case uint32_t(SimdOp::I32x4Add):
+ case uint32_t(SimdOp::I32x4Mul):
+ case uint32_t(SimdOp::I32x4MinS):
+ case uint32_t(SimdOp::I32x4MinU):
+ case uint32_t(SimdOp::I32x4MaxS):
+ case uint32_t(SimdOp::I32x4MaxU):
+ case uint32_t(SimdOp::I64x2Add):
+ case uint32_t(SimdOp::I64x2Mul):
+ case uint32_t(SimdOp::F32x4Add):
+ case uint32_t(SimdOp::F32x4Mul):
+ case uint32_t(SimdOp::F32x4Min):
+ case uint32_t(SimdOp::F32x4Max):
+ case uint32_t(SimdOp::F64x2Add):
+ case uint32_t(SimdOp::F64x2Mul):
+ case uint32_t(SimdOp::F64x2Min):
+ case uint32_t(SimdOp::F64x2Max):
+ case uint32_t(SimdOp::I8x16Eq):
+ case uint32_t(SimdOp::I8x16Ne):
+ case uint32_t(SimdOp::I16x8Eq):
+ case uint32_t(SimdOp::I16x8Ne):
+ case uint32_t(SimdOp::I32x4Eq):
+ case uint32_t(SimdOp::I32x4Ne):
+ case uint32_t(SimdOp::I64x2Eq):
+ case uint32_t(SimdOp::I64x2Ne):
+ case uint32_t(SimdOp::F32x4Eq):
+ case uint32_t(SimdOp::F32x4Ne):
+ case uint32_t(SimdOp::F64x2Eq):
+ case uint32_t(SimdOp::F64x2Ne):
+ case uint32_t(SimdOp::I32x4DotI16x8S):
+ case uint32_t(SimdOp::I16x8ExtmulLowI8x16S):
+ case uint32_t(SimdOp::I16x8ExtmulHighI8x16S):
+ case uint32_t(SimdOp::I16x8ExtmulLowI8x16U):
+ case uint32_t(SimdOp::I16x8ExtmulHighI8x16U):
+ case uint32_t(SimdOp::I32x4ExtmulLowI16x8S):
+ case uint32_t(SimdOp::I32x4ExtmulHighI16x8S):
+ case uint32_t(SimdOp::I32x4ExtmulLowI16x8U):
+ case uint32_t(SimdOp::I32x4ExtmulHighI16x8U):
+ case uint32_t(SimdOp::I64x2ExtmulLowI32x4S):
+ case uint32_t(SimdOp::I64x2ExtmulHighI32x4S):
+ case uint32_t(SimdOp::I64x2ExtmulLowI32x4U):
+ case uint32_t(SimdOp::I64x2ExtmulHighI32x4U):
+ case uint32_t(SimdOp::I16x8Q15MulrSatS):
+ CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1)));
+ case uint32_t(SimdOp::V128AndNot):
+ case uint32_t(SimdOp::I8x16Sub):
+ case uint32_t(SimdOp::I8x16SubSatS):
+ case uint32_t(SimdOp::I8x16SubSatU):
+ case uint32_t(SimdOp::I16x8Sub):
+ case uint32_t(SimdOp::I16x8SubSatS):
+ case uint32_t(SimdOp::I16x8SubSatU):
+ case uint32_t(SimdOp::I32x4Sub):
+ case uint32_t(SimdOp::I64x2Sub):
+ case uint32_t(SimdOp::F32x4Sub):
+ case uint32_t(SimdOp::F32x4Div):
+ case uint32_t(SimdOp::F64x2Sub):
+ case uint32_t(SimdOp::F64x2Div):
+ case uint32_t(SimdOp::I8x16NarrowI16x8S):
+ case uint32_t(SimdOp::I8x16NarrowI16x8U):
+ case uint32_t(SimdOp::I16x8NarrowI32x4S):
+ case uint32_t(SimdOp::I16x8NarrowI32x4U):
+ case uint32_t(SimdOp::I8x16LtS):
+ case uint32_t(SimdOp::I8x16LtU):
+ case uint32_t(SimdOp::I8x16GtS):
+ case uint32_t(SimdOp::I8x16GtU):
+ case uint32_t(SimdOp::I8x16LeS):
+ case uint32_t(SimdOp::I8x16LeU):
+ case uint32_t(SimdOp::I8x16GeS):
+ case uint32_t(SimdOp::I8x16GeU):
+ case uint32_t(SimdOp::I16x8LtS):
+ case uint32_t(SimdOp::I16x8LtU):
+ case uint32_t(SimdOp::I16x8GtS):
+ case uint32_t(SimdOp::I16x8GtU):
+ case uint32_t(SimdOp::I16x8LeS):
+ case uint32_t(SimdOp::I16x8LeU):
+ case uint32_t(SimdOp::I16x8GeS):
+ case uint32_t(SimdOp::I16x8GeU):
+ case uint32_t(SimdOp::I32x4LtS):
+ case uint32_t(SimdOp::I32x4LtU):
+ case uint32_t(SimdOp::I32x4GtS):
+ case uint32_t(SimdOp::I32x4GtU):
+ case uint32_t(SimdOp::I32x4LeS):
+ case uint32_t(SimdOp::I32x4LeU):
+ case uint32_t(SimdOp::I32x4GeS):
+ case uint32_t(SimdOp::I32x4GeU):
+ case uint32_t(SimdOp::I64x2LtS):
+ case uint32_t(SimdOp::I64x2GtS):
+ case uint32_t(SimdOp::I64x2LeS):
+ case uint32_t(SimdOp::I64x2GeS):
+ case uint32_t(SimdOp::F32x4Lt):
+ case uint32_t(SimdOp::F32x4Gt):
+ case uint32_t(SimdOp::F32x4Le):
+ case uint32_t(SimdOp::F32x4Ge):
+ case uint32_t(SimdOp::F64x2Lt):
+ case uint32_t(SimdOp::F64x2Gt):
+ case uint32_t(SimdOp::F64x2Le):
+ case uint32_t(SimdOp::F64x2Ge):
+ case uint32_t(SimdOp::I8x16Swizzle):
+ case uint32_t(SimdOp::F32x4PMax):
+ case uint32_t(SimdOp::F32x4PMin):
+ case uint32_t(SimdOp::F64x2PMax):
+ case uint32_t(SimdOp::F64x2PMin):
+ CHECK(
+ EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16Splat):
+ case uint32_t(SimdOp::I16x8Splat):
+ case uint32_t(SimdOp::I32x4Splat):
+ CHECK(EmitSplatSimd128(f, ValType::I32, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I64x2Splat):
+ CHECK(EmitSplatSimd128(f, ValType::I64, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F32x4Splat):
+ CHECK(EmitSplatSimd128(f, ValType::F32, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F64x2Splat):
+ CHECK(EmitSplatSimd128(f, ValType::F64, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16Neg):
+ case uint32_t(SimdOp::I16x8Neg):
+ case uint32_t(SimdOp::I16x8ExtendLowI8x16S):
+ case uint32_t(SimdOp::I16x8ExtendHighI8x16S):
+ case uint32_t(SimdOp::I16x8ExtendLowI8x16U):
+ case uint32_t(SimdOp::I16x8ExtendHighI8x16U):
+ case uint32_t(SimdOp::I32x4Neg):
+ case uint32_t(SimdOp::I32x4ExtendLowI16x8S):
+ case uint32_t(SimdOp::I32x4ExtendHighI16x8S):
+ case uint32_t(SimdOp::I32x4ExtendLowI16x8U):
+ case uint32_t(SimdOp::I32x4ExtendHighI16x8U):
+ case uint32_t(SimdOp::I32x4TruncSatF32x4S):
+ case uint32_t(SimdOp::I32x4TruncSatF32x4U):
+ case uint32_t(SimdOp::I64x2Neg):
+ case uint32_t(SimdOp::I64x2ExtendLowI32x4S):
+ case uint32_t(SimdOp::I64x2ExtendHighI32x4S):
+ case uint32_t(SimdOp::I64x2ExtendLowI32x4U):
+ case uint32_t(SimdOp::I64x2ExtendHighI32x4U):
+ case uint32_t(SimdOp::F32x4Abs):
+ case uint32_t(SimdOp::F32x4Neg):
+ case uint32_t(SimdOp::F32x4Sqrt):
+ case uint32_t(SimdOp::F32x4ConvertI32x4S):
+ case uint32_t(SimdOp::F32x4ConvertI32x4U):
+ case uint32_t(SimdOp::F64x2Abs):
+ case uint32_t(SimdOp::F64x2Neg):
+ case uint32_t(SimdOp::F64x2Sqrt):
+ case uint32_t(SimdOp::V128Not):
+ case uint32_t(SimdOp::I8x16Popcnt):
+ case uint32_t(SimdOp::I8x16Abs):
+ case uint32_t(SimdOp::I16x8Abs):
+ case uint32_t(SimdOp::I32x4Abs):
+ case uint32_t(SimdOp::I64x2Abs):
+ case uint32_t(SimdOp::F32x4Ceil):
+ case uint32_t(SimdOp::F32x4Floor):
+ case uint32_t(SimdOp::F32x4Trunc):
+ case uint32_t(SimdOp::F32x4Nearest):
+ case uint32_t(SimdOp::F64x2Ceil):
+ case uint32_t(SimdOp::F64x2Floor):
+ case uint32_t(SimdOp::F64x2Trunc):
+ case uint32_t(SimdOp::F64x2Nearest):
+ case uint32_t(SimdOp::F32x4DemoteF64x2Zero):
+ case uint32_t(SimdOp::F64x2PromoteLowF32x4):
+ case uint32_t(SimdOp::F64x2ConvertLowI32x4S):
+ case uint32_t(SimdOp::F64x2ConvertLowI32x4U):
+ case uint32_t(SimdOp::I32x4TruncSatF64x2SZero):
+ case uint32_t(SimdOp::I32x4TruncSatF64x2UZero):
+ case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S):
+ case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U):
+ case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S):
+ case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U):
+ CHECK(EmitUnarySimd128(f, SimdOp(op.b1)));
+ case uint32_t(SimdOp::V128AnyTrue):
+ case uint32_t(SimdOp::I8x16AllTrue):
+ case uint32_t(SimdOp::I16x8AllTrue):
+ case uint32_t(SimdOp::I32x4AllTrue):
+ case uint32_t(SimdOp::I64x2AllTrue):
+ case uint32_t(SimdOp::I8x16Bitmask):
+ case uint32_t(SimdOp::I16x8Bitmask):
+ case uint32_t(SimdOp::I32x4Bitmask):
+ case uint32_t(SimdOp::I64x2Bitmask):
+ CHECK(EmitReduceSimd128(f, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16Shl):
+ case uint32_t(SimdOp::I8x16ShrS):
+ case uint32_t(SimdOp::I8x16ShrU):
+ case uint32_t(SimdOp::I16x8Shl):
+ case uint32_t(SimdOp::I16x8ShrS):
+ case uint32_t(SimdOp::I16x8ShrU):
+ case uint32_t(SimdOp::I32x4Shl):
+ case uint32_t(SimdOp::I32x4ShrS):
+ case uint32_t(SimdOp::I32x4ShrU):
+ case uint32_t(SimdOp::I64x2Shl):
+ case uint32_t(SimdOp::I64x2ShrS):
+ case uint32_t(SimdOp::I64x2ShrU):
+ CHECK(EmitShiftSimd128(f, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16ExtractLaneS):
+ case uint32_t(SimdOp::I8x16ExtractLaneU):
+ CHECK(EmitExtractLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I16x8ExtractLaneS):
+ case uint32_t(SimdOp::I16x8ExtractLaneU):
+ CHECK(EmitExtractLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I32x4ExtractLane):
+ CHECK(EmitExtractLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I64x2ExtractLane):
+ CHECK(EmitExtractLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F32x4ExtractLane):
+ CHECK(EmitExtractLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F64x2ExtractLane):
+ CHECK(EmitExtractLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I16x8ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I32x4ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I64x2ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F32x4ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1)));
+ case uint32_t(SimdOp::F64x2ReplaceLane):
+ CHECK(EmitReplaceLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1)));
+ case uint32_t(SimdOp::V128Bitselect):
+ CHECK(EmitTernarySimd128(f, SimdOp(op.b1)));
+ case uint32_t(SimdOp::I8x16Shuffle):
+ CHECK(EmitShuffleSimd128(f));
+ case uint32_t(SimdOp::V128Load8Splat):
+ CHECK(EmitLoadSplatSimd128(f, Scalar::Uint8, SimdOp::I8x16Splat));
+ case uint32_t(SimdOp::V128Load16Splat):
+ CHECK(EmitLoadSplatSimd128(f, Scalar::Uint16, SimdOp::I16x8Splat));
+ case uint32_t(SimdOp::V128Load32Splat):
+ CHECK(EmitLoadSplatSimd128(f, Scalar::Float32, SimdOp::I32x4Splat));
+ case uint32_t(SimdOp::V128Load64Splat):
+ CHECK(EmitLoadSplatSimd128(f, Scalar::Float64, SimdOp::I64x2Splat));
+ case uint32_t(SimdOp::V128Load8x8S):
+ case uint32_t(SimdOp::V128Load8x8U):
+ case uint32_t(SimdOp::V128Load16x4S):
+ case uint32_t(SimdOp::V128Load16x4U):
+ case uint32_t(SimdOp::V128Load32x2S):
+ case uint32_t(SimdOp::V128Load32x2U):
+ CHECK(EmitLoadExtendSimd128(f, SimdOp(op.b1)));
+ case uint32_t(SimdOp::V128Load32Zero):
+ CHECK(EmitLoadZeroSimd128(f, Scalar::Float32, 4));
+ case uint32_t(SimdOp::V128Load64Zero):
+ CHECK(EmitLoadZeroSimd128(f, Scalar::Float64, 8));
+ case uint32_t(SimdOp::V128Load8Lane):
+ CHECK(EmitLoadLaneSimd128(f, 1));
+ case uint32_t(SimdOp::V128Load16Lane):
+ CHECK(EmitLoadLaneSimd128(f, 2));
+ case uint32_t(SimdOp::V128Load32Lane):
+ CHECK(EmitLoadLaneSimd128(f, 4));
+ case uint32_t(SimdOp::V128Load64Lane):
+ CHECK(EmitLoadLaneSimd128(f, 8));
+ case uint32_t(SimdOp::V128Store8Lane):
+ CHECK(EmitStoreLaneSimd128(f, 1));
+ case uint32_t(SimdOp::V128Store16Lane):
+ CHECK(EmitStoreLaneSimd128(f, 2));
+ case uint32_t(SimdOp::V128Store32Lane):
+ CHECK(EmitStoreLaneSimd128(f, 4));
+ case uint32_t(SimdOp::V128Store64Lane):
+ CHECK(EmitStoreLaneSimd128(f, 8));
+# ifdef ENABLE_WASM_RELAXED_SIMD
+ case uint32_t(SimdOp::F32x4RelaxedFma):
+ case uint32_t(SimdOp::F32x4RelaxedFnma):
+ case uint32_t(SimdOp::F64x2RelaxedFma):
+ case uint32_t(SimdOp::F64x2RelaxedFnma):
+ case uint32_t(SimdOp::I8x16RelaxedLaneSelect):
+ case uint32_t(SimdOp::I16x8RelaxedLaneSelect):
+ case uint32_t(SimdOp::I32x4RelaxedLaneSelect):
+ case uint32_t(SimdOp::I64x2RelaxedLaneSelect):
+ case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): {
+ if (!f.moduleEnv().v128RelaxedEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitTernarySimd128(f, SimdOp(op.b1)));
+ }
+ case uint32_t(SimdOp::F32x4RelaxedMin):
+ case uint32_t(SimdOp::F32x4RelaxedMax):
+ case uint32_t(SimdOp::F64x2RelaxedMin):
+ case uint32_t(SimdOp::F64x2RelaxedMax):
+ case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): {
+ if (!f.moduleEnv().v128RelaxedEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1)));
+ }
+ case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S):
+ case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U):
+ case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero):
+ case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): {
+ if (!f.moduleEnv().v128RelaxedEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitUnarySimd128(f, SimdOp(op.b1)));
+ }
+ case uint32_t(SimdOp::I8x16RelaxedSwizzle):
+ case uint32_t(SimdOp::I16x8DotI8x16I7x16S): {
+ if (!f.moduleEnv().v128RelaxedEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(
+ EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1)));
+ }
+# endif
+
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ } // switch (op.b1)
+ break;
+ }
+#endif
+
+ // Miscellaneous operations
+ case uint16_t(Op::MiscPrefix): {
+ switch (op.b1) {
+ case uint32_t(MiscOp::I32TruncSatF32S):
+ case uint32_t(MiscOp::I32TruncSatF32U):
+ CHECK(EmitTruncate(f, ValType::F32, ValType::I32,
+ MiscOp(op.b1) == MiscOp::I32TruncSatF32U, true));
+ case uint32_t(MiscOp::I32TruncSatF64S):
+ case uint32_t(MiscOp::I32TruncSatF64U):
+ CHECK(EmitTruncate(f, ValType::F64, ValType::I32,
+ MiscOp(op.b1) == MiscOp::I32TruncSatF64U, true));
+ case uint32_t(MiscOp::I64TruncSatF32S):
+ case uint32_t(MiscOp::I64TruncSatF32U):
+ CHECK(EmitTruncate(f, ValType::F32, ValType::I64,
+ MiscOp(op.b1) == MiscOp::I64TruncSatF32U, true));
+ case uint32_t(MiscOp::I64TruncSatF64S):
+ case uint32_t(MiscOp::I64TruncSatF64U):
+ CHECK(EmitTruncate(f, ValType::F64, ValType::I64,
+ MiscOp(op.b1) == MiscOp::I64TruncSatF64U, true));
+ case uint32_t(MiscOp::MemoryCopy):
+ CHECK(EmitMemCopy(f));
+ case uint32_t(MiscOp::DataDrop):
+ CHECK(EmitDataOrElemDrop(f, /*isData=*/true));
+ case uint32_t(MiscOp::MemoryFill):
+ CHECK(EmitMemFill(f));
+ case uint32_t(MiscOp::MemoryInit):
+ CHECK(EmitMemOrTableInit(f, /*isMem=*/true));
+ case uint32_t(MiscOp::TableCopy):
+ CHECK(EmitTableCopy(f));
+ case uint32_t(MiscOp::ElemDrop):
+ CHECK(EmitDataOrElemDrop(f, /*isData=*/false));
+ case uint32_t(MiscOp::TableInit):
+ CHECK(EmitMemOrTableInit(f, /*isMem=*/false));
+ case uint32_t(MiscOp::TableFill):
+ CHECK(EmitTableFill(f));
+#if ENABLE_WASM_MEMORY_CONTROL
+ case uint32_t(MiscOp::MemoryDiscard): {
+ if (!f.moduleEnv().memoryControlEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitMemDiscard(f));
+ }
+#endif
+ case uint32_t(MiscOp::TableGrow):
+ CHECK(EmitTableGrow(f));
+ case uint32_t(MiscOp::TableSize):
+ CHECK(EmitTableSize(f));
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ break;
+ }
+
+ // Thread operations
+ case uint16_t(Op::ThreadPrefix): {
+ // Though thread ops can be used on nonshared memories, we make them
+ // unavailable if shared memory has been disabled in the prefs, for
+ // maximum predictability and safety and consistency with JS.
+ if (f.moduleEnv().sharedMemoryEnabled() == Shareable::False) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(ThreadOp::Wake):
+ CHECK(EmitWake(f));
+
+ case uint32_t(ThreadOp::I32Wait):
+ CHECK(EmitWait(f, ValType::I32, 4));
+ case uint32_t(ThreadOp::I64Wait):
+ CHECK(EmitWait(f, ValType::I64, 8));
+ case uint32_t(ThreadOp::Fence):
+ CHECK(EmitFence(f));
+
+ case uint32_t(ThreadOp::I32AtomicLoad):
+ CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicLoad):
+ CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicLoad8U):
+ CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicLoad16U):
+ CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicLoad8U):
+ CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicLoad16U):
+ CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicLoad32U):
+ CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicStore):
+ CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicStore):
+ CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicStore8U):
+ CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicStore16U):
+ CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicStore8U):
+ CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicStore16U):
+ CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicStore32U):
+ CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicAdd):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I32AtomicAdd8U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I32AtomicAdd16U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd8U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd16U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16,
+ AtomicFetchAddOp));
+ case uint32_t(ThreadOp::I64AtomicAdd32U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32,
+ AtomicFetchAddOp));
+
+ case uint32_t(ThreadOp::I32AtomicSub):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I32AtomicSub8U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I32AtomicSub16U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub8U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub16U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16,
+ AtomicFetchSubOp));
+ case uint32_t(ThreadOp::I64AtomicSub32U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32,
+ AtomicFetchSubOp));
+
+ case uint32_t(ThreadOp::I32AtomicAnd):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I32AtomicAnd8U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I32AtomicAnd16U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd8U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd16U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16,
+ AtomicFetchAndOp));
+ case uint32_t(ThreadOp::I64AtomicAnd32U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32,
+ AtomicFetchAndOp));
+
+ case uint32_t(ThreadOp::I32AtomicOr):
+ CHECK(
+ EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr):
+ CHECK(
+ EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I32AtomicOr8U):
+ CHECK(
+ EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I32AtomicOr16U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16,
+ AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr8U):
+ CHECK(
+ EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr16U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16,
+ AtomicFetchOrOp));
+ case uint32_t(ThreadOp::I64AtomicOr32U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32,
+ AtomicFetchOrOp));
+
+ case uint32_t(ThreadOp::I32AtomicXor):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I32AtomicXor8U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I32AtomicXor16U):
+ CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor8U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor16U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16,
+ AtomicFetchXorOp));
+ case uint32_t(ThreadOp::I64AtomicXor32U):
+ CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32,
+ AtomicFetchXorOp));
+
+ case uint32_t(ThreadOp::I32AtomicXchg):
+ CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicXchg):
+ CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicXchg8U):
+ CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicXchg16U):
+ CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicXchg8U):
+ CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicXchg16U):
+ CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicXchg32U):
+ CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint32));
+
+ case uint32_t(ThreadOp::I32AtomicCmpXchg):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Int32));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Int64));
+ case uint32_t(ThreadOp::I32AtomicCmpXchg8U):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint8));
+ case uint32_t(ThreadOp::I32AtomicCmpXchg16U):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg8U):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint8));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg16U):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint16));
+ case uint32_t(ThreadOp::I64AtomicCmpXchg32U):
+ CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint32));
+
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ break;
+ }
+
+ // asm.js-specific operators
+ case uint16_t(Op::MozPrefix): {
+ if (op.b1 == uint32_t(MozOp::Intrinsic)) {
+ if (!f.moduleEnv().intrinsicsEnabled()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ CHECK(EmitIntrinsic(f));
+ }
+
+ if (!f.moduleEnv().isAsmJS()) {
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ switch (op.b1) {
+ case uint32_t(MozOp::TeeGlobal):
+ CHECK(EmitTeeGlobal(f));
+ case uint32_t(MozOp::I32Min):
+ case uint32_t(MozOp::I32Max):
+ CHECK(EmitMinMax(f, ValType::I32, MIRType::Int32,
+ MozOp(op.b1) == MozOp::I32Max));
+ case uint32_t(MozOp::I32Neg):
+ CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::I32, MIRType::Int32));
+ case uint32_t(MozOp::I32BitNot):
+ CHECK(EmitBitNot(f, ValType::I32));
+ case uint32_t(MozOp::I32Abs):
+ CHECK(EmitUnaryWithType<MAbs>(f, ValType::I32, MIRType::Int32));
+ case uint32_t(MozOp::F32TeeStoreF64):
+ CHECK(EmitTeeStoreWithCoercion(f, ValType::F32, Scalar::Float64));
+ case uint32_t(MozOp::F64TeeStoreF32):
+ CHECK(EmitTeeStoreWithCoercion(f, ValType::F64, Scalar::Float32));
+ case uint32_t(MozOp::I32TeeStore8):
+ CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int8));
+ case uint32_t(MozOp::I32TeeStore16):
+ CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int16));
+ case uint32_t(MozOp::I64TeeStore8):
+ CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int8));
+ case uint32_t(MozOp::I64TeeStore16):
+ CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int16));
+ case uint32_t(MozOp::I64TeeStore32):
+ CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int32));
+ case uint32_t(MozOp::I32TeeStore):
+ CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int32));
+ case uint32_t(MozOp::I64TeeStore):
+ CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int64));
+ case uint32_t(MozOp::F32TeeStore):
+ CHECK(EmitTeeStore(f, ValType::F32, Scalar::Float32));
+ case uint32_t(MozOp::F64TeeStore):
+ CHECK(EmitTeeStore(f, ValType::F64, Scalar::Float64));
+ case uint32_t(MozOp::F64Mod):
+ CHECK(EmitRem(f, ValType::F64, MIRType::Double,
+ /* isUnsigned = */ false));
+ case uint32_t(MozOp::F64SinNative):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigSinNativeD));
+ case uint32_t(MozOp::F64SinFdlibm):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigSinFdlibmD));
+ case uint32_t(MozOp::F64CosNative):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigCosNativeD));
+ case uint32_t(MozOp::F64CosFdlibm):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigCosFdlibmD));
+ case uint32_t(MozOp::F64TanNative):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigTanNativeD));
+ case uint32_t(MozOp::F64TanFdlibm):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigTanFdlibmD));
+ case uint32_t(MozOp::F64Asin):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigASinD));
+ case uint32_t(MozOp::F64Acos):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigACosD));
+ case uint32_t(MozOp::F64Atan):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigATanD));
+ case uint32_t(MozOp::F64Exp):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigExpD));
+ case uint32_t(MozOp::F64Log):
+ CHECK(EmitUnaryMathBuiltinCall(f, SASigLogD));
+ case uint32_t(MozOp::F64Pow):
+ CHECK(EmitBinaryMathBuiltinCall(f, SASigPowD));
+ case uint32_t(MozOp::F64Atan2):
+ CHECK(EmitBinaryMathBuiltinCall(f, SASigATan2D));
+ case uint32_t(MozOp::OldCallDirect):
+ CHECK(EmitCall(f, /* asmJSFuncDef = */ true));
+ case uint32_t(MozOp::OldCallIndirect):
+ CHECK(EmitCallIndirect(f, /* oldStyle = */ true));
+
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ break;
+ }
+
+ default:
+ return f.iter().unrecognizedOpcode(&op);
+ }
+ }
+
+ MOZ_CRASH("unreachable");
+
+#undef CHECK
+}
+
+bool wasm::IonCompileFunctions(const ModuleEnvironment& moduleEnv,
+ const CompilerEnvironment& compilerEnv,
+ LifoAlloc& lifo,
+ const FuncCompileInputVector& inputs,
+ CompiledCode* code, UniqueChars* error) {
+ MOZ_ASSERT(compilerEnv.tier() == Tier::Optimized);
+ MOZ_ASSERT(compilerEnv.debug() == DebugEnabled::False);
+
+ TempAllocator alloc(&lifo);
+ JitContext jitContext;
+ MOZ_ASSERT(IsCompilingWasm());
+ WasmMacroAssembler masm(alloc, moduleEnv);
+#if defined(JS_CODEGEN_ARM64)
+ masm.SetStackPointer64(PseudoStackPointer64);
+#endif
+
+ // Swap in already-allocated empty vectors to avoid malloc/free.
+ MOZ_ASSERT(code->empty());
+ if (!code->swap(masm)) {
+ return false;
+ }
+
+ // Create a description of the stack layout created by GenerateTrapExit().
+ RegisterOffsets trapExitLayout;
+ size_t trapExitLayoutNumWords;
+ GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords);
+
+ for (const FuncCompileInput& func : inputs) {
+ JitSpewCont(JitSpew_Codegen, "\n");
+ JitSpew(JitSpew_Codegen,
+ "# ================================"
+ "==================================");
+ JitSpew(JitSpew_Codegen, "# ==");
+ JitSpew(JitSpew_Codegen,
+ "# wasm::IonCompileFunctions: starting on function index %d",
+ (int)func.index);
+
+ Decoder d(func.begin, func.end, func.lineOrBytecode, error);
+
+ // Build the local types vector.
+
+ const FuncType& funcType = *moduleEnv.funcs[func.index].type;
+ ValTypeVector locals;
+ if (!locals.appendAll(funcType.args())) {
+ return false;
+ }
+ if (!DecodeLocalEntries(d, *moduleEnv.types, moduleEnv.features, &locals)) {
+ return false;
+ }
+
+ // Set up for Ion compilation.
+
+ const JitCompileOptions options;
+ MIRGraph graph(&alloc);
+ CompileInfo compileInfo(locals.length());
+ MIRGenerator mir(nullptr, options, &alloc, &graph, &compileInfo,
+ IonOptimizations.get(OptimizationLevel::Wasm));
+ if (moduleEnv.usesMemory()) {
+ if (moduleEnv.memory->indexType() == IndexType::I32) {
+ mir.initMinWasmHeapLength(moduleEnv.memory->initialLength32());
+ } else {
+ mir.initMinWasmHeapLength(moduleEnv.memory->initialLength64());
+ }
+ }
+
+ // Build MIR graph
+ {
+ FunctionCompiler f(moduleEnv, d, func, locals, mir, masm.tryNotes());
+ if (!f.init()) {
+ return false;
+ }
+
+ if (!f.startBlock()) {
+ return false;
+ }
+
+ if (!EmitBodyExprs(f)) {
+ return false;
+ }
+
+ f.finish();
+ }
+
+ // Compile MIR graph
+ {
+ jit::SpewBeginWasmFunction(&mir, func.index);
+ jit::AutoSpewEndFunction spewEndFunction(&mir);
+
+ if (!OptimizeMIR(&mir)) {
+ return false;
+ }
+
+ LIRGraph* lir = GenerateLIR(&mir);
+ if (!lir) {
+ return false;
+ }
+
+ CodeGenerator codegen(&mir, lir, &masm);
+
+ BytecodeOffset prologueTrapOffset(func.lineOrBytecode);
+ FuncOffsets offsets;
+ ArgTypeVector args(funcType);
+ if (!codegen.generateWasm(CallIndirectId::forFunc(moduleEnv, func.index),
+ prologueTrapOffset, args, trapExitLayout,
+ trapExitLayoutNumWords, &offsets,
+ &code->stackMaps, &d)) {
+ return false;
+ }
+
+ if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode,
+ offsets)) {
+ return false;
+ }
+ }
+
+ JitSpew(JitSpew_Codegen,
+ "# wasm::IonCompileFunctions: completed function index %d",
+ (int)func.index);
+ JitSpew(JitSpew_Codegen, "# ==");
+ JitSpew(JitSpew_Codegen,
+ "# ================================"
+ "==================================");
+ JitSpewCont(JitSpew_Codegen, "\n");
+ }
+
+ masm.finish();
+ if (masm.oom()) {
+ return false;
+ }
+
+ return code->swap(masm);
+}
+
+bool js::wasm::IonPlatformSupport() {
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \
+ defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64) || \
+ defined(JS_CODEGEN_RISCV64)
+ return true;
+#else
+ return false;
+#endif
+}