summaryrefslogtreecommitdiffstats
path: root/js/src/jit/WarpBuilder.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/WarpBuilder.cpp')
-rw-r--r--js/src/jit/WarpBuilder.cpp3576
1 files changed, 3576 insertions, 0 deletions
diff --git a/js/src/jit/WarpBuilder.cpp b/js/src/jit/WarpBuilder.cpp
new file mode 100644
index 0000000000..f005a6ea5e
--- /dev/null
+++ b/js/src/jit/WarpBuilder.cpp
@@ -0,0 +1,3576 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpBuilder.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/CacheIR.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/WarpCacheIRTranspiler.h"
+#include "jit/WarpSnapshot.h"
+#include "js/friend/ErrorMessages.h" // JSMSG_BAD_CONST_ASSIGN
+#include "vm/GeneratorObject.h"
+#include "vm/Interpreter.h"
+#include "vm/Opcodes.h"
+
+#include "gc/ObjectKind-inl.h"
+#include "vm/BytecodeIterator-inl.h"
+#include "vm/BytecodeLocation-inl.h"
+#include "vm/JSObject-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// Used for building the outermost script.
+WarpBuilder::WarpBuilder(WarpSnapshot& snapshot, MIRGenerator& mirGen,
+ WarpCompilation* warpCompilation)
+ : WarpBuilderShared(snapshot, mirGen, nullptr),
+ warpCompilation_(warpCompilation),
+ graph_(mirGen.graph()),
+ info_(mirGen.outerInfo()),
+ scriptSnapshot_(snapshot.rootScript()),
+ script_(snapshot.rootScript()->script()),
+ loopStack_(mirGen.alloc()) {
+ opSnapshotIter_ = scriptSnapshot_->opSnapshots().getFirst();
+}
+
+// Used for building inlined scripts.
+WarpBuilder::WarpBuilder(WarpBuilder* caller, WarpScriptSnapshot* snapshot,
+ CompileInfo& compileInfo, CallInfo* inlineCallInfo,
+ MResumePoint* callerResumePoint)
+ : WarpBuilderShared(caller->snapshot(), caller->mirGen(), nullptr),
+ warpCompilation_(caller->warpCompilation()),
+ graph_(caller->mirGen().graph()),
+ info_(compileInfo),
+ scriptSnapshot_(snapshot),
+ script_(snapshot->script()),
+ loopStack_(caller->mirGen().alloc()),
+ callerBuilder_(caller),
+ callerResumePoint_(callerResumePoint),
+ inlineCallInfo_(inlineCallInfo) {
+ opSnapshotIter_ = snapshot->opSnapshots().getFirst();
+}
+
+BytecodeSite* WarpBuilder::newBytecodeSite(BytecodeLocation loc) {
+ jsbytecode* pc = loc.toRawBytecode();
+ MOZ_ASSERT(info().inlineScriptTree()->script()->containsPC(pc));
+ return new (alloc()) BytecodeSite(info().inlineScriptTree(), pc);
+}
+
+const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(
+ BytecodeLocation loc, WarpOpSnapshot::Kind kind) {
+ uint32_t offset = loc.bytecodeToOffset(script_);
+
+ // Skip snapshots until we get to a snapshot with offset >= offset. This is
+ // a loop because WarpBuilder can skip unreachable bytecode ops.
+ while (opSnapshotIter_ && opSnapshotIter_->offset() < offset) {
+ opSnapshotIter_ = opSnapshotIter_->getNext();
+ }
+
+ if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset ||
+ opSnapshotIter_->kind() != kind) {
+ return nullptr;
+ }
+
+ return opSnapshotIter_;
+}
+
+void WarpBuilder::initBlock(MBasicBlock* block) {
+ graph().addBlock(block);
+
+ block->setLoopDepth(loopDepth());
+
+ current = block;
+}
+
+bool WarpBuilder::startNewBlock(MBasicBlock* predecessor, BytecodeLocation loc,
+ size_t numToPop) {
+ MBasicBlock* block =
+ MBasicBlock::NewPopN(graph(), info(), predecessor, newBytecodeSite(loc),
+ MBasicBlock::NORMAL, numToPop);
+ if (!block) {
+ return false;
+ }
+
+ initBlock(block);
+ return true;
+}
+
+bool WarpBuilder::startNewEntryBlock(size_t stackDepth, BytecodeLocation loc) {
+ MBasicBlock* block =
+ MBasicBlock::New(graph(), stackDepth, info(), /* maybePred = */ nullptr,
+ newBytecodeSite(loc), MBasicBlock::NORMAL);
+ if (!block) {
+ return false;
+ }
+
+ initBlock(block);
+ return true;
+}
+
+bool WarpBuilder::startNewLoopHeaderBlock(BytecodeLocation loopHead) {
+ MBasicBlock* header = MBasicBlock::NewPendingLoopHeader(
+ graph(), info(), current, newBytecodeSite(loopHead));
+ if (!header) {
+ return false;
+ }
+
+ initBlock(header);
+ return loopStack_.emplaceBack(header);
+}
+
+bool WarpBuilder::startNewOsrPreHeaderBlock(BytecodeLocation loopHead) {
+ MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
+ MOZ_ASSERT(loopHead.toRawBytecode() == info().osrPc());
+
+ // Create two blocks:
+ // * The OSR entry block. This is always the graph's second block and has no
+ // predecessors. This is the entry point for OSR from the Baseline JIT.
+ // * The OSR preheader block. This has two predecessors: the OSR entry block
+ // and the current block.
+
+ MBasicBlock* pred = current;
+
+ // Create the OSR entry block.
+ if (!startNewEntryBlock(pred->stackDepth(), loopHead)) {
+ return false;
+ }
+
+ MBasicBlock* osrBlock = current;
+ graph().setOsrBlock(osrBlock);
+ graph().moveBlockAfter(*graph().begin(), osrBlock);
+
+ MOsrEntry* entry = MOsrEntry::New(alloc());
+ osrBlock->add(entry);
+
+ // Initialize environment chain.
+ {
+ uint32_t slot = info().environmentChainSlot();
+ MInstruction* envv;
+ if (usesEnvironmentChain()) {
+ envv = MOsrEnvironmentChain::New(alloc(), entry);
+ } else {
+ // Use an undefined value if the script does not need its environment
+ // chain, to match the main entry point.
+ envv = MConstant::New(alloc(), UndefinedValue());
+ }
+ osrBlock->add(envv);
+ osrBlock->initSlot(slot, envv);
+ }
+
+ // Initialize return value.
+ {
+ MInstruction* returnValue;
+ if (!script_->noScriptRval()) {
+ returnValue = MOsrReturnValue::New(alloc(), entry);
+ } else {
+ returnValue = MConstant::New(alloc(), UndefinedValue());
+ }
+ osrBlock->add(returnValue);
+ osrBlock->initSlot(info().returnValueSlot(), returnValue);
+ }
+
+ // Initialize arguments object.
+ MInstruction* argsObj = nullptr;
+ if (info().needsArgsObj()) {
+ argsObj = MOsrArgumentsObject::New(alloc(), entry);
+ osrBlock->add(argsObj);
+ osrBlock->initSlot(info().argsObjSlot(), argsObj);
+ }
+
+ if (info().hasFunMaybeLazy()) {
+ // Initialize |this| parameter.
+ MParameter* thisv = MParameter::New(alloc(), MParameter::THIS_SLOT);
+ osrBlock->add(thisv);
+ osrBlock->initSlot(info().thisSlot(), thisv);
+
+ // Initialize arguments. There are three cases:
+ //
+ // 1) There's no ArgumentsObject or it doesn't alias formals. In this case
+ // we can just use the frame's argument slot.
+ // 2) The ArgumentsObject aliases formals and the argument is stored in the
+ // CallObject. Use |undefined| because we can't load from the arguments
+ // object and code will use the CallObject anyway.
+ // 3) The ArgumentsObject aliases formals and the argument isn't stored in
+ // the CallObject. We have to load it from the ArgumentsObject.
+ for (uint32_t i = 0; i < info().nargs(); i++) {
+ uint32_t slot = info().argSlotUnchecked(i);
+ MInstruction* osrv;
+ if (!info().argsObjAliasesFormals()) {
+ osrv = MParameter::New(alloc().fallible(), i);
+ } else if (script_->formalIsAliased(i)) {
+ osrv = MConstant::New(alloc().fallible(), UndefinedValue());
+ } else {
+ osrv = MGetArgumentsObjectArg::New(alloc().fallible(), argsObj, i);
+ }
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+ }
+
+ // Initialize locals.
+ uint32_t nlocals = info().nlocals();
+ for (uint32_t i = 0; i < nlocals; i++) {
+ uint32_t slot = info().localSlot(i);
+ ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(i);
+ MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+
+ // Initialize expression stack slots.
+ uint32_t numStackSlots = current->stackDepth() - info().firstStackSlot();
+ for (uint32_t i = 0; i < numStackSlots; i++) {
+ uint32_t slot = info().stackSlot(i);
+ ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(nlocals + i);
+ MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+
+ MStart* start = MStart::New(alloc());
+ current->add(start);
+
+ // Note: phi specialization can add type guard instructions to the OSR entry
+ // block if needed. See TypeAnalyzer::shouldSpecializeOsrPhis.
+
+ // Create the preheader block, with the predecessor block and OSR block as
+ // predecessors.
+ if (!startNewBlock(pred, loopHead)) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+ osrBlock->end(MGoto::New(alloc(), current));
+
+ if (!current->addPredecessor(alloc(), osrBlock)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool WarpBuilder::addPendingEdge(BytecodeLocation target, MBasicBlock* block,
+ uint32_t successor, uint32_t numToPop) {
+ MOZ_ASSERT(successor < block->lastIns()->numSuccessors());
+ MOZ_ASSERT(numToPop <= block->stackDepth());
+
+ jsbytecode* targetPC = target.toRawBytecode();
+ PendingEdgesMap::AddPtr p = pendingEdges_.lookupForAdd(targetPC);
+ if (p) {
+ return p->value().emplaceBack(block, successor, numToPop);
+ }
+
+ PendingEdges edges;
+ static_assert(PendingEdges::InlineLength >= 1,
+ "Appending one element should be infallible");
+ MOZ_ALWAYS_TRUE(edges.emplaceBack(block, successor, numToPop));
+
+ return pendingEdges_.add(p, targetPC, std::move(edges));
+}
+
+bool WarpBuilder::build() {
+ if (!buildPrologue()) {
+ return false;
+ }
+
+ if (!buildBody()) {
+ return false;
+ }
+
+ if (!MPhi::markIteratorPhis(*iterators())) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(info().osrPc(), graph().osrBlock());
+ MOZ_ASSERT(loopStack_.empty());
+ MOZ_ASSERT(loopDepth() == 0);
+
+ return true;
+}
+
+bool WarpBuilder::buildInline() {
+ if (!buildInlinePrologue()) {
+ return false;
+ }
+
+ if (!buildBody()) {
+ return false;
+ }
+
+ MOZ_ASSERT(loopStack_.empty());
+ return true;
+}
+
+MInstruction* WarpBuilder::buildNamedLambdaEnv(MDefinition* callee,
+ MDefinition* env,
+ NamedLambdaObject* templateObj) {
+ MOZ_ASSERT(templateObj->numDynamicSlots() == 0);
+
+ MInstruction* namedLambda = MNewNamedLambdaObject::New(alloc(), templateObj);
+ current->add(namedLambda);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barriers.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, env));
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, callee));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here:
+ // the object will be allocated in the nursery if possible, and if the
+ // tenured heap is used instead, a minor collection will have been performed
+ // that moved env/callee to the tenured heap.
+ size_t enclosingSlot = NamedLambdaObject::enclosingEnvironmentSlot();
+ size_t lambdaSlot = NamedLambdaObject::lambdaSlot();
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda,
+ enclosingSlot, env));
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda, lambdaSlot,
+ callee));
+
+ return namedLambda;
+}
+
+MInstruction* WarpBuilder::buildCallObject(MDefinition* callee,
+ MDefinition* env,
+ CallObject* templateObj) {
+ MConstant* templateCst = constant(ObjectValue(*templateObj));
+
+ MNewCallObject* callObj = MNewCallObject::New(alloc(), templateCst);
+ current->add(callObj);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barriers.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, env));
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, callee));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ size_t enclosingSlot = CallObject::enclosingEnvironmentSlot();
+ size_t calleeSlot = CallObject::calleeSlot();
+ current->add(
+ MStoreFixedSlot::NewUnbarriered(alloc(), callObj, enclosingSlot, env));
+ current->add(
+ MStoreFixedSlot::NewUnbarriered(alloc(), callObj, calleeSlot, callee));
+
+ return callObj;
+}
+
+bool WarpBuilder::buildEnvironmentChain() {
+ const WarpEnvironment& env = scriptSnapshot()->environment();
+
+ if (env.is<NoEnvironment>()) {
+ return true;
+ }
+
+ MInstruction* envDef = env.match(
+ [](const NoEnvironment&) -> MInstruction* {
+ MOZ_CRASH("Already handled");
+ },
+ [this](JSObject* obj) -> MInstruction* {
+ return constant(ObjectValue(*obj));
+ },
+ [this](const FunctionEnvironment& env) -> MInstruction* {
+ MDefinition* callee = getCallee();
+ MInstruction* envDef = MFunctionEnvironment::New(alloc(), callee);
+ current->add(envDef);
+ if (NamedLambdaObject* obj = env.namedLambdaTemplate) {
+ envDef = buildNamedLambdaEnv(callee, envDef, obj);
+ }
+ if (CallObject* obj = env.callObjectTemplate) {
+ envDef = buildCallObject(callee, envDef, obj);
+ if (!envDef) {
+ return nullptr;
+ }
+ }
+ return envDef;
+ });
+ if (!envDef) {
+ return false;
+ }
+
+ // Update the environment slot from UndefinedValue only after the initial
+ // environment is created so that bailout doesn't see a partial environment.
+ // See: |BaselineStackBuilder::buildBaselineFrame|
+ current->setEnvironmentChain(envDef);
+ return true;
+}
+
+bool WarpBuilder::buildPrologue() {
+ BytecodeLocation startLoc(script_, script_->code());
+ if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
+ return false;
+ }
+
+ if (info().hasFunMaybeLazy()) {
+ // Initialize |this|.
+ MParameter* param = MParameter::New(alloc(), MParameter::THIS_SLOT);
+ current->add(param);
+ current->initSlot(info().thisSlot(), param);
+
+ // Initialize arguments.
+ for (uint32_t i = 0; i < info().nargs(); i++) {
+ MParameter* param = MParameter::New(alloc().fallible(), i);
+ if (!param) {
+ return false;
+ }
+ current->add(param);
+ current->initSlot(info().argSlotUnchecked(i), param);
+ }
+ }
+
+ MConstant* undef = constant(UndefinedValue());
+
+ // Initialize local slots.
+ for (uint32_t i = 0; i < info().nlocals(); i++) {
+ current->initSlot(info().localSlot(i), undef);
+ }
+
+ // Initialize the environment chain, return value, and arguments object slots.
+ current->initSlot(info().environmentChainSlot(), undef);
+ current->initSlot(info().returnValueSlot(), undef);
+ if (info().needsArgsObj()) {
+ current->initSlot(info().argsObjSlot(), undef);
+ }
+
+ current->add(MStart::New(alloc()));
+
+ // Guard against over-recursion.
+ MCheckOverRecursed* check = MCheckOverRecursed::New(alloc());
+ current->add(check);
+
+ if (!buildEnvironmentChain()) {
+ return false;
+ }
+
+#ifdef JS_CACHEIR_SPEW
+ if (snapshot().needsFinalWarmUpCount()) {
+ MIncrementWarmUpCounter* ins =
+ MIncrementWarmUpCounter::New(alloc(), script_);
+ current->add(ins);
+ }
+#endif
+
+ return true;
+}
+
+bool WarpBuilder::buildInlinePrologue() {
+ // Generate entry block.
+ BytecodeLocation startLoc(script_, script_->code());
+ if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
+ return false;
+ }
+ current->setCallerResumePoint(callerResumePoint());
+
+ // Connect the entry block to the last block in the caller's graph.
+ MBasicBlock* pred = callerBuilder()->current;
+ MOZ_ASSERT(pred == callerResumePoint()->block());
+
+ pred->end(MGoto::New(alloc(), current));
+ if (!current->addPredecessorWithoutPhis(pred)) {
+ return false;
+ }
+
+ MConstant* undef = constant(UndefinedValue());
+
+ // Initialize env chain slot to Undefined. It's set later by
+ // |buildEnvironmentChain|.
+ current->initSlot(info().environmentChainSlot(), undef);
+
+ // Initialize |return value| slot.
+ current->initSlot(info().returnValueSlot(), undef);
+
+ // Initialize |arguments| slot if needed.
+ if (info().needsArgsObj()) {
+ current->initSlot(info().argsObjSlot(), undef);
+ }
+
+ // Initialize |this| slot.
+ current->initSlot(info().thisSlot(), inlineCallInfo()->thisArg());
+
+ uint32_t callerArgs = inlineCallInfo()->argc();
+ uint32_t actualArgs = info().nargs();
+ uint32_t passedArgs = std::min<uint32_t>(callerArgs, actualArgs);
+
+ // Initialize actually set arguments.
+ for (uint32_t i = 0; i < passedArgs; i++) {
+ MDefinition* arg = inlineCallInfo()->getArg(i);
+ current->initSlot(info().argSlotUnchecked(i), arg);
+ }
+
+ // Pass undefined for missing arguments.
+ for (uint32_t i = passedArgs; i < actualArgs; i++) {
+ current->initSlot(info().argSlotUnchecked(i), undef);
+ }
+
+ // Initialize local slots.
+ for (uint32_t i = 0; i < info().nlocals(); i++) {
+ current->initSlot(info().localSlot(i), undef);
+ }
+
+ MOZ_ASSERT(current->entryResumePoint()->stackDepth() == info().totalSlots());
+
+ if (!buildEnvironmentChain()) {
+ return false;
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+// In debug builds, after compiling a bytecode op, this class is used to check
+// that all values popped by this opcode either:
+//
+// (1) Have the ImplicitlyUsed flag set on them.
+// (2) Have more uses than before compiling this op (the value is
+// used as operand of a new MIR instruction).
+//
+// This is used to catch problems where WarpBuilder pops a value without
+// adding any SSA uses and doesn't call setImplicitlyUsedUnchecked on it.
+class MOZ_RAII WarpPoppedValueUseChecker {
+ Vector<MDefinition*, 4, SystemAllocPolicy> popped_;
+ Vector<size_t, 4, SystemAllocPolicy> poppedUses_;
+ MBasicBlock* current_;
+ BytecodeLocation loc_;
+
+ public:
+ WarpPoppedValueUseChecker(MBasicBlock* current, BytecodeLocation loc)
+ : current_(current), loc_(loc) {}
+
+ [[nodiscard]] bool init() {
+ // Don't require SSA uses for values popped by these ops.
+ switch (loc_.getOp()) {
+ case JSOp::Pop:
+ case JSOp::PopN:
+ case JSOp::DupAt:
+ case JSOp::Dup:
+ case JSOp::Dup2:
+ case JSOp::Pick:
+ case JSOp::Unpick:
+ case JSOp::Swap:
+ case JSOp::SetArg:
+ case JSOp::SetLocal:
+ case JSOp::InitLexical:
+ case JSOp::SetRval:
+ case JSOp::Void:
+ // Basic stack/local/argument management opcodes.
+ return true;
+
+ case JSOp::Case:
+ case JSOp::Default:
+ // These ops have to pop the switch value when branching but don't
+ // actually use it.
+ return true;
+
+ default:
+ break;
+ }
+
+ unsigned nuses = loc_.useCount();
+
+ for (unsigned i = 0; i < nuses; i++) {
+ MDefinition* def = current_->peek(-int32_t(i + 1));
+ if (!popped_.append(def) || !poppedUses_.append(def->defUseCount())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ void checkAfterOp() {
+ for (size_t i = 0; i < popped_.length(); i++) {
+ // First value popped by JSOp::EndIter is not used at all, it's similar
+ // to JSOp::Pop above.
+ if (loc_.is(JSOp::EndIter) && i == 0) {
+ continue;
+ }
+ MOZ_ASSERT(popped_[i]->isImplicitlyUsed() ||
+ popped_[i]->defUseCount() > poppedUses_[i]);
+ }
+ }
+};
+#endif
+
+bool WarpBuilder::buildBody() {
+ for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
+ if (mirGen().shouldCancel("WarpBuilder (opcode loop)")) {
+ return false;
+ }
+
+ // Skip unreachable ops (for example code after a 'return' or 'throw') until
+ // we get to the next jump target.
+ if (hasTerminatedBlock()) {
+ // Finish any "broken" loops with an unreachable backedge. For example:
+ //
+ // do {
+ // ...
+ // return;
+ // ...
+ // } while (x);
+ //
+ // This loop never actually loops.
+ if (loc.isBackedge() && !loopStack_.empty()) {
+ BytecodeLocation loopHead(script_, loopStack_.back().header()->pc());
+ if (loc.isBackedgeForLoophead(loopHead)) {
+ decLoopDepth();
+ loopStack_.popBack();
+ }
+ }
+ if (!loc.isJumpTarget()) {
+ continue;
+ }
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+#ifdef DEBUG
+ WarpPoppedValueUseChecker useChecker(current, loc);
+ if (!useChecker.init()) {
+ return false;
+ }
+#endif
+
+ JSOp op = loc.getOp();
+
+#define BUILD_OP(OP, ...) \
+ case JSOp::OP: \
+ if (MOZ_UNLIKELY(!this->build_##OP(loc))) { \
+ return false; \
+ } \
+ break;
+ switch (op) { FOR_EACH_OPCODE(BUILD_OP) }
+#undef BUILD_OP
+
+#ifdef DEBUG
+ useChecker.checkAfterOp();
+#endif
+ }
+
+ return true;
+}
+
+#define DEF_OP(OP) \
+ bool WarpBuilder::build_##OP(BytecodeLocation) { \
+ MOZ_CRASH("Unsupported op"); \
+ }
+WARP_UNSUPPORTED_OPCODE_LIST(DEF_OP)
+#undef DEF_OP
+
+bool WarpBuilder::build_Nop(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_NopDestructuring(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_TryDestructuring(BytecodeLocation) {
+ // Set the hasTryBlock flag to turn off optimizations that eliminate dead
+ // resume points operands because the exception handler code for
+ // TryNoteKind::Destructuring is effectively a (specialized) catch-block.
+ graph().setHasTryBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Lineno(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_DebugLeaveLexicalEnv(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_Undefined(BytecodeLocation) {
+ pushConstant(UndefinedValue());
+ return true;
+}
+
+bool WarpBuilder::build_Void(BytecodeLocation) {
+ current->pop();
+ pushConstant(UndefinedValue());
+ return true;
+}
+
+bool WarpBuilder::build_Null(BytecodeLocation) {
+ pushConstant(NullValue());
+ return true;
+}
+
+bool WarpBuilder::build_Hole(BytecodeLocation) {
+ pushConstant(MagicValue(JS_ELEMENTS_HOLE));
+ return true;
+}
+
+bool WarpBuilder::build_Uninitialized(BytecodeLocation) {
+ pushConstant(MagicValue(JS_UNINITIALIZED_LEXICAL));
+ return true;
+}
+
+bool WarpBuilder::build_IsConstructing(BytecodeLocation) {
+ pushConstant(MagicValue(JS_IS_CONSTRUCTING));
+ return true;
+}
+
+bool WarpBuilder::build_False(BytecodeLocation) {
+ pushConstant(BooleanValue(false));
+ return true;
+}
+
+bool WarpBuilder::build_True(BytecodeLocation) {
+ pushConstant(BooleanValue(true));
+ return true;
+}
+
+bool WarpBuilder::build_Pop(BytecodeLocation) {
+ current->pop();
+ return true;
+}
+
+bool WarpBuilder::build_PopN(BytecodeLocation loc) {
+ for (uint32_t i = 0, n = loc.getPopCount(); i < n; i++) {
+ current->pop();
+ }
+ return true;
+}
+
+bool WarpBuilder::build_Dup(BytecodeLocation) {
+ current->pushSlot(current->stackDepth() - 1);
+ return true;
+}
+
+bool WarpBuilder::build_Dup2(BytecodeLocation) {
+ uint32_t lhsSlot = current->stackDepth() - 2;
+ uint32_t rhsSlot = current->stackDepth() - 1;
+ current->pushSlot(lhsSlot);
+ current->pushSlot(rhsSlot);
+ return true;
+}
+
+bool WarpBuilder::build_DupAt(BytecodeLocation loc) {
+ current->pushSlot(current->stackDepth() - 1 - loc.getDupAtIndex());
+ return true;
+}
+
+bool WarpBuilder::build_Swap(BytecodeLocation) {
+ current->swapAt(-1);
+ return true;
+}
+
+bool WarpBuilder::build_Pick(BytecodeLocation loc) {
+ int32_t depth = -int32_t(loc.getPickDepth());
+ current->pick(depth);
+ return true;
+}
+
+bool WarpBuilder::build_Unpick(BytecodeLocation loc) {
+ int32_t depth = -int32_t(loc.getUnpickDepth());
+ current->unpick(depth);
+ return true;
+}
+
+bool WarpBuilder::build_Zero(BytecodeLocation) {
+ pushConstant(Int32Value(0));
+ return true;
+}
+
+bool WarpBuilder::build_One(BytecodeLocation) {
+ pushConstant(Int32Value(1));
+ return true;
+}
+
+bool WarpBuilder::build_Int8(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getInt8()));
+ return true;
+}
+
+bool WarpBuilder::build_Uint16(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getUint16()));
+ return true;
+}
+
+bool WarpBuilder::build_Uint24(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getUint24()));
+ return true;
+}
+
+bool WarpBuilder::build_Int32(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getInt32()));
+ return true;
+}
+
+bool WarpBuilder::build_Double(BytecodeLocation loc) {
+ pushConstant(loc.getInlineValue());
+ return true;
+}
+
+bool WarpBuilder::build_BigInt(BytecodeLocation loc) {
+ BigInt* bi = loc.getBigInt(script_);
+ pushConstant(BigIntValue(bi));
+ return true;
+}
+
+bool WarpBuilder::build_String(BytecodeLocation loc) {
+ JSString* str = loc.getString(script_);
+ pushConstant(StringValue(str));
+ return true;
+}
+
+bool WarpBuilder::build_Symbol(BytecodeLocation loc) {
+ uint32_t which = loc.getSymbolIndex();
+ JS::Symbol* sym = mirGen().runtime->wellKnownSymbols().get(which);
+ pushConstant(SymbolValue(sym));
+ return true;
+}
+
+bool WarpBuilder::build_RegExp(BytecodeLocation loc) {
+ RegExpObject* reObj = loc.getRegExp(script_);
+
+ auto* snapshot = getOpSnapshot<WarpRegExp>(loc);
+
+ MRegExp* regexp = MRegExp::New(alloc(), reObj, snapshot->hasShared());
+ current->add(regexp);
+ current->push(regexp);
+
+ return true;
+}
+
+bool WarpBuilder::build_Return(BytecodeLocation) {
+ MDefinition* def = current->pop();
+
+ MReturn* ret = MReturn::New(alloc(), def);
+ current->end(ret);
+
+ if (!graph().addReturn(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_RetRval(BytecodeLocation) {
+ MDefinition* rval;
+ if (script_->noScriptRval()) {
+ rval = constant(UndefinedValue());
+ } else {
+ rval = current->getSlot(info().returnValueSlot());
+ }
+
+ MReturn* ret = MReturn::New(alloc(), rval);
+ current->end(ret);
+
+ if (!graph().addReturn(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_SetRval(BytecodeLocation) {
+ MOZ_ASSERT(!script_->noScriptRval());
+ MDefinition* rval = current->pop();
+ current->setSlot(info().returnValueSlot(), rval);
+ return true;
+}
+
+bool WarpBuilder::build_GetRval(BytecodeLocation) {
+ MOZ_ASSERT(!script_->noScriptRval());
+ MDefinition* rval = current->getSlot(info().returnValueSlot());
+ current->push(rval);
+ return true;
+}
+
+bool WarpBuilder::build_GetLocal(BytecodeLocation loc) {
+ current->pushLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_SetLocal(BytecodeLocation loc) {
+ current->setLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_InitLexical(BytecodeLocation loc) {
+ current->setLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_GetArg(BytecodeLocation loc) {
+ uint32_t arg = loc.arg();
+ if (info().argsObjAliasesFormals()) {
+ MDefinition* argsObj = current->argumentsObject();
+ auto* getArg = MGetArgumentsObjectArg::New(alloc(), argsObj, arg);
+ current->add(getArg);
+ current->push(getArg);
+ } else {
+ current->pushArg(arg);
+ }
+ return true;
+}
+
+bool WarpBuilder::build_GetFrameArg(BytecodeLocation loc) {
+ current->pushArgUnchecked(loc.arg());
+ return true;
+}
+
+bool WarpBuilder::build_SetArg(BytecodeLocation loc) {
+ uint32_t arg = loc.arg();
+ MDefinition* val = current->peek(-1);
+
+ if (!info().argsObjAliasesFormals()) {
+ // Either |arguments| is never referenced within this function, or
+ // it doesn't map to the actual arguments values. Either way, we
+ // don't need to worry about synchronizing the argument values
+ // when writing to them.
+ current->setArg(arg);
+ return true;
+ }
+
+ // If an arguments object is in use, and it aliases formals, then all SetArgs
+ // must go through the arguments object.
+ MDefinition* argsObj = current->argumentsObject();
+ current->add(MPostWriteBarrier::New(alloc(), argsObj, val));
+ auto* ins = MSetArgumentsObjectArg::New(alloc(), argsObj, val, arg);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ArgumentsLength(BytecodeLocation) {
+ if (inlineCallInfo()) {
+ pushConstant(Int32Value(inlineCallInfo()->argc()));
+ } else {
+ auto* argsLength = MArgumentsLength::New(alloc());
+ current->add(argsLength);
+ current->push(argsLength);
+ }
+ return true;
+}
+
+bool WarpBuilder::build_GetActualArg(BytecodeLocation) {
+ MDefinition* index = current->pop();
+ MInstruction* arg;
+ if (inlineCallInfo()) {
+ arg = MGetInlinedArgument::New(alloc(), index, *inlineCallInfo());
+ } else {
+ arg = MGetFrameArgument::New(alloc(), index);
+ }
+ current->add(arg);
+ current->push(arg);
+ return true;
+}
+
+bool WarpBuilder::build_ToNumeric(BytecodeLocation loc) {
+ return buildUnaryOp(loc);
+}
+
+bool WarpBuilder::buildUnaryOp(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::UnaryArith, {value});
+}
+
+bool WarpBuilder::build_Inc(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Dec(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Pos(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Neg(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_BitNot(BytecodeLocation loc) {
+ return buildUnaryOp(loc);
+}
+
+bool WarpBuilder::buildBinaryOp(BytecodeLocation loc) {
+ MDefinition* right = current->pop();
+ MDefinition* left = current->pop();
+ return buildIC(loc, CacheKind::BinaryArith, {left, right});
+}
+
+bool WarpBuilder::build_Add(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Sub(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Mul(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Div(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Mod(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Pow(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_BitAnd(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_BitOr(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_BitXor(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_Lsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Rsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Ursh(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::buildCompareOp(BytecodeLocation loc) {
+ MDefinition* right = current->pop();
+ MDefinition* left = current->pop();
+ return buildIC(loc, CacheKind::Compare, {left, right});
+}
+
+bool WarpBuilder::build_Eq(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Ne(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Lt(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Le(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Gt(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Ge(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_StrictEq(BytecodeLocation loc) {
+ return buildCompareOp(loc);
+}
+
+bool WarpBuilder::build_StrictNe(BytecodeLocation loc) {
+ return buildCompareOp(loc);
+}
+
+// Returns true iff the MTest added for |op| has a true-target corresponding
+// with the join point in the bytecode.
+static bool TestTrueTargetIsJoinPoint(JSOp op) {
+ switch (op) {
+ case JSOp::JumpIfTrue:
+ case JSOp::Or:
+ case JSOp::Case:
+ return true;
+
+ case JSOp::JumpIfFalse:
+ case JSOp::And:
+ case JSOp::Coalesce:
+ return false;
+
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+}
+
+bool WarpBuilder::build_JumpTarget(BytecodeLocation loc) {
+ PendingEdgesMap::Ptr p = pendingEdges_.lookup(loc.toRawBytecode());
+ if (!p) {
+ // No (reachable) jumps so this is just a no-op.
+ return true;
+ }
+
+ PendingEdges edges(std::move(p->value()));
+ pendingEdges_.remove(p);
+
+ MOZ_ASSERT(!edges.empty());
+
+ // Create join block if there's fall-through from the previous bytecode op.
+ if (!hasTerminatedBlock()) {
+ MBasicBlock* pred = current;
+ if (!startNewBlock(pred, loc)) {
+ return false;
+ }
+ pred->end(MGoto::New(alloc(), current));
+ }
+
+ for (const PendingEdge& edge : edges) {
+ MBasicBlock* source = edge.block();
+ uint32_t numToPop = edge.numToPop();
+
+ if (hasTerminatedBlock()) {
+ if (!startNewBlock(source, loc, numToPop)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(source->stackDepth() - numToPop == current->stackDepth());
+ if (!current->addPredecessorPopN(alloc(), source, numToPop)) {
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(source->lastIns()->isTest() || source->lastIns()->isGoto() ||
+ source->lastIns()->isTableSwitch());
+ source->lastIns()->initSuccessor(edge.successor(), current);
+ }
+
+ MOZ_ASSERT(!hasTerminatedBlock());
+ return true;
+}
+
+bool WarpBuilder::addIteratorLoopPhis(BytecodeLocation loopHead) {
+ // When unwinding the stack for a thrown exception, the exception handler must
+ // close live iterators. For ForIn and Destructuring loops, the exception
+ // handler needs access to values on the stack. To prevent them from being
+ // optimized away (and replaced with the JS_OPTIMIZED_OUT MagicValue), we need
+ // to mark the phis (and phis they flow into) as having implicit uses.
+ // See ProcessTryNotes in vm/Interpreter.cpp and CloseLiveIteratorIon in
+ // jit/JitFrames.cpp
+
+ MOZ_ASSERT(current->stackDepth() >= info().firstStackSlot());
+
+ bool emptyStack = current->stackDepth() == info().firstStackSlot();
+ if (emptyStack) {
+ return true;
+ }
+
+ jsbytecode* loopHeadPC = loopHead.toRawBytecode();
+
+ for (TryNoteIterAllNoGC tni(script_, loopHeadPC); !tni.done(); ++tni) {
+ const TryNote& tn = **tni;
+
+ // Stop if we reach an outer loop because outer loops were already
+ // processed when we visited their loop headers.
+ if (tn.isLoop()) {
+ BytecodeLocation tnStart = script_->offsetToLocation(tn.start);
+ if (tnStart != loopHead) {
+ MOZ_ASSERT(tnStart.is(JSOp::LoopHead));
+ MOZ_ASSERT(tnStart < loopHead);
+ return true;
+ }
+ }
+
+ switch (tn.kind()) {
+ case TryNoteKind::Destructuring:
+ case TryNoteKind::ForIn: {
+ // For for-in loops we add the iterator object to iterators(). For
+ // destructuring loops we add the "done" value that's on top of the
+ // stack and used in the exception handler.
+ MOZ_ASSERT(tn.stackDepth >= 1);
+ uint32_t slot = info().stackSlot(tn.stackDepth - 1);
+ MPhi* phi = current->getSlot(slot)->toPhi();
+ if (!iterators()->append(phi)) {
+ return false;
+ }
+ break;
+ }
+ case TryNoteKind::Loop:
+ case TryNoteKind::ForOf:
+ // Regular loops do not have iterators to close. ForOf loops handle
+ // unwinding using catch blocks.
+ break;
+ default:
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_LoopHead(BytecodeLocation loc) {
+ // All loops have the following bytecode structure:
+ //
+ // LoopHead
+ // ...
+ // JumpIfTrue/Goto to LoopHead
+
+ if (hasTerminatedBlock()) {
+ // The whole loop is unreachable.
+ return true;
+ }
+
+ // Handle OSR from Baseline JIT code.
+ if (loc.toRawBytecode() == info().osrPc()) {
+ if (!startNewOsrPreHeaderBlock(loc)) {
+ return false;
+ }
+ }
+
+ incLoopDepth();
+
+ MBasicBlock* pred = current;
+ if (!startNewLoopHeaderBlock(loc)) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+
+ if (!addIteratorLoopPhis(loc)) {
+ return false;
+ }
+
+ MInterruptCheck* check = MInterruptCheck::New(alloc());
+ current->add(check);
+
+#ifdef JS_CACHEIR_SPEW
+ if (snapshot().needsFinalWarmUpCount()) {
+ MIncrementWarmUpCounter* ins =
+ MIncrementWarmUpCounter::New(alloc(), script_);
+ current->add(ins);
+ }
+#endif
+
+ return true;
+}
+
+bool WarpBuilder::buildTestOp(BytecodeLocation loc) {
+ MDefinition* originalValue = current->peek(-1);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ // If we have CacheIR, we can use it to refine the input. Note that
+ // the transpiler doesn't generate any control instructions. Instead,
+ // we fall through and generate them below.
+ MDefinition* value = current->pop();
+ if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
+ return false;
+ }
+ }
+
+ if (loc.isBackedge()) {
+ return buildTestBackedge(loc);
+ }
+
+ JSOp op = loc.getOp();
+ BytecodeLocation target1 = loc.next();
+ BytecodeLocation target2 = loc.getJumpTarget();
+
+ if (TestTrueTargetIsJoinPoint(op)) {
+ std::swap(target1, target2);
+ }
+
+ MDefinition* value = current->pop();
+
+ // JSOp::And and JSOp::Or leave the top stack value unchanged. The
+ // top stack value may have been converted to bool by a transpiled
+ // ToBool IC, so we push the original value.
+ bool mustKeepCondition = (op == JSOp::And || op == JSOp::Or);
+ if (mustKeepCondition) {
+ current->push(originalValue);
+ }
+
+ // If this op always branches to the same location we treat this as a
+ // JSOp::Goto.
+ if (target1 == target2) {
+ value->setImplicitlyUsedUnchecked();
+ return buildForwardGoto(target1);
+ }
+
+ MTest* test = MTest::New(alloc(), value, /* ifTrue = */ nullptr,
+ /* ifFalse = */ nullptr);
+ current->end(test);
+
+ // JSOp::Case must pop a second value on the true-branch (the input to the
+ // switch-statement).
+ uint32_t numToPop = (loc.getOp() == JSOp::Case) ? 1 : 0;
+
+ if (!addPendingEdge(target1, current, MTest::TrueBranchIndex, numToPop)) {
+ return false;
+ }
+ if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ test->setObservedTypes(typesSnapshot->list());
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildTestBackedge(BytecodeLocation loc) {
+ MOZ_ASSERT(loc.is(JSOp::JumpIfTrue));
+ MOZ_ASSERT(loopDepth() > 0);
+
+ MDefinition* value = current->pop();
+
+ BytecodeLocation loopHead = loc.getJumpTarget();
+ MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
+
+ BytecodeLocation successor = loc.next();
+
+ // We can finish the loop now. Use the loophead pc instead of the current pc
+ // because the stack depth at the start of that op matches the current stack
+ // depth (after popping our operand).
+ MBasicBlock* pred = current;
+ if (!startNewBlock(current, loopHead)) {
+ return false;
+ }
+
+ MTest* test = MTest::New(alloc(), value, /* ifTrue = */ current,
+ /* ifFalse = */ nullptr);
+ pred->end(test);
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ test->setObservedTypes(typesSnapshot->list());
+ }
+
+ if (!addPendingEdge(successor, pred, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ return buildBackedge();
+}
+
+bool WarpBuilder::build_JumpIfFalse(BytecodeLocation loc) {
+ return buildTestOp(loc);
+}
+
+bool WarpBuilder::build_JumpIfTrue(BytecodeLocation loc) {
+ return buildTestOp(loc);
+}
+
+bool WarpBuilder::build_And(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Or(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Case(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Default(BytecodeLocation loc) {
+ current->pop();
+ return buildForwardGoto(loc.getJumpTarget());
+}
+
+bool WarpBuilder::build_Coalesce(BytecodeLocation loc) {
+ BytecodeLocation target1 = loc.next();
+ BytecodeLocation target2 = loc.getJumpTarget();
+ MOZ_ASSERT(target2 > target1);
+
+ MDefinition* value = current->peek(-1);
+
+ MInstruction* isNullOrUndefined = MIsNullOrUndefined::New(alloc(), value);
+ current->add(isNullOrUndefined);
+
+ current->end(MTest::New(alloc(), isNullOrUndefined, /* ifTrue = */ nullptr,
+ /* ifFalse = */ nullptr));
+
+ if (!addPendingEdge(target1, current, MTest::TrueBranchIndex)) {
+ return false;
+ }
+ if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildBackedge() {
+ decLoopDepth();
+
+ MBasicBlock* header = loopStack_.popCopy().header();
+ current->end(MGoto::New(alloc(), header));
+
+ if (!header->setBackedge(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildForwardGoto(BytecodeLocation target) {
+ current->end(MGoto::New(alloc(), nullptr));
+
+ if (!addPendingEdge(target, current, MGoto::TargetIndex)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Goto(BytecodeLocation loc) {
+ if (loc.isBackedge()) {
+ return buildBackedge();
+ }
+
+ return buildForwardGoto(loc.getJumpTarget());
+}
+
+bool WarpBuilder::build_IsNullOrUndefined(BytecodeLocation loc) {
+ MDefinition* value = current->peek(-1);
+ auto* isNullOrUndef = MIsNullOrUndefined::New(alloc(), value);
+ current->add(isNullOrUndef);
+ current->push(isNullOrUndef);
+ return true;
+}
+
+bool WarpBuilder::build_DebugCheckSelfHosted(BytecodeLocation loc) {
+#ifdef DEBUG
+ MDefinition* val = current->pop();
+ MDebugCheckSelfHosted* check = MDebugCheckSelfHosted::New(alloc(), val);
+ current->add(check);
+ current->push(check);
+ if (!resumeAfter(check, loc)) {
+ return false;
+ }
+#endif
+ return true;
+}
+
+bool WarpBuilder::build_DynamicImport(BytecodeLocation loc) {
+ MDefinition* options = current->pop();
+ MDefinition* specifier = current->pop();
+ MDynamicImport* ins = MDynamicImport::New(alloc(), specifier, options);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Not(BytecodeLocation loc) {
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ // If we have CacheIR, we can use it to refine the input before
+ // emitting the MNot.
+ MDefinition* value = current->pop();
+ if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
+ return false;
+ }
+ }
+
+ MDefinition* value = current->pop();
+ MNot* ins = MNot::New(alloc(), value);
+ current->add(ins);
+ current->push(ins);
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ ins->setObservedTypes(typesSnapshot->list());
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_ToString(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+
+ if (value->type() == MIRType::String) {
+ value->setImplicitlyUsedUnchecked();
+ current->push(value);
+ return true;
+ }
+
+ MToString* ins =
+ MToString::New(alloc(), value, MToString::SideEffectHandling::Supported);
+ current->add(ins);
+ current->push(ins);
+ if (ins->isEffectful()) {
+ return resumeAfter(ins, loc);
+ }
+ return true;
+}
+
+bool WarpBuilder::usesEnvironmentChain() const {
+ return script_->jitScript()->usesEnvironmentChain();
+}
+
+bool WarpBuilder::build_GlobalOrEvalDeclInstantiation(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->isForEval(), "Eval scripts not supported");
+ auto* redeclCheck = MGlobalDeclInstantiation::New(alloc());
+ current->add(redeclCheck);
+ return resumeAfter(redeclCheck, loc);
+}
+
+bool WarpBuilder::build_BindVar(BytecodeLocation) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+ MCallBindVar* ins = MCallBindVar::New(alloc(), env);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_MutateProto(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ MDefinition* obj = current->peek(-1);
+ MMutateProto* mutate = MMutateProto::New(alloc(), obj, value);
+ current->add(mutate);
+ return resumeAfter(mutate, loc);
+}
+
+MDefinition* WarpBuilder::getCallee() {
+ if (inlineCallInfo()) {
+ return inlineCallInfo()->callee();
+ }
+
+ MInstruction* callee = MCallee::New(alloc());
+ current->add(callee);
+ return callee;
+}
+
+bool WarpBuilder::build_Callee(BytecodeLocation) {
+ MDefinition* callee = getCallee();
+ current->push(callee);
+ return true;
+}
+
+bool WarpBuilder::build_ToAsyncIter(BytecodeLocation loc) {
+ MDefinition* nextMethod = current->pop();
+ MDefinition* iterator = current->pop();
+ MToAsyncIter* ins = MToAsyncIter::New(alloc(), iterator, nextMethod);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ToPropertyKey(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::ToPropertyKey, {value});
+}
+
+bool WarpBuilder::build_Typeof(BytecodeLocation loc) {
+ MDefinition* input = current->pop();
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ auto* typeOf = MTypeOf::New(alloc(), input);
+ typeOf->setObservedTypes(typesSnapshot->list());
+ current->add(typeOf);
+
+ auto* ins = MTypeOfName::New(alloc(), typeOf);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+
+ return buildIC(loc, CacheKind::TypeOf, {input});
+}
+
+bool WarpBuilder::build_TypeofExpr(BytecodeLocation loc) {
+ return build_Typeof(loc);
+}
+
+bool WarpBuilder::build_Arguments(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpArguments>(loc);
+ MOZ_ASSERT(info().needsArgsObj());
+ MOZ_ASSERT(snapshot);
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ ArgumentsObject* templateObj = snapshot->templateObj();
+ MDefinition* env = current->environmentChain();
+
+ MInstruction* argsObj;
+ if (inlineCallInfo()) {
+ argsObj = MCreateInlinedArgumentsObject::New(
+ alloc(), env, getCallee(), inlineCallInfo()->argv(), templateObj);
+ if (!argsObj) {
+ return false;
+ }
+ } else {
+ argsObj = MCreateArgumentsObject::New(alloc(), env, templateObj);
+ }
+ current->add(argsObj);
+ current->setArgumentsObject(argsObj);
+ current->push(argsObj);
+
+ return true;
+}
+
+bool WarpBuilder::build_ObjWithProto(BytecodeLocation loc) {
+ MDefinition* proto = current->pop();
+ MInstruction* ins = MObjectWithProto::New(alloc(), proto);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+MDefinition* WarpBuilder::walkEnvironmentChain(uint32_t numHops) {
+ MDefinition* env = current->environmentChain();
+
+ for (uint32_t i = 0; i < numHops; i++) {
+ if (!alloc().ensureBallast()) {
+ return nullptr;
+ }
+
+ MInstruction* ins = MEnclosingEnvironment::New(alloc(), env);
+ current->add(ins);
+ env = ins;
+ }
+
+ return env;
+}
+
+bool WarpBuilder::build_GetAliasedVar(BytecodeLocation loc) {
+ EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
+ MDefinition* obj = walkEnvironmentChain(ec.hops());
+ if (!obj) {
+ return false;
+ }
+
+ MInstruction* load;
+ if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
+ load = MLoadFixedSlot::New(alloc(), obj, ec.slot());
+ } else {
+ MInstruction* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
+ load = MLoadDynamicSlot::New(alloc(), slots, slot);
+ }
+
+ current->add(load);
+ current->push(load);
+ return true;
+}
+
+bool WarpBuilder::build_SetAliasedVar(BytecodeLocation loc) {
+ EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
+ MDefinition* val = current->peek(-1);
+ MDefinition* obj = walkEnvironmentChain(ec.hops());
+ if (!obj) {
+ return false;
+ }
+
+ current->add(MPostWriteBarrier::New(alloc(), obj, val));
+
+ MInstruction* store;
+ if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
+ store = MStoreFixedSlot::NewBarriered(alloc(), obj, ec.slot(), val);
+ } else {
+ MInstruction* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
+ store = MStoreDynamicSlot::NewBarriered(alloc(), slots, slot, val);
+ }
+
+ current->add(store);
+ return resumeAfter(store, loc);
+}
+
+bool WarpBuilder::build_InitAliasedLexical(BytecodeLocation loc) {
+ return build_SetAliasedVar(loc);
+}
+
+bool WarpBuilder::build_EnvCallee(BytecodeLocation loc) {
+ uint32_t numHops = loc.getEnvCalleeNumHops();
+ MDefinition* env = walkEnvironmentChain(numHops);
+ if (!env) {
+ return false;
+ }
+
+ auto* callee = MLoadFixedSlot::New(alloc(), env, CallObject::calleeSlot());
+ current->add(callee);
+ current->push(callee);
+ return true;
+}
+
+bool WarpBuilder::build_Iter(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ return buildIC(loc, CacheKind::GetIterator, {obj});
+}
+
+bool WarpBuilder::build_MoreIter(BytecodeLocation loc) {
+ MDefinition* iter = current->peek(-1);
+ MInstruction* ins = MIteratorMore::New(alloc(), iter);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_EndIter(BytecodeLocation loc) {
+ current->pop(); // Iterator value is not used.
+ MDefinition* iter = current->pop();
+ MInstruction* ins = MIteratorEnd::New(alloc(), iter);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CloseIter(BytecodeLocation loc) {
+ MDefinition* iter = current->pop();
+ return buildIC(loc, CacheKind::CloseIter, {iter});
+}
+
+bool WarpBuilder::build_IsNoIter(BytecodeLocation) {
+ MDefinition* def = current->peek(-1);
+ MOZ_ASSERT(def->isIteratorMore());
+ MInstruction* ins = MIsNoIter::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::transpileCall(BytecodeLocation loc,
+ const WarpCacheIR* cacheIRSnapshot,
+ CallInfo* callInfo) {
+ // Synthesize the constant number of arguments for this call op.
+ auto* argc = MConstant::New(alloc(), Int32Value(callInfo->argc()));
+ current->add(argc);
+
+ return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {argc}, callInfo);
+}
+
+void WarpBuilder::buildCreateThis(CallInfo& callInfo) {
+ MOZ_ASSERT(callInfo.constructing());
+
+ // Inline the this-object allocation on the caller-side.
+ MDefinition* callee = callInfo.callee();
+ MDefinition* newTarget = callInfo.getNewTarget();
+ auto* createThis = MCreateThis::New(alloc(), callee, newTarget);
+ current->add(createThis);
+ callInfo.thisArg()->setImplicitlyUsedUnchecked();
+ callInfo.setThis(createThis);
+}
+
+bool WarpBuilder::buildCallOp(BytecodeLocation loc) {
+ uint32_t argc = loc.getCallArgc();
+ JSOp op = loc.getOp();
+ bool constructing = IsConstructOp(op);
+ bool ignoresReturnValue = (op == JSOp::CallIgnoresRv || loc.resultIsPopped());
+
+ CallInfo callInfo(alloc(), constructing, ignoresReturnValue);
+ if (!callInfo.init(current, argc)) {
+ return false;
+ }
+
+ if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
+ // Transpile the CacheIR to generate the correct guards before
+ // inlining. In this case, CacheOp::CallInlinedFunction updates
+ // the CallInfo, but does not generate a call.
+ callInfo.markAsInlined();
+ if (!transpileCall(loc, inliningSnapshot->cacheIRSnapshot(), &callInfo)) {
+ return false;
+ }
+
+ // Generate the body of the inlined function.
+ return buildInlinedCall(loc, inliningSnapshot, callInfo);
+ }
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ if (getOpSnapshot<WarpBailout>(loc)) {
+ callInfo.setImplicitlyUsedUnchecked();
+ return buildBailoutForColdIC(loc, CacheKind::Call);
+ }
+
+ bool needsThisCheck = false;
+ if (callInfo.constructing()) {
+ buildCreateThis(callInfo);
+ needsThisCheck = true;
+ }
+
+ MCall* call = makeCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_Call(BytecodeLocation loc) { return buildCallOp(loc); }
+
+bool WarpBuilder::build_CallContent(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallIgnoresRv(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallIter(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallContentIter(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_New(BytecodeLocation loc) { return buildCallOp(loc); }
+
+bool WarpBuilder::build_NewContent(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_SuperCall(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_FunctionThis(BytecodeLocation loc) {
+ MOZ_ASSERT(info().hasFunMaybeLazy());
+
+ if (script_->strict()) {
+ // No need to wrap primitive |this| in strict mode.
+ current->pushSlot(info().thisSlot());
+ return true;
+ }
+
+ MOZ_ASSERT(!script_->hasNonSyntacticScope(),
+ "WarpOracle should have aborted compilation");
+
+ MDefinition* def = current->getSlot(info().thisSlot());
+ JSObject* globalThis = snapshot().globalLexicalEnvThis();
+
+ auto* thisObj = MBoxNonStrictThis::New(alloc(), def, globalThis);
+ current->add(thisObj);
+ current->push(thisObj);
+
+ return true;
+}
+
+bool WarpBuilder::build_GlobalThis(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+ JSObject* obj = snapshot().globalLexicalEnvThis();
+ pushConstant(ObjectValue(*obj));
+ return true;
+}
+
+MConstant* WarpBuilder::globalLexicalEnvConstant() {
+ JSObject* globalLexical = snapshot().globalLexicalEnv();
+ return constant(ObjectValue(*globalLexical));
+}
+
+bool WarpBuilder::build_GetName(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+ return buildIC(loc, CacheKind::GetName, {env});
+}
+
+bool WarpBuilder::build_GetGName(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ MDefinition* env = globalLexicalEnvConstant();
+ return buildIC(loc, CacheKind::GetName, {env});
+}
+
+bool WarpBuilder::build_BindName(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+ return buildIC(loc, CacheKind::BindName, {env});
+}
+
+bool WarpBuilder::build_BindGName(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ if (const auto* snapshot = getOpSnapshot<WarpBindGName>(loc)) {
+ JSObject* globalEnv = snapshot->globalEnv();
+ pushConstant(ObjectValue(*globalEnv));
+ return true;
+ }
+
+ MDefinition* env = globalLexicalEnvConstant();
+ return buildIC(loc, CacheKind::BindName, {env});
+}
+
+bool WarpBuilder::build_GetProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ return buildIC(loc, CacheKind::GetProp, {val});
+}
+
+bool WarpBuilder::build_GetElem(BytecodeLocation loc) {
+ MDefinition* id = current->pop();
+ MDefinition* val = current->pop();
+ return buildIC(loc, CacheKind::GetElem, {val, id});
+}
+
+bool WarpBuilder::build_SetProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->pop();
+ current->push(val);
+ return buildIC(loc, CacheKind::SetProp, {obj, val});
+}
+
+bool WarpBuilder::build_StrictSetProp(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_SetName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_StrictSetName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_SetGName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_StrictSetGName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_InitGLexical(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ MDefinition* globalLexical = globalLexicalEnvConstant();
+ MDefinition* val = current->peek(-1);
+
+ return buildIC(loc, CacheKind::SetProp, {globalLexical, val});
+}
+
+bool WarpBuilder::build_SetElem(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->pop();
+ current->push(val);
+ return buildIC(loc, CacheKind::SetElem, {obj, id, val});
+}
+
+bool WarpBuilder::build_StrictSetElem(BytecodeLocation loc) {
+ return build_SetElem(loc);
+}
+
+bool WarpBuilder::build_DelProp(BytecodeLocation loc) {
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* obj = current->pop();
+ bool strict = loc.getOp() == JSOp::StrictDelProp;
+
+ MInstruction* ins = MDeleteProperty::New(alloc(), obj, name, strict);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_StrictDelProp(BytecodeLocation loc) {
+ return build_DelProp(loc);
+}
+
+bool WarpBuilder::build_DelElem(BytecodeLocation loc) {
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->pop();
+ bool strict = loc.getOp() == JSOp::StrictDelElem;
+
+ MInstruction* ins = MDeleteElement::New(alloc(), obj, id, strict);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_StrictDelElem(BytecodeLocation loc) {
+ return build_DelElem(loc);
+}
+
+bool WarpBuilder::build_SetFunName(BytecodeLocation loc) {
+ FunctionPrefixKind prefixKind = loc.getFunctionPrefixKind();
+ MDefinition* name = current->pop();
+ MDefinition* fun = current->pop();
+
+ MSetFunName* ins = MSetFunName::New(alloc(), fun, name, uint8_t(prefixKind));
+ current->add(ins);
+ current->push(fun);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_PushLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PushClassBodyEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpClassBodyEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewClassBodyEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PopLexicalEnv(BytecodeLocation) {
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+ current->setEnvironmentChain(enclosingEnv);
+ return true;
+}
+
+bool WarpBuilder::build_FreshenLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* templateObj = snapshot->templateObj();
+ auto* scope = &templateObj->scope();
+ MOZ_ASSERT(scope->hasEnvironment());
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
+ enclosingEnv));
+
+ // Copy environment slots.
+ MSlots* envSlots = nullptr;
+ MSlots* slots = nullptr;
+ for (BindingIter iter(scope); iter; iter++) {
+ auto loc = iter.location();
+ if (loc.kind() != BindingLocation::Kind::Environment) {
+ MOZ_ASSERT(loc.kind() == BindingLocation::Kind::Frame);
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ uint32_t slot = loc.slot();
+ uint32_t numFixedSlots = templateObj->numFixedSlots();
+ if (slot >= numFixedSlots) {
+ if (!envSlots) {
+ envSlots = MSlots::New(alloc(), env);
+ current->add(envSlots);
+ }
+ if (!slots) {
+ slots = MSlots::New(alloc(), ins);
+ current->add(slots);
+ }
+
+ uint32_t dynamicSlot = slot - numFixedSlots;
+
+ auto* load = MLoadDynamicSlot::New(alloc(), envSlots, dynamicSlot);
+ current->add(load);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
+#endif
+
+ current->add(
+ MStoreDynamicSlot::NewUnbarriered(alloc(), slots, dynamicSlot, load));
+ } else {
+ auto* load = MLoadFixedSlot::New(alloc(), env, slot);
+ current->add(load);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
+#endif
+
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), ins, slot, load));
+ }
+ }
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_RecreateLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
+ enclosingEnv));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PushVarEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpVarEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewVarEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_ImplicitThis(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* env = current->environmentChain();
+
+ auto* ins = MImplicitThis::New(alloc(), env, name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckClassHeritage(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckClassHeritage::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckThis(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckThis::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckThisReinit(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckThisReinit::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Generator(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* callee = getCallee();
+ MDefinition* environmentChain = current->environmentChain();
+ MDefinition* argsObj = info().needsArgsObj() ? current->argumentsObject()
+ : constant(Int32Value(0));
+
+ MGenerator* generator =
+ MGenerator::New(alloc(), callee, environmentChain, argsObj);
+
+ current->add(generator);
+ current->push(generator);
+ return resumeAfter(generator, loc);
+}
+
+bool WarpBuilder::build_AfterYield(BytecodeLocation loc) {
+ // Unreachable blocks don't need to generate a bail.
+ if (hasTerminatedBlock()) {
+ return true;
+ }
+
+ // This comes after a yield, which we generate as a return,
+ // so we know this should be unreachable code.
+ //
+ // We emit an unreachable bail for this, which will assert if we
+ // ever execute this.
+ //
+ // An Unreachable bail, instead of MUnreachable, because MUnreachable
+ // is a control instruction, and injecting it in the middle of a block
+ // causes various graph state assertions to fail.
+ MBail* bail = MBail::New(alloc(), BailoutKind::Unreachable);
+ current->add(bail);
+
+ return true;
+}
+
+bool WarpBuilder::build_FinalYieldRval(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+
+ auto setSlotNull = [this, gen](size_t slot) {
+ auto* ins = MStoreFixedSlot::NewBarriered(alloc(), gen, slot,
+ constant(NullValue()));
+ current->add(ins);
+ };
+
+ // Close the generator
+ setSlotNull(AbstractGeneratorObject::calleeSlot());
+ setSlotNull(AbstractGeneratorObject::envChainSlot());
+ setSlotNull(AbstractGeneratorObject::argsObjectSlot());
+ setSlotNull(AbstractGeneratorObject::stackStorageSlot());
+ setSlotNull(AbstractGeneratorObject::resumeIndexSlot());
+
+ // Return
+ return build_RetRval(loc);
+}
+
+bool WarpBuilder::build_AsyncResolve(BytecodeLocation loc) {
+ MDefinition* generator = current->pop();
+ MDefinition* valueOrReason = current->pop();
+ auto resolveKind = loc.getAsyncFunctionResolveKind();
+
+ MAsyncResolve* resolve =
+ MAsyncResolve::New(alloc(), generator, valueOrReason, resolveKind);
+ current->add(resolve);
+ current->push(resolve);
+ return resumeAfter(resolve, loc);
+}
+
+bool WarpBuilder::build_ResumeKind(BytecodeLocation loc) {
+ GeneratorResumeKind resumeKind = loc.resumeKind();
+
+ current->push(constant(Int32Value(static_cast<int32_t>(resumeKind))));
+ return true;
+}
+
+bool WarpBuilder::build_CheckResumeKind(BytecodeLocation loc) {
+ // Outside of `yield*`, this is normally unreachable code in Warp,
+ // so we just manipulate the stack appropriately to ensure correct
+ // MIR generation.
+ //
+ // However, `yield*` emits a forced generator return which can be
+ // warp compiled, so in order to correctly handle these semantics
+ // we also generate a bailout, so that the forced generator return
+ // runs in baseline.
+ MDefinition* resumeKind = current->pop();
+ MDefinition* gen = current->pop();
+ MDefinition* rval = current->peek(-1);
+
+ // Mark operands as implicitly used.
+ resumeKind->setImplicitlyUsedUnchecked();
+ gen->setImplicitlyUsedUnchecked();
+ rval->setImplicitlyUsedUnchecked();
+
+ // Bail out if we encounter CheckResumeKind.
+ MBail* bail = MBail::New(alloc(), BailoutKind::Inevitable);
+ current->add(bail);
+ current->setAlwaysBails();
+
+ return true;
+}
+
+bool WarpBuilder::build_CanSkipAwait(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+
+ MCanSkipAwait* canSkip = MCanSkipAwait::New(alloc(), val);
+ current->add(canSkip);
+
+ current->push(val);
+ current->push(canSkip);
+
+ return resumeAfter(canSkip, loc);
+}
+
+bool WarpBuilder::build_MaybeExtractAwaitValue(BytecodeLocation loc) {
+ MDefinition* canSkip = current->pop();
+ MDefinition* value = current->pop();
+
+ MMaybeExtractAwaitValue* extracted =
+ MMaybeExtractAwaitValue::New(alloc(), value, canSkip);
+ current->add(extracted);
+
+ current->push(extracted);
+ current->push(canSkip);
+
+ return resumeAfter(extracted, loc);
+}
+
+bool WarpBuilder::build_InitialYield(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ return buildSuspend(loc, gen, gen);
+}
+
+bool WarpBuilder::build_Await(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ MDefinition* promiseOrGenerator = current->pop();
+
+ return buildSuspend(loc, gen, promiseOrGenerator);
+}
+bool WarpBuilder::build_Yield(BytecodeLocation loc) { return build_Await(loc); }
+
+bool WarpBuilder::buildSuspend(BytecodeLocation loc, MDefinition* gen,
+ MDefinition* retVal) {
+ // If required, unbox the generator object explicitly and infallibly.
+ //
+ // This is done to avoid fuzz-bugs where ApplyTypeInformation does the
+ // unboxing, and generates fallible unboxes which can lead to torn object
+ // state due to `bailAfter`.
+ MDefinition* genObj = gen;
+ if (genObj->type() != MIRType::Object) {
+ auto* unbox =
+ MUnbox::New(alloc(), gen, MIRType::Object, MUnbox::Mode::Infallible);
+ current->add(unbox);
+
+ genObj = unbox;
+ }
+
+ int32_t slotsToCopy = current->stackDepth() - info().firstLocalSlot();
+ MOZ_ASSERT(slotsToCopy >= 0);
+ if (slotsToCopy > 0) {
+ auto* arrayObj = MLoadFixedSlotAndUnbox::New(
+ alloc(), genObj, AbstractGeneratorObject::stackStorageSlot(),
+ MUnbox::Mode::Infallible, MIRType::Object);
+ current->add(arrayObj);
+
+ auto* stackStorage = MElements::New(alloc(), arrayObj);
+ current->add(stackStorage);
+
+ for (int32_t i = 0; i < slotsToCopy; i++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ // Use peekUnchecked because we're also writing out the argument slots
+ int32_t peek = -slotsToCopy + i;
+ MDefinition* stackElem = current->peekUnchecked(peek);
+ auto* store = MStoreElement::NewUnbarriered(
+ alloc(), stackStorage, constant(Int32Value(i)), stackElem,
+ /* needsHoleCheck = */ false);
+
+ current->add(store);
+ current->add(MPostWriteBarrier::New(alloc(), arrayObj, stackElem));
+ }
+
+ auto* len = constant(Int32Value(slotsToCopy - 1));
+
+ auto* setInitLength =
+ MSetInitializedLength::New(alloc(), stackStorage, len);
+ current->add(setInitLength);
+
+ auto* setLength = MSetArrayLength::New(alloc(), stackStorage, len);
+ current->add(setLength);
+ }
+
+ // Update Generator Object state
+ uint32_t resumeIndex = loc.getResumeIndex();
+
+ // This store is unbarriered, as it's only ever storing an integer, and as
+ // such doesn't partake of object tracing.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), genObj, AbstractGeneratorObject::resumeIndexSlot(),
+ constant(Int32Value(resumeIndex))));
+
+ // This store is barriered because it stores an object value.
+ current->add(MStoreFixedSlot::NewBarriered(
+ alloc(), genObj, AbstractGeneratorObject::envChainSlot(),
+ current->environmentChain()));
+
+ current->add(
+ MPostWriteBarrier::New(alloc(), genObj, current->environmentChain()));
+
+ // GeneratorReturn will return from the method, however to support MIR
+ // generation isn't treated like the end of a block
+ MGeneratorReturn* ret = MGeneratorReturn::New(alloc(), retVal);
+ current->add(ret);
+
+ // To ensure the rest of the MIR generation looks correct, fill the stack with
+ // the appropriately typed MUnreachable's for the stack pushes from this
+ // opcode.
+ auto* unreachableResumeKind =
+ MUnreachableResult::New(alloc(), MIRType::Int32);
+ current->add(unreachableResumeKind);
+ current->push(unreachableResumeKind);
+
+ auto* unreachableGenerator =
+ MUnreachableResult::New(alloc(), MIRType::Object);
+ current->add(unreachableGenerator);
+ current->push(unreachableGenerator);
+
+ auto* unreachableRval = MUnreachableResult::New(alloc(), MIRType::Value);
+ current->add(unreachableRval);
+ current->push(unreachableRval);
+
+ return true;
+}
+
+bool WarpBuilder::build_AsyncAwait(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ MDefinition* value = current->pop();
+
+ MAsyncAwait* asyncAwait = MAsyncAwait::New(alloc(), value, gen);
+ current->add(asyncAwait);
+ current->push(asyncAwait);
+ return resumeAfter(asyncAwait, loc);
+}
+
+bool WarpBuilder::build_CheckReturn(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->noScriptRval());
+
+ MDefinition* returnValue = current->getSlot(info().returnValueSlot());
+ MDefinition* thisValue = current->pop();
+
+ auto* ins = MCheckReturn::New(alloc(), returnValue, thisValue);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+void WarpBuilder::buildCheckLexicalOp(BytecodeLocation loc) {
+ JSOp op = loc.getOp();
+ MOZ_ASSERT(op == JSOp::CheckLexical || op == JSOp::CheckAliasedLexical);
+
+ MDefinition* input = current->pop();
+ MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), input);
+ current->add(lexicalCheck);
+ current->push(lexicalCheck);
+
+ if (snapshot().bailoutInfo().failedLexicalCheck()) {
+ // If we have previously had a failed lexical check in Ion, we want to avoid
+ // hoisting any lexical checks, which can cause spurious failures. In this
+ // case, we also have to be careful not to hoist any loads of this lexical
+ // past the check. For unaliased lexical variables, we can set the local
+ // slot to create a dependency (see below). For aliased lexicals, that
+ // doesn't work, so we disable LICM instead.
+ lexicalCheck->setNotMovable();
+ if (op == JSOp::CheckAliasedLexical) {
+ mirGen().disableLICM();
+ }
+ }
+
+ if (op == JSOp::CheckLexical) {
+ // Set the local slot so that a subsequent GetLocal without a CheckLexical
+ // (the frontend can elide lexical checks) doesn't let a definition with
+ // MIRType::MagicUninitializedLexical escape to arbitrary MIR instructions.
+ // Note that in this case the GetLocal would be unreachable because we throw
+ // an exception here, but we still generate MIR instructions for it.
+ uint32_t slot = info().localSlot(loc.local());
+ current->setSlot(slot, lexicalCheck);
+ }
+}
+
+bool WarpBuilder::build_CheckLexical(BytecodeLocation loc) {
+ buildCheckLexicalOp(loc);
+ return true;
+}
+
+bool WarpBuilder::build_CheckAliasedLexical(BytecodeLocation loc) {
+ buildCheckLexicalOp(loc);
+ return true;
+}
+
+bool WarpBuilder::build_InitHomeObject(BytecodeLocation loc) {
+ MDefinition* homeObject = current->pop();
+ MDefinition* function = current->pop();
+
+ current->add(MPostWriteBarrier::New(alloc(), function, homeObject));
+
+ auto* ins = MInitHomeObject::New(alloc(), function, homeObject);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_SuperBase(BytecodeLocation) {
+ MDefinition* callee = current->pop();
+
+ auto* homeObject = MHomeObject::New(alloc(), callee);
+ current->add(homeObject);
+
+ auto* superBase = MHomeObjectSuperBase::New(alloc(), homeObject);
+ current->add(superBase);
+ current->push(superBase);
+ return true;
+}
+
+bool WarpBuilder::build_SuperFun(BytecodeLocation) {
+ MDefinition* callee = current->pop();
+ auto* ins = MSuperFunction::New(alloc(), callee);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_BuiltinObject(BytecodeLocation loc) {
+ if (auto* snapshot = getOpSnapshot<WarpBuiltinObject>(loc)) {
+ JSObject* builtin = snapshot->builtin();
+ pushConstant(ObjectValue(*builtin));
+ return true;
+ }
+
+ auto kind = loc.getBuiltinObjectKind();
+ auto* ins = MBuiltinObject::New(alloc(), kind);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_GetIntrinsic(BytecodeLocation loc) {
+ if (auto* snapshot = getOpSnapshot<WarpGetIntrinsic>(loc)) {
+ Value intrinsic = snapshot->intrinsic();
+ pushConstant(intrinsic);
+ return true;
+ }
+
+ PropertyName* name = loc.getPropertyName(script_);
+ MCallGetIntrinsicValue* ins = MCallGetIntrinsicValue::New(alloc(), name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ImportMeta(BytecodeLocation loc) {
+ ModuleObject* moduleObj = scriptSnapshot()->moduleObject();
+ MOZ_ASSERT(moduleObj);
+
+ MModuleMetadata* ins = MModuleMetadata::New(alloc(), moduleObj);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CallSiteObj(BytecodeLocation loc) {
+ return build_Object(loc);
+}
+
+bool WarpBuilder::build_NewArray(BytecodeLocation loc) {
+ return buildIC(loc, CacheKind::NewArray, {});
+}
+
+bool WarpBuilder::build_NewObject(BytecodeLocation loc) {
+ return buildIC(loc, CacheKind::NewObject, {});
+}
+
+bool WarpBuilder::build_NewInit(BytecodeLocation loc) {
+ return build_NewObject(loc);
+}
+
+bool WarpBuilder::build_Object(BytecodeLocation loc) {
+ JSObject* obj = loc.getObject(script_);
+ MConstant* objConst = constant(ObjectValue(*obj));
+
+ current->push(objConst);
+ return true;
+}
+
+bool WarpBuilder::buildInitPropGetterSetterOp(BytecodeLocation loc) {
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* value = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ auto* ins = MInitPropGetterSetter::New(alloc(), obj, value, name);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_InitPropGetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitPropSetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenPropGetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenPropSetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::buildInitElemGetterSetterOp(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ auto* ins = MInitElemGetterSetter::New(alloc(), obj, id, value);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_InitElemGetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitElemSetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElemGetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElemSetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_In(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ return buildIC(loc, CacheKind::In, {id, obj});
+}
+
+bool WarpBuilder::build_HasOwn(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ return buildIC(loc, CacheKind::HasOwn, {id, obj});
+}
+
+bool WarpBuilder::build_CheckPrivateField(BytecodeLocation loc) {
+ MDefinition* id = current->peek(-1);
+ MDefinition* obj = current->peek(-2);
+ return buildIC(loc, CacheKind::CheckPrivateField, {obj, id});
+}
+
+bool WarpBuilder::build_NewPrivateName(BytecodeLocation loc) {
+ JSAtom* name = loc.getAtom(script_);
+
+ auto* ins = MNewPrivateName::New(alloc(), name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Instanceof(BytecodeLocation loc) {
+ MDefinition* rhs = current->pop();
+ MDefinition* obj = current->pop();
+ return buildIC(loc, CacheKind::InstanceOf, {obj, rhs});
+}
+
+bool WarpBuilder::build_NewTarget(BytecodeLocation loc) {
+ MOZ_ASSERT(script_->isFunction());
+ MOZ_ASSERT(info().hasFunMaybeLazy());
+ MOZ_ASSERT(!scriptSnapshot()->isArrowFunction());
+
+ if (inlineCallInfo()) {
+ if (inlineCallInfo()->constructing()) {
+ current->push(inlineCallInfo()->getNewTarget());
+ } else {
+ pushConstant(UndefinedValue());
+ }
+ return true;
+ }
+
+ MNewTarget* ins = MNewTarget::New(alloc());
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_CheckIsObj(BytecodeLocation loc) {
+ CheckIsObjectKind kind = loc.getCheckIsObjectKind();
+
+ MDefinition* toCheck = current->peek(-1);
+ if (toCheck->type() == MIRType::Object) {
+ toCheck->setImplicitlyUsedUnchecked();
+ return true;
+ }
+
+ MDefinition* val = current->pop();
+ MCheckIsObj* ins = MCheckIsObj::New(alloc(), val, uint8_t(kind));
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_CheckObjCoercible(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MCheckObjCoercible* ins = MCheckObjCoercible::New(alloc(), val);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+MInstruction* WarpBuilder::buildLoadSlot(MDefinition* obj,
+ uint32_t numFixedSlots,
+ uint32_t slot) {
+ if (slot < numFixedSlots) {
+ MLoadFixedSlot* load = MLoadFixedSlot::New(alloc(), obj, slot);
+ current->add(load);
+ return load;
+ }
+
+ MSlots* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ MLoadDynamicSlot* load =
+ MLoadDynamicSlot::New(alloc(), slots, slot - numFixedSlots);
+ current->add(load);
+ return load;
+}
+
+bool WarpBuilder::build_GetImport(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpGetImport>(loc);
+
+ ModuleEnvironmentObject* targetEnv = snapshot->targetEnv();
+
+ // Load the target environment slot.
+ MConstant* obj = constant(ObjectValue(*targetEnv));
+ auto* load = buildLoadSlot(obj, snapshot->numFixedSlots(), snapshot->slot());
+
+ if (snapshot->needsLexicalCheck()) {
+ // TODO: IonBuilder has code to mark non-movable. See buildCheckLexicalOp.
+ MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), load);
+ current->add(lexicalCheck);
+ current->push(lexicalCheck);
+ } else {
+ current->push(load);
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_GetPropSuper(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* receiver = current->pop();
+ return buildIC(loc, CacheKind::GetPropSuper, {obj, receiver});
+}
+
+bool WarpBuilder::build_GetElemSuper(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* receiver = current->pop();
+ return buildIC(loc, CacheKind::GetElemSuper, {obj, id, receiver});
+}
+
+bool WarpBuilder::build_InitProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->peek(-1);
+ return buildIC(loc, CacheKind::SetProp, {obj, val});
+}
+
+bool WarpBuilder::build_InitLockedProp(BytecodeLocation loc) {
+ return build_InitProp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenProp(BytecodeLocation loc) {
+ return build_InitProp(loc);
+}
+
+bool WarpBuilder::build_InitElem(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->peek(-1);
+ return buildIC(loc, CacheKind::SetElem, {obj, id, val});
+}
+
+bool WarpBuilder::build_InitLockedElem(BytecodeLocation loc) {
+ return build_InitElem(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElem(BytecodeLocation loc) {
+ return build_InitElem(loc);
+}
+
+bool WarpBuilder::build_InitElemArray(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ // Note: getInitElemArrayIndex asserts the index fits in int32_t.
+ uint32_t index = loc.getInitElemArrayIndex();
+ MConstant* indexConst = constant(Int32Value(index));
+
+ // Note: InitArrayElemOperation asserts the index does not exceed the array's
+ // dense element capacity.
+
+ auto* elements = MElements::New(alloc(), obj);
+ current->add(elements);
+
+ if (val->type() == MIRType::MagicHole) {
+ val->setImplicitlyUsedUnchecked();
+ auto* store = MStoreHoleValueElement::New(alloc(), elements, indexConst);
+ current->add(store);
+ } else {
+ current->add(MPostWriteBarrier::New(alloc(), obj, val));
+ auto* store =
+ MStoreElement::NewUnbarriered(alloc(), elements, indexConst, val,
+ /* needsHoleCheck = */ false);
+ current->add(store);
+ }
+
+ auto* setLength = MSetInitializedLength::New(alloc(), elements, indexConst);
+ current->add(setLength);
+
+ return resumeAfter(setLength, loc);
+}
+
+bool WarpBuilder::build_InitElemInc(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* index = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ // Push index + 1.
+ MConstant* constOne = constant(Int32Value(1));
+ MAdd* nextIndex = MAdd::New(alloc(), index, constOne, TruncateKind::Truncate);
+ current->add(nextIndex);
+ current->push(nextIndex);
+
+ return buildIC(loc, CacheKind::SetElem, {obj, index, val});
+}
+
+bool WarpBuilder::build_Lambda(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+
+ JSFunction* fun = loc.getFunction(script_);
+ MConstant* funConst = constant(ObjectValue(*fun));
+
+ auto* ins = MLambda::New(alloc(), env, funConst);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_FunWithProto(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* proto = current->pop();
+ MDefinition* env = current->environmentChain();
+
+ JSFunction* fun = loc.getFunction(script_);
+ MConstant* funConst = constant(ObjectValue(*fun));
+
+ auto* ins = MFunctionWithProto::New(alloc(), env, proto, funConst);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_SpreadCall(BytecodeLocation loc) {
+ bool constructing = false;
+ CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
+ callInfo.initForSpreadCall(current);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ bool needsThisCheck = false;
+ MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+ call->setBailoutKind(BailoutKind::TooManyArguments);
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_SpreadNew(BytecodeLocation loc) {
+ bool constructing = true;
+ CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
+ callInfo.initForSpreadCall(current);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ buildCreateThis(callInfo);
+
+ bool needsThisCheck = true;
+ MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+ call->setBailoutKind(BailoutKind::TooManyArguments);
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_SpreadSuperCall(BytecodeLocation loc) {
+ return build_SpreadNew(loc);
+}
+
+bool WarpBuilder::build_OptimizeSpreadCall(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::OptimizeSpreadCall, {value});
+}
+
+bool WarpBuilder::build_Debugger(BytecodeLocation loc) {
+ // The |debugger;| statement will bail out to Baseline if the realm is a
+ // debuggee realm with an onDebuggerStatement hook.
+ MDebugger* debugger = MDebugger::New(alloc());
+ current->add(debugger);
+ return resumeAfter(debugger, loc);
+}
+
+bool WarpBuilder::build_TableSwitch(BytecodeLocation loc) {
+ int32_t low = loc.getTableSwitchLow();
+ int32_t high = loc.getTableSwitchHigh();
+ size_t numCases = high - low + 1;
+
+ MDefinition* input = current->pop();
+ MTableSwitch* tableswitch = MTableSwitch::New(alloc(), input, low, high);
+ current->end(tableswitch);
+
+ // Table mapping from target bytecode offset to MTableSwitch successor index.
+ // This prevents adding multiple predecessor/successor edges to the same
+ // target block, which isn't valid in MIR.
+ using TargetToSuccessorMap =
+ InlineMap<uint32_t, uint32_t, 8, DefaultHasher<uint32_t>,
+ SystemAllocPolicy>;
+ TargetToSuccessorMap targetToSuccessor;
+
+ // Create |default| edge.
+ {
+ BytecodeLocation defaultLoc = loc.getTableSwitchDefaultTarget();
+ uint32_t defaultOffset = defaultLoc.bytecodeToOffset(script_);
+
+ size_t index;
+ if (!tableswitch->addDefault(nullptr, &index)) {
+ return false;
+ }
+ if (!addPendingEdge(defaultLoc, current, index)) {
+ return false;
+ }
+ if (!targetToSuccessor.put(defaultOffset, index)) {
+ return false;
+ }
+ }
+
+ // Add all cases.
+ for (size_t i = 0; i < numCases; i++) {
+ BytecodeLocation caseLoc = loc.getTableSwitchCaseTarget(script_, i);
+ uint32_t caseOffset = caseLoc.bytecodeToOffset(script_);
+
+ size_t index;
+ if (auto p = targetToSuccessor.lookupForAdd(caseOffset)) {
+ index = p->value();
+ } else {
+ if (!tableswitch->addSuccessor(nullptr, &index)) {
+ return false;
+ }
+ if (!addPendingEdge(caseLoc, current, index)) {
+ return false;
+ }
+ if (!targetToSuccessor.add(p, caseOffset, index)) {
+ return false;
+ }
+ }
+ if (!tableswitch->addCase(index)) {
+ return false;
+ }
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Rest(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpRest>(loc);
+ Shape* shape = snapshot ? snapshot->shape() : nullptr;
+
+ // NOTE: Keep this code in sync with |ArgumentsReplacer|.
+
+ if (inlineCallInfo()) {
+ // If we are inlining, we know the actual arguments.
+ unsigned numActuals = inlineCallInfo()->argc();
+ unsigned numFormals = info().nargs() - 1;
+ unsigned numRest = numActuals > numFormals ? numActuals - numFormals : 0;
+
+ // TODO: support pre-tenuring.
+ gc::Heap heap = gc::Heap::Default;
+
+ // Allocate an array of the correct size.
+ MInstruction* newArray;
+ if (shape && gc::CanUseFixedElementsForArray(numRest)) {
+ auto* shapeConstant = MConstant::NewShape(alloc(), shape);
+ current->add(shapeConstant);
+ newArray = MNewArrayObject::New(alloc(), shapeConstant, numRest, heap);
+ } else {
+ MConstant* templateConst = constant(NullValue());
+ newArray = MNewArray::NewVM(alloc(), numRest, templateConst, heap);
+ }
+ current->add(newArray);
+ current->push(newArray);
+
+ if (numRest == 0) {
+ // No more updating to do.
+ return true;
+ }
+
+ MElements* elements = MElements::New(alloc(), newArray);
+ current->add(elements);
+
+ // Unroll the argument copy loop. We don't need to do any bounds or hole
+ // checking here.
+ MConstant* index = nullptr;
+ for (uint32_t i = numFormals; i < numActuals; i++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ index = MConstant::New(alloc(), Int32Value(i - numFormals));
+ current->add(index);
+
+ MDefinition* arg = inlineCallInfo()->argv()[i];
+ MStoreElement* store =
+ MStoreElement::NewUnbarriered(alloc(), elements, index, arg,
+ /* needsHoleCheck = */ false);
+ current->add(store);
+ current->add(MPostWriteBarrier::New(alloc(), newArray, arg));
+ }
+
+ // Update the initialized length for all the (necessarily non-hole)
+ // elements added.
+ MSetInitializedLength* initLength =
+ MSetInitializedLength::New(alloc(), elements, index);
+ current->add(initLength);
+
+ return true;
+ }
+
+ MArgumentsLength* numActuals = MArgumentsLength::New(alloc());
+ current->add(numActuals);
+
+ // Pass in the number of actual arguments, the number of formals (not
+ // including the rest parameter slot itself), and the shape.
+ unsigned numFormals = info().nargs() - 1;
+ MRest* rest = MRest::New(alloc(), numActuals, numFormals, shape);
+ current->add(rest);
+ current->push(rest);
+ return true;
+}
+
+bool WarpBuilder::build_Try(BytecodeLocation loc) {
+ graph().setHasTryBlock();
+
+ MBasicBlock* pred = current;
+ if (!startNewBlock(pred, loc.next())) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+ return true;
+}
+
+bool WarpBuilder::build_Finally(BytecodeLocation loc) {
+ MOZ_ASSERT(graph().hasTryBlock());
+ return true;
+}
+
+bool WarpBuilder::build_Exception(BytecodeLocation) {
+ MOZ_CRASH("Unreachable because we skip catch-blocks");
+}
+
+bool WarpBuilder::build_Throw(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+
+ MThrow* ins = MThrow::New(alloc(), def);
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_ThrowSetConst(BytecodeLocation loc) {
+ auto* ins = MThrowRuntimeLexicalError::New(alloc(), JSMSG_BAD_CONST_ASSIGN);
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_ThrowMsg(BytecodeLocation loc) {
+ auto* ins = MThrowMsg::New(alloc(), loc.throwMsgKind());
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
+ std::initializer_list<MDefinition*> inputs) {
+ MOZ_ASSERT(loc.opHasIC());
+
+ mozilla::DebugOnly<size_t> numInputs = inputs.size();
+ MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind));
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, inputs);
+ }
+
+ if (getOpSnapshot<WarpBailout>(loc)) {
+ for (MDefinition* input : inputs) {
+ input->setImplicitlyUsedUnchecked();
+ }
+ return buildBailoutForColdIC(loc, kind);
+ }
+
+ if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
+ // The CallInfo will be initialized by the transpiler.
+ bool ignoresRval = BytecodeIsPopped(loc.toRawBytecode());
+ CallInfo callInfo(alloc(), /*constructing =*/false, ignoresRval);
+ callInfo.markAsInlined();
+
+ if (!TranspileCacheIRToMIR(this, loc, inliningSnapshot->cacheIRSnapshot(),
+ inputs, &callInfo)) {
+ return false;
+ }
+ return buildInlinedCall(loc, inliningSnapshot, callInfo);
+ }
+
+ // Work around std::initializer_list not defining operator[].
+ auto getInput = [&](size_t index) -> MDefinition* {
+ MOZ_ASSERT(index < numInputs);
+ return inputs.begin()[index];
+ };
+
+ switch (kind) {
+ case CacheKind::UnaryArith: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MUnaryCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::ToPropertyKey: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MToPropertyKeyCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::BinaryArith: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins =
+ MBinaryCache::New(alloc(), getInput(0), getInput(1), MIRType::Value);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::Compare: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MBinaryCache::New(alloc(), getInput(0), getInput(1),
+ MIRType::Boolean);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::In: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MInCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::HasOwn: {
+ MOZ_ASSERT(numInputs == 2);
+ // Note: the MHasOwnCache constructor takes obj/id instead of id/obj.
+ auto* ins = MHasOwnCache::New(alloc(), getInput(1), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::CheckPrivateField: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins =
+ MCheckPrivateFieldCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::InstanceOf: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MInstanceOfCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::BindName: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MBindNameCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetIterator: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MGetIteratorCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetName: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MGetNameCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetProp: {
+ MOZ_ASSERT(numInputs == 1);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ MDefinition* val = getInput(0);
+ auto* ins = MGetPropertyCache::New(alloc(), val, id);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetElem: {
+ MOZ_ASSERT(numInputs == 2);
+ MDefinition* val = getInput(0);
+ auto* ins = MGetPropertyCache::New(alloc(), val, getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::SetProp: {
+ MOZ_ASSERT(numInputs == 2);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ bool strict = loc.isStrictSetOp();
+ auto* ins =
+ MSetPropertyCache::New(alloc(), getInput(0), id, getInput(1), strict);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::SetElem: {
+ MOZ_ASSERT(numInputs == 3);
+ bool strict = loc.isStrictSetOp();
+ auto* ins = MSetPropertyCache::New(alloc(), getInput(0), getInput(1),
+ getInput(2), strict);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetPropSuper: {
+ MOZ_ASSERT(numInputs == 2);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ auto* ins =
+ MGetPropSuperCache::New(alloc(), getInput(0), getInput(1), id);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetElemSuper: {
+ MOZ_ASSERT(numInputs == 3);
+ // Note: CacheIR expects obj/id/receiver but MGetPropSuperCache takes
+ // obj/receiver/id so swap the last two inputs.
+ auto* ins = MGetPropSuperCache::New(alloc(), getInput(0), getInput(2),
+ getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::OptimizeSpreadCall: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MOptimizeSpreadCallCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::TypeOf: {
+ // Note: Warp does not have a TypeOf IC, it just inlines the operation.
+ MOZ_ASSERT(numInputs == 1);
+ auto* typeOf = MTypeOf::New(alloc(), getInput(0));
+ current->add(typeOf);
+
+ auto* ins = MTypeOfName::New(alloc(), typeOf);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+ case CacheKind::NewObject: {
+ auto* templateConst = constant(NullValue());
+ MNewObject* ins = MNewObject::NewVM(
+ alloc(), templateConst, gc::Heap::Default, MNewObject::ObjectLiteral);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::NewArray: {
+ uint32_t length = loc.getNewArrayLength();
+ MConstant* templateConst = constant(NullValue());
+ MNewArray* ins =
+ MNewArray::NewVM(alloc(), length, templateConst, gc::Heap::Default);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+ case CacheKind::CloseIter: {
+ MOZ_ASSERT(numInputs == 1);
+ static_assert(sizeof(CompletionKind) == sizeof(uint8_t));
+ CompletionKind kind = loc.getCompletionKind();
+ auto* ins = MCloseIterCache::New(alloc(), getInput(0), uint8_t(kind));
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetIntrinsic:
+ case CacheKind::ToBool:
+ case CacheKind::Call:
+ // We're currently not using an IC or transpiling CacheIR for these kinds.
+ MOZ_CRASH("Unexpected kind");
+ }
+
+ return true;
+}
+
+bool WarpBuilder::buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind) {
+ MOZ_ASSERT(loc.opHasIC());
+
+ MBail* bail = MBail::New(alloc(), BailoutKind::FirstExecution);
+ current->add(bail);
+ current->setAlwaysBails();
+
+ MIRType resultType;
+ switch (kind) {
+ case CacheKind::UnaryArith:
+ case CacheKind::BinaryArith:
+ case CacheKind::GetName:
+ case CacheKind::GetProp:
+ case CacheKind::GetElem:
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::Call:
+ case CacheKind::ToPropertyKey:
+ case CacheKind::OptimizeSpreadCall:
+ resultType = MIRType::Value;
+ break;
+ case CacheKind::BindName:
+ case CacheKind::GetIterator:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ resultType = MIRType::Object;
+ break;
+ case CacheKind::TypeOf:
+ resultType = MIRType::String;
+ break;
+ case CacheKind::ToBool:
+ case CacheKind::Compare:
+ case CacheKind::In:
+ case CacheKind::HasOwn:
+ case CacheKind::CheckPrivateField:
+ case CacheKind::InstanceOf:
+ resultType = MIRType::Boolean;
+ break;
+ case CacheKind::SetProp:
+ case CacheKind::SetElem:
+ case CacheKind::CloseIter:
+ return true; // No result.
+ }
+
+ auto* ins = MUnreachableResult::New(alloc(), resultType);
+ current->add(ins);
+ current->push(ins);
+
+ return true;
+}
+
+class MOZ_RAII AutoAccumulateReturns {
+ MIRGraph& graph_;
+ MIRGraphReturns* prev_;
+
+ public:
+ AutoAccumulateReturns(MIRGraph& graph, MIRGraphReturns& returns)
+ : graph_(graph) {
+ prev_ = graph_.returnAccumulator();
+ graph_.setReturnAccumulator(&returns);
+ }
+ ~AutoAccumulateReturns() { graph_.setReturnAccumulator(prev_); }
+};
+
+bool WarpBuilder::buildInlinedCall(BytecodeLocation loc,
+ const WarpInlinedCall* inlineSnapshot,
+ CallInfo& callInfo) {
+ jsbytecode* pc = loc.toRawBytecode();
+
+ if (callInfo.isSetter()) {
+ // build_SetProp pushes the rhs argument onto the stack. Remove it
+ // in preparation for pushCallStack.
+ current->pop();
+ }
+
+ callInfo.setImplicitlyUsedUnchecked();
+
+ // Capture formals in the outer resume point.
+ if (!callInfo.pushCallStack(current)) {
+ return false;
+ }
+ MResumePoint* outerResumePoint =
+ MResumePoint::New(alloc(), current, pc, callInfo.inliningResumeMode());
+ if (!outerResumePoint) {
+ return false;
+ }
+ current->setOuterResumePoint(outerResumePoint);
+
+ // Pop formals again, except leave |callee| on stack for duration of call.
+ callInfo.popCallStack(current);
+ current->push(callInfo.callee());
+
+ // Build the graph.
+ CompileInfo* calleeCompileInfo = inlineSnapshot->info();
+ MIRGraphReturns returns(alloc());
+ AutoAccumulateReturns aar(graph(), returns);
+ WarpBuilder inlineBuilder(this, inlineSnapshot->scriptSnapshot(),
+ *calleeCompileInfo, &callInfo, outerResumePoint);
+ if (!inlineBuilder.buildInline()) {
+ // Note: Inlining only aborts on OOM. If inlining would fail for
+ // any other reason, we detect it in advance and don't inline.
+ return false;
+ }
+
+ // We mark scripts as uninlineable in BytecodeAnalysis if we cannot
+ // reach a return statement (without going through a catch/finally).
+ MOZ_ASSERT(!returns.empty());
+
+ // Create return block
+ BytecodeLocation postCall = loc.next();
+ MBasicBlock* prev = current;
+ if (!startNewEntryBlock(prev->stackDepth(), postCall)) {
+ return false;
+ }
+ // Restore previous value of callerResumePoint.
+ current->setCallerResumePoint(callerResumePoint());
+ current->inheritSlots(prev);
+
+ // Pop |callee|.
+ current->pop();
+
+ // Accumulate return values.
+ MDefinition* returnValue =
+ patchInlinedReturns(calleeCompileInfo, callInfo, returns, current);
+ if (!returnValue) {
+ return false;
+ }
+ current->push(returnValue);
+
+ // Initialize entry slots
+ if (!current->initEntrySlots(alloc())) {
+ return false;
+ }
+
+ return true;
+}
+
+MDefinition* WarpBuilder::patchInlinedReturns(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo,
+ MIRGraphReturns& exits,
+ MBasicBlock* returnBlock) {
+ if (exits.length() == 1) {
+ return patchInlinedReturn(calleeCompileInfo, callInfo, exits[0],
+ returnBlock);
+ }
+
+ // Accumulate multiple returns with a phi.
+ MPhi* phi = MPhi::New(alloc());
+ if (!phi->reserveLength(exits.length())) {
+ return nullptr;
+ }
+
+ for (auto* exit : exits) {
+ MDefinition* rdef =
+ patchInlinedReturn(calleeCompileInfo, callInfo, exit, returnBlock);
+ if (!rdef) {
+ return nullptr;
+ }
+ phi->addInput(rdef);
+ }
+ returnBlock->addPhi(phi);
+ return phi;
+}
+
+MDefinition* WarpBuilder::patchInlinedReturn(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo,
+ MBasicBlock* exit,
+ MBasicBlock* returnBlock) {
+ // Replace the MReturn in the exit block with an MGoto branching to
+ // the return block.
+ MDefinition* rdef = exit->lastIns()->toReturn()->input();
+ exit->discardLastIns();
+
+ // Constructors must be patched by the caller to always return an object.
+ // Derived class constructors contain extra bytecode to ensure an object
+ // is always returned, so no additional patching is needed.
+ if (callInfo.constructing() &&
+ !calleeCompileInfo->isDerivedClassConstructor()) {
+ auto* filter = MReturnFromCtor::New(alloc(), rdef, callInfo.thisArg());
+ exit->add(filter);
+ rdef = filter;
+ } else if (callInfo.isSetter()) {
+ // Setters return the rhs argument, not whatever value is returned.
+ rdef = callInfo.getArg(0);
+ }
+
+ exit->end(MGoto::New(alloc(), returnBlock));
+ if (!returnBlock->addPredecessorWithoutPhis(exit)) {
+ return nullptr;
+ }
+
+ return rdef;
+}