summaryrefslogtreecommitdiffstats
path: root/js/src/jit/shared/CodeGenerator-shared.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/shared/CodeGenerator-shared.cpp')
-rw-r--r--js/src/jit/shared/CodeGenerator-shared.cpp1198
1 files changed, 1198 insertions, 0 deletions
diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp
new file mode 100644
index 0000000000..70c75ddb59
--- /dev/null
+++ b/js/src/jit/shared/CodeGenerator-shared.cpp
@@ -0,0 +1,1198 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include <utility>
+
+#include "jit/CodeGenerator.h"
+#include "jit/CompactBuffer.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitSpewer.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/SafepointIndex.h"
+#include "js/Conversions.h"
+#include "util/Memory.h"
+#include "vm/TraceLogging.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::BitwiseCast;
+using mozilla::DebugOnly;
+
+namespace js {
+namespace jit {
+
+MacroAssembler& CodeGeneratorShared::ensureMasm(MacroAssembler* masmArg) {
+ if (masmArg) {
+ return *masmArg;
+ }
+ maybeMasm_.emplace();
+ return *maybeMasm_;
+}
+
+CodeGeneratorShared::CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masmArg)
+ : maybeMasm_(),
+ useWasmStackArgumentAbi_(false),
+ masm(ensureMasm(masmArg)),
+ gen(gen),
+ graph(*graph),
+ current(nullptr),
+ snapshots_(),
+ recovers_(),
+ deoptTable_(),
+#ifdef DEBUG
+ pushedArgs_(0),
+#endif
+ lastOsiPointOffset_(0),
+ safepoints_(graph->totalSlotCount(),
+ (gen->outerInfo().nargs() + 1) * sizeof(Value)),
+ returnLabel_(),
+ nativeToBytecodeMap_(nullptr),
+ nativeToBytecodeMapSize_(0),
+ nativeToBytecodeTableOffset_(0),
+ nativeToBytecodeNumRegions_(0),
+ nativeToBytecodeScriptList_(nullptr),
+ nativeToBytecodeScriptListLength_(0),
+#ifdef CHECK_OSIPOINT_REGISTERS
+ checkOsiPointRegisters(JitOptions.checkOsiPointRegisters),
+#endif
+ frameDepth_(graph->paddedLocalSlotsSize() + graph->argumentsSize()),
+ frameClass_(FrameSizeClass::None()) {
+ if (gen->isProfilerInstrumentationEnabled()) {
+ masm.enableProfilingInstrumentation();
+ }
+
+ if (gen->compilingWasm()) {
+ // Since wasm uses the system ABI which does not necessarily use a
+ // regular array where all slots are sizeof(Value), it maintains the max
+ // argument stack depth separately.
+ MOZ_ASSERT(graph->argumentSlotCount() == 0);
+ frameDepth_ += gen->wasmMaxStackArgBytes();
+
+#ifdef ENABLE_WASM_SIMD
+# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \
+ defined(JS_CODEGEN_ARM64)
+ // On X64/x86 and ARM64, we don't need alignment for Wasm SIMD at this time.
+# else
+# error \
+ "we may need padding so that local slots are SIMD-aligned and the stack must be kept SIMD-aligned too."
+# endif
+#endif
+
+ if (gen->needsStaticStackAlignment()) {
+ // An MWasmCall does not align the stack pointer at calls sites but
+ // instead relies on the a priori stack adjustment. This must be the
+ // last adjustment of frameDepth_.
+ frameDepth_ += ComputeByteAlignment(sizeof(wasm::Frame) + frameDepth_,
+ WasmStackAlignment);
+ }
+
+ // FrameSizeClass is only used for bailing, which cannot happen in
+ // wasm code.
+ MOZ_ASSERT(frameClass_ == FrameSizeClass::None());
+ } else {
+ frameClass_ = FrameSizeClass::FromDepth(frameDepth_);
+ }
+}
+
+bool CodeGeneratorShared::generatePrologue() {
+ MOZ_ASSERT(masm.framePushed() == 0);
+ MOZ_ASSERT(!gen->compilingWasm());
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+
+ // If profiling, save the current frame pointer to a per-thread global field.
+ if (isProfilerInstrumentationEnabled()) {
+ masm.profilerEnterFrame(masm.getStackPointer(), CallTempReg0);
+ }
+
+ // Ensure that the Ion frame is properly aligned.
+ masm.assertStackAlignment(JitStackAlignment, 0);
+
+ // Note that this automatically sets MacroAssembler::framePushed().
+ masm.reserveStack(frameSize());
+ masm.checkStackAlignment();
+
+ if (JS::TraceLoggerSupported()) {
+ emitTracelogIonStart();
+ }
+
+ return true;
+}
+
+bool CodeGeneratorShared::generateEpilogue() {
+ MOZ_ASSERT(!gen->compilingWasm());
+ masm.bind(&returnLabel_);
+
+ if (JS::TraceLoggerSupported()) {
+ emitTracelogIonStop();
+ }
+
+ masm.freeStack(frameSize());
+ MOZ_ASSERT(masm.framePushed() == 0);
+
+ // If profiling, reset the per-thread global lastJitFrame to point to
+ // the previous frame.
+ if (isProfilerInstrumentationEnabled()) {
+ masm.profilerExitFrame();
+ }
+
+ masm.ret();
+
+ // On systems that use a constant pool, this is a good time to emit.
+ masm.flushBuffer();
+ return true;
+}
+
+bool CodeGeneratorShared::generateOutOfLineCode() {
+ AutoCreatedBy acb(masm, "CodeGeneratorShared::generateOutOfLineCode");
+
+ // OOL paths should not attempt to use |current| as it's the last block
+ // instead of the block corresponding to the OOL path.
+ current = nullptr;
+
+ for (size_t i = 0; i < outOfLineCode_.length(); i++) {
+ // Add native => bytecode mapping entries for OOL sites.
+ // Not enabled on wasm yet since it doesn't contain bytecode mappings.
+ if (!gen->compilingWasm()) {
+ if (!addNativeToBytecodeEntry(outOfLineCode_[i]->bytecodeSite())) {
+ return false;
+ }
+ }
+
+ if (!gen->alloc().ensureBallast()) {
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting out of line code");
+
+ masm.setFramePushed(outOfLineCode_[i]->framePushed());
+ outOfLineCode_[i]->bind(&masm);
+
+ outOfLineCode_[i]->generate(this);
+ }
+
+ return !masm.oom();
+}
+
+void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code,
+ const MInstruction* mir) {
+ MOZ_ASSERT(mir);
+ addOutOfLineCode(code, mir->trackedSite());
+}
+
+void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code,
+ const BytecodeSite* site) {
+ MOZ_ASSERT_IF(!gen->compilingWasm(), site->script()->containsPC(site->pc()));
+ code->setFramePushed(masm.framePushed());
+ code->setBytecodeSite(site);
+ masm.propagateOOM(outOfLineCode_.append(code));
+}
+
+bool CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite* site) {
+ MOZ_ASSERT(site);
+ MOZ_ASSERT(site->tree());
+ MOZ_ASSERT(site->pc());
+
+ // Skip the table entirely if profiling is not enabled.
+ if (!isProfilerInstrumentationEnabled()) {
+ return true;
+ }
+
+ // Fails early if the last added instruction caused the macro assembler to
+ // run out of memory as continuity assumption below do not hold.
+ if (masm.oom()) {
+ return false;
+ }
+
+ InlineScriptTree* tree = site->tree();
+ jsbytecode* pc = site->pc();
+ uint32_t nativeOffset = masm.currentOffset();
+
+ MOZ_ASSERT_IF(nativeToBytecodeList_.empty(), nativeOffset == 0);
+
+ if (!nativeToBytecodeList_.empty()) {
+ size_t lastIdx = nativeToBytecodeList_.length() - 1;
+ NativeToBytecode& lastEntry = nativeToBytecodeList_[lastIdx];
+
+ MOZ_ASSERT(nativeOffset >= lastEntry.nativeOffset.offset());
+
+ // If the new entry is for the same inlineScriptTree and same
+ // bytecodeOffset, but the nativeOffset has changed, do nothing.
+ // The same site just generated some more code.
+ if (lastEntry.tree == tree && lastEntry.pc == pc) {
+ JitSpew(JitSpew_Profiling, " => In-place update [%zu-%" PRIu32 "]",
+ lastEntry.nativeOffset.offset(), nativeOffset);
+ return true;
+ }
+
+ // If the new entry is for the same native offset, then update the
+ // previous entry with the new bytecode site, since the previous
+ // bytecode site did not generate any native code.
+ if (lastEntry.nativeOffset.offset() == nativeOffset) {
+ lastEntry.tree = tree;
+ lastEntry.pc = pc;
+ JitSpew(JitSpew_Profiling, " => Overwriting zero-length native region.");
+
+ // This overwrite might have made the entry merge-able with a
+ // previous one. If so, merge it.
+ if (lastIdx > 0) {
+ NativeToBytecode& nextToLastEntry = nativeToBytecodeList_[lastIdx - 1];
+ if (nextToLastEntry.tree == lastEntry.tree &&
+ nextToLastEntry.pc == lastEntry.pc) {
+ JitSpew(JitSpew_Profiling, " => Merging with previous region");
+ nativeToBytecodeList_.erase(&lastEntry);
+ }
+ }
+
+ dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
+ return true;
+ }
+ }
+
+ // Otherwise, some native code was generated for the previous bytecode site.
+ // Add a new entry for code that is about to be generated.
+ NativeToBytecode entry;
+ entry.nativeOffset = CodeOffset(nativeOffset);
+ entry.tree = tree;
+ entry.pc = pc;
+ if (!nativeToBytecodeList_.append(entry)) {
+ return false;
+ }
+
+ JitSpew(JitSpew_Profiling, " => Push new entry.");
+ dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
+ return true;
+}
+
+void CodeGeneratorShared::dumpNativeToBytecodeEntries() {
+#ifdef JS_JITSPEW
+ InlineScriptTree* topTree = gen->outerInfo().inlineScriptTree();
+ JitSpewStart(JitSpew_Profiling, "Native To Bytecode Entries for %s:%u:%u\n",
+ topTree->script()->filename(), topTree->script()->lineno(),
+ topTree->script()->column());
+ for (unsigned i = 0; i < nativeToBytecodeList_.length(); i++) {
+ dumpNativeToBytecodeEntry(i);
+ }
+#endif
+}
+
+void CodeGeneratorShared::dumpNativeToBytecodeEntry(uint32_t idx) {
+#ifdef JS_JITSPEW
+ NativeToBytecode& ref = nativeToBytecodeList_[idx];
+ InlineScriptTree* tree = ref.tree;
+ JSScript* script = tree->script();
+ uint32_t nativeOffset = ref.nativeOffset.offset();
+ unsigned nativeDelta = 0;
+ unsigned pcDelta = 0;
+ if (idx + 1 < nativeToBytecodeList_.length()) {
+ NativeToBytecode* nextRef = &ref + 1;
+ nativeDelta = nextRef->nativeOffset.offset() - nativeOffset;
+ if (nextRef->tree == ref.tree) {
+ pcDelta = nextRef->pc - ref.pc;
+ }
+ }
+ JitSpewStart(
+ JitSpew_Profiling, " %08zx [+%-6u] => %-6ld [%-4u] {%-10s} (%s:%u:%u",
+ ref.nativeOffset.offset(), nativeDelta, (long)(ref.pc - script->code()),
+ pcDelta, CodeName(JSOp(*ref.pc)), script->filename(), script->lineno(),
+ script->column());
+
+ for (tree = tree->caller(); tree; tree = tree->caller()) {
+ JitSpewCont(JitSpew_Profiling, " <= %s:%u:%u", tree->script()->filename(),
+ tree->script()->lineno(), tree->script()->column());
+ }
+ JitSpewCont(JitSpew_Profiling, ")");
+ JitSpewFin(JitSpew_Profiling);
+#endif
+}
+
+// see OffsetOfFrameSlot
+static inline int32_t ToStackIndex(LAllocation* a) {
+ if (a->isStackSlot()) {
+ MOZ_ASSERT(a->toStackSlot()->slot() >= 1);
+ return a->toStackSlot()->slot();
+ }
+ return -int32_t(sizeof(JitFrameLayout) + a->toArgument()->index());
+}
+
+void CodeGeneratorShared::encodeAllocation(LSnapshot* snapshot,
+ MDefinition* mir,
+ uint32_t* allocIndex) {
+ if (mir->isBox()) {
+ mir = mir->toBox()->getOperand(0);
+ }
+
+ MIRType type = mir->isRecoveredOnBailout() ? MIRType::None
+ : mir->isUnused() ? MIRType::MagicOptimizedOut
+ : mir->type();
+
+ RValueAllocation alloc;
+
+ switch (type) {
+ case MIRType::None: {
+ MOZ_ASSERT(mir->isRecoveredOnBailout());
+ uint32_t index = 0;
+ LRecoverInfo* recoverInfo = snapshot->recoverInfo();
+ MNode** it = recoverInfo->begin();
+ MNode** end = recoverInfo->end();
+ while (it != end && mir != *it) {
+ ++it;
+ ++index;
+ }
+
+ // This MDefinition is recovered, thus it should be listed in the
+ // LRecoverInfo.
+ MOZ_ASSERT(it != end && mir == *it);
+
+ // Lambda should have a default value readable for iterating over the
+ // inner frames.
+ MConstant* functionOperand = nullptr;
+ if (mir->isLambda()) {
+ functionOperand = mir->toLambda()->functionOperand();
+ } else if (mir->isFunctionWithProto()) {
+ functionOperand = mir->toFunctionWithProto()->functionOperand();
+ }
+ if (functionOperand) {
+ uint32_t cstIndex;
+ masm.propagateOOM(
+ graph.addConstantToPool(functionOperand->toJSValue(), &cstIndex));
+ alloc = RValueAllocation::RecoverInstruction(index, cstIndex);
+ break;
+ }
+
+ alloc = RValueAllocation::RecoverInstruction(index);
+ break;
+ }
+ case MIRType::Undefined:
+ alloc = RValueAllocation::Undefined();
+ break;
+ case MIRType::Null:
+ alloc = RValueAllocation::Null();
+ break;
+ case MIRType::Int32:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ case MIRType::Shape:
+ case MIRType::Boolean:
+ case MIRType::Double: {
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+ if (payload->isConstant()) {
+ MConstant* constant = mir->toConstant();
+ uint32_t index;
+ masm.propagateOOM(
+ graph.addConstantToPool(constant->toJSValue(), &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+
+ JSValueType valueType = ValueTypeFromMIRType(type);
+
+ MOZ_DIAGNOSTIC_ASSERT(payload->isMemory() || payload->isRegister());
+ if (payload->isMemory()) {
+ alloc = RValueAllocation::Typed(valueType, ToStackIndex(payload));
+ } else if (payload->isGeneralReg()) {
+ alloc = RValueAllocation::Typed(valueType, ToRegister(payload));
+ } else if (payload->isFloatReg()) {
+ alloc = RValueAllocation::Double(ToFloatRegister(payload));
+ } else {
+ MOZ_CRASH("Unexpected payload type.");
+ }
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Simd128: {
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+ if (payload->isConstant()) {
+ MConstant* constant = mir->toConstant();
+ uint32_t index;
+ masm.propagateOOM(
+ graph.addConstantToPool(constant->toJSValue(), &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+
+ MOZ_ASSERT(payload->isMemory() || payload->isFloatReg());
+ if (payload->isFloatReg()) {
+ alloc = RValueAllocation::AnyFloat(ToFloatRegister(payload));
+ } else {
+ alloc = RValueAllocation::AnyFloat(ToStackIndex(payload));
+ }
+ break;
+ }
+ case MIRType::MagicOptimizedOut:
+ case MIRType::MagicUninitializedLexical:
+ case MIRType::MagicIsConstructing: {
+ uint32_t index;
+ JSWhyMagic why = JS_GENERIC_MAGIC;
+ switch (type) {
+ case MIRType::MagicOptimizedOut:
+ why = JS_OPTIMIZED_OUT;
+ break;
+ case MIRType::MagicUninitializedLexical:
+ why = JS_UNINITIALIZED_LEXICAL;
+ break;
+ case MIRType::MagicIsConstructing:
+ why = JS_IS_CONSTRUCTING;
+ break;
+ default:
+ MOZ_CRASH("Invalid Magic MIRType");
+ }
+
+ Value v = MagicValue(why);
+ masm.propagateOOM(graph.addConstantToPool(v, &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+ default: {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+#ifdef JS_NUNBOX32
+ LAllocation* type = snapshot->typeOfSlot(*allocIndex);
+ if (type->isRegister()) {
+ if (payload->isRegister()) {
+ alloc =
+ RValueAllocation::Untyped(ToRegister(type), ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToRegister(type),
+ ToStackIndex(payload));
+ }
+ } else {
+ if (payload->isRegister()) {
+ alloc = RValueAllocation::Untyped(ToStackIndex(type),
+ ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToStackIndex(type),
+ ToStackIndex(payload));
+ }
+ }
+#elif JS_PUNBOX64
+ if (payload->isRegister()) {
+ alloc = RValueAllocation::Untyped(ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToStackIndex(payload));
+ }
+#endif
+ break;
+ }
+ }
+ MOZ_DIAGNOSTIC_ASSERT(alloc.valid());
+
+ // This set an extra bit as part of the RValueAllocation, such that we know
+ // that recover instruction have to be executed without wrapping the
+ // instruction in a no-op recover instruction.
+ if (mir->isIncompleteObject()) {
+ alloc.setNeedSideEffect();
+ }
+
+ masm.propagateOOM(snapshots_.add(alloc));
+
+ *allocIndex += mir->isRecoveredOnBailout() ? 0 : 1;
+}
+
+void CodeGeneratorShared::encode(LRecoverInfo* recover) {
+ if (recover->recoverOffset() != INVALID_RECOVER_OFFSET) {
+ return;
+ }
+
+ uint32_t numInstructions = recover->numInstructions();
+ JitSpew(JitSpew_IonSnapshots,
+ "Encoding LRecoverInfo %p (frameCount %u, instructions %u)",
+ (void*)recover, recover->mir()->frameCount(), numInstructions);
+
+ RecoverOffset offset = recovers_.startRecover(numInstructions);
+
+ for (MNode* insn : *recover) {
+ recovers_.writeInstruction(insn);
+ }
+
+ recovers_.endRecover();
+ recover->setRecoverOffset(offset);
+ masm.propagateOOM(!recovers_.oom());
+}
+
+void CodeGeneratorShared::encode(LSnapshot* snapshot) {
+ if (snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET) {
+ return;
+ }
+
+ LRecoverInfo* recoverInfo = snapshot->recoverInfo();
+ encode(recoverInfo);
+
+ RecoverOffset recoverOffset = recoverInfo->recoverOffset();
+ MOZ_ASSERT(recoverOffset != INVALID_RECOVER_OFFSET);
+
+ JitSpew(JitSpew_IonSnapshots, "Encoding LSnapshot %p (LRecover %p)",
+ (void*)snapshot, (void*)recoverInfo);
+
+ SnapshotOffset offset =
+ snapshots_.startSnapshot(recoverOffset, snapshot->bailoutKind());
+
+#ifdef TRACK_SNAPSHOTS
+ uint32_t pcOpcode = 0;
+ uint32_t lirOpcode = 0;
+ uint32_t lirId = 0;
+ uint32_t mirOpcode = 0;
+ uint32_t mirId = 0;
+
+ if (LInstruction* ins = instruction()) {
+ lirOpcode = uint32_t(ins->op());
+ lirId = ins->id();
+ if (MDefinition* mir = ins->mirRaw()) {
+ mirOpcode = uint32_t(mir->op());
+ mirId = mir->id();
+ if (jsbytecode* pc = mir->trackedSite()->pc()) {
+ pcOpcode = *pc;
+ }
+ }
+ }
+ snapshots_.trackSnapshot(pcOpcode, mirOpcode, mirId, lirOpcode, lirId);
+#endif
+
+ uint32_t allocIndex = 0;
+ for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
+ DebugOnly<uint32_t> allocWritten = snapshots_.allocWritten();
+ encodeAllocation(snapshot, *it, &allocIndex);
+ MOZ_ASSERT_IF(!snapshots_.oom(),
+ allocWritten + 1 == snapshots_.allocWritten());
+ }
+
+ MOZ_ASSERT(allocIndex == snapshot->numSlots());
+ snapshots_.endSnapshot();
+ snapshot->setSnapshotOffset(offset);
+ masm.propagateOOM(!snapshots_.oom());
+}
+
+bool CodeGeneratorShared::assignBailoutId(LSnapshot* snapshot) {
+ MOZ_ASSERT(snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET);
+
+ // Can we not use bailout tables at all?
+ if (!deoptTable_) {
+ return false;
+ }
+
+ MOZ_ASSERT(frameClass_ != FrameSizeClass::None());
+
+ if (snapshot->bailoutId() != INVALID_BAILOUT_ID) {
+ return true;
+ }
+
+ // Is the bailout table full?
+ if (bailouts_.length() >= BAILOUT_TABLE_SIZE) {
+ return false;
+ }
+
+ unsigned bailoutId = bailouts_.length();
+ snapshot->setBailoutId(bailoutId);
+ JitSpew(JitSpew_IonSnapshots, "Assigned snapshot bailout id %u", bailoutId);
+ masm.propagateOOM(bailouts_.append(snapshot->snapshotOffset()));
+ return true;
+}
+
+bool CodeGeneratorShared::encodeSafepoints() {
+ for (CodegenSafepointIndex& index : safepointIndices_) {
+ LSafepoint* safepoint = index.safepoint();
+
+ if (!safepoint->encoded()) {
+ safepoints_.encode(safepoint);
+ }
+ }
+
+ return !safepoints_.oom();
+}
+
+bool CodeGeneratorShared::createNativeToBytecodeScriptList(JSContext* cx) {
+ js::Vector<JSScript*, 0, SystemAllocPolicy> scriptList;
+ InlineScriptTree* tree = gen->outerInfo().inlineScriptTree();
+ for (;;) {
+ // Add script from current tree.
+ bool found = false;
+ for (uint32_t i = 0; i < scriptList.length(); i++) {
+ if (scriptList[i] == tree->script()) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ if (!scriptList.append(tree->script())) {
+ return false;
+ }
+ }
+
+ // Process rest of tree
+
+ // If children exist, emit children.
+ if (tree->hasChildren()) {
+ tree = tree->firstChild();
+ continue;
+ }
+
+ // Otherwise, find the first tree up the chain (including this one)
+ // that contains a next sibling.
+ while (!tree->hasNextCallee() && tree->hasCaller()) {
+ tree = tree->caller();
+ }
+
+ // If we found a sibling, use it.
+ if (tree->hasNextCallee()) {
+ tree = tree->nextCallee();
+ continue;
+ }
+
+ // Otherwise, we must have reached the top without finding any siblings.
+ MOZ_ASSERT(tree->isOutermostCaller());
+ break;
+ }
+
+ // Allocate array for list.
+ JSScript** data = cx->pod_malloc<JSScript*>(scriptList.length());
+ if (!data) {
+ return false;
+ }
+
+ for (uint32_t i = 0; i < scriptList.length(); i++) {
+ data[i] = scriptList[i];
+ }
+
+ // Success.
+ nativeToBytecodeScriptListLength_ = scriptList.length();
+ nativeToBytecodeScriptList_ = data;
+ return true;
+}
+
+bool CodeGeneratorShared::generateCompactNativeToBytecodeMap(JSContext* cx,
+ JitCode* code) {
+ MOZ_ASSERT(nativeToBytecodeScriptListLength_ == 0);
+ MOZ_ASSERT(nativeToBytecodeScriptList_ == nullptr);
+ MOZ_ASSERT(nativeToBytecodeMap_ == nullptr);
+ MOZ_ASSERT(nativeToBytecodeMapSize_ == 0);
+ MOZ_ASSERT(nativeToBytecodeTableOffset_ == 0);
+ MOZ_ASSERT(nativeToBytecodeNumRegions_ == 0);
+
+ if (!createNativeToBytecodeScriptList(cx)) {
+ return false;
+ }
+
+ MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0);
+ MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr);
+
+ CompactBufferWriter writer;
+ uint32_t tableOffset = 0;
+ uint32_t numRegions = 0;
+
+ if (!JitcodeIonTable::WriteIonTable(
+ writer, nativeToBytecodeScriptList_,
+ nativeToBytecodeScriptListLength_, &nativeToBytecodeList_[0],
+ &nativeToBytecodeList_[0] + nativeToBytecodeList_.length(),
+ &tableOffset, &numRegions)) {
+ js_free(nativeToBytecodeScriptList_);
+ return false;
+ }
+
+ MOZ_ASSERT(tableOffset > 0);
+ MOZ_ASSERT(numRegions > 0);
+
+ // Writer is done, copy it to sized buffer.
+ uint8_t* data = cx->pod_malloc<uint8_t>(writer.length());
+ if (!data) {
+ js_free(nativeToBytecodeScriptList_);
+ return false;
+ }
+
+ memcpy(data, writer.buffer(), writer.length());
+ nativeToBytecodeMap_ = data;
+ nativeToBytecodeMapSize_ = writer.length();
+ nativeToBytecodeTableOffset_ = tableOffset;
+ nativeToBytecodeNumRegions_ = numRegions;
+
+ verifyCompactNativeToBytecodeMap(code);
+
+ JitSpew(JitSpew_Profiling, "Compact Native To Bytecode Map [%p-%p]", data,
+ data + nativeToBytecodeMapSize_);
+
+ return true;
+}
+
+void CodeGeneratorShared::verifyCompactNativeToBytecodeMap(JitCode* code) {
+#ifdef DEBUG
+ MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0);
+ MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr);
+ MOZ_ASSERT(nativeToBytecodeMap_ != nullptr);
+ MOZ_ASSERT(nativeToBytecodeMapSize_ > 0);
+ MOZ_ASSERT(nativeToBytecodeTableOffset_ > 0);
+ MOZ_ASSERT(nativeToBytecodeNumRegions_ > 0);
+
+ // The pointer to the table must be 4-byte aligned
+ const uint8_t* tablePtr = nativeToBytecodeMap_ + nativeToBytecodeTableOffset_;
+ MOZ_ASSERT(uintptr_t(tablePtr) % sizeof(uint32_t) == 0);
+
+ // Verify that numRegions was encoded correctly.
+ const JitcodeIonTable* ionTable =
+ reinterpret_cast<const JitcodeIonTable*>(tablePtr);
+ MOZ_ASSERT(ionTable->numRegions() == nativeToBytecodeNumRegions_);
+
+ // Region offset for first region should be at the start of the payload
+ // region. Since the offsets are backward from the start of the table, the
+ // first entry backoffset should be equal to the forward table offset from the
+ // start of the allocated data.
+ MOZ_ASSERT(ionTable->regionOffset(0) == nativeToBytecodeTableOffset_);
+
+ // Verify each region.
+ for (uint32_t i = 0; i < ionTable->numRegions(); i++) {
+ // Back-offset must point into the payload region preceding the table, not
+ // before it.
+ MOZ_ASSERT(ionTable->regionOffset(i) <= nativeToBytecodeTableOffset_);
+
+ // Back-offset must point to a later area in the payload region than
+ // previous back-offset. This means that back-offsets decrease
+ // monotonically.
+ MOZ_ASSERT_IF(i > 0,
+ ionTable->regionOffset(i) < ionTable->regionOffset(i - 1));
+
+ JitcodeRegionEntry entry = ionTable->regionEntry(i);
+
+ // Ensure native code offset for region falls within jitcode.
+ MOZ_ASSERT(entry.nativeOffset() <= code->instructionsSize());
+
+ // Read out script/pc stack and verify.
+ JitcodeRegionEntry::ScriptPcIterator scriptPcIter =
+ entry.scriptPcIterator();
+ while (scriptPcIter.hasMore()) {
+ uint32_t scriptIdx = 0, pcOffset = 0;
+ scriptPcIter.readNext(&scriptIdx, &pcOffset);
+
+ // Ensure scriptIdx refers to a valid script in the list.
+ MOZ_ASSERT(scriptIdx < nativeToBytecodeScriptListLength_);
+ JSScript* script = nativeToBytecodeScriptList_[scriptIdx];
+
+ // Ensure pcOffset falls within the script.
+ MOZ_ASSERT(pcOffset < script->length());
+ }
+
+ // Obtain the original nativeOffset and pcOffset and script.
+ uint32_t curNativeOffset = entry.nativeOffset();
+ JSScript* script = nullptr;
+ uint32_t curPcOffset = 0;
+ {
+ uint32_t scriptIdx = 0;
+ scriptPcIter.reset();
+ scriptPcIter.readNext(&scriptIdx, &curPcOffset);
+ script = nativeToBytecodeScriptList_[scriptIdx];
+ }
+
+ // Read out nativeDeltas and pcDeltas and verify.
+ JitcodeRegionEntry::DeltaIterator deltaIter = entry.deltaIterator();
+ while (deltaIter.hasMore()) {
+ uint32_t nativeDelta = 0;
+ int32_t pcDelta = 0;
+ deltaIter.readNext(&nativeDelta, &pcDelta);
+
+ curNativeOffset += nativeDelta;
+ curPcOffset = uint32_t(int32_t(curPcOffset) + pcDelta);
+
+ // Ensure that nativeOffset still falls within jitcode after delta.
+ MOZ_ASSERT(curNativeOffset <= code->instructionsSize());
+
+ // Ensure that pcOffset still falls within bytecode after delta.
+ MOZ_ASSERT(curPcOffset < script->length());
+ }
+ }
+#endif // DEBUG
+}
+
+void CodeGeneratorShared::markSafepoint(LInstruction* ins) {
+ markSafepointAt(masm.currentOffset(), ins);
+}
+
+void CodeGeneratorShared::markSafepointAt(uint32_t offset, LInstruction* ins) {
+ MOZ_ASSERT_IF(
+ !safepointIndices_.empty() && !masm.oom(),
+ offset - safepointIndices_.back().displacement() >= sizeof(uint32_t));
+ masm.propagateOOM(safepointIndices_.append(
+ CodegenSafepointIndex(offset, ins->safepoint())));
+}
+
+void CodeGeneratorShared::ensureOsiSpace() {
+ // For a refresher, an invalidation point is of the form:
+ // 1: call <target>
+ // 2: ...
+ // 3: <osipoint>
+ //
+ // The four bytes *before* instruction 2 are overwritten with an offset.
+ // Callers must ensure that the instruction itself has enough bytes to
+ // support this.
+ //
+ // The bytes *at* instruction 3 are overwritten with an invalidation jump.
+ // jump. These bytes may be in a completely different IR sequence, but
+ // represent the join point of the call out of the function.
+ //
+ // At points where we want to ensure that invalidation won't corrupt an
+ // important instruction, we make sure to pad with nops.
+ if (masm.currentOffset() - lastOsiPointOffset_ <
+ Assembler::PatchWrite_NearCallSize()) {
+ int32_t paddingSize = Assembler::PatchWrite_NearCallSize();
+ paddingSize -= masm.currentOffset() - lastOsiPointOffset_;
+ for (int32_t i = 0; i < paddingSize; ++i) {
+ masm.nop();
+ }
+ }
+ MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() - lastOsiPointOffset_ >=
+ Assembler::PatchWrite_NearCallSize());
+}
+
+uint32_t CodeGeneratorShared::markOsiPoint(LOsiPoint* ins) {
+ encode(ins->snapshot());
+ ensureOsiSpace();
+
+ uint32_t offset = masm.currentOffset();
+ SnapshotOffset so = ins->snapshot()->snapshotOffset();
+ masm.propagateOOM(osiIndices_.append(OsiIndex(offset, so)));
+ lastOsiPointOffset_ = offset;
+
+ return offset;
+}
+
+class OutOfLineTruncateSlow : public OutOfLineCodeBase<CodeGeneratorShared> {
+ FloatRegister src_;
+ Register dest_;
+ bool widenFloatToDouble_;
+ wasm::BytecodeOffset bytecodeOffset_;
+ bool preserveInstance_;
+
+ public:
+ OutOfLineTruncateSlow(
+ FloatRegister src, Register dest, bool widenFloatToDouble = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset(),
+ bool preserveInstance = false)
+ : src_(src),
+ dest_(dest),
+ widenFloatToDouble_(widenFloatToDouble),
+ bytecodeOffset_(bytecodeOffset),
+ preserveInstance_(preserveInstance) {}
+
+ void accept(CodeGeneratorShared* codegen) override {
+ codegen->visitOutOfLineTruncateSlow(this);
+ }
+ FloatRegister src() const { return src_; }
+ Register dest() const { return dest_; }
+ bool widenFloatToDouble() const { return widenFloatToDouble_; }
+ bool preserveInstance() const { return preserveInstance_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+OutOfLineCode* CodeGeneratorShared::oolTruncateDouble(
+ FloatRegister src, Register dest, MInstruction* mir,
+ wasm::BytecodeOffset bytecodeOffset, bool preserveInstance) {
+ MOZ_ASSERT_IF(IsCompilingWasm(), bytecodeOffset.isValid());
+
+ OutOfLineTruncateSlow* ool = new (alloc()) OutOfLineTruncateSlow(
+ src, dest, /* float32 */ false, bytecodeOffset, preserveInstance);
+ addOutOfLineCode(ool, mir);
+ return ool;
+}
+
+void CodeGeneratorShared::emitTruncateDouble(FloatRegister src, Register dest,
+ MInstruction* mir) {
+ MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32());
+ wasm::BytecodeOffset bytecodeOffset =
+ mir->isTruncateToInt32()
+ ? mir->toTruncateToInt32()->bytecodeOffset()
+ : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset();
+ OutOfLineCode* ool = oolTruncateDouble(src, dest, mir, bytecodeOffset);
+
+ masm.branchTruncateDoubleMaybeModUint32(src, dest, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorShared::emitTruncateFloat32(FloatRegister src, Register dest,
+ MInstruction* mir) {
+ MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32());
+ wasm::BytecodeOffset bytecodeOffset =
+ mir->isTruncateToInt32()
+ ? mir->toTruncateToInt32()->bytecodeOffset()
+ : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset();
+ OutOfLineTruncateSlow* ool = new (alloc())
+ OutOfLineTruncateSlow(src, dest, /* float32 */ true, bytecodeOffset);
+ addOutOfLineCode(ool, mir);
+
+ masm.branchTruncateFloat32MaybeModUint32(src, dest, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorShared::visitOutOfLineTruncateSlow(
+ OutOfLineTruncateSlow* ool) {
+ FloatRegister src = ool->src();
+ Register dest = ool->dest();
+
+ saveVolatile(dest);
+ masm.outOfLineTruncateSlow(src, dest, ool->widenFloatToDouble(),
+ gen->compilingWasm(), ool->bytecodeOffset());
+ restoreVolatile(dest);
+
+ masm.jump(ool->rejoin());
+}
+
+bool CodeGeneratorShared::omitOverRecursedCheck() const {
+ // If the current function makes no calls (which means it isn't recursive)
+ // and it uses only a small amount of stack space, it doesn't need a
+ // stack overflow check. Note that the actual number here is somewhat
+ // arbitrary, and codegen actually uses small bounded amounts of
+ // additional stack space in some cases too.
+ return frameSize() < MAX_UNCHECKED_LEAF_FRAME_SIZE &&
+ !gen->needsOverrecursedCheck();
+}
+
+void CodeGeneratorShared::emitPreBarrier(Register elements,
+ const LAllocation* index) {
+ if (index->isConstant()) {
+ Address address(elements, ToInt32(index) * sizeof(Value));
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+ } else {
+ BaseObjectElementIndex address(elements, ToRegister(index));
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+ }
+}
+
+void CodeGeneratorShared::emitPreBarrier(Address address) {
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+}
+
+void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir) {
+ // Skip past trivial blocks.
+ mir = skipTrivialBlocks(mir);
+
+ // No jump necessary if we can fall through to the next block.
+ if (isNextBlock(mir->lir())) {
+ return;
+ }
+
+ masm.jump(mir->lir()->label());
+}
+
+Label* CodeGeneratorShared::getJumpLabelForBranch(MBasicBlock* block) {
+ // Skip past trivial blocks.
+ return skipTrivialBlocks(block)->lir()->label();
+}
+
+// This function is not used for MIPS/MIPS64/LOONG64. They have
+// branchToBlock.
+#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && \
+ !defined(JS_CODEGEN_LOONG64)
+void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir,
+ Assembler::Condition cond) {
+ // Skip past trivial blocks.
+ masm.j(cond, skipTrivialBlocks(mir)->lir()->label());
+}
+#endif
+
+ReciprocalMulConstants CodeGeneratorShared::computeDivisionConstants(
+ uint32_t d, int maxLog) {
+ MOZ_ASSERT(maxLog >= 2 && maxLog <= 32);
+ // In what follows, 0 < d < 2^maxLog and d is not a power of 2.
+ MOZ_ASSERT(d < (uint64_t(1) << maxLog) && (d & (d - 1)) != 0);
+
+ // Speeding up division by non power-of-2 constants is possible by
+ // calculating, during compilation, a value M such that high-order
+ // bits of M*n correspond to the result of the division of n by d.
+ // No value of M can serve this purpose for arbitrarily big values
+ // of n but, for optimizing integer division, we're just concerned
+ // with values of n whose absolute value is bounded (by fitting in
+ // an integer type, say). With this in mind, we'll find a constant
+ // M as above that works for -2^maxLog <= n < 2^maxLog; maxLog can
+ // then be 31 for signed division or 32 for unsigned division.
+ //
+ // The original presentation of this technique appears in Hacker's
+ // Delight, a book by Henry S. Warren, Jr.. A proof of correctness
+ // for our version follows; we'll denote maxLog by L in the proof,
+ // for conciseness.
+ //
+ // Formally, for |d| < 2^L, we'll compute two magic values M and s
+ // in the ranges 0 <= M < 2^(L+1) and 0 <= s <= L such that
+ // (M * n) >> (32 + s) = floor(n/d) if 0 <= n < 2^L
+ // (M * n) >> (32 + s) = ceil(n/d) - 1 if -2^L <= n < 0.
+ //
+ // Define p = 32 + s, M = ceil(2^p/d), and assume that s satisfies
+ // M - 2^p/d <= 2^(p-L)/d. (1)
+ // (Observe that p = CeilLog32(d) + L satisfies this, as the right
+ // side of (1) is at least one in this case). Then,
+ //
+ // a) If p <= CeilLog32(d) + L, then M < 2^(L+1) - 1.
+ // Proof: Indeed, M is monotone in p and, for p equal to the above
+ // value, the bounds 2^L > d >= 2^(p-L-1) + 1 readily imply that
+ // 2^p / d < 2^p/(d - 1) * (d - 1)/d
+ // <= 2^(L+1) * (1 - 1/d) < 2^(L+1) - 2.
+ // The claim follows by applying the ceiling function.
+ //
+ // b) For any 0 <= n < 2^L, floor(Mn/2^p) = floor(n/d).
+ // Proof: Put x = floor(Mn/2^p); it's the unique integer for which
+ // Mn/2^p - 1 < x <= Mn/2^p. (2)
+ // Using M >= 2^p/d on the LHS and (1) on the RHS, we get
+ // n/d - 1 < x <= n/d + n/(2^L d) < n/d + 1/d.
+ // Since x is an integer, it's not in the interval (n/d, (n+1)/d),
+ // and so n/d - 1 < x <= n/d, which implies x = floor(n/d).
+ //
+ // c) For any -2^L <= n < 0, floor(Mn/2^p) + 1 = ceil(n/d).
+ // Proof: The proof is similar. Equation (2) holds as above. Using
+ // M > 2^p/d (d isn't a power of 2) on the RHS and (1) on the LHS,
+ // n/d + n/(2^L d) - 1 < x < n/d.
+ // Using n >= -2^L and summing 1,
+ // n/d - 1/d < x + 1 < n/d + 1.
+ // Since x + 1 is an integer, this implies n/d <= x + 1 < n/d + 1.
+ // In other words, x + 1 = ceil(n/d).
+ //
+ // Condition (1) isn't necessary for the existence of M and s with
+ // the properties above. Hacker's Delight provides a slightly less
+ // restrictive condition when d >= 196611, at the cost of a 3-page
+ // proof of correctness, for the case L = 31.
+ //
+ // Note that, since d*M - 2^p = d - (2^p)%d, (1) can be written as
+ // 2^(p-L) >= d - (2^p)%d.
+ // In order to avoid overflow in the (2^p) % d calculation, we can
+ // compute it as (2^p-1) % d + 1, where 2^p-1 can then be computed
+ // without overflow as UINT64_MAX >> (64-p).
+
+ // We now compute the least p >= 32 with the property above...
+ int32_t p = 32;
+ while ((uint64_t(1) << (p - maxLog)) + (UINT64_MAX >> (64 - p)) % d + 1 < d) {
+ p++;
+ }
+
+ // ...and the corresponding M. For either the signed (L=31) or the
+ // unsigned (L=32) case, this value can be too large (cf. item a).
+ // Codegen can still multiply by M by multiplying by (M - 2^L) and
+ // adjusting the value afterwards, if this is the case.
+ ReciprocalMulConstants rmc;
+ rmc.multiplier = (UINT64_MAX >> (64 - p)) / d + 1;
+ rmc.shiftAmount = p - 32;
+
+ return rmc;
+}
+
+#ifdef JS_TRACE_LOGGING
+
+void CodeGeneratorShared::emitTracelogScript(bool isStart) {
+ if (!TraceLogTextIdEnabled(TraceLogger_Scripts)) {
+ return;
+ }
+
+ Label done;
+
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ Register logger = regs.takeAnyGeneral();
+ Register script = regs.takeAnyGeneral();
+
+ masm.Push(logger);
+
+ masm.loadTraceLogger(logger);
+ masm.branchTestPtr(Assembler::Zero, logger, logger, &done);
+
+ Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled());
+ masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
+
+ masm.Push(script);
+
+ CodeOffset patchScript = masm.movWithPatch(ImmWord(0), script);
+ masm.propagateOOM(patchableTLScripts_.append(patchScript));
+
+ if (isStart) {
+ masm.tracelogStartId(logger, script);
+ } else {
+ masm.tracelogStopId(logger, script);
+ }
+
+ masm.Pop(script);
+
+ masm.bind(&done);
+
+ masm.Pop(logger);
+}
+
+void CodeGeneratorShared::emitTracelogTree(bool isStart, uint32_t textId) {
+ if (!TraceLogTextIdEnabled(textId)) {
+ return;
+ }
+
+ Label done;
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ Register logger = regs.takeAnyGeneral();
+
+ masm.Push(logger);
+
+ masm.loadTraceLogger(logger);
+ masm.branchTestPtr(Assembler::Zero, logger, logger, &done);
+
+ Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled());
+ masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
+
+ if (isStart) {
+ masm.tracelogStartId(logger, textId);
+ } else {
+ masm.tracelogStopId(logger, textId);
+ }
+
+ masm.bind(&done);
+
+ masm.Pop(logger);
+}
+
+void CodeGeneratorShared::emitTracelogTree(bool isStart, const char* text,
+ TraceLoggerTextId enabledTextId) {
+ if (!TraceLogTextIdEnabled(enabledTextId)) {
+ return;
+ }
+
+ Label done;
+
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ Register loggerReg = regs.takeAnyGeneral();
+ Register eventReg = regs.takeAnyGeneral();
+
+ masm.Push(loggerReg);
+
+ masm.loadTraceLogger(loggerReg);
+ masm.branchTestPtr(Assembler::Zero, loggerReg, loggerReg, &done);
+
+ Address enabledAddress(loggerReg, TraceLoggerThread::offsetOfEnabled());
+ masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
+
+ masm.Push(eventReg);
+
+ PatchableTLEvent patchEvent(masm.movWithPatch(ImmWord(0), eventReg), text);
+ masm.propagateOOM(patchableTLEvents_.append(std::move(patchEvent)));
+
+ if (isStart) {
+ masm.tracelogStartId(loggerReg, eventReg);
+ } else {
+ masm.tracelogStopId(loggerReg, eventReg);
+ }
+
+ masm.Pop(eventReg);
+
+ masm.bind(&done);
+
+ masm.Pop(loggerReg);
+}
+#endif
+
+} // namespace jit
+} // namespace js