diff options
Diffstat (limited to 'js/src/jit/Ion.cpp')
| -rw-r--r-- | js/src/jit/Ion.cpp | 2631 |
1 files changed, 2631 insertions, 0 deletions
diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp new file mode 100644 index 0000000000..3d472aa85f --- /dev/null +++ b/js/src/jit/Ion.cpp @@ -0,0 +1,2631 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Ion.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/ThreadLocal.h" + +#include "gc/GCContext.h" +#include "gc/PublicIterators.h" +#include "jit/AliasAnalysis.h" +#include "jit/AlignmentMaskAnalysis.h" +#include "jit/AutoWritableJitCode.h" +#include "jit/BacktrackingAllocator.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineJIT.h" +#include "jit/CodeGenerator.h" +#include "jit/CompileInfo.h" +#include "jit/EdgeCaseAnalysis.h" +#include "jit/EffectiveAddressAnalysis.h" +#include "jit/ExecutableAllocator.h" +#include "jit/FoldLinearArithConstants.h" +#include "jit/InlineScriptTree.h" +#include "jit/InstructionReordering.h" +#include "jit/Invalidation.h" +#include "jit/IonAnalysis.h" +#include "jit/IonCompileTask.h" +#include "jit/IonIC.h" +#include "jit/IonOptimizationLevels.h" +#include "jit/IonScript.h" +#include "jit/JitcodeMap.h" +#include "jit/JitFrames.h" +#include "jit/JitRealm.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/JitZone.h" +#include "jit/LICM.h" +#include "jit/Linker.h" +#include "jit/LIR.h" +#include "jit/Lowering.h" +#include "jit/PerfSpewer.h" +#include "jit/RangeAnalysis.h" +#include "jit/ScalarReplacement.h" +#include "jit/ScriptFromCalleeToken.h" +#include "jit/Sink.h" +#include "jit/ValueNumbering.h" +#include "jit/WarpBuilder.h" +#include "jit/WarpOracle.h" +#include "jit/WasmBCE.h" +#include "js/Printf.h" +#include "js/UniquePtr.h" +#include "util/Memory.h" +#include "util/WindowsWrapper.h" +#include "vm/HelperThreads.h" +#include "vm/Realm.h" +#ifdef MOZ_VTUNE +# include "vtune/VTuneWrapper.h" +#endif + +#include "gc/GC-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "jit/InlineScriptTree-inl.h" +#include "jit/MacroAssembler-inl.h" +#include "jit/SafepointIndex-inl.h" +#include "vm/GeckoProfiler-inl.h" +#include "vm/JSScript-inl.h" +#include "vm/Realm-inl.h" + +#if defined(ANDROID) +# include <sys/system_properties.h> +#endif + +using mozilla::CheckedInt; +using mozilla::DebugOnly; + +using namespace js; +using namespace js::jit; + +JitRuntime::~JitRuntime() { + MOZ_ASSERT(numFinishedOffThreadTasks_ == 0); + MOZ_ASSERT(ionLazyLinkListSize_ == 0); + MOZ_ASSERT(ionLazyLinkList_.ref().isEmpty()); + + // By this point, the jitcode global table should be empty. + MOZ_ASSERT_IF(jitcodeGlobalTable_, jitcodeGlobalTable_->empty()); + js_delete(jitcodeGlobalTable_.ref()); + + // interpreterEntryMap should be cleared out during finishRoots() + MOZ_ASSERT_IF(interpreterEntryMap_, interpreterEntryMap_->empty()); + js_delete(interpreterEntryMap_.ref()); + + js_delete(jitHintsMap_.ref()); +} + +uint32_t JitRuntime::startTrampolineCode(MacroAssembler& masm) { + AutoCreatedBy acb(masm, "startTrampolineCode"); + + masm.assumeUnreachable("Shouldn't get here"); + masm.flushBuffer(); + masm.haltingAlign(CodeAlignment); + masm.setFramePushed(0); + return masm.currentOffset(); +} + +bool JitRuntime::initialize(JSContext* cx) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime())); + + AutoAllocInAtomsZone az(cx); + JitContext jctx(cx); + + if (!generateTrampolines(cx)) { + return false; + } + + if (!generateBaselineICFallbackCode(cx)) { + return false; + } + + jitcodeGlobalTable_ = cx->new_<JitcodeGlobalTable>(); + if (!jitcodeGlobalTable_) { + return false; + } + + if (!JitOptions.disableJitHints) { + jitHintsMap_ = cx->new_<JitHintsMap>(); + if (!jitHintsMap_) { + return false; + } + } + + if (JitOptions.emitInterpreterEntryTrampoline) { + interpreterEntryMap_ = cx->new_<EntryTrampolineMap>(); + if (!interpreterEntryMap_) { + return false; + } + } + + if (!GenerateBaselineInterpreter(cx, baselineInterpreter_)) { + return false; + } + + // Initialize the jitCodeRaw of the Runtime's canonical SelfHostedLazyScript + // to point to the interpreter trampoline. + cx->runtime()->selfHostedLazyScript.ref().jitCodeRaw_ = + interpreterStub().value; + + return true; +} + +bool JitRuntime::generateTrampolines(JSContext* cx) { + TempAllocator temp(&cx->tempLifoAlloc()); + StackMacroAssembler masm(cx, temp); + PerfSpewerRangeRecorder rangeRecorder(masm); + + Label bailoutTail; + JitSpew(JitSpew_Codegen, "# Emitting bailout tail stub"); + generateBailoutTailStub(masm, &bailoutTail); + + JitSpew(JitSpew_Codegen, "# Emitting bailout handler"); + generateBailoutHandler(masm, &bailoutTail); + rangeRecorder.recordOffset("Trampoline: Bailout"); + + JitSpew(JitSpew_Codegen, "# Emitting invalidator"); + generateInvalidator(masm, &bailoutTail); + rangeRecorder.recordOffset("Trampoline: Invalidator"); + + // The arguments rectifier has to use the same frame layout as the function + // frames it rectifies. + static_assert(std::is_base_of_v<JitFrameLayout, RectifierFrameLayout>, + "a rectifier frame can be used with jit frame"); + static_assert(std::is_base_of_v<JitFrameLayout, WasmToJSJitFrameLayout>, + "wasm frames simply are jit frames"); + static_assert(sizeof(JitFrameLayout) == sizeof(WasmToJSJitFrameLayout), + "thus a rectifier frame can be used with a wasm frame"); + + JitSpew(JitSpew_Codegen, "# Emitting arguments rectifier"); + generateArgumentsRectifier(masm, ArgumentsRectifierKind::Normal); + rangeRecorder.recordOffset("Trampoline: Arguments Rectifier"); + + JitSpew(JitSpew_Codegen, "# Emitting trial inlining arguments rectifier"); + generateArgumentsRectifier(masm, ArgumentsRectifierKind::TrialInlining); + rangeRecorder.recordOffset( + "Trampoline: Arguments Rectifier (Trial Inlining)"); + + JitSpew(JitSpew_Codegen, "# Emitting EnterJIT sequence"); + generateEnterJIT(cx, masm); + rangeRecorder.recordOffset("Trampoline: EnterJIT"); + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Value"); + valuePreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Value); + rangeRecorder.recordOffset("Trampoline: PreBarrier Value"); + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for String"); + stringPreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::String); + rangeRecorder.recordOffset("Trampoline: PreBarrier String"); + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Object"); + objectPreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Object); + rangeRecorder.recordOffset("Trampoline: PreBarrier Object"); + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Shape"); + shapePreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Shape); + rangeRecorder.recordOffset("Trampoline: PreBarrier Shape"); + + JitSpew(JitSpew_Codegen, "# Emitting free stub"); + generateFreeStub(masm); + rangeRecorder.recordOffset("Trampoline: FreeStub"); + + JitSpew(JitSpew_Codegen, "# Emitting lazy link stub"); + generateLazyLinkStub(masm); + rangeRecorder.recordOffset("Trampoline: LazyLinkStub"); + + JitSpew(JitSpew_Codegen, "# Emitting interpreter stub"); + generateInterpreterStub(masm); + rangeRecorder.recordOffset("Trampoline: Interpreter"); + + JitSpew(JitSpew_Codegen, "# Emitting double-to-int32-value stub"); + generateDoubleToInt32ValueStub(masm); + rangeRecorder.recordOffset("Trampoline: DoubleToInt32ValueStub"); + + JitSpew(JitSpew_Codegen, "# Emitting VM function wrappers"); + if (!generateVMWrappers(cx, masm)) { + return false; + } + rangeRecorder.recordOffset("Trampoline: VM Wrapper"); + + JitSpew(JitSpew_Codegen, "# Emitting profiler exit frame tail stub"); + Label profilerExitTail; + generateProfilerExitFrameTailStub(masm, &profilerExitTail); + rangeRecorder.recordOffset("Trampoline: ProfilerExitFrameTailStub"); + + JitSpew(JitSpew_Codegen, "# Emitting exception tail stub"); + generateExceptionTailStub(masm, &profilerExitTail, &bailoutTail); + rangeRecorder.recordOffset("Trampoline: ExceptionTailStub"); + + Linker linker(masm); + trampolineCode_ = linker.newCode(cx, CodeKind::Other); + if (!trampolineCode_) { + return false; + } + + rangeRecorder.collectRangesForJitCode(trampolineCode_); +#ifdef MOZ_VTUNE + vtune::MarkStub(trampolineCode_, "Trampolines"); +#endif + + return true; +} + +JitCode* JitRuntime::debugTrapHandler(JSContext* cx, + DebugTrapHandlerKind kind) { + if (!debugTrapHandlers_[kind]) { + // JitRuntime code stubs are shared across compartments and have to + // be allocated in the atoms zone. + mozilla::Maybe<AutoAllocInAtomsZone> az; + if (!cx->zone()->isAtomsZone()) { + az.emplace(cx); + } + debugTrapHandlers_[kind] = generateDebugTrapHandler(cx, kind); + } + return debugTrapHandlers_[kind]; +} + +JitRuntime::IonCompileTaskList& JitRuntime::ionLazyLinkList(JSRuntime* rt) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt), + "Should only be mutated by the main thread."); + return ionLazyLinkList_.ref(); +} + +void JitRuntime::ionLazyLinkListRemove(JSRuntime* rt, + jit::IonCompileTask* task) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt), + "Should only be mutated by the main thread."); + MOZ_ASSERT(rt == task->script()->runtimeFromMainThread()); + MOZ_ASSERT(ionLazyLinkListSize_ > 0); + + task->removeFrom(ionLazyLinkList(rt)); + ionLazyLinkListSize_--; + + MOZ_ASSERT(ionLazyLinkList(rt).isEmpty() == (ionLazyLinkListSize_ == 0)); +} + +void JitRuntime::ionLazyLinkListAdd(JSRuntime* rt, jit::IonCompileTask* task) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt), + "Should only be mutated by the main thread."); + MOZ_ASSERT(rt == task->script()->runtimeFromMainThread()); + ionLazyLinkList(rt).insertFront(task); + ionLazyLinkListSize_++; +} + +uint8_t* JitRuntime::allocateIonOsrTempData(size_t size) { + // Free the old buffer (if needed) before allocating a new one. Note that we + // could use realloc here but it's likely not worth the complexity. + freeIonOsrTempData(); + ionOsrTempData_.ref().reset(static_cast<uint8_t*>(js_malloc(size))); + return ionOsrTempData_.ref().get(); +} + +void JitRuntime::freeIonOsrTempData() { ionOsrTempData_.ref().reset(); } + +JitRealm::JitRealm() : initialStringHeap(gc::Heap::Tenured) {} + +void JitRealm::initialize(bool zoneHasNurseryStrings) { + setStringsCanBeInNursery(zoneHasNurseryStrings); +} + +template <typename T> +static T PopNextBitmaskValue(uint32_t* bitmask) { + MOZ_ASSERT(*bitmask); + uint32_t index = mozilla::CountTrailingZeroes32(*bitmask); + *bitmask ^= 1 << index; + + MOZ_ASSERT(index < uint32_t(T::Count)); + return T(index); +} + +void JitRealm::performStubReadBarriers(uint32_t stubsToBarrier) const { + while (stubsToBarrier) { + auto stub = PopNextBitmaskValue<StubIndex>(&stubsToBarrier); + const WeakHeapPtr<JitCode*>& jitCode = stubs_[stub]; + MOZ_ASSERT(jitCode); + jitCode.get(); + } +} + +static bool LinkCodeGen(JSContext* cx, CodeGenerator* codegen, + HandleScript script, const WarpSnapshot* snapshot) { + if (!codegen->link(cx, snapshot)) { + return false; + } + + return true; +} + +static bool LinkBackgroundCodeGen(JSContext* cx, IonCompileTask* task) { + CodeGenerator* codegen = task->backgroundCodegen(); + if (!codegen) { + return false; + } + + JitContext jctx(cx); + RootedScript script(cx, task->script()); + return LinkCodeGen(cx, codegen, script, task->snapshot()); +} + +void jit::LinkIonScript(JSContext* cx, HandleScript calleeScript) { + // Get the pending IonCompileTask from the script. + MOZ_ASSERT(calleeScript->hasBaselineScript()); + IonCompileTask* task = + calleeScript->baselineScript()->pendingIonCompileTask(); + calleeScript->baselineScript()->removePendingIonCompileTask(cx->runtime(), + calleeScript); + + // Remove from pending. + cx->runtime()->jitRuntime()->ionLazyLinkListRemove(cx->runtime(), task); + + { + gc::AutoSuppressGC suppressGC(cx); + if (!LinkBackgroundCodeGen(cx, task)) { + // Silently ignore OOM during code generation. The assembly code + // doesn't have code to handle it after linking happened. So it's + // not OK to throw a catchable exception from there. + cx->clearPendingException(); + } + } + + { + AutoLockHelperThreadState lock; + FinishOffThreadTask(cx->runtime(), task, lock); + } +} + +uint8_t* jit::LazyLinkTopActivation(JSContext* cx, + LazyLinkExitFrameLayout* frame) { + RootedScript calleeScript( + cx, ScriptFromCalleeToken(frame->jsFrame()->calleeToken())); + + LinkIonScript(cx, calleeScript); + + MOZ_ASSERT(calleeScript->hasBaselineScript()); + MOZ_ASSERT(calleeScript->jitCodeRaw()); + + return calleeScript->jitCodeRaw(); +} + +/* static */ +void JitRuntime::TraceAtomZoneRoots(JSTracer* trc) { + MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting()); + + // Shared stubs are allocated in the atoms zone, so do not iterate + // them after the atoms heap after it has been "finished." + if (trc->runtime()->atomsAreFinished()) { + return; + } + + Zone* zone = trc->runtime()->atomsZone(); + for (auto i = zone->cellIterUnsafe<JitCode>(); !i.done(); i.next()) { + JitCode* code = i; + TraceRoot(trc, &code, "wrapper"); + } +} + +/* static */ +bool JitRuntime::MarkJitcodeGlobalTableIteratively(GCMarker* marker) { + if (marker->runtime()->hasJitRuntime() && + marker->runtime()->jitRuntime()->hasJitcodeGlobalTable()) { + return marker->runtime() + ->jitRuntime() + ->getJitcodeGlobalTable() + ->markIteratively(marker); + } + return false; +} + +/* static */ +void JitRuntime::TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc) { + if (rt->hasJitRuntime() && rt->jitRuntime()->hasJitcodeGlobalTable()) { + rt->jitRuntime()->getJitcodeGlobalTable()->traceWeak(rt, trc); + } +} + +void JitRealm::traceWeak(JSTracer* trc, JS::Realm* realm) { + // Any outstanding compilations should have been cancelled by the GC. + MOZ_ASSERT(!HasOffThreadIonCompile(realm)); + + for (WeakHeapPtr<JitCode*>& stub : stubs_) { + TraceWeakEdge(trc, &stub, "JitRealm::stubs_"); + } +} + +bool JitZone::addInlinedCompilation(const RecompileInfo& info, + JSScript* inlined) { + MOZ_ASSERT(inlined != info.script()); + + auto p = inlinedCompilations_.lookupForAdd(inlined); + if (p) { + auto& compilations = p->value(); + if (!compilations.empty() && compilations.back() == info) { + return true; + } + return compilations.append(info); + } + + RecompileInfoVector compilations; + if (!compilations.append(info)) { + return false; + } + return inlinedCompilations_.add(p, inlined, std::move(compilations)); +} + +void jit::AddPendingInvalidation(RecompileInfoVector& invalid, + JSScript* script) { + MOZ_ASSERT(script); + + CancelOffThreadIonCompile(script); + + // Let the script warm up again before attempting another compile. + script->resetWarmUpCounterToDelayIonCompilation(); + + JitScript* jitScript = script->maybeJitScript(); + if (!jitScript) { + return; + } + + auto addPendingInvalidation = [&invalid](const RecompileInfo& info) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!invalid.append(info)) { + // BUG 1536159: For diagnostics, compute the size of the failed + // allocation. This presumes the vector growth strategy is to double. This + // is only used for crash reporting so not a problem if we get it wrong. + size_t allocSize = 2 * sizeof(RecompileInfo) * invalid.capacity(); + oomUnsafe.crash(allocSize, "Could not update RecompileInfoVector"); + } + }; + + // Trigger invalidation of the IonScript. + if (jitScript->hasIonScript()) { + RecompileInfo info(script, jitScript->ionScript()->compilationId()); + addPendingInvalidation(info); + } + + // Trigger invalidation of any callers inlining this script. + auto* inlinedCompilations = + script->zone()->jitZone()->maybeInlinedCompilations(script); + if (inlinedCompilations) { + for (const RecompileInfo& info : *inlinedCompilations) { + addPendingInvalidation(info); + } + script->zone()->jitZone()->removeInlinedCompilations(script); + } +} + +IonScript* RecompileInfo::maybeIonScriptToInvalidate() const { + // Make sure this is not called under CodeGenerator::link (before the + // IonScript is created). + MOZ_ASSERT_IF( + script_->zone()->jitZone()->currentCompilationId(), + script_->zone()->jitZone()->currentCompilationId().ref() != id_); + + if (!script_->hasIonScript() || + script_->ionScript()->compilationId() != id_) { + return nullptr; + } + + return script_->ionScript(); +} + +bool RecompileInfo::traceWeak(JSTracer* trc) { + // Sweep the RecompileInfo if either the script is dead or the IonScript has + // been invalidated. + + if (!TraceManuallyBarrieredWeakEdge(trc, &script_, "RecompileInfo::script")) { + return false; + } + + return maybeIonScriptToInvalidate() != nullptr; +} + +void JitZone::traceWeak(JSTracer* trc) { + baselineCacheIRStubCodes_.traceWeak(trc); + inlinedCompilations_.traceWeak(trc); + + TraceWeakEdge(trc, &lastStubFoldingBailoutChild_, + "JitZone::lastStubFoldingBailoutChild_"); + TraceWeakEdge(trc, &lastStubFoldingBailoutParent_, + "JitZone::lastStubFoldingBailoutParent_"); +} + +size_t JitRealm::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return mallocSizeOf(this); +} + +void JitZone::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, + JS::CodeSizes* code, size_t* jitZone, + size_t* baselineStubsOptimized) const { + *jitZone += mallocSizeOf(this); + *jitZone += + baselineCacheIRStubCodes_.shallowSizeOfExcludingThis(mallocSizeOf); + *jitZone += ionCacheIRStubInfoSet_.shallowSizeOfExcludingThis(mallocSizeOf); + + execAlloc().addSizeOfCode(code); + + *baselineStubsOptimized += + optimizedStubSpace_.sizeOfExcludingThis(mallocSizeOf); +} + +void JitCodeHeader::init(JitCode* jitCode) { + // As long as JitCode isn't moveable, we can avoid tracing this and + // mutating executable data. + MOZ_ASSERT(!gc::IsMovableKind(gc::AllocKind::JITCODE)); + jitCode_ = jitCode; +} + +template <AllowGC allowGC> +JitCode* JitCode::New(JSContext* cx, uint8_t* code, uint32_t totalSize, + uint32_t headerSize, ExecutablePool* pool, + CodeKind kind) { + uint32_t bufferSize = totalSize - headerSize; + JitCode* codeObj = + cx->newCell<JitCode, allowGC>(code, bufferSize, headerSize, pool, kind); + if (!codeObj) { + // The caller already allocated `totalSize` bytes of executable memory. + pool->release(totalSize, kind); + return nullptr; + } + + cx->zone()->incJitMemory(totalSize); + + return codeObj; +} + +template JitCode* JitCode::New<CanGC>(JSContext* cx, uint8_t* code, + uint32_t bufferSize, uint32_t headerSize, + ExecutablePool* pool, CodeKind kind); + +template JitCode* JitCode::New<NoGC>(JSContext* cx, uint8_t* code, + uint32_t bufferSize, uint32_t headerSize, + ExecutablePool* pool, CodeKind kind); + +void JitCode::copyFrom(MacroAssembler& masm) { + // Store the JitCode pointer in the JitCodeHeader so we can recover the + // gcthing from relocation tables. + JitCodeHeader::FromExecutable(raw())->init(this); + + insnSize_ = masm.instructionsSize(); + masm.executableCopy(raw()); + + jumpRelocTableBytes_ = masm.jumpRelocationTableBytes(); + masm.copyJumpRelocationTable(raw() + jumpRelocTableOffset()); + + dataRelocTableBytes_ = masm.dataRelocationTableBytes(); + masm.copyDataRelocationTable(raw() + dataRelocTableOffset()); + + masm.processCodeLabels(raw()); +} + +void JitCode::traceChildren(JSTracer* trc) { + // Note that we cannot mark invalidated scripts, since we've basically + // corrupted the code stream by injecting bailouts. + if (invalidated()) { + return; + } + + if (jumpRelocTableBytes_) { + uint8_t* start = raw() + jumpRelocTableOffset(); + CompactBufferReader reader(start, start + jumpRelocTableBytes_); + MacroAssembler::TraceJumpRelocations(trc, this, reader); + } + if (dataRelocTableBytes_) { + uint8_t* start = raw() + dataRelocTableOffset(); + CompactBufferReader reader(start, start + dataRelocTableBytes_); + MacroAssembler::TraceDataRelocations(trc, this, reader); + } +} + +void JitCode::finalize(JS::GCContext* gcx) { + // If this jitcode had a bytecode map, it must have already been removed. +#ifdef DEBUG + JSRuntime* rt = gcx->runtime(); + if (hasBytecodeMap_) { + MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable()); + MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw())); + } +#endif + +#ifdef MOZ_VTUNE + vtune::UnmarkCode(this); +#endif + + MOZ_ASSERT(pool_); + + // With W^X JIT code, reprotecting memory for each JitCode instance is + // slow, so we record the ranges and poison them later all at once. It's + // safe to ignore OOM here, it just means we won't poison the code. + if (gcx->appendJitPoisonRange(JitPoisonRange(pool_, raw() - headerSize_, + headerSize_ + bufferSize_))) { + pool_->addRef(); + } + setHeaderPtr(nullptr); + +#ifdef JS_ION_PERF + // Code buffers are stored inside ExecutablePools. Pools are refcounted. + // Releasing the pool may free it. Horrible hack: if we are using perf + // integration, we don't want to reuse code addresses, so we just leak the + // memory instead. + if (!PerfEnabled()) { + pool_->release(headerSize_ + bufferSize_, CodeKind(kind_)); + } +#else + pool_->release(headerSize_ + bufferSize_, CodeKind(kind_)); +#endif + + zone()->decJitMemory(headerSize_ + bufferSize_); + + pool_ = nullptr; +} + +IonScript::IonScript(IonCompilationId compilationId, uint32_t localSlotsSize, + uint32_t argumentSlotsSize, uint32_t frameSize) + : localSlotsSize_(localSlotsSize), + argumentSlotsSize_(argumentSlotsSize), + frameSize_(frameSize), + compilationId_(compilationId) {} + +IonScript* IonScript::New(JSContext* cx, IonCompilationId compilationId, + uint32_t localSlotsSize, uint32_t argumentSlotsSize, + uint32_t frameSize, size_t snapshotsListSize, + size_t snapshotsRVATableSize, size_t recoversSize, + size_t constants, size_t nurseryObjects, + size_t safepointIndices, size_t osiIndices, + size_t icEntries, size_t runtimeSize, + size_t safepointsSize) { + if (snapshotsListSize >= MAX_BUFFER_SIZE) { + ReportOutOfMemory(cx); + return nullptr; + } + + // Verify the hardcoded sizes in header are accurate. + static_assert(SizeOf_OsiIndex == sizeof(OsiIndex), + "IonScript has wrong size for OsiIndex"); + static_assert(SizeOf_SafepointIndex == sizeof(SafepointIndex), + "IonScript has wrong size for SafepointIndex"); + + CheckedInt<Offset> allocSize = sizeof(IonScript); + allocSize += CheckedInt<Offset>(constants) * sizeof(Value); + allocSize += CheckedInt<Offset>(runtimeSize); + allocSize += CheckedInt<Offset>(nurseryObjects) * sizeof(HeapPtr<JSObject*>); + allocSize += CheckedInt<Offset>(osiIndices) * sizeof(OsiIndex); + allocSize += CheckedInt<Offset>(safepointIndices) * sizeof(SafepointIndex); + allocSize += CheckedInt<Offset>(icEntries) * sizeof(uint32_t); + allocSize += CheckedInt<Offset>(safepointsSize); + allocSize += CheckedInt<Offset>(snapshotsListSize); + allocSize += CheckedInt<Offset>(snapshotsRVATableSize); + allocSize += CheckedInt<Offset>(recoversSize); + + if (!allocSize.isValid()) { + ReportAllocationOverflow(cx); + return nullptr; + } + + void* raw = cx->pod_malloc<uint8_t>(allocSize.value()); + MOZ_ASSERT(uintptr_t(raw) % alignof(IonScript) == 0); + if (!raw) { + return nullptr; + } + IonScript* script = new (raw) + IonScript(compilationId, localSlotsSize, argumentSlotsSize, frameSize); + + Offset offsetCursor = sizeof(IonScript); + + MOZ_ASSERT(offsetCursor % alignof(Value) == 0); + script->constantTableOffset_ = offsetCursor; + offsetCursor += constants * sizeof(Value); + + MOZ_ASSERT(offsetCursor % alignof(uint64_t) == 0); + script->runtimeDataOffset_ = offsetCursor; + offsetCursor += runtimeSize; + + MOZ_ASSERT(offsetCursor % alignof(HeapPtr<JSObject*>) == 0); + script->initElements<HeapPtr<JSObject*>>(offsetCursor, nurseryObjects); + script->nurseryObjectsOffset_ = offsetCursor; + offsetCursor += nurseryObjects * sizeof(HeapPtr<JSObject*>); + + MOZ_ASSERT(offsetCursor % alignof(OsiIndex) == 0); + script->osiIndexOffset_ = offsetCursor; + offsetCursor += osiIndices * sizeof(OsiIndex); + + MOZ_ASSERT(offsetCursor % alignof(SafepointIndex) == 0); + script->safepointIndexOffset_ = offsetCursor; + offsetCursor += safepointIndices * sizeof(SafepointIndex); + + MOZ_ASSERT(offsetCursor % alignof(uint32_t) == 0); + script->icIndexOffset_ = offsetCursor; + offsetCursor += icEntries * sizeof(uint32_t); + + script->safepointsOffset_ = offsetCursor; + offsetCursor += safepointsSize; + + script->snapshotsOffset_ = offsetCursor; + offsetCursor += snapshotsListSize; + + script->rvaTableOffset_ = offsetCursor; + offsetCursor += snapshotsRVATableSize; + + script->recoversOffset_ = offsetCursor; + offsetCursor += recoversSize; + + script->allocBytes_ = offsetCursor; + + MOZ_ASSERT(script->numConstants() == constants); + MOZ_ASSERT(script->runtimeSize() == runtimeSize); + MOZ_ASSERT(script->numNurseryObjects() == nurseryObjects); + MOZ_ASSERT(script->numOsiIndices() == osiIndices); + MOZ_ASSERT(script->numSafepointIndices() == safepointIndices); + MOZ_ASSERT(script->numICs() == icEntries); + MOZ_ASSERT(script->safepointsSize() == safepointsSize); + MOZ_ASSERT(script->snapshotsListSize() == snapshotsListSize); + MOZ_ASSERT(script->snapshotsRVATableSize() == snapshotsRVATableSize); + MOZ_ASSERT(script->recoversSize() == recoversSize); + MOZ_ASSERT(script->endOffset() == offsetCursor); + + return script; +} + +void IonScript::trace(JSTracer* trc) { + if (method_) { + TraceEdge(trc, &method_, "method"); + } + + for (size_t i = 0; i < numConstants(); i++) { + TraceEdge(trc, &getConstant(i), "constant"); + } + + for (size_t i = 0; i < numNurseryObjects(); i++) { + TraceEdge(trc, &nurseryObjects()[i], "nursery-object"); + } + + // Trace caches so that the JSScript pointer can be updated if moved. + for (size_t i = 0; i < numICs(); i++) { + getICFromIndex(i).trace(trc, this); + } +} + +/* static */ +void IonScript::preWriteBarrier(Zone* zone, IonScript* ionScript) { + PreWriteBarrier(zone, ionScript); +} + +void IonScript::copySnapshots(const SnapshotWriter* writer) { + MOZ_ASSERT(writer->listSize() == snapshotsListSize()); + memcpy(offsetToPointer<uint8_t>(snapshotsOffset()), writer->listBuffer(), + snapshotsListSize()); + + MOZ_ASSERT(snapshotsRVATableSize()); + MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize()); + memcpy(offsetToPointer<uint8_t>(rvaTableOffset()), writer->RVATableBuffer(), + snapshotsRVATableSize()); +} + +void IonScript::copyRecovers(const RecoverWriter* writer) { + MOZ_ASSERT(writer->size() == recoversSize()); + memcpy(offsetToPointer<uint8_t>(recoversOffset()), writer->buffer(), + recoversSize()); +} + +void IonScript::copySafepoints(const SafepointWriter* writer) { + MOZ_ASSERT(writer->size() == safepointsSize()); + memcpy(offsetToPointer<uint8_t>(safepointsOffset()), writer->buffer(), + safepointsSize()); +} + +void IonScript::copyConstants(const Value* vp) { + for (size_t i = 0; i < numConstants(); i++) { + constants()[i].init(vp[i]); + } +} + +void IonScript::copySafepointIndices(const CodegenSafepointIndex* si) { + // Convert CodegenSafepointIndex to more compact form. + SafepointIndex* table = safepointIndices(); + for (size_t i = 0; i < numSafepointIndices(); ++i) { + table[i] = SafepointIndex(si[i]); + } +} + +void IonScript::copyOsiIndices(const OsiIndex* oi) { + memcpy(osiIndices(), oi, numOsiIndices() * sizeof(OsiIndex)); +} + +void IonScript::copyRuntimeData(const uint8_t* data) { + memcpy(runtimeData(), data, runtimeSize()); +} + +void IonScript::copyICEntries(const uint32_t* icEntries) { + memcpy(icIndex(), icEntries, numICs() * sizeof(uint32_t)); + + // Update the codeRaw_ field in the ICs now that we know the code address. + for (size_t i = 0; i < numICs(); i++) { + getICFromIndex(i).resetCodeRaw(this); + } +} + +const SafepointIndex* IonScript::getSafepointIndex(uint32_t disp) const { + MOZ_ASSERT(numSafepointIndices() > 0); + + const SafepointIndex* table = safepointIndices(); + if (numSafepointIndices() == 1) { + MOZ_ASSERT(disp == table[0].displacement()); + return &table[0]; + } + + size_t minEntry = 0; + size_t maxEntry = numSafepointIndices() - 1; + uint32_t min = table[minEntry].displacement(); + uint32_t max = table[maxEntry].displacement(); + + // Raise if the element is not in the list. + MOZ_ASSERT(min <= disp && disp <= max); + + // Approximate the location of the FrameInfo. + size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry; + uint32_t guessDisp = table[guess].displacement(); + + if (table[guess].displacement() == disp) { + return &table[guess]; + } + + // Doing a linear scan from the guess should be more efficient in case of + // small group which are equally distributed on the code. + // + // such as: <... ... ... ... . ... ...> + if (guessDisp > disp) { + while (--guess >= minEntry) { + guessDisp = table[guess].displacement(); + MOZ_ASSERT(guessDisp >= disp); + if (guessDisp == disp) { + return &table[guess]; + } + } + } else { + while (++guess <= maxEntry) { + guessDisp = table[guess].displacement(); + MOZ_ASSERT(guessDisp <= disp); + if (guessDisp == disp) { + return &table[guess]; + } + } + } + + MOZ_CRASH("displacement not found."); +} + +const OsiIndex* IonScript::getOsiIndex(uint32_t disp) const { + const OsiIndex* end = osiIndices() + numOsiIndices(); + for (const OsiIndex* it = osiIndices(); it != end; ++it) { + if (it->returnPointDisplacement() == disp) { + return it; + } + } + + MOZ_CRASH("Failed to find OSI point return address"); +} + +const OsiIndex* IonScript::getOsiIndex(uint8_t* retAddr) const { + JitSpew(JitSpew_IonInvalidate, "IonScript %p has method %p raw %p", + (void*)this, (void*)method(), method()->raw()); + + MOZ_ASSERT(containsCodeAddress(retAddr)); + uint32_t disp = retAddr - method()->raw(); + return getOsiIndex(disp); +} + +void IonScript::Destroy(JS::GCContext* gcx, IonScript* script) { + // Make sure there are no pointers into the IonScript's nursery objects list + // in the store buffer. Because this can be called during sweeping when + // discarding JIT code, we have to lock the store buffer when we find an + // object that's (still) in the nursery. + mozilla::Maybe<gc::AutoLockStoreBuffer> lock; + for (size_t i = 0, len = script->numNurseryObjects(); i < len; i++) { + JSObject* obj = script->nurseryObjects()[i]; + if (!IsInsideNursery(obj)) { + continue; + } + if (lock.isNothing()) { + lock.emplace(&gcx->runtime()->gc.storeBuffer()); + } + script->nurseryObjects()[i] = HeapPtr<JSObject*>(); + } + + // This allocation is tracked by JSScript::setIonScriptImpl. + gcx->deleteUntracked(script); +} + +void JS::DeletePolicy<js::jit::IonScript>::operator()( + const js::jit::IonScript* script) { + IonScript::Destroy(rt_->gcContext(), const_cast<IonScript*>(script)); +} + +void IonScript::purgeICs(Zone* zone) { + for (size_t i = 0; i < numICs(); i++) { + getICFromIndex(i).reset(zone, this); + } +} + +namespace js { +namespace jit { + +bool OptimizeMIR(MIRGenerator* mir) { + MIRGraph& graph = mir->graph(); + GraphSpewer& gs = mir->graphSpewer(); + + if (mir->shouldCancel("Start")) { + return false; + } + + gs.spewPass("BuildSSA"); + AssertBasicGraphCoherency(graph); + + if (JitSpewEnabled(JitSpew_MIRExpressions)) { + JitSpewCont(JitSpew_MIRExpressions, "\n"); + DumpMIRExpressions(JitSpewPrinter(), graph, mir->outerInfo(), + "BuildSSA (== input to OptimizeMIR)"); + } + + if (!JitOptions.disablePruning && !mir->compilingWasm()) { + JitSpewCont(JitSpew_Prune, "\n"); + if (!PruneUnusedBranches(mir, graph)) { + return false; + } + gs.spewPass("Prune Unused Branches"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Prune Unused Branches")) { + return false; + } + } + + { + if (!FoldEmptyBlocks(graph)) { + return false; + } + gs.spewPass("Fold Empty Blocks"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Fold Empty Blocks")) { + return false; + } + } + + // Remove trivially dead resume point operands before folding tests, so the + // latter pass can optimize more aggressively. + if (!mir->compilingWasm()) { + if (!EliminateTriviallyDeadResumePointOperands(mir, graph)) { + return false; + } + gs.spewPass("Eliminate trivially dead resume point operands"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Eliminate trivially dead resume point operands")) { + return false; + } + } + + { + if (!FoldTests(graph)) { + return false; + } + gs.spewPass("Fold Tests"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Fold Tests")) { + return false; + } + } + + { + if (!SplitCriticalEdges(graph)) { + return false; + } + gs.spewPass("Split Critical Edges"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Split Critical Edges")) { + return false; + } + } + + { + RenumberBlocks(graph); + gs.spewPass("Renumber Blocks"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Renumber Blocks")) { + return false; + } + } + + { + if (!BuildDominatorTree(graph)) { + return false; + } + // No spew: graph not changed. + + if (mir->shouldCancel("Dominator Tree")) { + return false; + } + } + + { + // Aggressive phi elimination must occur before any code elimination. If the + // script contains a try-statement, we only compiled the try block and not + // the catch or finally blocks, so in this case it's also invalid to use + // aggressive phi elimination. + Observability observability = graph.hasTryBlock() + ? ConservativeObservability + : AggressiveObservability; + if (!EliminatePhis(mir, graph, observability)) { + return false; + } + gs.spewPass("Eliminate phis"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Eliminate phis")) { + return false; + } + + if (!BuildPhiReverseMapping(graph)) { + return false; + } + AssertExtendedGraphCoherency(graph); + // No spew: graph not changed. + + if (mir->shouldCancel("Phi reverse mapping")) { + return false; + } + } + + if (!mir->compilingWasm() && !JitOptions.disableIteratorIndices) { + if (!OptimizeIteratorIndices(mir, graph)) { + return false; + } + gs.spewPass("Iterator Indices"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Iterator Indices")) { + return false; + } + } + + if (!JitOptions.disableRecoverIns && + mir->optimizationInfo().scalarReplacementEnabled()) { + JitSpewCont(JitSpew_Escape, "\n"); + if (!ScalarReplacement(mir, graph)) { + return false; + } + gs.spewPass("Scalar Replacement"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Scalar Replacement")) { + return false; + } + } + + if (!mir->compilingWasm()) { + if (!ApplyTypeInformation(mir, graph)) { + return false; + } + gs.spewPass("Apply types"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Apply types")) { + return false; + } + } + + if (mir->optimizationInfo().amaEnabled()) { + AlignmentMaskAnalysis ama(graph); + if (!ama.analyze()) { + return false; + } + gs.spewPass("Alignment Mask Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Alignment Mask Analysis")) { + return false; + } + } + + ValueNumberer gvn(mir, graph); + + // Alias analysis is required for LICM and GVN so that we don't move + // loads across stores. We also use alias information when removing + // redundant shapeguards. + if (mir->optimizationInfo().licmEnabled() || + mir->optimizationInfo().gvnEnabled() || + mir->optimizationInfo().eliminateRedundantShapeGuardsEnabled()) { + { + AliasAnalysis analysis(mir, graph); + JitSpewCont(JitSpew_Alias, "\n"); + if (!analysis.analyze()) { + return false; + } + + gs.spewPass("Alias analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Alias analysis")) { + return false; + } + } + + if (!mir->compilingWasm()) { + // Eliminating dead resume point operands requires basic block + // instructions to be numbered. Reuse the numbering computed during + // alias analysis. + if (!EliminateDeadResumePointOperands(mir, graph)) { + return false; + } + + gs.spewPass("Eliminate dead resume point operands"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Eliminate dead resume point operands")) { + return false; + } + } + } + + if (mir->optimizationInfo().gvnEnabled()) { + JitSpewCont(JitSpew_GVN, "\n"); + if (!gvn.run(ValueNumberer::UpdateAliasAnalysis)) { + return false; + } + gs.spewPass("GVN"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("GVN")) { + return false; + } + } + + // LICM can hoist instructions from conditional branches and + // trigger bailouts. Disable it if bailing out of a hoisted + // instruction has previously invalidated this script. + if (mir->licmEnabled()) { + JitSpewCont(JitSpew_LICM, "\n"); + if (!LICM(mir, graph)) { + return false; + } + gs.spewPass("LICM"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("LICM")) { + return false; + } + } + + RangeAnalysis r(mir, graph); + if (mir->optimizationInfo().rangeAnalysisEnabled()) { + JitSpewCont(JitSpew_Range, "\n"); + if (!r.addBetaNodes()) { + return false; + } + gs.spewPass("Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA Beta")) { + return false; + } + + if (!r.analyze() || !r.addRangeAssertions()) { + return false; + } + gs.spewPass("Range Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Range Analysis")) { + return false; + } + + if (!r.removeBetaNodes()) { + return false; + } + gs.spewPass("De-Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA De-Beta")) { + return false; + } + + if (mir->optimizationInfo().gvnEnabled()) { + bool shouldRunUCE = false; + if (!r.prepareForUCE(&shouldRunUCE)) { + return false; + } + gs.spewPass("RA check UCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA check UCE")) { + return false; + } + + if (shouldRunUCE) { + if (!gvn.run(ValueNumberer::DontUpdateAliasAnalysis)) { + return false; + } + gs.spewPass("UCE After RA"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("UCE After RA")) { + return false; + } + } + } + + if (mir->optimizationInfo().autoTruncateEnabled()) { + if (!r.truncate()) { + return false; + } + gs.spewPass("Truncate Doubles"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Truncate Doubles")) { + return false; + } + } + } + + if (!JitOptions.disableRecoverIns) { + JitSpewCont(JitSpew_Sink, "\n"); + if (!Sink(mir, graph)) { + return false; + } + gs.spewPass("Sink"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Sink")) { + return false; + } + } + + if (!JitOptions.disableRecoverIns && + mir->optimizationInfo().rangeAnalysisEnabled()) { + JitSpewCont(JitSpew_Range, "\n"); + if (!r.removeUnnecessaryBitops()) { + return false; + } + gs.spewPass("Remove Unnecessary Bitops"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Remove Unnecessary Bitops")) { + return false; + } + } + + { + JitSpewCont(JitSpew_FLAC, "\n"); + if (!FoldLinearArithConstants(mir, graph)) { + return false; + } + gs.spewPass("Fold Linear Arithmetic Constants"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Fold Linear Arithmetic Constants")) { + return false; + } + } + + if (mir->optimizationInfo().eaaEnabled()) { + EffectiveAddressAnalysis eaa(mir, graph); + JitSpewCont(JitSpew_EAA, "\n"); + if (!eaa.analyze()) { + return false; + } + gs.spewPass("Effective Address Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Effective Address Analysis")) { + return false; + } + } + + // BCE marks bounds checks as dead, so do BCE before DCE. + if (mir->compilingWasm()) { + JitSpewCont(JitSpew_WasmBCE, "\n"); + if (!EliminateBoundsChecks(mir, graph)) { + return false; + } + gs.spewPass("Redundant Bounds Check Elimination"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("BCE")) { + return false; + } + } + + { + if (!EliminateDeadCode(mir, graph)) { + return false; + } + gs.spewPass("DCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("DCE")) { + return false; + } + } + + if (mir->optimizationInfo().instructionReorderingEnabled() && + !mir->outerInfo().hadReorderingBailout()) { + if (!ReorderInstructions(graph)) { + return false; + } + gs.spewPass("Reordering"); + + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Reordering")) { + return false; + } + } + + // Make loops contiguous. We do this after GVN/UCE and range analysis, + // which can remove CFG edges, exposing more blocks that can be moved. + { + if (!MakeLoopsContiguous(graph)) { + return false; + } + gs.spewPass("Make loops contiguous"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Make loops contiguous")) { + return false; + } + } + AssertExtendedGraphCoherency(graph, /* underValueNumberer = */ false, + /* force = */ true); + + // Remove unreachable blocks created by MBasicBlock::NewFakeLoopPredecessor + // to ensure every loop header has two predecessors. (This only happens due + // to OSR.) After this point, it is no longer possible to build the + // dominator tree. + if (!mir->compilingWasm() && graph.osrBlock()) { + graph.removeFakeLoopPredecessors(); + gs.spewPass("Remove fake loop predecessors"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Remove fake loop predecessors")) { + return false; + } + } + + // Passes after this point must not move instructions; these analyses + // depend on knowing the final order in which instructions will execute. + + if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) { + EdgeCaseAnalysis edgeCaseAnalysis(mir, graph); + if (!edgeCaseAnalysis.analyzeLate()) { + return false; + } + gs.spewPass("Edge Case Analysis (Late)"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Edge Case Analysis (Late)")) { + return false; + } + } + + if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) { + // Note: check elimination has to run after all other passes that move + // instructions. Since check uses are replaced with the actual index, + // code motion after this pass could incorrectly move a load or store + // before its bounds check. + if (!EliminateRedundantChecks(graph)) { + return false; + } + gs.spewPass("Bounds Check Elimination"); + AssertGraphCoherency(graph); + } + + if (mir->optimizationInfo().eliminateRedundantShapeGuardsEnabled()) { + if (!EliminateRedundantShapeGuards(graph)) { + return false; + } + gs.spewPass("Shape Guard Elimination"); + AssertGraphCoherency(graph); + } + + // Run the GC Barrier Elimination pass after instruction reordering, to + // ensure we don't move instructions that can trigger GC between stores we + // optimize here. + if (mir->optimizationInfo().eliminateRedundantGCBarriersEnabled()) { + if (!EliminateRedundantGCBarriers(graph)) { + return false; + } + gs.spewPass("GC Barrier Elimination"); + AssertGraphCoherency(graph); + } + + if (!mir->compilingWasm() && !mir->outerInfo().hadUnboxFoldingBailout()) { + if (!FoldLoadsWithUnbox(mir, graph)) { + return false; + } + gs.spewPass("FoldLoadsWithUnbox"); + AssertGraphCoherency(graph); + } + + if (!mir->compilingWasm()) { + if (!AddKeepAliveInstructions(graph)) { + return false; + } + gs.spewPass("Add KeepAlive Instructions"); + AssertGraphCoherency(graph); + } + + AssertGraphCoherency(graph, /* force = */ true); + + if (JitSpewEnabled(JitSpew_MIRExpressions)) { + JitSpewCont(JitSpew_MIRExpressions, "\n"); + DumpMIRExpressions(JitSpewPrinter(), graph, mir->outerInfo(), + "BeforeLIR (== result of OptimizeMIR)"); + } + + return true; +} + +LIRGraph* GenerateLIR(MIRGenerator* mir) { + MIRGraph& graph = mir->graph(); + GraphSpewer& gs = mir->graphSpewer(); + + LIRGraph* lir = mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph); + if (!lir || !lir->init()) { + return nullptr; + } + + LIRGenerator lirgen(mir, graph, *lir); + { + if (!lirgen.generate()) { + return nullptr; + } + gs.spewPass("Generate LIR"); + + if (mir->shouldCancel("Generate LIR")) { + return nullptr; + } + } + +#ifdef DEBUG + AllocationIntegrityState integrity(*lir); +#endif + + { + IonRegisterAllocator allocator = + mir->optimizationInfo().registerAllocator(); + + switch (allocator) { + case RegisterAllocator_Backtracking: + case RegisterAllocator_Testbed: { +#ifdef DEBUG + if (JitOptions.fullDebugChecks) { + if (!integrity.record()) { + return nullptr; + } + } +#endif + + BacktrackingAllocator regalloc(mir, &lirgen, *lir, + allocator == RegisterAllocator_Testbed); + if (!regalloc.go()) { + return nullptr; + } + +#ifdef DEBUG + if (JitOptions.fullDebugChecks) { + if (!integrity.check()) { + return nullptr; + } + } +#endif + + gs.spewPass("Allocate Registers [Backtracking]"); + break; + } + + default: + MOZ_CRASH("Bad regalloc"); + } + + if (mir->shouldCancel("Allocate Registers")) { + return nullptr; + } + } + + return lir; +} + +CodeGenerator* GenerateCode(MIRGenerator* mir, LIRGraph* lir) { + auto codegen = MakeUnique<CodeGenerator>(mir, lir); + if (!codegen) { + return nullptr; + } + + if (!codegen->generate()) { + return nullptr; + } + + return codegen.release(); +} + +CodeGenerator* CompileBackEnd(MIRGenerator* mir, WarpSnapshot* snapshot) { + // Everything in CompileBackEnd can potentially run on a helper thread. + AutoEnterIonBackend enter; + AutoSpewEndFunction spewEndFunction(mir); + + { + WarpCompilation comp(mir->alloc()); + WarpBuilder builder(*snapshot, *mir, &comp); + if (!builder.build()) { + return nullptr; + } + } + + if (!OptimizeMIR(mir)) { + return nullptr; + } + + LIRGraph* lir = GenerateLIR(mir); + if (!lir) { + return nullptr; + } + + return GenerateCode(mir, lir); +} + +static AbortReasonOr<WarpSnapshot*> CreateWarpSnapshot(JSContext* cx, + MIRGenerator* mirGen, + HandleScript script) { + // Suppress GC during compilation. + gc::AutoSuppressGC suppressGC(cx); + + SpewBeginFunction(mirGen, script); + + WarpOracle oracle(cx, *mirGen, script); + + AbortReasonOr<WarpSnapshot*> result = oracle.createSnapshot(); + + MOZ_ASSERT_IF(result.isErr(), result.unwrapErr() == AbortReason::Alloc || + result.unwrapErr() == AbortReason::Error || + result.unwrapErr() == AbortReason::Disable); + MOZ_ASSERT_IF(!result.isErr(), result.unwrap()); + + return result; +} + +static AbortReason IonCompile(JSContext* cx, HandleScript script, + jsbytecode* osrPc) { + cx->check(script); + + auto alloc = + cx->make_unique<LifoAlloc>(TempAllocator::PreferredLifoChunkSize); + if (!alloc) { + return AbortReason::Error; + } + + if (!cx->realm()->ensureJitRealmExists(cx)) { + return AbortReason::Error; + } + + if (!cx->realm()->jitRealm()->ensureIonStubsExist(cx)) { + return AbortReason::Error; + } + + TempAllocator* temp = alloc->new_<TempAllocator>(alloc.get()); + if (!temp) { + return AbortReason::Alloc; + } + + MIRGraph* graph = alloc->new_<MIRGraph>(temp); + if (!graph) { + return AbortReason::Alloc; + } + + InlineScriptTree* inlineScriptTree = + InlineScriptTree::New(temp, nullptr, nullptr, script); + if (!inlineScriptTree) { + return AbortReason::Alloc; + } + + CompileInfo* info = alloc->new_<CompileInfo>( + CompileRuntime::get(cx->runtime()), script, script->function(), osrPc, + script->needsArgsObj(), inlineScriptTree); + if (!info) { + return AbortReason::Alloc; + } + + const OptimizationInfo* optimizationInfo = + IonOptimizations.get(OptimizationLevel::Normal); + const JitCompileOptions options(cx); + + MIRGenerator* mirGen = + alloc->new_<MIRGenerator>(CompileRealm::get(cx->realm()), options, temp, + graph, info, optimizationInfo); + if (!mirGen) { + return AbortReason::Alloc; + } + + MOZ_ASSERT(!script->baselineScript()->hasPendingIonCompileTask()); + MOZ_ASSERT(!script->hasIonScript()); + MOZ_ASSERT(script->canIonCompile()); + + if (osrPc) { + script->jitScript()->setHadIonOSR(); + } + + AbortReasonOr<WarpSnapshot*> result = CreateWarpSnapshot(cx, mirGen, script); + if (result.isErr()) { + return result.unwrapErr(); + } + WarpSnapshot* snapshot = result.unwrap(); + + // If possible, compile the script off thread. + if (options.offThreadCompilationAvailable()) { + JitSpew(JitSpew_IonSyncLogs, + "Can't log script %s:%u:%u" + ". (Compiled on background thread.)", + script->filename(), script->lineno(), script->column()); + + IonCompileTask* task = alloc->new_<IonCompileTask>(cx, *mirGen, snapshot); + if (!task) { + return AbortReason::Alloc; + } + + AutoLockHelperThreadState lock; + if (!StartOffThreadIonCompile(task, lock)) { + JitSpew(JitSpew_IonAbort, "Unable to start off-thread ion compilation."); + mirGen->graphSpewer().endFunction(); + return AbortReason::Alloc; + } + + script->jitScript()->setIsIonCompilingOffThread(script); + + // The allocator and associated data will be destroyed after being + // processed in the finishedOffThreadCompilations list. + (void)alloc.release(); + + return AbortReason::NoAbort; + } + + bool succeeded = false; + { + gc::AutoSuppressGC suppressGC(cx); + JitContext jctx(cx); + UniquePtr<CodeGenerator> codegen(CompileBackEnd(mirGen, snapshot)); + if (!codegen) { + JitSpew(JitSpew_IonAbort, "Failed during back-end compilation."); + if (cx->isExceptionPending()) { + return AbortReason::Error; + } + return AbortReason::Disable; + } + + succeeded = LinkCodeGen(cx, codegen.get(), script, snapshot); + } + + if (succeeded) { + return AbortReason::NoAbort; + } + if (cx->isExceptionPending()) { + return AbortReason::Error; + } + return AbortReason::Disable; +} + +static bool CheckFrame(JSContext* cx, BaselineFrame* frame) { + MOZ_ASSERT(!frame->isDebuggerEvalFrame()); + MOZ_ASSERT(!frame->isEvalFrame()); + + // This check is to not overrun the stack. + if (frame->isFunctionFrame()) { + if (TooManyActualArguments(frame->numActualArgs())) { + JitSpew(JitSpew_IonAbort, "too many actual arguments"); + return false; + } + + if (TooManyFormalArguments(frame->numFormalArgs())) { + JitSpew(JitSpew_IonAbort, "too many arguments"); + return false; + } + } + + return true; +} + +static bool CanIonCompileOrInlineScript(JSScript* script, const char** reason) { + if (script->isForEval()) { + // Eval frames are not yet supported. Supporting this will require new + // logic in pushBailoutFrame to deal with linking prev. + // Additionally, JSOp::GlobalOrEvalDeclInstantiation support will require + // baking in isEvalFrame(). + *reason = "eval script"; + return false; + } + + if (script->isAsync()) { + if (script->isModule()) { + *reason = "async module"; + return false; + } + } + + if (script->hasNonSyntacticScope() && !script->function()) { + // Support functions with a non-syntactic global scope but not other + // scripts. For global scripts, WarpBuilder currently uses the global + // object as scope chain, this is not valid when the script has a + // non-syntactic global scope. + *reason = "has non-syntactic global scope"; + return false; + } + + return true; +} // namespace jit + +static bool ScriptIsTooLarge(JSContext* cx, JSScript* script) { + if (!JitOptions.limitScriptSize) { + return false; + } + + size_t numLocalsAndArgs = NumLocalsAndArgs(script); + + bool canCompileOffThread = OffThreadCompilationAvailable(cx); + size_t maxScriptSize = canCompileOffThread + ? JitOptions.ionMaxScriptSize + : JitOptions.ionMaxScriptSizeMainThread; + size_t maxLocalsAndArgs = canCompileOffThread + ? JitOptions.ionMaxLocalsAndArgs + : JitOptions.ionMaxLocalsAndArgsMainThread; + + if (script->length() > maxScriptSize || numLocalsAndArgs > maxLocalsAndArgs) { + JitSpew(JitSpew_IonAbort, + "Script too large (%zu bytes) (%zu locals/args) @ %s:%u:%u", + script->length(), numLocalsAndArgs, script->filename(), + script->lineno(), script->column()); + return true; + } + + return false; +} + +bool CanIonCompileScript(JSContext* cx, JSScript* script) { + if (!script->canIonCompile()) { + return false; + } + + const char* reason = nullptr; + if (!CanIonCompileOrInlineScript(script, &reason)) { + JitSpew(JitSpew_IonAbort, "%s", reason); + return false; + } + + if (ScriptIsTooLarge(cx, script)) { + return false; + } + + return true; +} + +bool CanIonInlineScript(JSScript* script) { + if (!script->canIonCompile()) { + return false; + } + + const char* reason = nullptr; + if (!CanIonCompileOrInlineScript(script, &reason)) { + JitSpew(JitSpew_Inlining, "Cannot Ion compile script (%s)", reason); + return false; + } + + return true; +} + +static MethodStatus Compile(JSContext* cx, HandleScript script, + BaselineFrame* osrFrame, jsbytecode* osrPc) { + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(jit::IsBaselineJitEnabled(cx)); + + MOZ_ASSERT(script->hasBaselineScript()); + MOZ_ASSERT(!script->baselineScript()->hasPendingIonCompileTask()); + MOZ_ASSERT(!script->hasIonScript()); + + AutoGeckoProfilerEntry pseudoFrame( + cx, "Ion script compilation", + JS::ProfilingCategoryPair::JS_IonCompilation); + + if (script->isDebuggee() || (osrFrame && osrFrame->isDebuggee())) { + JitSpew(JitSpew_IonAbort, "debugging"); + return Method_Skipped; + } + + if (!CanIonCompileScript(cx, script)) { + JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%u:%u", + script->filename(), script->lineno(), script->column()); + return Method_CantCompile; + } + + OptimizationLevel optimizationLevel = + IonOptimizations.levelForScript(script, osrPc); + if (optimizationLevel == OptimizationLevel::DontCompile) { + return Method_Skipped; + } + + MOZ_ASSERT(optimizationLevel == OptimizationLevel::Normal); + + if (!CanLikelyAllocateMoreExecutableMemory()) { + script->resetWarmUpCounterToDelayIonCompilation(); + return Method_Skipped; + } + + MOZ_ASSERT(!script->hasIonScript()); + + AbortReason reason = IonCompile(cx, script, osrPc); + if (reason == AbortReason::Error) { + MOZ_ASSERT(cx->isExceptionPending()); + return Method_Error; + } + + if (reason == AbortReason::Disable) { + return Method_CantCompile; + } + + if (reason == AbortReason::Alloc) { + ReportOutOfMemory(cx); + return Method_Error; + } + + // Compilation succeeded or we invalidated right away or an inlining/alloc + // abort + if (script->hasIonScript()) { + return Method_Compiled; + } + return Method_Skipped; +} + +} // namespace jit +} // namespace js + +bool jit::OffThreadCompilationAvailable(JSContext* cx) { + // Even if off thread compilation is enabled, compilation must still occur + // on the main thread in some cases. + // + // Require cpuCount > 1 so that Ion compilation jobs and active-thread + // execution are not competing for the same resources. + return cx->runtime()->canUseOffthreadIonCompilation() && + GetHelperThreadCPUCount() > 1 && CanUseExtraThreads(); +} + +MethodStatus jit::CanEnterIon(JSContext* cx, RunState& state) { + MOZ_ASSERT(jit::IsIonEnabled(cx)); + + HandleScript script = state.script(); + MOZ_ASSERT(!script->hasIonScript()); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) { + return Method_Skipped; + } + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) { + return Method_Skipped; + } + + if (state.isInvoke()) { + InvokeState& invoke = *state.asInvoke(); + + if (TooManyActualArguments(invoke.args().length())) { + JitSpew(JitSpew_IonAbort, "too many actual args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (TooManyFormalArguments( + invoke.args().callee().as<JSFunction>().nargs())) { + JitSpew(JitSpew_IonAbort, "too many args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + } + + // If --ion-eager is used, compile with Baseline first, so that we + // can directly enter IonMonkey. + if (JitOptions.eagerIonCompilation() && !script->hasBaselineScript()) { + MethodStatus status = + CanEnterBaselineMethod<BaselineTier::Compiler>(cx, state); + if (status != Method_Compiled) { + return status; + } + // Bytecode analysis may forbid compilation for a script. + if (!script->canIonCompile()) { + return Method_CantCompile; + } + } + + if (!script->hasBaselineScript()) { + return Method_Skipped; + } + + MOZ_ASSERT(!script->isIonCompilingOffThread()); + MOZ_ASSERT(script->canIonCompile()); + + // Attempt compilation. Returns Method_Compiled if already compiled. + MethodStatus status = Compile(cx, script, /* osrFrame = */ nullptr, + /* osrPc = */ nullptr); + if (status != Method_Compiled) { + if (status == Method_CantCompile) { + ForbidCompilation(cx, script); + } + return status; + } + + if (state.script()->baselineScript()->hasPendingIonCompileTask()) { + LinkIonScript(cx, state.script()); + if (!state.script()->hasIonScript()) { + return jit::Method_Skipped; + } + } + + return Method_Compiled; +} + +static MethodStatus BaselineCanEnterAtEntry(JSContext* cx, HandleScript script, + BaselineFrame* frame) { + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(script->canIonCompile()); + MOZ_ASSERT(!script->isIonCompilingOffThread()); + MOZ_ASSERT(!script->hasIonScript()); + MOZ_ASSERT(frame->isFunctionFrame()); + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(cx, frame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (script->baselineScript()->hasPendingIonCompileTask()) { + LinkIonScript(cx, script); + if (script->hasIonScript()) { + return Method_Compiled; + } + } + + // Attempt compilation. Returns Method_Compiled if already compiled. + MethodStatus status = Compile(cx, script, frame, nullptr); + if (status != Method_Compiled) { + if (status == Method_CantCompile) { + ForbidCompilation(cx, script); + } + return status; + } + + return Method_Compiled; +} + +// Decide if a transition from baseline execution to Ion code should occur. +// May compile or recompile the target JSScript. +static MethodStatus BaselineCanEnterAtBranch(JSContext* cx, HandleScript script, + BaselineFrame* osrFrame, + jsbytecode* pc) { + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT((JSOp)*pc == JSOp::LoopHead); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) { + return Method_Skipped; + } + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) { + return Method_Skipped; + } + + // Optionally ignore on user request. + if (!JitOptions.osr) { + return Method_Skipped; + } + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(cx, osrFrame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // Check if the jitcode still needs to get linked and do this + // to have a valid IonScript. + if (script->baselineScript()->hasPendingIonCompileTask()) { + LinkIonScript(cx, script); + } + + // By default a recompilation doesn't happen on osr mismatch. + // Decide if we want to force a recompilation if this happens too much. + if (script->hasIonScript()) { + if (pc == script->ionScript()->osrPc()) { + return Method_Compiled; + } + + uint32_t count = script->ionScript()->incrOsrPcMismatchCounter(); + if (count <= JitOptions.osrPcMismatchesBeforeRecompile && + !JitOptions.eagerIonCompilation()) { + return Method_Skipped; + } + + JitSpew(JitSpew_IonScripts, "Forcing OSR Mismatch Compilation"); + Invalidate(cx, script); + } + + // Attempt compilation. + // - Returns Method_Compiled if the right ionscript is present + // (Meaning it was present or a sequantial compile finished) + // - Returns Method_Skipped if pc doesn't match + // (This means a background thread compilation with that pc could have + // started or not.) + MethodStatus status = Compile(cx, script, osrFrame, pc); + if (status != Method_Compiled) { + if (status == Method_CantCompile) { + ForbidCompilation(cx, script); + } + return status; + } + + // Return the compilation was skipped when the osr pc wasn't adjusted. + // This can happen when there was still an IonScript available and a + // background compilation started, but hasn't finished yet. + // Or when we didn't force a recompile. + if (script->hasIonScript() && pc != script->ionScript()->osrPc()) { + return Method_Skipped; + } + + return Method_Compiled; +} + +static bool IonCompileScriptForBaseline(JSContext* cx, BaselineFrame* frame, + jsbytecode* pc) { + MOZ_ASSERT(IsIonEnabled(cx)); + + RootedScript script(cx, frame->script()); + bool isLoopHead = JSOp(*pc) == JSOp::LoopHead; + + // The Baseline JIT code checks for Ion disabled or compiling off-thread. + MOZ_ASSERT(script->canIonCompile()); + MOZ_ASSERT(!script->isIonCompilingOffThread()); + + // If Ion script exists, but PC is not at a loop entry, then Ion will be + // entered for this script at an appropriate LOOPENTRY or the next time this + // function is called. + if (script->hasIonScript() && !isLoopHead) { + JitSpew(JitSpew_BaselineOSR, "IonScript exists, but not at loop entry!"); + // TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist. + // TODO: Clear all optimized stubs. + // TODO: Add a ion-script-already-exists checker stub. + return true; + } + + // Ensure that Ion-compiled code is available. + JitSpew(JitSpew_BaselineOSR, + "WarmUpCounter for %s:%u:%u reached %d at pc %p, trying to switch to " + "Ion!", + script->filename(), script->lineno(), script->column(), + (int)script->getWarmUpCount(), (void*)pc); + + MethodStatus stat; + if (isLoopHead) { + JitSpew(JitSpew_BaselineOSR, " Compile at loop head!"); + stat = BaselineCanEnterAtBranch(cx, script, frame, pc); + } else if (frame->isFunctionFrame()) { + JitSpew(JitSpew_BaselineOSR, + " Compile function from top for later entry!"); + stat = BaselineCanEnterAtEntry(cx, script, frame); + } else { + return true; + } + + if (stat == Method_Error) { + JitSpew(JitSpew_BaselineOSR, " Compile with Ion errored!"); + return false; + } + + if (stat == Method_CantCompile) { + MOZ_ASSERT(!script->canIonCompile()); + JitSpew(JitSpew_BaselineOSR, " Can't compile with Ion!"); + } else if (stat == Method_Skipped) { + JitSpew(JitSpew_BaselineOSR, " Skipped compile with Ion!"); + } else if (stat == Method_Compiled) { + JitSpew(JitSpew_BaselineOSR, " Compiled with Ion!"); + } else { + MOZ_CRASH("Invalid MethodStatus!"); + } + + return true; +} + +bool jit::IonCompileScriptForBaselineAtEntry(JSContext* cx, + BaselineFrame* frame) { + JSScript* script = frame->script(); + return IonCompileScriptForBaseline(cx, frame, script->code()); +} + +/* clang-format off */ +// The following data is kept in a temporary heap-allocated buffer, stored in +// JitRuntime (high memory addresses at top, low at bottom): +// +// +----->+=================================+ -- <---- High Address +// | | | | +// | | ...BaselineFrame... | |-- Copy of BaselineFrame + stack values +// | | | | +// | +---------------------------------+ | +// | | | | +// | | ...Locals/Stack... | | +// | | | | +// | +=================================+ -- +// | | Padding(Maybe Empty) | +// | +=================================+ -- +// +------|-- baselineFrame | |-- IonOsrTempData +// | jitcode | | +// +=================================+ -- <---- Low Address +// +// A pointer to the IonOsrTempData is returned. +/* clang-format on */ + +static IonOsrTempData* PrepareOsrTempData(JSContext* cx, BaselineFrame* frame, + uint32_t frameSize, void* jitcode) { + uint32_t numValueSlots = frame->numValueSlots(frameSize); + + // Calculate the amount of space to allocate: + // BaselineFrame space: + // (sizeof(Value) * numValueSlots) + // + sizeof(BaselineFrame) + // + // IonOsrTempData space: + // sizeof(IonOsrTempData) + + size_t frameSpace = sizeof(BaselineFrame) + sizeof(Value) * numValueSlots; + size_t ionOsrTempDataSpace = sizeof(IonOsrTempData); + + size_t totalSpace = AlignBytes(frameSpace, sizeof(Value)) + + AlignBytes(ionOsrTempDataSpace, sizeof(Value)); + + JitRuntime* jrt = cx->runtime()->jitRuntime(); + uint8_t* buf = jrt->allocateIonOsrTempData(totalSpace); + if (!buf) { + ReportOutOfMemory(cx); + return nullptr; + } + + IonOsrTempData* info = new (buf) IonOsrTempData(); + info->jitcode = jitcode; + + // Copy the BaselineFrame + local/stack Values to the buffer. Arguments and + // |this| are not copied but left on the stack: the Baseline and Ion frame + // share the same frame prefix and Ion won't clobber these values. Note + // that info->baselineFrame will point to the *end* of the frame data, like + // the frame pointer register in baseline frames. + uint8_t* frameStart = + (uint8_t*)info + AlignBytes(ionOsrTempDataSpace, sizeof(Value)); + info->baselineFrame = frameStart + frameSpace; + + memcpy(frameStart, (uint8_t*)frame - numValueSlots * sizeof(Value), + frameSpace); + + JitSpew(JitSpew_BaselineOSR, "Allocated IonOsrTempData at %p", info); + JitSpew(JitSpew_BaselineOSR, "Jitcode is %p", info->jitcode); + + // All done. + return info; +} + +bool jit::IonCompileScriptForBaselineOSR(JSContext* cx, BaselineFrame* frame, + uint32_t frameSize, jsbytecode* pc, + IonOsrTempData** infoPtr) { + MOZ_ASSERT(infoPtr); + *infoPtr = nullptr; + + MOZ_ASSERT(frame->debugFrameSize() == frameSize); + MOZ_ASSERT(JSOp(*pc) == JSOp::LoopHead); + + if (!IonCompileScriptForBaseline(cx, frame, pc)) { + return false; + } + + RootedScript script(cx, frame->script()); + if (!script->hasIonScript() || script->ionScript()->osrPc() != pc || + frame->isDebuggee()) { + return true; + } + + IonScript* ion = script->ionScript(); + MOZ_ASSERT(cx->runtime()->geckoProfiler().enabled() == + ion->hasProfilingInstrumentation()); + MOZ_ASSERT(ion->osrPc() == pc); + + ion->resetOsrPcMismatchCounter(); + + JitSpew(JitSpew_BaselineOSR, " OSR possible!"); + void* jitcode = ion->method()->raw() + ion->osrEntryOffset(); + + // Prepare the temporary heap copy of the fake InterpreterFrame and actual + // args list. + JitSpew(JitSpew_BaselineOSR, "Got jitcode. Preparing for OSR into ion."); + IonOsrTempData* info = PrepareOsrTempData(cx, frame, frameSize, jitcode); + if (!info) { + return false; + } + + *infoPtr = info; + return true; +} + +static void InvalidateActivation(JS::GCContext* gcx, + const JitActivationIterator& activations, + bool invalidateAll) { + JitSpew(JitSpew_IonInvalidate, "BEGIN invalidating activation"); + +#ifdef CHECK_OSIPOINT_REGISTERS + if (JitOptions.checkOsiPointRegisters) { + activations->asJit()->setCheckRegs(false); + } +#endif + + size_t frameno = 1; + + for (OnlyJSJitFrameIter iter(activations); !iter.done(); ++iter, ++frameno) { + const JSJitFrameIter& frame = iter.frame(); + MOZ_ASSERT_IF(frameno == 1, frame.isExitFrame() || + frame.type() == FrameType::Bailout || + frame.type() == FrameType::JSJitToWasm); + +#ifdef JS_JITSPEW + switch (frame.type()) { + case FrameType::Exit: + JitSpew(JitSpew_IonInvalidate, "#%zu exit frame @ %p", frameno, + frame.fp()); + break; + case FrameType::JSJitToWasm: + JitSpew(JitSpew_IonInvalidate, "#%zu wasm exit frame @ %p", frameno, + frame.fp()); + break; + case FrameType::BaselineJS: + case FrameType::IonJS: + case FrameType::Bailout: { + MOZ_ASSERT(frame.isScripted()); + const char* type = "Unknown"; + if (frame.isIonJS()) { + type = "Optimized"; + } else if (frame.isBaselineJS()) { + type = "Baseline"; + } else if (frame.isBailoutJS()) { + type = "Bailing"; + } + JSScript* script = frame.maybeForwardedScript(); + JitSpew(JitSpew_IonInvalidate, + "#%zu %s JS frame @ %p, %s:%u:%u (fun: %p, script: %p, pc %p)", + frameno, type, frame.fp(), script->maybeForwardedFilename(), + script->lineno(), script->column(), frame.maybeCallee(), script, + frame.resumePCinCurrentFrame()); + break; + } + case FrameType::BaselineStub: + JitSpew(JitSpew_IonInvalidate, "#%zu baseline stub frame @ %p", frameno, + frame.fp()); + break; + case FrameType::BaselineInterpreterEntry: + JitSpew(JitSpew_IonInvalidate, + "#%zu baseline interpreter entry frame @ %p", frameno, + frame.fp()); + break; + case FrameType::Rectifier: + JitSpew(JitSpew_IonInvalidate, "#%zu rectifier frame @ %p", frameno, + frame.fp()); + break; + case FrameType::IonICCall: + JitSpew(JitSpew_IonInvalidate, "#%zu ion IC call frame @ %p", frameno, + frame.fp()); + break; + case FrameType::CppToJSJit: + JitSpew(JitSpew_IonInvalidate, "#%zu entry frame @ %p", frameno, + frame.fp()); + break; + case FrameType::WasmToJSJit: + JitSpew(JitSpew_IonInvalidate, "#%zu wasm frames @ %p", frameno, + frame.fp()); + break; + } +#endif // JS_JITSPEW + + if (!frame.isIonScripted()) { + continue; + } + + // See if the frame has already been invalidated. + if (frame.checkInvalidation()) { + continue; + } + + JSScript* script = frame.maybeForwardedScript(); + if (!script->hasIonScript()) { + continue; + } + + if (!invalidateAll && !script->ionScript()->invalidated()) { + continue; + } + + IonScript* ionScript = script->ionScript(); + + // Purge ICs before we mark this script as invalidated. This will + // prevent lastJump_ from appearing to be a bogus pointer, just + // in case anyone tries to read it. + ionScript->purgeICs(script->zone()); + + // This frame needs to be invalidated. We do the following: + // + // 1. Increment the reference counter to keep the ionScript alive + // for the invalidation bailout or for the exception handler. + // 2. Determine safepoint that corresponds to the current call. + // 3. From safepoint, get distance to the OSI-patchable offset. + // 4. From the IonScript, determine the distance between the + // call-patchable offset and the invalidation epilogue. + // 5. Patch the OSI point with a call-relative to the + // invalidation epilogue. + // + // The code generator ensures that there's enough space for us + // to patch in a call-relative operation at each invalidation + // point. + // + // Note: you can't simplify this mechanism to "just patch the + // instruction immediately after the call" because things may + // need to move into a well-defined register state (using move + // instructions after the call) in to capture an appropriate + // snapshot after the call occurs. + + ionScript->incrementInvalidationCount(); + + JitCode* ionCode = ionScript->method(); + + // We're about to remove edges from the JSScript to GC things embedded in + // the JitCode. Perform a barrier to let the GC know about those edges. + PreWriteBarrier(script->zone(), ionCode, [](JSTracer* trc, JitCode* code) { + code->traceChildren(trc); + }); + + ionCode->setInvalidated(); + + // Don't adjust OSI points in a bailout path. + if (frame.isBailoutJS()) { + continue; + } + + // Write the delta (from the return address offset to the + // IonScript pointer embedded into the invalidation epilogue) + // where the safepointed call instruction used to be. We rely on + // the call sequence causing the safepoint being >= the size of + // a uint32, which is checked during safepoint index + // construction. + AutoWritableJitCode awjc(ionCode); + const SafepointIndex* si = + ionScript->getSafepointIndex(frame.resumePCinCurrentFrame()); + CodeLocationLabel dataLabelToMunge(frame.resumePCinCurrentFrame()); + ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() - + (frame.resumePCinCurrentFrame() - ionCode->raw()); + Assembler::PatchWrite_Imm32(dataLabelToMunge, Imm32(delta)); + + CodeLocationLabel osiPatchPoint = + SafepointReader::InvalidationPatchPoint(ionScript, si); + CodeLocationLabel invalidateEpilogue( + ionCode, CodeOffset(ionScript->invalidateEpilogueOffset())); + + JitSpew( + JitSpew_IonInvalidate, + " ! Invalidate ionScript %p (inv count %zu) -> patching osipoint %p", + ionScript, ionScript->invalidationCount(), (void*)osiPatchPoint.raw()); + Assembler::PatchWrite_NearCall(osiPatchPoint, invalidateEpilogue); + } + + JitSpew(JitSpew_IonInvalidate, "END invalidating activation"); +} + +void jit::InvalidateAll(JS::GCContext* gcx, Zone* zone) { + // The caller should previously have cancelled off thread compilation. +#ifdef DEBUG + for (RealmsInZoneIter realm(zone); !realm.done(); realm.next()) { + MOZ_ASSERT(!HasOffThreadIonCompile(realm)); + } +#endif + if (zone->isAtomsZone()) { + return; + } + JSContext* cx = TlsContext.get(); + for (JitActivationIterator iter(cx); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) { + JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC"); + InvalidateActivation(gcx, iter, true); + } + } +} + +static void ClearIonScriptAfterInvalidation(JSContext* cx, JSScript* script, + IonScript* ionScript, + bool resetUses) { + // Null out the JitScript's IonScript pointer. The caller is responsible for + // destroying the IonScript using the invalidation count mechanism. + DebugOnly<IonScript*> clearedIonScript = + script->jitScript()->clearIonScript(cx->gcContext(), script); + MOZ_ASSERT(clearedIonScript == ionScript); + + // Wait for the scripts to get warm again before doing another + // compile, unless we are recompiling *because* a script got hot + // (resetUses is false). + if (resetUses) { + script->resetWarmUpCounterToDelayIonCompilation(); + } +} + +void jit::Invalidate(JSContext* cx, const RecompileInfoVector& invalid, + bool resetUses, bool cancelOffThread) { + JitSpew(JitSpew_IonInvalidate, "Start invalidation."); + + // Add an invalidation reference to all invalidated IonScripts to indicate + // to the traversal which frames have been invalidated. + size_t numInvalidations = 0; + for (const RecompileInfo& info : invalid) { + if (cancelOffThread) { + CancelOffThreadIonCompile(info.script()); + } + + IonScript* ionScript = info.maybeIonScriptToInvalidate(); + if (!ionScript) { + continue; + } + + JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%u:%u, IonScript %p", + info.script()->filename(), info.script()->lineno(), + info.script()->column(), ionScript); + + // Keep the ion script alive during the invalidation and flag this + // ionScript as being invalidated. This increment is removed by the + // loop after the calls to InvalidateActivation. + ionScript->incrementInvalidationCount(); + numInvalidations++; + } + + if (!numInvalidations) { + JitSpew(JitSpew_IonInvalidate, " No IonScript invalidation."); + return; + } + + JS::GCContext* gcx = cx->gcContext(); + for (JitActivationIterator iter(cx); !iter.done(); ++iter) { + InvalidateActivation(gcx, iter, false); + } + + // Drop the references added above. If a script was never active, its + // IonScript will be immediately destroyed. Otherwise, it will be held live + // until its last invalidated frame is destroyed. + for (const RecompileInfo& info : invalid) { + IonScript* ionScript = info.maybeIonScriptToInvalidate(); + if (!ionScript) { + continue; + } + + if (ionScript->invalidationCount() == 1) { + // decrementInvalidationCount will destroy the IonScript so null out + // jitScript->ionScript_ now. We don't want to do this unconditionally + // because maybeIonScriptToInvalidate depends on script->ionScript() (we + // would leak the IonScript if |invalid| contains duplicates). + ClearIonScriptAfterInvalidation(cx, info.script(), ionScript, resetUses); + } + + ionScript->decrementInvalidationCount(gcx); + numInvalidations--; + } + + // Make sure we didn't leak references by invalidating the same IonScript + // multiple times in the above loop. + MOZ_ASSERT(!numInvalidations); + + // Finally, null out jitScript->ionScript_ for IonScripts that are still on + // the stack. + for (const RecompileInfo& info : invalid) { + if (IonScript* ionScript = info.maybeIonScriptToInvalidate()) { + ClearIonScriptAfterInvalidation(cx, info.script(), ionScript, resetUses); + } + } +} + +void jit::IonScript::invalidate(JSContext* cx, JSScript* script, bool resetUses, + const char* reason) { + // Note: we could short circuit here if we already invalidated this + // IonScript, but jit::Invalidate also cancels off-thread compilations of + // |script|. + MOZ_RELEASE_ASSERT(invalidated() || script->ionScript() == this); + + JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason); + + // RecompileInfoVector has inline space for at least one element. + RecompileInfoVector list; + MOZ_RELEASE_ASSERT(list.reserve(1)); + list.infallibleEmplaceBack(script, compilationId()); + + Invalidate(cx, list, resetUses, true); +} + +void jit::Invalidate(JSContext* cx, JSScript* script, bool resetUses, + bool cancelOffThread) { + MOZ_ASSERT(script->hasIonScript()); + + if (cx->runtime()->geckoProfiler().enabled()) { + // Register invalidation with profiler. + // Format of event payload string: + // "<filename>:<lineno>" + + // Get the script filename, if any, and its length. + const char* filename = script->filename(); + if (filename == nullptr) { + filename = "<unknown>"; + } + + // Construct the descriptive string. + UniqueChars buf = + JS_smprintf("%s:%u:%u", filename, script->lineno(), script->column()); + + // Ignore the event on allocation failure. + if (buf) { + cx->runtime()->geckoProfiler().markEvent("Invalidate", buf.get()); + } + } + + // RecompileInfoVector has inline space for at least one element. + RecompileInfoVector scripts; + MOZ_ASSERT(script->hasIonScript()); + MOZ_RELEASE_ASSERT(scripts.reserve(1)); + scripts.infallibleEmplaceBack(script, script->ionScript()->compilationId()); + + Invalidate(cx, scripts, resetUses, cancelOffThread); +} + +void jit::FinishInvalidation(JS::GCContext* gcx, JSScript* script) { + if (!script->hasIonScript()) { + return; + } + + // In all cases, null out jitScript->ionScript_ to avoid re-entry. + IonScript* ion = script->jitScript()->clearIonScript(gcx, script); + + // If this script has Ion code on the stack, invalidated() will return + // true. In this case we have to wait until destroying it. + if (!ion->invalidated()) { + jit::IonScript::Destroy(gcx, ion); + } +} + +void jit::ForbidCompilation(JSContext* cx, JSScript* script) { + JitSpew(JitSpew_IonAbort, "Disabling Ion compilation of script %s:%u:%u", + script->filename(), script->lineno(), script->column()); + + CancelOffThreadIonCompile(script); + + if (script->hasIonScript()) { + Invalidate(cx, script, false); + } + + script->disableIon(); +} + +size_t jit::SizeOfIonData(JSScript* script, + mozilla::MallocSizeOf mallocSizeOf) { + size_t result = 0; + + if (script->hasIonScript()) { + result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf); + } + + return result; +} + +// If you change these, please also change the comment in TempAllocator. +/* static */ const size_t TempAllocator::BallastSize = 16 * 1024; +/* static */ const size_t TempAllocator::PreferredLifoChunkSize = 32 * 1024; |