diff options
Diffstat (limited to 'js/src/jit/JitScript.cpp')
| -rw-r--r-- | js/src/jit/JitScript.cpp | 913 |
1 files changed, 913 insertions, 0 deletions
diff --git a/js/src/jit/JitScript.cpp b/js/src/jit/JitScript.cpp new file mode 100644 index 0000000000..f2f6ee2c25 --- /dev/null +++ b/js/src/jit/JitScript.cpp @@ -0,0 +1,913 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/JitScript-inl.h" + +#include "mozilla/BinarySearch.h" +#include "mozilla/CheckedInt.h" + +#include <utility> + +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/BytecodeAnalysis.h" +#include "jit/IonScript.h" +#include "jit/JitFrames.h" +#include "jit/JitSpewer.h" +#include "jit/ScriptFromCalleeToken.h" +#include "jit/TrialInlining.h" +#include "js/ColumnNumber.h" // JS::LimitedColumnNumberOneOrigin +#include "vm/BytecodeUtil.h" +#include "vm/Compartment.h" +#include "vm/FrameIter.h" // js::OnlyJSJitFrameIter +#include "vm/JitActivation.h" +#include "vm/JSScript.h" + +#include "gc/GCContext-inl.h" +#include "jit/JSJitFrameIter-inl.h" +#include "vm/JSContext-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::CheckedInt; + +JitScript::JitScript(JSScript* script, Offset fallbackStubsOffset, + Offset endOffset, const char* profileString) + : profileString_(profileString), + owningScript_(script), + endOffset_(endOffset), + icScript_(script->getWarmUpCount(), + fallbackStubsOffset - offsetOfICScript(), + endOffset - offsetOfICScript(), + /*depth=*/0, script->length()) { + // Ensure the baselineScript_ and ionScript_ fields match the BaselineDisabled + // and IonDisabled script flags. + if (!script->canBaselineCompile()) { + setBaselineScriptImpl(script, BaselineDisabledScriptPtr); + } + if (!script->canIonCompile()) { + setIonScriptImpl(script, IonDisabledScriptPtr); + } +} + +ICScript::~ICScript() { + // The contents of the AllocSite LifoAlloc are removed and freed separately + // after the next minor GC. See prepareForDestruction. + MOZ_ASSERT(allocSitesSpace_.isEmpty()); +} + +#ifdef DEBUG +JitScript::~JitScript() { + // BaselineScript and IonScript must have been destroyed at this point. + MOZ_ASSERT(!hasBaselineScript()); + MOZ_ASSERT(!hasIonScript()); + + MOZ_ASSERT(!isInList()); +} +#else +JitScript::~JitScript() = default; +#endif + +bool JSScript::createJitScript(JSContext* cx) { + MOZ_ASSERT(!hasJitScript()); + cx->check(this); + + // Scripts with a JitScript can run in the Baseline Interpreter. Make sure + // we don't create a JitScript for scripts we shouldn't Baseline interpret. + MOZ_ASSERT_IF(IsBaselineInterpreterEnabled(), + CanBaselineInterpretScript(this)); + + // Store the profile string in the JitScript if the profiler is enabled. + const char* profileString = nullptr; + if (cx->runtime()->geckoProfiler().enabled()) { + profileString = cx->runtime()->geckoProfiler().profileString(cx, this); + if (!profileString) { + return false; + } + } + + static_assert(sizeof(JitScript) % sizeof(uintptr_t) == 0, + "Trailing arrays must be aligned properly"); + static_assert(sizeof(ICEntry) % sizeof(uintptr_t) == 0, + "Trailing arrays must be aligned properly"); + + static_assert( + sizeof(JitScript) == offsetof(JitScript, icScript_) + sizeof(ICScript), + "icScript_ must be the last field"); + + // Calculate allocation size. + CheckedInt<uint32_t> allocSize = sizeof(JitScript); + allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICEntry); + allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICFallbackStub); + if (!allocSize.isValid()) { + ReportAllocationOverflow(cx); + return false; + } + + void* raw = cx->pod_malloc<uint8_t>(allocSize.value()); + MOZ_ASSERT(uintptr_t(raw) % alignof(JitScript) == 0); + if (!raw) { + return false; + } + + size_t fallbackStubsOffset = + sizeof(JitScript) + numICEntries() * sizeof(ICEntry); + + UniquePtr<JitScript> jitScript(new (raw) JitScript( + this, fallbackStubsOffset, allocSize.value(), profileString)); + + // Sanity check the length computation. + MOZ_ASSERT(jitScript->numICEntries() == numICEntries()); + + jitScript->icScript()->initICEntries(cx, this); + + cx->zone()->jitZone()->registerJitScript(jitScript.get()); + + warmUpData_.initJitScript(jitScript.release()); + AddCellMemory(this, allocSize.value(), MemoryUse::JitScript); + + // We have a JitScript so we can set the script's jitCodeRaw pointer to the + // Baseline Interpreter code. + updateJitCodeRaw(cx->runtime()); + + return true; +} + +void JSScript::maybeReleaseJitScript(JS::GCContext* gcx) { + MOZ_ASSERT(hasJitScript()); + + if (zone()->jitZone()->keepJitScripts() || jitScript()->hasBaselineScript() || + jitScript()->icScript()->active()) { + return; + } + + releaseJitScript(gcx); +} + +void JSScript::releaseJitScript(JS::GCContext* gcx) { + MOZ_ASSERT(hasJitScript()); + MOZ_ASSERT(!hasBaselineScript()); + MOZ_ASSERT(!hasIonScript()); + + gcx->removeCellMemory(this, jitScript()->allocBytes(), MemoryUse::JitScript); + + JitScript::Destroy(zone(), jitScript()); + warmUpData_.clearJitScript(); + updateJitCodeRaw(gcx->runtime()); +} + +void JSScript::releaseJitScriptOnFinalize(JS::GCContext* gcx) { + MOZ_ASSERT(hasJitScript()); + + if (hasIonScript()) { + IonScript* ion = jitScript()->clearIonScript(gcx, this); + jit::IonScript::Destroy(gcx, ion); + } + + if (hasBaselineScript()) { + BaselineScript* baseline = jitScript()->clearBaselineScript(gcx, this); + jit::BaselineScript::Destroy(gcx, baseline); + } + + releaseJitScript(gcx); +} + +void JitScript::trace(JSTracer* trc) { + TraceEdge(trc, &owningScript_, "JitScript::owningScript_"); + + icScript_.trace(trc); + + if (hasBaselineScript()) { + baselineScript()->trace(trc); + } + + if (hasIonScript()) { + ionScript()->trace(trc); + } + + if (templateEnv_.isSome()) { + TraceNullableEdge(trc, templateEnv_.ptr(), "jitscript-template-env"); + } + + if (hasInliningRoot()) { + inliningRoot()->trace(trc); + } +} + +void JitScript::traceWeak(JSTracer* trc) { + if (!icScript_.traceWeak(trc)) { + notePurgedStubs(); + } + + if (hasInliningRoot()) { + if (!inliningRoot()->traceWeak(trc)) { + notePurgedStubs(); + } + } + + if (hasIonScript()) { + ionScript()->traceWeak(trc); + } +} + +void ICScript::trace(JSTracer* trc) { + // Mark all IC stub codes hanging off the IC stub entries. + for (size_t i = 0; i < numICEntries(); i++) { + ICEntry& ent = icEntry(i); + ent.trace(trc); + } + + for (gc::AllocSite* site : allocSites_) { + site->trace(trc); + } +} + +bool ICScript::traceWeak(JSTracer* trc) { + // Mark all IC stub codes hanging off the IC stub entries. + bool allSurvived = true; + for (size_t i = 0; i < numICEntries(); i++) { + ICEntry& ent = icEntry(i); + if (!ent.traceWeak(trc)) { + allSurvived = false; + } + } + + return allSurvived; +} + +bool ICScript::addInlinedChild(JSContext* cx, UniquePtr<ICScript> child, + uint32_t pcOffset) { + MOZ_ASSERT(!hasInlinedChild(pcOffset)); + + if (!inlinedChildren_) { + inlinedChildren_ = cx->make_unique<Vector<CallSite>>(cx); + if (!inlinedChildren_) { + return false; + } + } + + // First reserve space in inlinedChildren_ to ensure that if the ICScript is + // added to the inlining root, it can also be added to inlinedChildren_. + CallSite callsite(child.get(), pcOffset); + if (!inlinedChildren_->reserve(inlinedChildren_->length() + 1)) { + return false; + } + if (!inliningRoot()->addInlinedScript(std::move(child))) { + return false; + } + inlinedChildren_->infallibleAppend(callsite); + return true; +} + +ICScript* ICScript::findInlinedChild(uint32_t pcOffset) { + for (auto& callsite : *inlinedChildren_) { + if (callsite.pcOffset_ == pcOffset) { + return callsite.callee_; + } + } + MOZ_CRASH("Inlined child expected at pcOffset"); +} + +void ICScript::removeInlinedChild(uint32_t pcOffset) { + MOZ_ASSERT(inliningRoot()); + inlinedChildren_->eraseIf([pcOffset](const CallSite& callsite) -> bool { + return callsite.pcOffset_ == pcOffset; + }); +} + +bool ICScript::hasInlinedChild(uint32_t pcOffset) { + if (!inlinedChildren_) { + return false; + } + for (auto& callsite : *inlinedChildren_) { + if (callsite.pcOffset_ == pcOffset) { + return true; + } + } + return false; +} + +void ICScript::purgeInactiveICScripts() { + MOZ_ASSERT(inliningRoot()); + + if (!inlinedChildren_) { + return; + } + + inlinedChildren_->eraseIf( + [](const CallSite& callsite) { return !callsite.callee_->active(); }); + + if (inlinedChildren_->empty()) { + inlinedChildren_.reset(); + return; + } + + // We have an active callee ICScript. This means the current ICScript must be + // active too. + MOZ_ASSERT(active()); +} + +void JitScript::resetWarmUpCount(uint32_t count) { + forEachICScript([&](ICScript* script) { script->resetWarmUpCount(count); }); +} + +#ifdef DEBUG +bool JitScript::hasActiveICScript() const { + bool hasActive = false; + forEachICScript([&](const ICScript* script) { + if (script->active()) { + hasActive = true; + } + }); + return hasActive; +} +#endif + +void JitScript::resetAllActiveFlags() { + forEachICScript([](ICScript* script) { script->resetActive(); }); +} + +void JitScript::ensureProfileString(JSContext* cx, JSScript* script) { + MOZ_ASSERT(cx->runtime()->geckoProfiler().enabled()); + + if (profileString_) { + return; + } + + AutoEnterOOMUnsafeRegion oomUnsafe; + profileString_ = cx->runtime()->geckoProfiler().profileString(cx, script); + if (!profileString_) { + oomUnsafe.crash("Failed to allocate profile string"); + } +} + +/* static */ +void JitScript::Destroy(Zone* zone, JitScript* script) { + script->prepareForDestruction(zone); + + // Remove from JitZone's linked list of JitScripts. + script->remove(); + + js_delete(script); +} + +template <typename F> +void JitScript::forEachICScript(const F& f) { + f(&icScript_); + if (hasInliningRoot()) { + inliningRoot()->forEachInlinedScript(f); + } +} + +template <typename F> +void JitScript::forEachICScript(const F& f) const { + f(&icScript_); + if (hasInliningRoot()) { + inliningRoot()->forEachInlinedScript(f); + } +} + +void ICScript::prepareForDestruction(Zone* zone) { + // Defer freeing AllocSite memory until after the next minor GC, because the + // nursery can point to these alloc sites. + JSRuntime* rt = zone->runtimeFromMainThread(); + rt->gc.queueAllLifoBlocksForFreeAfterMinorGC(&allocSitesSpace_); + + // Trigger write barriers. + PreWriteBarrier(zone, this); +} + +void JitScript::prepareForDestruction(Zone* zone) { + forEachICScript( + [&](ICScript* script) { script->prepareForDestruction(zone); }); + + // Trigger write barriers. + owningScript_ = nullptr; + baselineScript_.set(zone, nullptr); + ionScript_.set(zone, nullptr); +} + +struct FallbackStubs { + ICScript* const icScript_; + + explicit FallbackStubs(ICScript* icScript) : icScript_(icScript) {} + + size_t numEntries() const { return icScript_->numICEntries(); } + ICFallbackStub* operator[](size_t index) const { + return icScript_->fallbackStub(index); + } +}; + +static bool ComputeBinarySearchMid(FallbackStubs stubs, uint32_t pcOffset, + size_t* loc) { + return mozilla::BinarySearchIf( + stubs, 0, stubs.numEntries(), + [pcOffset](const ICFallbackStub* stub) { + if (pcOffset < stub->pcOffset()) { + return -1; + } + if (stub->pcOffset() < pcOffset) { + return 1; + } + return 0; + }, + loc); +} + +ICEntry& ICScript::icEntryFromPCOffset(uint32_t pcOffset) { + size_t mid; + bool success = ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid); +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + if (!success) { + MOZ_CRASH_UNSAFE_PRINTF("Missing icEntry for offset %d (max offset: %d)", + int(pcOffset), + int(fallbackStub(numICEntries() - 1)->pcOffset())); + } +#endif + MOZ_ALWAYS_TRUE(success); + + MOZ_ASSERT(mid < numICEntries()); + + ICEntry& entry = icEntry(mid); + MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() == pcOffset); + return entry; +} + +ICEntry* ICScript::interpreterICEntryFromPCOffset(uint32_t pcOffset) { + // We have to return the entry to store in BaselineFrame::interpreterICEntry + // when resuming in the Baseline Interpreter at pcOffset. The bytecode op at + // pcOffset does not necessarily have an ICEntry, so we want to return the + // first ICEntry for which the following is true: + // + // entry.pcOffset() >= pcOffset + // + // Fortunately, ComputeBinarySearchMid returns exactly this entry. + + size_t mid; + ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid); + + if (mid < numICEntries()) { + ICEntry& entry = icEntry(mid); + MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() >= pcOffset); + return &entry; + } + + // Resuming at a pc after the last ICEntry. Just return nullptr: + // BaselineFrame::interpreterICEntry will never be used in this case. + return nullptr; +} + +void JitScript::purgeInactiveICScripts() { + if (!hasInliningRoot()) { + return; + } + + forEachICScript([](ICScript* script) { script->purgeInactiveICScripts(); }); + + inliningRoot()->purgeInactiveICScripts(); + if (inliningRoot()->numInlinedScripts() == 0) { + inliningRoot_.reset(); + icScript()->inliningRoot_ = nullptr; + } else { + // If a callee script is active on the stack, the root script must be active + // too. + MOZ_ASSERT(icScript()->active()); + } +} + +void JitScript::purgeStubs(JSScript* script, ICStubSpace& newStubSpace) { + MOZ_ASSERT(script->jitScript() == this); + + Zone* zone = script->zone(); + if (IsAboutToBeFinalizedUnbarriered(script)) { + // We're sweeping and the script is dead. Don't purge optimized stubs + // because (1) accessing CacheIRStubInfo pointers in ICStubs is invalid + // because we may have swept them already when we started (incremental) + // sweeping and (2) it's unnecessary because this script will be finalized + // soon anyway. + return; + } + + JitSpew(JitSpew_BaselineIC, "Purging optimized stubs"); + + forEachICScript( + [&](ICScript* script) { script->purgeStubs(zone, newStubSpace); }); + + notePurgedStubs(); +} + +void ICScript::purgeStubs(Zone* zone, ICStubSpace& newStubSpace) { + for (size_t i = 0; i < numICEntries(); i++) { + ICEntry& entry = icEntry(i); + ICFallbackStub* fallback = fallbackStub(i); + + // If this is a trial inlining call site and the callee's ICScript hasn't + // been discarded, clone the IC chain instead of purging stubs. In this case + // both the current ICScript and the callee's inlined ICScript must be + // active on the stack. + // + // We can't purge the IC stubs in this case because it'd confuse trial + // inlining if we try to inline again later and we already have an ICScript + // for this call site. + if (fallback->trialInliningState() == TrialInliningState::Inlined && + hasInlinedChild(fallback->pcOffset())) { + MOZ_ASSERT(active()); + MOZ_ASSERT(findInlinedChild(fallback->pcOffset())->active()); + + JSRuntime* rt = zone->runtimeFromMainThread(); + ICCacheIRStub* prev = nullptr; + ICStub* stub = entry.firstStub(); + while (stub != fallback) { + ICCacheIRStub* clone = stub->toCacheIRStub()->clone(rt, newStubSpace); + if (prev) { + prev->setNext(clone); + } else { + entry.setFirstStub(clone); + } + MOZ_ASSERT(stub->toCacheIRStub()->next() == clone->next()); + prev = clone; + stub = clone->next(); + } + continue; + } + + MOZ_ASSERT(!hasInlinedChild(fallback->pcOffset())); + + fallback->discardStubs(zone, &entry); + fallback->state().reset(); + } +} + +bool JitScript::ensureHasCachedBaselineJitData(JSContext* cx, + HandleScript script) { + if (templateEnv_.isSome()) { + return true; + } + + if (!script->function() || + !script->function()->needsFunctionEnvironmentObjects()) { + templateEnv_.emplace(); + return true; + } + + Rooted<EnvironmentObject*> templateEnv(cx); + Rooted<JSFunction*> fun(cx, script->function()); + + if (fun->needsNamedLambdaEnvironment()) { + templateEnv = NamedLambdaObject::createTemplateObject(cx, fun); + if (!templateEnv) { + return false; + } + } + + if (fun->needsCallObject()) { + templateEnv = CallObject::createTemplateObject(cx, script, templateEnv); + if (!templateEnv) { + return false; + } + } + + templateEnv_.emplace(templateEnv); + return true; +} + +bool JitScript::ensureHasCachedIonData(JSContext* cx, HandleScript script) { + MOZ_ASSERT(script->jitScript() == this); + + if (usesEnvironmentChain_.isSome()) { + return true; + } + + if (!ensureHasCachedBaselineJitData(cx, script)) { + return false; + } + + usesEnvironmentChain_.emplace(ScriptUsesEnvironmentChain(script)); + return true; +} + +void JitScript::setBaselineScriptImpl(JSScript* script, + BaselineScript* baselineScript) { + JSRuntime* rt = script->runtimeFromMainThread(); + setBaselineScriptImpl(rt->gcContext(), script, baselineScript); +} + +void JitScript::setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script, + BaselineScript* baselineScript) { + if (hasBaselineScript()) { + gcx->removeCellMemory(script, baselineScript_->allocBytes(), + MemoryUse::BaselineScript); + baselineScript_.set(script->zone(), nullptr); + } + + MOZ_ASSERT(ionScript_ == nullptr || ionScript_ == IonDisabledScriptPtr); + + baselineScript_.set(script->zone(), baselineScript); + if (hasBaselineScript()) { + AddCellMemory(script, baselineScript_->allocBytes(), + MemoryUse::BaselineScript); + } + + script->resetWarmUpResetCounter(); + script->updateJitCodeRaw(gcx->runtime()); +} + +void JitScript::setIonScriptImpl(JSScript* script, IonScript* ionScript) { + JSRuntime* rt = script->runtimeFromMainThread(); + setIonScriptImpl(rt->gcContext(), script, ionScript); +} + +void JitScript::setIonScriptImpl(JS::GCContext* gcx, JSScript* script, + IonScript* ionScript) { + MOZ_ASSERT_IF(ionScript != IonDisabledScriptPtr, + !baselineScript()->hasPendingIonCompileTask()); + + JS::Zone* zone = script->zone(); + if (hasIonScript()) { + gcx->removeCellMemory(script, ionScript_->allocBytes(), + MemoryUse::IonScript); + ionScript_.set(zone, nullptr); + } + + ionScript_.set(zone, ionScript); + MOZ_ASSERT_IF(hasIonScript(), hasBaselineScript()); + if (hasIonScript()) { + AddCellMemory(script, ionScript_->allocBytes(), MemoryUse::IonScript); + } + + script->updateJitCodeRaw(gcx->runtime()); +} + +#ifdef JS_STRUCTURED_SPEW +static bool HasEnteredCounters(ICEntry& entry) { + ICStub* stub = entry.firstStub(); + if (stub && !stub->isFallback()) { + return true; + } + return false; +} + +void jit::JitSpewBaselineICStats(JSScript* script, const char* dumpReason) { + MOZ_ASSERT(script->hasJitScript()); + JSContext* cx = TlsContext.get(); + AutoStructuredSpewer spew(cx, SpewChannel::BaselineICStats, script); + if (!spew) { + return; + } + + JitScript* jitScript = script->jitScript(); + spew->property("reason", dumpReason); + spew->beginListProperty("entries"); + for (size_t i = 0; i < jitScript->numICEntries(); i++) { + ICEntry& entry = jitScript->icEntry(i); + ICFallbackStub* fallback = jitScript->fallbackStub(i); + if (!HasEnteredCounters(entry)) { + continue; + } + + uint32_t pcOffset = fallback->pcOffset(); + jsbytecode* pc = script->offsetToPC(pcOffset); + + JS::LimitedColumnNumberOneOrigin column; + unsigned int line = PCToLineNumber(script, pc, &column); + + spew->beginObject(); + spew->property("op", CodeName(JSOp(*pc))); + spew->property("pc", pcOffset); + spew->property("line", line); + spew->property("column", column.oneOriginValue()); + + spew->beginListProperty("counts"); + ICStub* stub = entry.firstStub(); + while (stub && !stub->isFallback()) { + uint32_t count = stub->enteredCount(); + spew->value(count); + stub = stub->toCacheIRStub()->next(); + } + spew->endList(); + spew->property("fallback_count", fallback->enteredCount()); + spew->endObject(); + } + spew->endList(); +} +#endif + +static void MarkActiveICScriptsAndCopyStubs( + JSContext* cx, const JitActivationIterator& activation, + ICStubSpace& newStubSpace) { + for (OnlyJSJitFrameIter iter(activation); !iter.done(); ++iter) { + const JSJitFrameIter& frame = iter.frame(); + switch (frame.type()) { + case FrameType::BaselineJS: + frame.script()->jitScript()->icScript()->setActive(); + // If the frame is using a trial-inlining ICScript, we have to preserve + // it too. + if (frame.baselineFrame()->icScript()->isInlined()) { + frame.baselineFrame()->icScript()->setActive(); + } + break; + case FrameType::BaselineStub: { + auto* layout = reinterpret_cast<BaselineStubFrameLayout*>(frame.fp()); + if (layout->maybeStubPtr() && !layout->maybeStubPtr()->isFallback()) { + ICCacheIRStub* stub = layout->maybeStubPtr()->toCacheIRStub(); + ICCacheIRStub* newStub = stub->clone(cx->runtime(), newStubSpace); + layout->setStubPtr(newStub); + + JSJitFrameIter parentFrame(frame); + ++parentFrame; + BaselineFrame* blFrame = parentFrame.baselineFrame(); + jsbytecode* pc; + parentFrame.baselineScriptAndPc(nullptr, &pc); + uint32_t pcOffset = blFrame->script()->pcToOffset(pc); + if (blFrame->icScript()->hasInlinedChild(pcOffset)) { + blFrame->icScript()->findInlinedChild(pcOffset)->setActive(); + } + } + break; + } + case FrameType::Exit: + if (frame.exitFrame()->is<LazyLinkExitFrameLayout>()) { + LazyLinkExitFrameLayout* ll = + frame.exitFrame()->as<LazyLinkExitFrameLayout>(); + JSScript* script = + ScriptFromCalleeToken(ll->jsFrame()->calleeToken()); + script->jitScript()->icScript()->setActive(); + } + break; + case FrameType::Bailout: + case FrameType::IonJS: { + // Keep the JitScript and BaselineScript around, since bailouts from + // the ion jitcode need to re-enter into the Baseline code. + frame.script()->jitScript()->icScript()->setActive(); + for (InlineFrameIterator inlineIter(cx, &frame); inlineIter.more(); + ++inlineIter) { + inlineIter.script()->jitScript()->icScript()->setActive(); + } + // Because we're purging ICScripts, the bailout machinery should use + // the generic ICScript for inlined callees. + frame.ionScript()->notePurgedICScripts(); + break; + } + default:; + } + } +} + +void jit::MarkActiveICScriptsAndCopyStubs(Zone* zone, + ICStubSpace& newStubSpace) { + if (zone->isAtomsZone()) { + return; + } + JSContext* cx = TlsContext.get(); + for (JitActivationIterator iter(cx); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) { + MarkActiveICScriptsAndCopyStubs(cx, iter, newStubSpace); + } + } +} + +InliningRoot* JitScript::getOrCreateInliningRoot(JSContext* cx, + JSScript* script) { + if (!inliningRoot_) { + inliningRoot_ = js::MakeUnique<InliningRoot>(cx, script); + if (!inliningRoot_) { + ReportOutOfMemory(cx); + return nullptr; + } + icScript_.inliningRoot_ = inliningRoot_.get(); + } + return inliningRoot_.get(); +} + +gc::AllocSite* ICScript::getOrCreateAllocSite(JSScript* outerScript, + uint32_t pcOffset) { + // The script must be the outer script. + MOZ_ASSERT(outerScript->jitScript()->icScript() == this || + (inliningRoot() && inliningRoot()->owningScript() == outerScript)); + + // The pcOffset must be for this (maybe inlined) script. + MOZ_ASSERT(pcOffset < bytecodeSize()); + + for (gc::AllocSite* site : allocSites_) { + if (site->pcOffset() == pcOffset) { + MOZ_ASSERT(site->isNormal()); + MOZ_ASSERT(site->script() == outerScript); + MOZ_ASSERT(site->traceKind() == JS::TraceKind::Object); + return site; + } + } + + Nursery& nursery = outerScript->runtimeFromMainThread()->gc.nursery(); + if (!nursery.canCreateAllocSite()) { + // Don't block attaching an optimized stub, but don't process allocations + // for this site. + return outerScript->zone()->unknownAllocSite(JS::TraceKind::Object); + } + + if (!allocSites_.reserve(allocSites_.length() + 1)) { + return nullptr; + } + + auto* site = allocSitesSpace_.new_<gc::AllocSite>( + outerScript->zone(), outerScript, pcOffset, JS::TraceKind::Object); + if (!site) { + return nullptr; + } + + allocSites_.infallibleAppend(site); + + nursery.noteAllocSiteCreated(); + + return site; +} + +bool JitScript::resetAllocSites(bool resetNurserySites, + bool resetPretenuredSites) { + MOZ_ASSERT(resetNurserySites || resetPretenuredSites); + + bool anyReset = false; + + forEachICScript([&](ICScript* script) { + for (gc::AllocSite* site : script->allocSites_) { + if ((resetNurserySites && site->initialHeap() == gc::Heap::Default) || + (resetPretenuredSites && site->initialHeap() == gc::Heap::Tenured)) { + if (site->maybeResetState()) { + anyReset = true; + } + } + } + }); + + return anyReset; +} + +void JitScript::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, + size_t* data, size_t* allocSites) const { + *data += mallocSizeOf(this); + + forEachICScript([=](const ICScript* script) { + // |data| already includes the outer ICScript because it's part of the + // JitScript. + if (script != &icScript_) { + *data += mallocSizeOf(script); + } + + // |data| already includes the LifoAlloc and Vector, so use + // sizeOfExcludingThis. + *allocSites += script->allocSitesSpace_.sizeOfExcludingThis(mallocSizeOf); + *allocSites += script->allocSites_.sizeOfExcludingThis(mallocSizeOf); + }); +} + +JitScript* ICScript::outerJitScript() { + MOZ_ASSERT(!isInlined()); + uint8_t* ptr = reinterpret_cast<uint8_t*>(this); + return reinterpret_cast<JitScript*>(ptr - JitScript::offsetOfICScript()); +} + +#ifdef DEBUG +// This hash is used to verify that we do not recompile after a +// TranspiledCacheIR invalidation with the exact same ICs. +// +// It should change iff an ICEntry in this ICScript (or an ICScript +// inlined into this ICScript) is modified such that we will make a +// different decision in WarpScriptOracle::maybeInlineIC. This means: +// +// 1. The hash will change if we attach a new stub. +// 2. The hash will change if the entered count of any CacheIR stub +// other than the first changes from 0. +// 3. The hash will change if the entered count of the fallback stub +// changes from 0. +// 4. The hash will change if the failure count of the fallback stub +// changes from 0. +HashNumber ICScript::hash() { + HashNumber h = 0; + for (size_t i = 0; i < numICEntries(); i++) { + ICStub* stub = icEntry(i).firstStub(); + + // Hash the address of the first stub. + h = mozilla::AddToHash(h, stub); + + // Hash whether subsequent stubs have entry count 0. + if (!stub->isFallback()) { + stub = stub->toCacheIRStub()->next(); + while (!stub->isFallback()) { + h = mozilla::AddToHash(h, stub->enteredCount() == 0); + stub = stub->toCacheIRStub()->next(); + } + } + + // Hash whether the fallback has entry count 0 and failure count 0. + MOZ_ASSERT(stub->isFallback()); + h = mozilla::AddToHash(h, stub->enteredCount() == 0); + h = mozilla::AddToHash(h, stub->toFallbackStub()->state().hasFailures()); + } + + return h; +} +#endif |