diff options
Diffstat (limited to 'js/src/jit/TrialInlining.cpp')
| -rw-r--r-- | js/src/jit/TrialInlining.cpp | 928 |
1 files changed, 928 insertions, 0 deletions
diff --git a/js/src/jit/TrialInlining.cpp b/js/src/jit/TrialInlining.cpp new file mode 100644 index 0000000000..f4befca1fa --- /dev/null +++ b/js/src/jit/TrialInlining.cpp @@ -0,0 +1,928 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/TrialInlining.h" + +#include "jit/BaselineCacheIRCompiler.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineIC.h" +#include "jit/CacheIRCloner.h" +#include "jit/CacheIRHealth.h" +#include "jit/CacheIRWriter.h" +#include "jit/Ion.h" // TooManyFormalArguments + +#include "vm/BytecodeLocation-inl.h" + +using mozilla::Maybe; + +namespace js { +namespace jit { + +bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) { + RootedScript script(cx, frame->script()); + ICScript* icScript = frame->icScript(); + bool isRecursive = icScript->depth() > 0; + +#ifdef JS_CACHEIR_SPEW + if (cx->spewer().enabled(cx, script, SpewChannel::CacheIRHealthReport)) { + for (uint32_t i = 0; i < icScript->numICEntries(); i++) { + ICEntry& entry = icScript->icEntry(i); + ICFallbackStub* fallbackStub = icScript->fallbackStub(i); + + // If the IC is megamorphic or generic, then we have already + // spewed the IC report on transition. + if (!(uint8_t(fallbackStub->state().mode()) > 0)) { + jit::ICStub* stub = entry.firstStub(); + bool sawNonZeroCount = false; + while (!stub->isFallback()) { + uint32_t count = stub->enteredCount(); + if (count > 0 && sawNonZeroCount) { + CacheIRHealth cih; + cih.healthReportForIC(cx, &entry, fallbackStub, script, + SpewContext::TrialInlining); + break; + } + + if (count > 0 && !sawNonZeroCount) { + sawNonZeroCount = true; + } + + stub = stub->toCacheIRStub()->next(); + } + } + } + } +#endif + + if (!script->canIonCompile()) { + return true; + } + + // Baseline shouldn't attempt trial inlining in scripts that are too large. + MOZ_ASSERT_IF(JitOptions.limitScriptSize, + script->length() <= JitOptions.ionMaxScriptSize); + + const uint32_t MAX_INLINING_DEPTH = 4; + if (icScript->depth() > MAX_INLINING_DEPTH) { + return true; + } + + InliningRoot* root = isRecursive ? icScript->inliningRoot() + : script->jitScript()->inliningRoot(); + if (JitSpewEnabled(JitSpew_WarpTrialInlining)) { + // Eagerly create the inlining root when it's used in the spew output. + if (!root) { + MOZ_ASSERT(!isRecursive); + root = script->jitScript()->getOrCreateInliningRoot(cx, script); + if (!root) { + return false; + } + } + UniqueChars funName; + if (script->function() && script->function()->displayAtom()) { + funName = AtomToPrintableString(cx, script->function()->displayAtom()); + } + + JitSpew( + JitSpew_WarpTrialInlining, + "Trial inlining for %s script '%s' (%s:%u:%u (%p)) (inliningRoot=%p)", + (isRecursive ? "inner" : "outer"), + funName ? funName.get() : "<unnamed>", script->filename(), + script->lineno(), script->column(), frame->script(), root); + JitSpewIndent spewIndent(JitSpew_WarpTrialInlining); + } + + TrialInliner inliner(cx, script, icScript); + return inliner.tryInlining(); +} + +void TrialInliner::cloneSharedPrefix(ICCacheIRStub* stub, + const uint8_t* endOfPrefix, + CacheIRWriter& writer) { + CacheIRReader reader(stub->stubInfo()); + CacheIRCloner cloner(stub); + while (reader.currentPosition() < endOfPrefix) { + CacheOp op = reader.readOp(); + cloner.cloneOp(op, reader, writer); + } +} + +bool TrialInliner::replaceICStub(ICEntry& entry, ICFallbackStub* fallback, + CacheIRWriter& writer, CacheKind kind) { + MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate); + + fallback->discardStubs(cx(), &entry); + + // Note: AttachBaselineCacheIRStub never throws an exception. + ICAttachResult result = AttachBaselineCacheIRStub( + cx(), writer, kind, script_, icScript_, fallback, "TrialInline"); + if (result == ICAttachResult::Attached) { + MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Inlined); + return true; + } + + MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate); + icScript_->removeInlinedChild(fallback->pcOffset()); + + if (result == ICAttachResult::OOM) { + ReportOutOfMemory(cx()); + return false; + } + + // We failed to attach a new IC stub due to CacheIR size limits. Disable trial + // inlining for this location and return true. + MOZ_ASSERT(result == ICAttachResult::TooLarge); + fallback->setTrialInliningState(TrialInliningState::Failure); + return true; +} + +ICCacheIRStub* TrialInliner::maybeSingleStub(const ICEntry& entry) { + // Look for a single non-fallback stub followed by stubs with entered-count 0. + // Allow one optimized stub before the fallback stub to support the + // CallIRGenerator::emitCalleeGuard optimization where we first try a + // GuardSpecificFunction guard before falling back to GuardFunctionHasScript. + ICStub* stub = entry.firstStub(); + if (stub->isFallback()) { + return nullptr; + } + ICStub* next = stub->toCacheIRStub()->next(); + if (next->enteredCount() != 0) { + return nullptr; + } + + ICFallbackStub* fallback = nullptr; + if (next->isFallback()) { + fallback = next->toFallbackStub(); + } else { + ICStub* nextNext = next->toCacheIRStub()->next(); + if (!nextNext->isFallback() || nextNext->enteredCount() != 0) { + return nullptr; + } + fallback = nextNext->toFallbackStub(); + } + + if (fallback->trialInliningState() != TrialInliningState::Candidate) { + return nullptr; + } + + return stub->toCacheIRStub(); +} + +Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub, + BytecodeLocation loc) { + if (loc.isInvokeOp()) { + Maybe<InlinableCallData> call = FindInlinableCallData(stub); + if (call.isSome()) { + return call; + } + } + if (loc.isGetPropOp() || loc.isGetElemOp()) { + Maybe<InlinableGetterData> getter = FindInlinableGetterData(stub); + if (getter.isSome()) { + return getter; + } + } + if (loc.isSetPropOp()) { + Maybe<InlinableSetterData> setter = FindInlinableSetterData(stub); + if (setter.isSome()) { + return setter; + } + } + return mozilla::Nothing(); +} + +Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub) { + Maybe<InlinableCallData> data; + + const CacheIRStubInfo* stubInfo = stub->stubInfo(); + const uint8_t* stubData = stub->stubDataStart(); + + ObjOperandId calleeGuardOperand; + CallFlags flags; + JSFunction* target = nullptr; + + CacheIRReader reader(stubInfo); + while (reader.more()) { + const uint8_t* opStart = reader.currentPosition(); + + CacheOp op = reader.readOp(); + CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)]; + uint32_t argLength = opInfo.argLength; + mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition(); + + switch (op) { + case CacheOp::GuardSpecificFunction: { + // If we see a guard, remember which operand we are guarding. + MOZ_ASSERT(data.isNothing()); + calleeGuardOperand = reader.objOperandId(); + uint32_t targetOffset = reader.stubOffset(); + (void)reader.stubOffset(); // nargsAndFlags + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset); + target = reinterpret_cast<JSFunction*>(rawTarget); + break; + } + case CacheOp::GuardFunctionScript: { + MOZ_ASSERT(data.isNothing()); + calleeGuardOperand = reader.objOperandId(); + uint32_t targetOffset = reader.stubOffset(); + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset); + target = reinterpret_cast<BaseScript*>(rawTarget)->function(); + (void)reader.stubOffset(); // nargsAndFlags + break; + } + case CacheOp::CallScriptedFunction: { + // If we see a call, check if `callee` is the previously guarded + // operand. If it is, we know the target and can inline. + ObjOperandId calleeOperand = reader.objOperandId(); + mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId(); + flags = reader.callFlags(); + mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate(); + MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy); + + if (calleeOperand == calleeGuardOperand) { + MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0); + MOZ_ASSERT(data.isNothing()); + data.emplace(); + data->endOfSharedPrefix = opStart; + } + break; + } + case CacheOp::CallInlinedFunction: { + ObjOperandId calleeOperand = reader.objOperandId(); + mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId(); + uint32_t icScriptOffset = reader.stubOffset(); + flags = reader.callFlags(); + mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate(); + MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy); + + if (calleeOperand == calleeGuardOperand) { + MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0); + MOZ_ASSERT(data.isNothing()); + data.emplace(); + data->endOfSharedPrefix = opStart; + uintptr_t rawICScript = + stubInfo->getStubRawWord(stubData, icScriptOffset); + data->icScript = reinterpret_cast<ICScript*>(rawICScript); + } + break; + } + default: + if (!opInfo.transpile) { + return mozilla::Nothing(); + } + if (data.isSome()) { + MOZ_ASSERT(op == CacheOp::ReturnFromIC); + } + reader.skip(argLength); + break; + } + MOZ_ASSERT(argStart + argLength == reader.currentPosition()); + } + + if (data.isSome()) { + // Warp only supports inlining Standard and FunCall calls. + if (flags.getArgFormat() != CallFlags::Standard && + flags.getArgFormat() != CallFlags::FunCall) { + return mozilla::Nothing(); + } + data->calleeOperand = calleeGuardOperand; + data->callFlags = flags; + data->target = target; + } + return data; +} + +Maybe<InlinableGetterData> FindInlinableGetterData(ICCacheIRStub* stub) { + Maybe<InlinableGetterData> data; + + const CacheIRStubInfo* stubInfo = stub->stubInfo(); + const uint8_t* stubData = stub->stubDataStart(); + + CacheIRReader reader(stubInfo); + while (reader.more()) { + const uint8_t* opStart = reader.currentPosition(); + + CacheOp op = reader.readOp(); + CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)]; + uint32_t argLength = opInfo.argLength; + mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition(); + + switch (op) { + case CacheOp::CallScriptedGetterResult: { + data.emplace(); + data->receiverOperand = reader.valOperandId(); + + uint32_t getterOffset = reader.stubOffset(); + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset); + data->target = reinterpret_cast<JSFunction*>(rawTarget); + + data->sameRealm = reader.readBool(); + (void)reader.stubOffset(); // nargsAndFlags + + data->endOfSharedPrefix = opStart; + break; + } + case CacheOp::CallInlinedGetterResult: { + data.emplace(); + data->receiverOperand = reader.valOperandId(); + + uint32_t getterOffset = reader.stubOffset(); + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset); + data->target = reinterpret_cast<JSFunction*>(rawTarget); + + uint32_t icScriptOffset = reader.stubOffset(); + uintptr_t rawICScript = + stubInfo->getStubRawWord(stubData, icScriptOffset); + data->icScript = reinterpret_cast<ICScript*>(rawICScript); + + data->sameRealm = reader.readBool(); + (void)reader.stubOffset(); // nargsAndFlags + + data->endOfSharedPrefix = opStart; + break; + } + default: + if (!opInfo.transpile) { + return mozilla::Nothing(); + } + if (data.isSome()) { + MOZ_ASSERT(op == CacheOp::ReturnFromIC); + } + reader.skip(argLength); + break; + } + MOZ_ASSERT(argStart + argLength == reader.currentPosition()); + } + + return data; +} + +Maybe<InlinableSetterData> FindInlinableSetterData(ICCacheIRStub* stub) { + Maybe<InlinableSetterData> data; + + const CacheIRStubInfo* stubInfo = stub->stubInfo(); + const uint8_t* stubData = stub->stubDataStart(); + + CacheIRReader reader(stubInfo); + while (reader.more()) { + const uint8_t* opStart = reader.currentPosition(); + + CacheOp op = reader.readOp(); + CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)]; + uint32_t argLength = opInfo.argLength; + mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition(); + + switch (op) { + case CacheOp::CallScriptedSetter: { + data.emplace(); + data->receiverOperand = reader.objOperandId(); + + uint32_t setterOffset = reader.stubOffset(); + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset); + data->target = reinterpret_cast<JSFunction*>(rawTarget); + + data->rhsOperand = reader.valOperandId(); + data->sameRealm = reader.readBool(); + (void)reader.stubOffset(); // nargsAndFlags + + data->endOfSharedPrefix = opStart; + break; + } + case CacheOp::CallInlinedSetter: { + data.emplace(); + data->receiverOperand = reader.objOperandId(); + + uint32_t setterOffset = reader.stubOffset(); + uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset); + data->target = reinterpret_cast<JSFunction*>(rawTarget); + + data->rhsOperand = reader.valOperandId(); + + uint32_t icScriptOffset = reader.stubOffset(); + uintptr_t rawICScript = + stubInfo->getStubRawWord(stubData, icScriptOffset); + data->icScript = reinterpret_cast<ICScript*>(rawICScript); + + data->sameRealm = reader.readBool(); + (void)reader.stubOffset(); // nargsAndFlags + + data->endOfSharedPrefix = opStart; + break; + } + default: + if (!opInfo.transpile) { + return mozilla::Nothing(); + } + if (data.isSome()) { + MOZ_ASSERT(op == CacheOp::ReturnFromIC); + } + reader.skip(argLength); + break; + } + MOZ_ASSERT(argStart + argLength == reader.currentPosition()); + } + + return data; +} + +// Return the maximum number of actual arguments that will be passed to the +// target function. This may be an overapproximation, for example when inlining +// js::fun_call we may omit an argument. +static uint32_t GetMaxCalleeNumActuals(BytecodeLocation loc) { + switch (loc.getOp()) { + case JSOp::GetProp: + case JSOp::GetElem: + // Getters do not pass arguments. + return 0; + + case JSOp::SetProp: + case JSOp::StrictSetProp: + // Setters pass 1 argument. + return 1; + + case JSOp::Call: + case JSOp::CallContent: + case JSOp::CallIgnoresRv: + case JSOp::CallIter: + case JSOp::CallContentIter: + case JSOp::New: + case JSOp::NewContent: + case JSOp::SuperCall: + return loc.getCallArgc(); + + default: + MOZ_CRASH("Unsupported op"); + } +} + +/*static*/ +bool TrialInliner::canInline(JSFunction* target, HandleScript caller, + BytecodeLocation loc) { + if (!target->hasJitScript()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: no JIT script"); + return false; + } + JSScript* script = target->nonLazyScript(); + if (!script->jitScript()->hasBaselineScript()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: no BaselineScript"); + return false; + } + if (script->uninlineable()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag"); + return false; + } + if (!script->canIonCompile()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile"); + return false; + } + if (script->isDebuggee()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: is debuggee"); + return false; + } + // Don't inline cross-realm calls. + if (target->realm() != caller->realm()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: cross-realm call"); + return false; + } + if (JitOptions.onlyInlineSelfHosted && !script->selfHosted()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: only inlining self hosted"); + return false; + } + + uint32_t maxCalleeNumActuals = GetMaxCalleeNumActuals(loc); + if (maxCalleeNumActuals > ArgumentsObject::MaxInlinedArgs) { + if (script->needsArgsObj()) { + JitSpew(JitSpew_WarpTrialInlining, + "SKIP: needs arguments object with %u actual args (maximum %u)", + maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs); + return false; + } + // The GetArgument(n) intrinsic in self-hosted code uses MGetInlinedArgument + // too, so the same limit applies. + if (script->usesArgumentsIntrinsics()) { + JitSpew(JitSpew_WarpTrialInlining, + "SKIP: uses GetArgument(i) with %u actual args (maximum %u)", + maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs); + return false; + } + } + + if (TooManyFormalArguments(target->nargs())) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u", + unsigned(target->nargs())); + return false; + } + + if (TooManyFormalArguments(maxCalleeNumActuals)) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u", + unsigned(loc.getCallArgc())); + return false; + } + + return true; +} + +TrialInliningDecision TrialInliner::getInliningDecision(JSFunction* target, + ICCacheIRStub* stub, + BytecodeLocation loc) { +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_WarpTrialInlining)) { + BaseScript* baseScript = + target->hasBaseScript() ? target->baseScript() : nullptr; + + UniqueChars funName; + if (target->displayAtom()) { + funName = AtomToPrintableString(cx(), target->displayAtom()); + } + + JitSpew(JitSpew_WarpTrialInlining, + "Inlining candidate JSOp::%s (offset=%u): callee script '%s' " + "(%s:%u:%u)", + CodeName(loc.getOp()), loc.bytecodeToOffset(script_), + funName ? funName.get() : "<unnamed>", + baseScript ? baseScript->filename() : "<not-scripted>", + baseScript ? baseScript->lineno() : 0, + baseScript ? baseScript->column() : 0); + JitSpewIndent spewIndent(JitSpew_WarpTrialInlining); + } +#endif + + if (!canInline(target, script_, loc)) { + return TrialInliningDecision::NoInline; + } + + // Don't inline (direct) recursive calls. This still allows recursion if + // called through another function (f => g => f). + JSScript* targetScript = target->nonLazyScript(); + if (script_ == targetScript) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: recursion"); + return TrialInliningDecision::NoInline; + } + + // Don't inline if the callee has a loop that was hot enough to enter Warp + // via OSR. This helps prevent getting stuck in Baseline code for a long time. + if (targetScript->jitScript()->hadIonOSR()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: had OSR"); + return TrialInliningDecision::NoInline; + } + + // Ensure the total bytecode size does not exceed ionMaxScriptSize. + size_t newTotalSize = + inliningRootTotalBytecodeSize() + targetScript->length(); + if (newTotalSize > JitOptions.ionMaxScriptSize) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: total size too big"); + return TrialInliningDecision::NoInline; + } + + uint32_t entryCount = stub->enteredCount(); + if (entryCount < JitOptions.inliningEntryThreshold) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)", + unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold)); + return TrialInliningDecision::NoInline; + } + + if (!JitOptions.isSmallFunction(targetScript)) { + if (!targetScript->isInlinableLargeFunction()) { + JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)", + unsigned(targetScript->length()), + unsigned(JitOptions.smallFunctionMaxBytecodeLength)); + return TrialInliningDecision::NoInline; + } + + JitSpew(JitSpew_WarpTrialInlining, + "INFO: Ignored length (%u) of InlinableLargeFunction", + unsigned(targetScript->length())); + } + + JitScript* jitScript = targetScript->jitScript(); + ICScript* icScript = jitScript->icScript(); + + // Check for any ICs which are not monomorphic. The observation here is that + // trial inlining can help us a lot in cases where it lets us further + // specialize a script. But if it's already monomorphic, it's unlikely that + // we will see significant specialization wins from trial inlining, so we + // can use a cheaper and simpler inlining strategy. + for (size_t i = 0; i < icScript->numICEntries(); i++) { + ICEntry& entry = icScript->icEntry(i); + ICFallbackStub* fallback = icScript->fallbackStub(i); + if (fallback->enteredCount() > 0 || + fallback->state().mode() != ICState::Mode::Specialized) { + return TrialInliningDecision::Inline; + } + + ICStub* firstStub = entry.firstStub(); + if (firstStub != fallback) { + for (ICStub* next = firstStub->toCacheIRStub()->next(); next; + next = next->maybeNext()) { + if (next->enteredCount() != 0) { + return TrialInliningDecision::Inline; + } + } + } + } + + JitSpewIndent spewIndent(JitSpew_WarpTrialInlining); + JitSpew(JitSpew_WarpTrialInlining, "SUCCESS: Inlined monomorphically"); + return TrialInliningDecision::MonomorphicInline; +} + +ICScript* TrialInliner::createInlinedICScript(JSFunction* target, + BytecodeLocation loc) { + MOZ_ASSERT(target->hasJitEntry()); + MOZ_ASSERT(target->hasJitScript()); + + InliningRoot* root = getOrCreateInliningRoot(); + if (!root) { + return nullptr; + } + + JSScript* targetScript = target->baseScript()->asJSScript(); + + // We don't have to check for overflow here because we have already + // successfully allocated an ICScript with this number of entries + // when creating the JitScript for the target function, and we + // checked for overflow then. + uint32_t fallbackStubsOffset = + sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry); + uint32_t allocSize = fallbackStubsOffset + + targetScript->numICEntries() * sizeof(ICFallbackStub); + + void* raw = cx()->pod_malloc<uint8_t>(allocSize); + MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0); + if (!raw) { + return nullptr; + } + + uint32_t initialWarmUpCount = JitOptions.trialInliningInitialWarmUpCount; + + uint32_t depth = icScript_->depth() + 1; + UniquePtr<ICScript> inlinedICScript(new (raw) ICScript( + initialWarmUpCount, fallbackStubsOffset, allocSize, depth, root)); + + inlinedICScript->initICEntries(cx(), targetScript); + + uint32_t pcOffset = loc.bytecodeToOffset(script_); + ICScript* result = inlinedICScript.get(); + if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) { + return nullptr; + } + MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries()); + + root->addToTotalBytecodeSize(targetScript->length()); + + JitSpewIndent spewIndent(JitSpew_WarpTrialInlining); + JitSpew(JitSpew_WarpTrialInlining, + "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_, result); + + return result; +} + +bool TrialInliner::maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback, + BytecodeLocation loc) { + ICCacheIRStub* stub = maybeSingleStub(entry); + if (!stub) { +#ifdef JS_JITSPEW + if (fallback->numOptimizedStubs() > 1) { + JitSpew(JitSpew_WarpTrialInlining, + "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc.getOp()), + fallback->pcOffset()); + JitSpewIndent spewIndent(JitSpew_WarpTrialInlining); + JitSpew(JitSpew_WarpTrialInlining, "SKIP: Polymorphic (%u stubs)", + (unsigned)fallback->numOptimizedStubs()); + } +#endif + return true; + } + + MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset())); + + // Look for a CallScriptedFunction with a known target. + Maybe<InlinableCallData> data = FindInlinableCallData(stub); + if (data.isNothing()) { + return true; + } + + MOZ_ASSERT(!data->icScript); + + TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc); + // Decide whether to inline the target. + if (inlining == TrialInliningDecision::NoInline) { + return true; + } + + if (inlining == TrialInliningDecision::MonomorphicInline) { + fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined); + return true; + } + + ICScript* newICScript = createInlinedICScript(data->target, loc); + if (!newICScript) { + return false; + } + + CacheIRWriter writer(cx()); + Int32OperandId argcId(writer.setInputOperandId(0)); + cloneSharedPrefix(stub, data->endOfSharedPrefix, writer); + + writer.callInlinedFunction(data->calleeOperand, argcId, newICScript, + data->callFlags, + ClampFixedArgc(loc.getCallArgc())); + writer.returnFromIC(); + + return replaceICStub(entry, fallback, writer, CacheKind::Call); +} + +bool TrialInliner::maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback, + BytecodeLocation loc, CacheKind kind) { + ICCacheIRStub* stub = maybeSingleStub(entry); + if (!stub) { + return true; + } + + MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset())); + + Maybe<InlinableGetterData> data = FindInlinableGetterData(stub); + if (data.isNothing()) { + return true; + } + + MOZ_ASSERT(!data->icScript); + + TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc); + // Decide whether to inline the target. + if (inlining == TrialInliningDecision::NoInline) { + return true; + } + + if (inlining == TrialInliningDecision::MonomorphicInline) { + fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined); + return true; + } + + ICScript* newICScript = createInlinedICScript(data->target, loc); + if (!newICScript) { + return false; + } + + CacheIRWriter writer(cx()); + ValOperandId valId(writer.setInputOperandId(0)); + if (kind == CacheKind::GetElem) { + // Register the key operand. + writer.setInputOperandId(1); + } + cloneSharedPrefix(stub, data->endOfSharedPrefix, writer); + + writer.callInlinedGetterResult(data->receiverOperand, data->target, + newICScript, data->sameRealm); + writer.returnFromIC(); + + return replaceICStub(entry, fallback, writer, kind); +} + +bool TrialInliner::maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback, + BytecodeLocation loc, CacheKind kind) { + ICCacheIRStub* stub = maybeSingleStub(entry); + if (!stub) { + return true; + } + + MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset())); + + Maybe<InlinableSetterData> data = FindInlinableSetterData(stub); + if (data.isNothing()) { + return true; + } + + MOZ_ASSERT(!data->icScript); + + TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc); + // Decide whether to inline the target. + if (inlining == TrialInliningDecision::NoInline) { + return true; + } + + if (inlining == TrialInliningDecision::MonomorphicInline) { + fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined); + return true; + } + + ICScript* newICScript = createInlinedICScript(data->target, loc); + if (!newICScript) { + return false; + } + + CacheIRWriter writer(cx()); + ValOperandId objValId(writer.setInputOperandId(0)); + ValOperandId rhsValId(writer.setInputOperandId(1)); + cloneSharedPrefix(stub, data->endOfSharedPrefix, writer); + + writer.callInlinedSetter(data->receiverOperand, data->target, + data->rhsOperand, newICScript, data->sameRealm); + writer.returnFromIC(); + + return replaceICStub(entry, fallback, writer, kind); +} + +bool TrialInliner::tryInlining() { + uint32_t numICEntries = icScript_->numICEntries(); + BytecodeLocation startLoc = script_->location(); + + for (uint32_t icIndex = 0; icIndex < numICEntries; icIndex++) { + ICEntry& entry = icScript_->icEntry(icIndex); + ICFallbackStub* fallback = icScript_->fallbackStub(icIndex); + + if (!TryFoldingStubs(cx(), fallback, script_, icScript_)) { + return false; + } + + BytecodeLocation loc = + startLoc + BytecodeLocationOffset(fallback->pcOffset()); + JSOp op = loc.getOp(); + switch (op) { + case JSOp::Call: + case JSOp::CallContent: + case JSOp::CallIgnoresRv: + case JSOp::CallIter: + case JSOp::CallContentIter: + case JSOp::New: + case JSOp::NewContent: + case JSOp::SuperCall: + if (!maybeInlineCall(entry, fallback, loc)) { + return false; + } + break; + case JSOp::GetProp: + if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetProp)) { + return false; + } + break; + case JSOp::GetElem: + if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetElem)) { + return false; + } + break; + case JSOp::SetProp: + case JSOp::StrictSetProp: + if (!maybeInlineSetter(entry, fallback, loc, CacheKind::SetProp)) { + return false; + } + break; + default: + break; + } + } + + return true; +} + +InliningRoot* TrialInliner::maybeGetInliningRoot() const { + if (auto* root = icScript_->inliningRoot()) { + return root; + } + + MOZ_ASSERT(!icScript_->isInlined()); + return script_->jitScript()->inliningRoot(); +} + +InliningRoot* TrialInliner::getOrCreateInliningRoot() { + if (auto* root = maybeGetInliningRoot()) { + return root; + } + return script_->jitScript()->getOrCreateInliningRoot(cx(), script_); +} + +size_t TrialInliner::inliningRootTotalBytecodeSize() const { + if (auto* root = maybeGetInliningRoot()) { + return root->totalBytecodeSize(); + } + return script_->length(); +} + +bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) { + return inlinedScripts_.append(std::move(icScript)); +} + +void InliningRoot::trace(JSTracer* trc) { + TraceEdge(trc, &owningScript_, "inlining-root-owning-script"); + for (auto& inlinedScript : inlinedScripts_) { + inlinedScript->trace(trc); + } +} + +void InliningRoot::purgeOptimizedStubs(Zone* zone) { + for (auto& inlinedScript : inlinedScripts_) { + inlinedScript->purgeOptimizedStubs(zone); + } +} + +void InliningRoot::resetWarmUpCounts(uint32_t count) { + for (auto& inlinedScript : inlinedScripts_) { + inlinedScript->resetWarmUpCount(count); + } +} + +} // namespace jit +} // namespace js |