From 0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:47:29 +0200 Subject: Adding upstream version 115.8.0esr. Signed-off-by: Daniel Baumann --- js/src/jit/JitRuntime.h | 451 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 451 insertions(+) create mode 100644 js/src/jit/JitRuntime.h (limited to 'js/src/jit/JitRuntime.h') diff --git a/js/src/jit/JitRuntime.h b/js/src/jit/JitRuntime.h new file mode 100644 index 0000000000..cc3bd231c4 --- /dev/null +++ b/js/src/jit/JitRuntime.h @@ -0,0 +1,451 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_JitRuntime_h +#define jit_JitRuntime_h + +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/LinkedList.h" + +#include +#include + +#include "jstypes.h" + +#include "jit/ABIFunctions.h" +#include "jit/BaselineICList.h" +#include "jit/BaselineJIT.h" +#include "jit/CalleeToken.h" +#include "jit/InterpreterEntryTrampoline.h" +#include "jit/IonCompileTask.h" +#include "jit/IonTypes.h" +#include "jit/JitCode.h" +#include "jit/JitHints.h" +#include "jit/shared/Assembler-shared.h" +#include "js/AllocPolicy.h" +#include "js/ProfilingFrameIterator.h" +#include "js/TypeDecls.h" +#include "js/UniquePtr.h" +#include "js/Vector.h" +#include "threading/ProtectedData.h" +#include "vm/GeckoProfiler.h" +#include "vm/Runtime.h" + +class JS_PUBLIC_API JSTracer; + +namespace js { + +class AutoLockHelperThreadState; +class GCMarker; + +namespace jit { + +class FrameSizeClass; +class JitRealm; +class Label; +class MacroAssembler; +struct VMFunctionData; + +enum class TailCallVMFunctionId; +enum class VMFunctionId; + +enum class BaselineICFallbackKind : uint8_t { +#define DEF_ENUM_KIND(kind) kind, + IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND) +#undef DEF_ENUM_KIND + Count +}; + +enum class BailoutReturnKind { + GetProp, + GetPropSuper, + SetProp, + GetElem, + GetElemSuper, + Call, + New, + Count +}; + +// Class storing code and offsets for all Baseline IC fallback trampolines. This +// is stored in JitRuntime and generated when creating the JitRuntime. +class BaselineICFallbackCode { + JitCode* code_ = nullptr; + using OffsetArray = + mozilla::EnumeratedArray; + OffsetArray offsets_ = {}; + + // Keep track of offset into various baseline stubs' code at return + // point from called script. + using BailoutReturnArray = + mozilla::EnumeratedArray; + BailoutReturnArray bailoutReturnOffsets_ = {}; + + public: + BaselineICFallbackCode() = default; + BaselineICFallbackCode(const BaselineICFallbackCode&) = delete; + void operator=(const BaselineICFallbackCode&) = delete; + + void initOffset(BaselineICFallbackKind kind, uint32_t offset) { + offsets_[kind] = offset; + } + void initCode(JitCode* code) { code_ = code; } + void initBailoutReturnOffset(BailoutReturnKind kind, uint32_t offset) { + bailoutReturnOffsets_[kind] = offset; + } + TrampolinePtr addr(BaselineICFallbackKind kind) const { + return TrampolinePtr(code_->raw() + offsets_[kind]); + } + uint8_t* bailoutReturnAddr(BailoutReturnKind kind) const { + return code_->raw() + bailoutReturnOffsets_[kind]; + } +}; + +enum class ArgumentsRectifierKind { Normal, TrialInlining }; + +enum class DebugTrapHandlerKind { Interpreter, Compiler, Count }; + +using EnterJitCode = void (*)(void*, unsigned int, Value*, InterpreterFrame*, + CalleeToken, JSObject*, size_t, Value*); + +class JitcodeGlobalTable; + +class JitRuntime { + private: + friend class JitRealm; + + MainThreadData nextCompilationId_{0}; + + // Buffer for OSR from baseline to Ion. To avoid holding on to this for too + // long it's also freed in EnterBaseline and EnterJit (after returning from + // JIT code). + MainThreadData> ionOsrTempData_{nullptr}; + + // Shared exception-handler tail. + WriteOnceData exceptionTailOffset_{0}; + + // Shared profiler exit frame tail. + WriteOnceData profilerExitFrameTailOffset_{0}; + + // Trampoline for entering JIT code. + WriteOnceData enterJITOffset_{0}; + + // Generic bailout table; used if the bailout table overflows. + WriteOnceData bailoutHandlerOffset_{0}; + + // Argument-rectifying thunks, in the case of insufficient arguments passed + // to a function call site. The return offset is used to rebuild stack frames + // when bailing out. + WriteOnceData argumentsRectifierOffset_{0}; + WriteOnceData trialInliningArgumentsRectifierOffset_{0}; + WriteOnceData argumentsRectifierReturnOffset_{0}; + + // Thunk that invalides an (Ion compiled) caller on the Ion stack. + WriteOnceData invalidatorOffset_{0}; + + // Thunk that calls the GC pre barrier. + WriteOnceData valuePreBarrierOffset_{0}; + WriteOnceData stringPreBarrierOffset_{0}; + WriteOnceData objectPreBarrierOffset_{0}; + WriteOnceData shapePreBarrierOffset_{0}; + + // Thunk to call malloc/free. + WriteOnceData freeStubOffset_{0}; + + // Thunk called to finish compilation of an IonScript. + WriteOnceData lazyLinkStubOffset_{0}; + + // Thunk to enter the interpreter from JIT code. + WriteOnceData interpreterStubOffset_{0}; + + // Thunk to convert the value in R0 to int32 if it's a double. + // Note: this stub treats -0 as +0 and may clobber R1.scratchReg(). + WriteOnceData doubleToInt32ValueStubOffset_{0}; + + // Thunk used by the debugger for breakpoint and step mode. + mozilla::EnumeratedArray> + debugTrapHandlers_; + + // BaselineInterpreter state. + BaselineInterpreter baselineInterpreter_; + + // Code for trampolines and VMFunction wrappers. + WriteOnceData trampolineCode_{nullptr}; + + // Thunk that calls into the C++ interpreter from the interpreter + // entry trampoline that is generated with --emit-interpreter-entry + WriteOnceData vmInterpreterEntryOffset_{0}; + + // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_. + using VMWrapperOffsets = Vector; + VMWrapperOffsets functionWrapperOffsets_; + + // Maps TailCallVMFunctionId to the offset of the wrapper code in + // trampolineCode_. + VMWrapperOffsets tailCallFunctionWrapperOffsets_; + + MainThreadData baselineICFallbackCode_; + + // Global table of jitcode native address => bytecode address mappings. + UnprotectedData jitcodeGlobalTable_{nullptr}; + + // Map that stores Jit Hints for each script. + MainThreadData jitHintsMap_{nullptr}; + + // Map used to collect entry trampolines for the Interpreters which is used + // for external profiling to identify which functions are being interpreted. + MainThreadData interpreterEntryMap_{nullptr}; + +#ifdef DEBUG + // The number of possible bailing places encountered before forcefully bailing + // in that place if the counter reaches zero. Note that zero also means + // inactive. + MainThreadData ionBailAfterCounter_{0}; + + // Whether the bailAfter mechanism is enabled. Used to avoid generating the + // Ion code instrumentation for ionBailAfterCounter_ if the testing function + // isn't used. + MainThreadData ionBailAfterEnabled_{false}; +#endif + + // Number of Ion compilations which were finished off thread and are + // waiting to be lazily linked. This is only set while holding the helper + // thread state lock, but may be read from at other times. + typedef mozilla::Atomic + NumFinishedOffThreadTasksType; + NumFinishedOffThreadTasksType numFinishedOffThreadTasks_{0}; + + // List of Ion compilation waiting to get linked. + using IonCompileTaskList = mozilla::LinkedList; + MainThreadData ionLazyLinkList_; + MainThreadData ionLazyLinkListSize_{0}; + +#ifdef DEBUG + // Flag that can be set from JIT code to indicate it's invalid to call + // arbitrary JS code in a particular region. This is checked in RunScript. + MainThreadData disallowArbitraryCode_{false}; +#endif + + bool generateTrampolines(JSContext* cx); + bool generateBaselineICFallbackCode(JSContext* cx); + + void generateLazyLinkStub(MacroAssembler& masm); + void generateInterpreterStub(MacroAssembler& masm); + void generateDoubleToInt32ValueStub(MacroAssembler& masm); + void generateProfilerExitFrameTailStub(MacroAssembler& masm, + Label* profilerExitTail); + void generateExceptionTailStub(MacroAssembler& masm, Label* profilerExitTail, + Label* bailoutTail); + void generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail); + void generateEnterJIT(JSContext* cx, MacroAssembler& masm); + void generateArgumentsRectifier(MacroAssembler& masm, + ArgumentsRectifierKind kind); + void generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail); + void generateInvalidator(MacroAssembler& masm, Label* bailoutTail); + uint32_t generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type); + void generateFreeStub(MacroAssembler& masm); + JitCode* generateDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind); + + bool generateVMWrapper(JSContext* cx, MacroAssembler& masm, + const VMFunctionData& f, DynFn nativeFun, + uint32_t* wrapperOffset); + + template + bool generateVMWrappers(JSContext* cx, MacroAssembler& masm, + VMWrapperOffsets& offsets); + bool generateVMWrappers(JSContext* cx, MacroAssembler& masm); + + uint32_t startTrampolineCode(MacroAssembler& masm); + + TrampolinePtr trampolineCode(uint32_t offset) const { + MOZ_ASSERT(offset > 0); + MOZ_ASSERT(offset < trampolineCode_->instructionsSize()); + return TrampolinePtr(trampolineCode_->raw() + offset); + } + + void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm); + void generateInterpreterEntryTrampoline(MacroAssembler& masm); + + public: + JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script); + + JitRuntime() = default; + ~JitRuntime(); + [[nodiscard]] bool initialize(JSContext* cx); + + static void TraceAtomZoneRoots(JSTracer* trc); + [[nodiscard]] static bool MarkJitcodeGlobalTableIteratively(GCMarker* marker); + static void TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc); + + const BaselineICFallbackCode& baselineICFallbackCode() const { + return baselineICFallbackCode_.ref(); + } + + IonCompilationId nextCompilationId() { + return IonCompilationId(nextCompilationId_++); + } + +#ifdef DEBUG + bool disallowArbitraryCode() const { return disallowArbitraryCode_; } + void clearDisallowArbitraryCode() { disallowArbitraryCode_ = false; } + const void* addressOfDisallowArbitraryCode() const { + return &disallowArbitraryCode_.refNoCheck(); + } +#endif + + uint8_t* allocateIonOsrTempData(size_t size); + void freeIonOsrTempData(); + + TrampolinePtr getVMWrapper(VMFunctionId funId) const { + MOZ_ASSERT(trampolineCode_); + return trampolineCode(functionWrapperOffsets_[size_t(funId)]); + } + TrampolinePtr getVMWrapper(TailCallVMFunctionId funId) const { + MOZ_ASSERT(trampolineCode_); + return trampolineCode(tailCallFunctionWrapperOffsets_[size_t(funId)]); + } + + JitCode* debugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind); + + BaselineInterpreter& baselineInterpreter() { return baselineInterpreter_; } + + TrampolinePtr getGenericBailoutHandler() const { + return trampolineCode(bailoutHandlerOffset_); + } + + TrampolinePtr getExceptionTail() const { + return trampolineCode(exceptionTailOffset_); + } + + TrampolinePtr getProfilerExitFrameTail() const { + return trampolineCode(profilerExitFrameTailOffset_); + } + + TrampolinePtr getArgumentsRectifier( + ArgumentsRectifierKind kind = ArgumentsRectifierKind::Normal) const { + if (kind == ArgumentsRectifierKind::TrialInlining) { + return trampolineCode(trialInliningArgumentsRectifierOffset_); + } + return trampolineCode(argumentsRectifierOffset_); + } + + uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; } + + TrampolinePtr getArgumentsRectifierReturnAddr() const { + return trampolineCode(argumentsRectifierReturnOffset_); + } + + TrampolinePtr getInvalidationThunk() const { + return trampolineCode(invalidatorOffset_); + } + + EnterJitCode enterJit() const { + return JS_DATA_TO_FUNC_PTR(EnterJitCode, + trampolineCode(enterJITOffset_).value); + } + + // Return the registers from the native caller frame of the given JIT frame. + // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry + // frame, or if the information is not accessible/implemented on this + // platform. + static mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> + getCppEntryRegisters(JitFrameLayout* frameStackAddress); + + TrampolinePtr preBarrier(MIRType type) const { + switch (type) { + case MIRType::Value: + return trampolineCode(valuePreBarrierOffset_); + case MIRType::String: + return trampolineCode(stringPreBarrierOffset_); + case MIRType::Object: + return trampolineCode(objectPreBarrierOffset_); + case MIRType::Shape: + return trampolineCode(shapePreBarrierOffset_); + default: + MOZ_CRASH(); + } + } + + TrampolinePtr freeStub() const { return trampolineCode(freeStubOffset_); } + + TrampolinePtr lazyLinkStub() const { + return trampolineCode(lazyLinkStubOffset_); + } + TrampolinePtr interpreterStub() const { + return trampolineCode(interpreterStubOffset_); + } + + TrampolinePtr getDoubleToInt32ValueStub() const { + return trampolineCode(doubleToInt32ValueStubOffset_); + } + + bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_ != nullptr; } + + JitcodeGlobalTable* getJitcodeGlobalTable() { + MOZ_ASSERT(hasJitcodeGlobalTable()); + return jitcodeGlobalTable_; + } + + bool hasJitHintsMap() const { return jitHintsMap_ != nullptr; } + + JitHintsMap* getJitHintsMap() { + MOZ_ASSERT(hasJitHintsMap()); + return jitHintsMap_; + } + + bool hasInterpreterEntryMap() const { + return interpreterEntryMap_ != nullptr; + } + + EntryTrampolineMap* getInterpreterEntryMap() { + MOZ_ASSERT(hasInterpreterEntryMap()); + return interpreterEntryMap_; + } + + bool isProfilerInstrumentationEnabled(JSRuntime* rt) { + return rt->geckoProfiler().enabled(); + } + + bool isOptimizationTrackingEnabled(JSRuntime* rt) { + return isProfilerInstrumentationEnabled(rt); + } + +#ifdef DEBUG + void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_; } + + // Set after how many bailing places we should forcefully bail. + // Zero disables this feature. + void setIonBailAfterCounter(uint32_t after) { ionBailAfterCounter_ = after; } + bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; } + void setIonBailAfterEnabled(bool enabled) { ionBailAfterEnabled_ = enabled; } +#endif + + size_t numFinishedOffThreadTasks() const { + return numFinishedOffThreadTasks_; + } + NumFinishedOffThreadTasksType& numFinishedOffThreadTasksRef( + const AutoLockHelperThreadState& locked) { + return numFinishedOffThreadTasks_; + } + + IonCompileTaskList& ionLazyLinkList(JSRuntime* rt); + + size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_; } + + void ionLazyLinkListRemove(JSRuntime* rt, js::jit::IonCompileTask* task); + void ionLazyLinkListAdd(JSRuntime* rt, js::jit::IonCompileTask* task); +}; + +} // namespace jit +} // namespace js + +#endif /* jit_JitRuntime_h */ -- cgit v1.2.3