path: root/js/src/jit/JitScript.h

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * vim: set ts=8 sts=2 et sw=2 tw=80:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_JitScript_h
#define jit_JitScript_h

#include "mozilla/Assertions.h"
#include "mozilla/Atomics.h"
#include "mozilla/Attributes.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/LinkedList.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"

#include <stddef.h>
#include <stdint.h>

#include "jstypes.h"
#include "NamespaceImports.h"

#include "ds/LifoAlloc.h"
#include "gc/Barrier.h"
#include "jit/BaselineIC.h"
#include "js/TypeDecls.h"
#include "js/UniquePtr.h"
#include "js/Vector.h"
#include "util/TrailingArray.h"
#include "vm/EnvironmentObject.h"

class JS_PUBLIC_API JSScript;
class JS_PUBLIC_API JSTracer;
struct JS_PUBLIC_API JSContext;

namespace JS {
class Zone;
}

namespace js {

class SystemAllocPolicy;

namespace gc {
class AllocSite;
}

namespace jit {

class BaselineScript;
class ICStubSpace;
class InliningRoot;
class IonScript;
class JitScript;
class JitZone;

// Magic BaselineScript value indicating Baseline compilation has been disabled.
static constexpr uintptr_t BaselineDisabledScript = 0x1;

static BaselineScript* const BaselineDisabledScriptPtr =
    reinterpret_cast<BaselineScript*>(BaselineDisabledScript);

// Magic IonScript values indicating Ion compilation has been disabled or the
// script is being Ion-compiled off-thread.
static constexpr uintptr_t IonDisabledScript = 0x1;
static constexpr uintptr_t IonCompilingScript = 0x2;

static IonScript* const IonDisabledScriptPtr =
    reinterpret_cast<IonScript*>(IonDisabledScript);
static IonScript* const IonCompilingScriptPtr =
    reinterpret_cast<IonScript*>(IonCompilingScript);

/* [SMDOC] ICScript Lifetimes
 *
 * An ICScript owns an array of ICEntries, each of which owns a linked
 * list of ICStubs.
 *
 * A JitScript contains an embedded ICScript. If it has done any trial
 * inlining, it also owns an InliningRoot. The InliningRoot owns all
 * of the ICScripts that have been created for inlining into the
 * corresponding JitScript. This ties the lifetime of the inlined
 * ICScripts to the lifetime of the JitScript itself.
 *
 * We store pointers to ICScripts in two other places: on the stack in
 * BaselineFrame, and in IC stubs for CallInlinedFunction.
 *
 * The ICScript pointer in a BaselineFrame either points to the
 * ICScript embedded in the JitScript for that frame, or to an inlined
 * ICScript owned by a caller. In each case, there must be a frame on
 * the stack corresponding to the JitScript that owns the current
 * ICScript, which will keep the ICScript alive.
 *
 * Each ICStub is owned by an ICScript and, indirectly, a
 * JitScript. An ICStub that uses CallInlinedFunction contains an
 * ICScript for use by the callee. The ICStub and the callee ICScript
 * are always owned by the same JitScript, so the callee ICScript will
 * not be freed while the ICStub is alive.
 *
 * The lifetime of an ICScript is independent of the lifetimes of the
 * BaselineScript and IonScript/WarpScript to which it
 * corresponds. They can be destroyed and recreated, and the ICScript
 * will remain valid.
 *
 * When we discard JIT code, we mark ICScripts that are active on the stack as
 * active and then purge all of the inactive ICScripts. We also purge ICStubs,
 * including the CallInlinedFunction stub at the trial inining call site, and
 * reset the ICStates to allow trial inlining again later.
 *
 * If there's a BaselineFrame for an inlined ICScript, we'll preserve both this
 * ICScript and the IC chain for the call site in the caller's ICScript.
 * See ICScript::purgeStubs and ICScript::purgeInactiveICScripts.
 */

class alignas(uintptr_t) ICScript final : public TrailingArray<ICScript> {
 public:
  ICScript(uint32_t warmUpCount, Offset fallbackStubsOffset, Offset endOffset,
           uint32_t depth, uint32_t bytecodeSize,
           InliningRoot* inliningRoot = nullptr)
      : inliningRoot_(inliningRoot),
        warmUpCount_(warmUpCount),
        fallbackStubsOffset_(fallbackStubsOffset),
        endOffset_(endOffset),
        depth_(depth),
        bytecodeSize_(bytecodeSize) {}

  ~ICScript();

  bool isInlined() const { return depth_ > 0; }

  void initICEntries(JSContext* cx, JSScript* script);

  ICEntry& icEntry(size_t index) {
    MOZ_ASSERT(index < numICEntries());
    return icEntries()[index];
  }

  ICFallbackStub* fallbackStub(size_t index) {
    MOZ_ASSERT(index < numICEntries());
    return fallbackStubs() + index;
  }

  ICEntry* icEntryForStub(const ICFallbackStub* stub) {
    size_t index = stub - fallbackStubs();
    MOZ_ASSERT(index < numICEntries());
    return &icEntry(index);
  }
  ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
    size_t index = entry - icEntries();
    MOZ_ASSERT(index < numICEntries());
    return fallbackStub(index);
  }

  InliningRoot* inliningRoot() const { return inliningRoot_; }
  uint32_t depth() const { return depth_; }

  uint32_t bytecodeSize() const { return bytecodeSize_; }

  void resetWarmUpCount(uint32_t count) { warmUpCount_ = count; }

  static constexpr size_t offsetOfFirstStub(uint32_t entryIndex) {
    return sizeof(ICScript) + entryIndex * sizeof(ICEntry) +
           ICEntry::offsetOfFirstStub();
  }

  static constexpr Offset offsetOfWarmUpCount() {
    return offsetof(ICScript, warmUpCount_);
  }
  static constexpr Offset offsetOfDepth() { return offsetof(ICScript, depth_); }

  static constexpr Offset offsetOfICEntries() { return sizeof(ICScript); }
  uint32_t numICEntries() const {
    return numElements<ICEntry>(icEntriesOffset(), fallbackStubsOffset());
  }

  ICEntry* interpreterICEntryFromPCOffset(uint32_t pcOffset);

  ICEntry& icEntryFromPCOffset(uint32_t pcOffset);

  [[nodiscard]] bool addInlinedChild(JSContext* cx,
                                     js::UniquePtr<ICScript> child,
                                     uint32_t pcOffset);
  ICScript* findInlinedChild(uint32_t pcOffset);
  void removeInlinedChild(uint32_t pcOffset);
  bool hasInlinedChild(uint32_t pcOffset);

  void purgeStubs(Zone* zone, ICStubSpace& newStubSpace);

  void purgeInactiveICScripts();

  bool active() const { return active_; }
  void setActive() { active_ = true; }
  void resetActive() { active_ = false; }

  gc::AllocSite* getOrCreateAllocSite(JSScript* outerScript, uint32_t pcOffset);

  void prepareForDestruction(Zone* zone);

  void trace(JSTracer* trc);
  bool traceWeak(JSTracer* trc);

#ifdef DEBUG
  mozilla::HashNumber hash();
#endif

 private:
  class CallSite {
   public:
    CallSite(ICScript* callee, uint32_t pcOffset)
        : callee_(callee), pcOffset_(pcOffset) {}
    ICScript* callee_;
    uint32_t pcOffset_;
  };

  // If this ICScript was created for trial inlining or has another
  // ICScript inlined into it, a pointer to the root of the inlining
  // tree. Otherwise, nullptr.
  InliningRoot* inliningRoot_ = nullptr;

  // ICScripts that have been inlined into this ICScript.
  js::UniquePtr<Vector<CallSite>> inlinedChildren_;

  // List of allocation sites referred to by ICs in this script.
  static constexpr size_t AllocSiteChunkSize = 256;
  LifoAlloc allocSitesSpace_{AllocSiteChunkSize};
  Vector<gc::AllocSite*, 0, SystemAllocPolicy> allocSites_;

  // Number of times this copy of the script has been called or has had
  // backedges taken.  Reset if the script's JIT code is forcibly discarded.
  // See also the ScriptWarmUpData class.
  mozilla::Atomic<uint32_t, mozilla::Relaxed> warmUpCount_ = {};

  // The offset of the ICFallbackStub array.
  Offset fallbackStubsOffset_;

  // The size of this allocation.
  Offset endOffset_;

  // The inlining depth of this ICScript. 0 for the inlining root.
  uint32_t depth_;

  // Bytecode size of the JSScript corresponding to this ICScript.
  uint32_t bytecodeSize_;

  // Flag set when discarding JIT code to indicate this script is on the stack
  // and should not be discarded.
  bool active_ = false;

  Offset icEntriesOffset() const { return offsetOfICEntries(); }
  Offset fallbackStubsOffset() const { return fallbackStubsOffset_; }
  Offset endOffset() const { return endOffset_; }

 public:
  ICEntry* icEntries() { return offsetToPointer<ICEntry>(icEntriesOffset()); }

 private:
  ICFallbackStub* fallbackStubs() {
    return offsetToPointer<ICFallbackStub>(fallbackStubsOffset());
  }

  JitScript* outerJitScript();

  friend class JitScript;
};

// [SMDOC] JitScript
//
// JitScript stores type inference data, Baseline ICs and other JIT-related data
// for a script. Scripts with a JitScript can run in the Baseline Interpreter.
//
// IC Data
// =======
// All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
// JitScript contains an ICScript as the last field. Additional free-standing
// ICScripts may be created during trial inlining. Ion has its own IC chains
// stored in IonScript.
//
// For each IC we store an ICEntry, which points to the first ICStub in the
// chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
// share Baseline IC code. This works because the stub data is stored in the
// ICStub instead of baked in in the stub code.
//
// Storing this separate from BaselineScript allows us to use the same ICs in
// the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
// because the JitScript can be reused when we have to recompile the
// BaselineScript.
//
// An ICScript contains a list of IC entries and a list of fallback stubs.
// There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
//
// The ICScript also contains the warmUpCount for the script.
//
// Inlining Data
// =============
// JitScript also contains a list of Warp compilations inlining this script, for
// invalidation.
//
// Memory Layout
// =============
// JitScript contains an ICScript as the last field. ICScript has trailing
// (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
// as follows:
//
//  Item                    | Offset
//  ------------------------+------------------------
//  JitScript               | 0
//  -->ICScript  (field)    |
//     ICEntry[]            | icEntriesOffset()
//     ICFallbackStub[]     | fallbackStubsOffset()
//
// These offsets are also used to compute numICEntries.
class alignas(uintptr_t) JitScript final
    : public mozilla::LinkedListElement<JitScript>,
      public TrailingArray<JitScript> {
  friend class ::JSScript;

  // Profile string used by the profiler for Baseline Interpreter frames.
  const char* profileString_ = nullptr;

  HeapPtr<JSScript*> owningScript_;

  // Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr or
  // a valid BaselineScript*.
  GCStructPtr<BaselineScript*> baselineScript_;

  // Ion code for this script. Either nullptr, IonDisabledScriptPtr,
  // IonCompilingScriptPtr or a valid IonScript*.
  GCStructPtr<IonScript*> ionScript_;

  // For functions that need a CallObject and/or NamedLambdaObject, the template
  // objects used by the Baseline JIT and Ion. If the function needs both a
  // named lambda object and a call object, the named lambda object template is
  // linked via the call object's enclosing environment. This field is set the
  // first time the Baseline JIT compiles this script.
  mozilla::Maybe<HeapPtr<EnvironmentObject*>> templateEnv_;

  // Analysis data computed lazily the first time this script is compiled or
  // inlined by WarpBuilder.
  mozilla::Maybe<bool> usesEnvironmentChain_;

  // The size of this allocation.
  Offset endOffset_ = 0;

  struct Flags {
    // True if this script entered Ion via OSR at a loop header.
    bool hadIonOSR : 1;
  };
  Flags flags_ = {};  // Zero-initialize flags.

  js::UniquePtr<InliningRoot> inliningRoot_;

#ifdef DEBUG
  // If the last warp compilation invalidated because of TranspiledCacheIR
  // bailouts, this is a hash of the ICScripts used in that compilation.
  // When recompiling, we assert that the hash has changed.
  mozilla::Maybe<mozilla::HashNumber> failedICHash_;

  // To avoid pathological cases, we skip the check if we have purged
  // stubs due to GC pressure.
  bool hasPurgedStubs_ = false;
#endif

  // Value of the warmup counter when the last IC stub was attached,
  // used for Ion hints.
  uint32_t warmUpCountAtLastICStub_ = 0;

  ICScript icScript_;
  // End of fields.

  Offset endOffset() const { return endOffset_; }

 public:
  JitScript(JSScript* script, Offset fallbackStubsOffset, Offset endOffset,
            const char* profileString);

  ~JitScript();

  JSScript* owningScript() const { return owningScript_; }

  [[nodiscard]] bool ensureHasCachedBaselineJitData(JSContext* cx,
                                                    HandleScript script);
  [[nodiscard]] bool ensureHasCachedIonData(JSContext* cx, HandleScript script);

  void setHadIonOSR() { flags_.hadIonOSR = true; }
  bool hadIonOSR() const { return flags_.hadIonOSR; }

  uint32_t numICEntries() const { return icScript_.numICEntries(); }

#ifdef DEBUG
  bool hasActiveICScript() const;
#endif
  void resetAllActiveFlags();

  void ensureProfileString(JSContext* cx, JSScript* script);

  const char* profileString() const {
    MOZ_ASSERT(profileString_);
    return profileString_;
  }

  static void Destroy(Zone* zone, JitScript* script);

  static constexpr Offset offsetOfICEntries() { return sizeof(JitScript); }

  static constexpr size_t offsetOfBaselineScript() {
    return offsetof(JitScript, baselineScript_);
  }
  static constexpr size_t offsetOfIonScript() {
    return offsetof(JitScript, ionScript_);
  }
  static constexpr size_t offsetOfICScript() {
    return offsetof(JitScript, icScript_);
  }
  static constexpr size_t offsetOfWarmUpCount() {
    return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
  }

  uint32_t warmUpCount() const { return icScript_.warmUpCount_; }
  void incWarmUpCount() { icScript_.warmUpCount_++; }
  void resetWarmUpCount(uint32_t count);

  void prepareForDestruction(Zone* zone);

  void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, size_t* data,
                              size_t* allocSites) const;

  ICEntry& icEntry(size_t index) { return icScript_.icEntry(index); }

  ICFallbackStub* fallbackStub(size_t index) {
    return icScript_.fallbackStub(index);
  }

  ICEntry* icEntryForStub(const ICFallbackStub* stub) {
    return icScript_.icEntryForStub(stub);
  }
  ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
    return icScript_.fallbackStubForICEntry(entry);
  }

  void trace(JSTracer* trc);
  void traceWeak(JSTracer* trc);
  void purgeStubs(JSScript* script, ICStubSpace& newStubSpace);

  void purgeInactiveICScripts();

  ICEntry& icEntryFromPCOffset(uint32_t pcOffset) {
    return icScript_.icEntryFromPCOffset(pcOffset);
  };

  size_t allocBytes() const { return endOffset(); }

  EnvironmentObject* templateEnvironment() const { return templateEnv_.ref(); }

  bool usesEnvironmentChain() const { return *usesEnvironmentChain_; }

  bool resetAllocSites(bool resetNurserySites, bool resetPretenuredSites);

  void updateLastICStubCounter() { warmUpCountAtLastICStub_ = warmUpCount(); }
  uint32_t warmUpCountAtLastICStub() const { return warmUpCountAtLastICStub_; }

 private:
  // Methods to set baselineScript_ to a BaselineScript*, nullptr, or
  // BaselineDisabledScriptPtr.
  void setBaselineScriptImpl(JSScript* script, BaselineScript* baselineScript);
  void setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
                             BaselineScript* baselineScript);

 public:
  // Methods for getting/setting/clearing a BaselineScript*.
  bool hasBaselineScript() const {
    bool res = baselineScript_ && baselineScript_ != BaselineDisabledScriptPtr;
    MOZ_ASSERT_IF(!res, !hasIonScript());
    return res;
  }
  BaselineScript* baselineScript() const {
    MOZ_ASSERT(hasBaselineScript());
    return baselineScript_;
  }
  void setBaselineScript(JSScript* script, BaselineScript* baselineScript) {
    MOZ_ASSERT(!hasBaselineScript());
    setBaselineScriptImpl(script, baselineScript);
    MOZ_ASSERT(hasBaselineScript());
  }
  [[nodiscard]] BaselineScript* clearBaselineScript(JS::GCContext* gcx,
                                                    JSScript* script) {
    BaselineScript* baseline = baselineScript();
    setBaselineScriptImpl(gcx, script, nullptr);
    return baseline;
  }

 private:
  // Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
  // Ion{Disabled,Compiling}ScriptPtr values.
  void setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
                        IonScript* ionScript);
  void setIonScriptImpl(JSScript* script, IonScript* ionScript);

  // Helper that calls the passed function for the outer ICScript and for each
  // inlined ICScript.
  template <typename F>
  void forEachICScript(const F& f);
  template <typename F>
  void forEachICScript(const F& f) const;

 public:
  // Methods for getting/setting/clearing an IonScript*.
  bool hasIonScript() const {
    bool res = ionScript_ && ionScript_ != IonDisabledScriptPtr &&
               ionScript_ != IonCompilingScriptPtr;
    MOZ_ASSERT_IF(res, baselineScript_);
    return res;
  }
  IonScript* ionScript() const {
    MOZ_ASSERT(hasIonScript());
    return ionScript_;
  }
  void setIonScript(JSScript* script, IonScript* ionScript) {
    MOZ_ASSERT(!hasIonScript());
    setIonScriptImpl(script, ionScript);
    MOZ_ASSERT(hasIonScript());
  }
  [[nodiscard]] IonScript* clearIonScript(JS::GCContext* gcx,
                                          JSScript* script) {
    IonScript* ion = ionScript();
    setIonScriptImpl(gcx, script, nullptr);
    return ion;
  }

  // Methods for off-thread compilation.
  bool isIonCompilingOffThread() const {
    return ionScript_ == IonCompilingScriptPtr;
  }
  void setIsIonCompilingOffThread(JSScript* script) {
    MOZ_ASSERT(ionScript_ == nullptr);
    setIonScriptImpl(script, IonCompilingScriptPtr);
  }
  void clearIsIonCompilingOffThread(JSScript* script) {
    MOZ_ASSERT(isIonCompilingOffThread());
    setIonScriptImpl(script, nullptr);
  }
  ICScript* icScript() { return &icScript_; }

  bool hasInliningRoot() const { return !!inliningRoot_; }
  InliningRoot* inliningRoot() const { return inliningRoot_.get(); }
  InliningRoot* getOrCreateInliningRoot(JSContext* cx, JSScript* script);

  inline void notePurgedStubs() {
#ifdef DEBUG
    failedICHash_.reset();
    hasPurgedStubs_ = true;
#endif
  }

#ifdef DEBUG
  bool hasPurgedStubs() const { return hasPurgedStubs_; }
  bool hasFailedICHash() const { return failedICHash_.isSome(); }
  mozilla::HashNumber getFailedICHash() { return failedICHash_.extract(); }
  void setFailedICHash(mozilla::HashNumber hash) {
    MOZ_ASSERT(failedICHash_.isNothing());
    if (!hasPurgedStubs_) {
      failedICHash_.emplace(hash);
    }
  }
#endif
};

// Ensures no JitScripts are purged in the current zone.
class MOZ_RAII AutoKeepJitScripts {
  jit::JitZone* zone_;
  bool prev_;

  AutoKeepJitScripts(const AutoKeepJitScripts&) = delete;
  void operator=(const AutoKeepJitScripts&) = delete;

 public:
  explicit inline AutoKeepJitScripts(JSContext* cx);
  inline ~AutoKeepJitScripts();
};

// Mark ICScripts on the stack as active, so that they are not discarded
// during GC, and copy active Baseline IC stubs to the new stub space.
void MarkActiveICScriptsAndCopyStubs(Zone* zone, ICStubSpace& newStubSpace);

#ifdef JS_STRUCTURED_SPEW
void JitSpewBaselineICStats(JSScript* script, const char* dumpReason);
#endif

}  // namespace jit
}  // namespace js

#endif /* jit_JitScript_h */