1 files changed, 616 insertions, 0 deletions
diff --git a/js/src/jit/IonScript.h b/js/src/jit/IonScript.h
new file mode 100644
index 0000000000..a2e289d3ad
--- /dev/null
+++ b/js/src/jit/IonScript.h
@@ -0,0 +1,616 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonScript_h
+#define jit_IonScript_h
+
+#include "mozilla/MemoryReporting.h"  // MallocSizeOf
+
+#include <stddef.h>  // size_t
+#include <stdint.h>  // uint8_t, uint32_t
+
+#include "jstypes.h"
+
+#include "gc/Barrier.h"          // HeapPtr{JitCode,Object}, PreBarrieredValue
+#include "jit/IonTypes.h"        // IonCompilationId
+#include "jit/JitCode.h"         // JitCode
+#include "jit/JitOptions.h"      // JitOptions
+#include "js/TypeDecls.h"        // jsbytecode
+#include "util/TrailingArray.h"  // TrailingArray
+#include "vm/TraceLogging.h"     // TraceLoggerEvent
+
+namespace js {
+namespace jit {
+
+using TraceLoggerEventVector = Vector<TraceLoggerEvent, 0, SystemAllocPolicy>;
+
+class SnapshotWriter;
+class RecoverWriter;
+class SafepointWriter;
+class CodegenSafepointIndex;
+class SafepointIndex;
+class OsiIndex;
+class IonIC;
+
+// An IonScript attaches Ion-generated information to a JSScript. The header
+// structure is followed by several arrays of data. These trailing arrays have a
+// layout based on offsets (bytes from 'this') stored in the IonScript header.
+//
+//    <IonScript itself>
+//    --
+//    PreBarrieredValue[]   constantTable()
+//    uint8_t[]             runtimeData()
+//    OsiIndex[]            osiIndex()
+//    SafepointIndex[]      safepointIndex()
+//    SnapshotOffset[]      bailoutTable()
+//    uint32_t[]            icIndex()
+//    --
+//    uint8_t[]             safepoints()
+//    uint8_t[]             snapshots()
+//    uint8_t[]             snapshotsRVATable()
+//    uint8_t[]             recovers()
+//
+// Note: These are arranged in order of descending alignment requirements to
+// avoid the need for padding. The `runtimeData` uses uint64_t alignement due to
+// its use of mozilla::AlignedStorage2.
+class alignas(8) IonScript final : public TrailingArray {
+ private:
+  // Offset (in bytes) from `this` to the start of each trailing array. Each
+  // array ends where following one begins. There is no implicit padding (except
+  // possible at very end).
+  Offset constantTableOffset_ = 0;   // JS::Value aligned
+  Offset runtimeDataOffset_ = 0;     // uint64_t aligned
+  Offset nurseryObjectsOffset_ = 0;  // pointer aligned
+  Offset osiIndexOffset_ = 0;
+  Offset safepointIndexOffset_ = 0;
+  Offset bailoutTableOffset_ = 0;
+  Offset icIndexOffset_ = 0;
+  Offset safepointsOffset_ = 0;
+  Offset snapshotsOffset_ = 0;
+  Offset rvaTableOffset_ = 0;
+  Offset recoversOffset_ = 0;
+  Offset allocBytes_ = 0;
+
+  // Code pointer containing the actual method.
+  HeapPtrJitCode method_ = nullptr;
+
+  // Entrypoint for OSR, or nullptr.
+  jsbytecode* osrPc_ = nullptr;
+
+  // Offset to OSR entrypoint from method_->raw(), or 0.
+  uint32_t osrEntryOffset_ = 0;
+
+  // Offset of the invalidation epilogue (which pushes this IonScript
+  // and calls the invalidation thunk).
+  uint32_t invalidateEpilogueOffset_ = 0;
+
+  // The offset immediately after the IonScript immediate.
+  // NOTE: technically a constant delta from
+  // |invalidateEpilogueOffset_|, so we could hard-code this
+  // per-platform if we want.
+  uint32_t invalidateEpilogueDataOffset_ = 0;
+
+  // Number of bailouts that have occurred for reasons that could be
+  // fixed if we invalidated and recompiled.
+  uint16_t numFixableBailouts_ = 0;
+
+  // Number of bailouts that have occurred for reasons that can't be
+  // fixed by recompiling: for example, bailing out to catch an exception.
+  uint16_t numUnfixableBailouts_ = 0;
+
+ public:
+  enum class LICMState : uint8_t { NeverBailed, Bailed, BailedAndHitFallback };
+
+ private:
+  // Tracks the state of LICM bailouts.
+  LICMState licmState_ = LICMState::NeverBailed;
+
+  // Flag set if IonScript was compiled with profiling enabled.
+  bool hasProfilingInstrumentation_ = false;
+
+  // Number of bytes this function reserves on the stack.
+  uint32_t frameSlots_ = 0;
+
+  // Number of bytes used passed in as formal arguments or |this|.
+  uint32_t argumentSlots_ = 0;
+
+  // Frame size is the value that can be added to the StackPointer along
+  // with the frame prefix to get a valid JitFrameLayout.
+  uint32_t frameSize_ = 0;
+
+  // Number of references from invalidation records.
+  uint32_t invalidationCount_ = 0;
+
+  // Identifier of the compilation which produced this code.
+  IonCompilationId compilationId_;
+
+  // Number of times we tried to enter this script via OSR but failed due to
+  // a LOOPENTRY pc other than osrPc_.
+  uint32_t osrPcMismatchCounter_ = 0;
+
+  // TraceLogger events that are baked into the IonScript.
+  TraceLoggerEventVector traceLoggerEvents_;
+
+#ifdef DEBUG
+  // A hash of the ICScripts used in this compilation.
+  mozilla::HashNumber icHash_ = 0;
+#endif
+
+  // End of fields.
+
+ private:
+  // Layout helpers
+  Offset constantTableOffset() const { return constantTableOffset_; }
+  Offset runtimeDataOffset() const { return runtimeDataOffset_; }
+  Offset nurseryObjectsOffset() const { return nurseryObjectsOffset_; }
+  Offset osiIndexOffset() const { return osiIndexOffset_; }
+  Offset safepointIndexOffset() const { return safepointIndexOffset_; }
+  Offset bailoutTableOffset() const { return bailoutTableOffset_; }
+  Offset icIndexOffset() const { return icIndexOffset_; }
+  Offset safepointsOffset() const { return safepointsOffset_; }
+  Offset snapshotsOffset() const { return snapshotsOffset_; }
+  Offset rvaTableOffset() const { return rvaTableOffset_; }
+  Offset recoversOffset() const { return recoversOffset_; }
+  Offset endOffset() const { return allocBytes_; }
+
+  // Hardcode size of incomplete types. These are verified in Ion.cpp.
+  static constexpr size_t SizeOf_OsiIndex = 2 * sizeof(uint32_t);
+  static constexpr size_t SizeOf_SafepointIndex = 2 * sizeof(uint32_t);
+  static constexpr size_t SizeOf_SnapshotOffset = sizeof(uint32_t);
+
+ public:
+  //
+  // Table of constants referenced in snapshots. (JS::Value alignment)
+  //
+  PreBarrieredValue* constants() {
+    // Nursery constants are manually barriered in CodeGenerator::link() so a
+    // post barrier is not required..
+    return offsetToPointer<PreBarrieredValue>(constantTableOffset());
+  }
+  size_t numConstants() const {
+    return numElements<PreBarrieredValue>(constantTableOffset(),
+                                          runtimeDataOffset());
+  }
+
+  //
+  // IonIC data structures. (uint64_t alignment)
+  //
+  uint8_t* runtimeData() {
+    return offsetToPointer<uint8_t>(runtimeDataOffset());
+  }
+  size_t runtimeSize() const {
+    return numElements<uint8_t>(runtimeDataOffset(), nurseryObjectsOffset());
+  }
+
+  //
+  // List of (originally) nursery-allocated objects referenced from JIT code.
+  // (JSObject* alignment)
+  //
+  HeapPtrObject* nurseryObjects() {
+    return offsetToPointer<HeapPtrObject>(nurseryObjectsOffset());
+  }
+  size_t numNurseryObjects() const {
+    return numElements<HeapPtrObject>(nurseryObjectsOffset(), osiIndexOffset());
+  }
+  void* addressOfNurseryObject(uint32_t index) {
+    MOZ_ASSERT(index < numNurseryObjects());
+    return &nurseryObjects()[index];
+  }
+
+  //
+  // Map OSI-point displacement to snapshot.
+  //
+  OsiIndex* osiIndices() { return offsetToPointer<OsiIndex>(osiIndexOffset()); }
+  const OsiIndex* osiIndices() const {
+    return offsetToPointer<OsiIndex>(osiIndexOffset());
+  }
+  size_t numOsiIndices() const {
+    return numElements<SizeOf_OsiIndex>(osiIndexOffset(),
+                                        safepointIndexOffset());
+  }
+
+  //
+  // Map code displacement to safepoint / OSI-patch-delta.
+  //
+  SafepointIndex* safepointIndices() {
+    return offsetToPointer<SafepointIndex>(safepointIndexOffset());
+  }
+  const SafepointIndex* safepointIndices() const {
+    return offsetToPointer<SafepointIndex>(safepointIndexOffset());
+  }
+  size_t numSafepointIndices() const {
+    return numElements<SizeOf_SafepointIndex>(safepointIndexOffset(),
+                                              bailoutTableOffset());
+  }
+
+  //
+  // Table mapping bailout IDs to snapshot offsets.
+  //
+  SnapshotOffset* bailoutTable() {
+    return offsetToPointer<SnapshotOffset>(bailoutTableOffset());
+  }
+  size_t numBailoutEntries() const {
+    return numElements<SizeOf_SnapshotOffset>(bailoutTableOffset(),
+                                              icIndexOffset());
+  }
+
+  //
+  // Offset into `runtimeData` for each (variable-length) IonIC.
+  //
+  uint32_t* icIndex() { return offsetToPointer<uint32_t>(icIndexOffset()); }
+  size_t numICs() const {
+    return numElements<uint32_t>(icIndexOffset(), safepointsOffset());
+  }
+
+  //
+  // Safepoint table as a CompactBuffer.
+  //
+  const uint8_t* safepoints() const {
+    return offsetToPointer<uint8_t>(safepointsOffset());
+  }
+  size_t safepointsSize() const {
+    return numElements<uint8_t>(safepointsOffset(), snapshotsOffset());
+  }
+
+  //
+  // Snapshot and RValueAllocation tables as CompactBuffers.
+  //
+  const uint8_t* snapshots() const {
+    return offsetToPointer<uint8_t>(snapshotsOffset());
+  }
+  size_t snapshotsListSize() const {
+    return numElements<uint8_t>(snapshotsOffset(), rvaTableOffset());
+  }
+  size_t snapshotsRVATableSize() const {
+    return numElements<uint8_t>(rvaTableOffset(), recoversOffset());
+  }
+
+  //
+  // Recover instruction table as a CompactBuffer.
+  //
+  const uint8_t* recovers() const {
+    return offsetToPointer<uint8_t>(recoversOffset());
+  }
+  size_t recoversSize() const {
+    return numElements<uint8_t>(recoversOffset(), endOffset());
+  }
+
+ private:
+  IonScript(IonCompilationId compilationId, uint32_t frameSlots,
+            uint32_t argumentSlots, uint32_t frameSize);
+
+ public:
+  static IonScript* New(JSContext* cx, IonCompilationId compilationId,
+                        uint32_t frameSlots, uint32_t argumentSlots,
+                        uint32_t frameSize, size_t snapshotsListSize,
+                        size_t snapshotsRVATableSize, size_t recoversSize,
+                        size_t bailoutEntries, size_t constants,
+                        size_t nurseryObjects, size_t safepointIndices,
+                        size_t osiIndices, size_t icEntries, size_t runtimeSize,
+                        size_t safepointsSize);
+
+  static void Destroy(JSFreeOp* fop, IonScript* script);
+
+  void trace(JSTracer* trc);
+
+  static inline size_t offsetOfInvalidationCount() {
+    return offsetof(IonScript, invalidationCount_);
+  }
+
+ public:
+  JitCode* method() const { return method_; }
+  void setMethod(JitCode* code) {
+    MOZ_ASSERT(!invalidated());
+    method_ = code;
+  }
+  void setOsrPc(jsbytecode* osrPc) { osrPc_ = osrPc; }
+  jsbytecode* osrPc() const { return osrPc_; }
+  void setOsrEntryOffset(uint32_t offset) {
+    MOZ_ASSERT(!osrEntryOffset_);
+    osrEntryOffset_ = offset;
+  }
+  uint32_t osrEntryOffset() const { return osrEntryOffset_; }
+  bool containsCodeAddress(uint8_t* addr) const {
+    return method()->raw() <= addr &&
+           addr <= method()->raw() + method()->instructionsSize();
+  }
+  bool containsReturnAddress(uint8_t* addr) const {
+    // This accounts for an off by one error caused by the return address of a
+    // bailout sitting outside the range of the containing function.
+    return method()->raw() <= addr &&
+           addr <= method()->raw() + method()->instructionsSize();
+  }
+  void setInvalidationEpilogueOffset(uint32_t offset) {
+    MOZ_ASSERT(!invalidateEpilogueOffset_);
+    invalidateEpilogueOffset_ = offset;
+  }
+  uint32_t invalidateEpilogueOffset() const {
+    MOZ_ASSERT(invalidateEpilogueOffset_);
+    return invalidateEpilogueOffset_;
+  }
+  void setInvalidationEpilogueDataOffset(uint32_t offset) {
+    MOZ_ASSERT(!invalidateEpilogueDataOffset_);
+    invalidateEpilogueDataOffset_ = offset;
+  }
+  uint32_t invalidateEpilogueDataOffset() const {
+    MOZ_ASSERT(invalidateEpilogueDataOffset_);
+    return invalidateEpilogueDataOffset_;
+  }
+
+  void incNumFixableBailouts() { numFixableBailouts_++; }
+  void incNumUnfixableBailouts() { numUnfixableBailouts_++; }
+
+  bool shouldInvalidate() const {
+    return numFixableBailouts_ >= JitOptions.frequentBailoutThreshold;
+  }
+  bool shouldInvalidateAndDisable() const {
+    return numUnfixableBailouts_ >= JitOptions.frequentBailoutThreshold * 5;
+  }
+
+  LICMState licmState() const { return licmState_; }
+  void setHadLICMBailout() {
+    if (licmState_ == LICMState::NeverBailed) {
+      licmState_ = LICMState::Bailed;
+    }
+  }
+  void noteBaselineFallback() {
+    if (licmState_ == LICMState::Bailed) {
+      licmState_ = LICMState::BailedAndHitFallback;
+    }
+  }
+
+  void setHasProfilingInstrumentation() { hasProfilingInstrumentation_ = true; }
+  void clearHasProfilingInstrumentation() {
+    hasProfilingInstrumentation_ = false;
+  }
+  bool hasProfilingInstrumentation() const {
+    return hasProfilingInstrumentation_;
+  }
+  [[nodiscard]] bool addTraceLoggerEvent(TraceLoggerEvent& event) {
+    MOZ_ASSERT(event.hasTextId());
+    return traceLoggerEvents_.append(std::move(event));
+  }
+  size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+    return mallocSizeOf(this);
+  }
+  PreBarrieredValue& getConstant(size_t index) {
+    MOZ_ASSERT(index < numConstants());
+    return constants()[index];
+  }
+  uint32_t frameSlots() const { return frameSlots_; }
+  uint32_t argumentSlots() const { return argumentSlots_; }
+  uint32_t frameSize() const { return frameSize_; }
+  SnapshotOffset bailoutToSnapshot(uint32_t bailoutId) {
+    MOZ_ASSERT(bailoutId < numBailoutEntries());
+    return bailoutTable()[bailoutId];
+  }
+  const SafepointIndex* getSafepointIndex(uint32_t disp) const;
+  const SafepointIndex* getSafepointIndex(uint8_t* retAddr) const {
+    MOZ_ASSERT(containsCodeAddress(retAddr));
+    return getSafepointIndex(retAddr - method()->raw());
+  }
+  const OsiIndex* getOsiIndex(uint32_t disp) const;
+  const OsiIndex* getOsiIndex(uint8_t* retAddr) const;
+
+  IonIC& getICFromIndex(uint32_t index) {
+    MOZ_ASSERT(index < numICs());
+    uint32_t offset = icIndex()[index];
+    return getIC(offset);
+  }
+  inline IonIC& getIC(uint32_t offset) {
+    MOZ_ASSERT(offset < runtimeSize());
+    return *reinterpret_cast<IonIC*>(runtimeData() + offset);
+  }
+  void purgeICs(Zone* zone);
+  void copySnapshots(const SnapshotWriter* writer);
+  void copyRecovers(const RecoverWriter* writer);
+  void copyBailoutTable(const SnapshotOffset* table);
+  void copyConstants(const Value* vp);
+  void copySafepointIndices(const CodegenSafepointIndex* si);
+  void copyOsiIndices(const OsiIndex* oi);
+  void copyRuntimeData(const uint8_t* data);
+  void copyICEntries(const uint32_t* icEntries);
+  void copySafepoints(const SafepointWriter* writer);
+
+  bool invalidated() const { return invalidationCount_ != 0; }
+
+  // Invalidate the current compilation.
+  void invalidate(JSContext* cx, JSScript* script, bool resetUses,
+                  const char* reason);
+
+  size_t invalidationCount() const { return invalidationCount_; }
+  void incrementInvalidationCount() { invalidationCount_++; }
+  void decrementInvalidationCount(JSFreeOp* fop) {
+    MOZ_ASSERT(invalidationCount_);
+    invalidationCount_--;
+    if (!invalidationCount_) {
+      Destroy(fop, this);
+    }
+  }
+  IonCompilationId compilationId() const { return compilationId_; }
+  uint32_t incrOsrPcMismatchCounter() { return ++osrPcMismatchCounter_; }
+  void resetOsrPcMismatchCounter() { osrPcMismatchCounter_ = 0; }
+
+  size_t allocBytes() const { return allocBytes_; }
+
+  static void preWriteBarrier(Zone* zone, IonScript* ionScript);
+
+#ifdef DEBUG
+  mozilla::HashNumber icHash() const { return icHash_; }
+  void setICHash(mozilla::HashNumber hash) { icHash_ = hash; }
+#endif
+};
+
+// Execution information for a basic block which may persist after the
+// accompanying IonScript is destroyed, for use during profiling.
+struct IonBlockCounts {
+ private:
+  uint32_t id_;
+
+  // Approximate bytecode in the outer (not inlined) script this block
+  // was generated from.
+  uint32_t offset_;
+
+  // File and line of the inner script this block was generated from.
+  char* description_;
+
+  // ids for successors of this block.
+  uint32_t numSuccessors_;
+  uint32_t* successors_;
+
+  // Hit count for this block.
+  uint64_t hitCount_;
+
+  // Text information about the code generated for this block.
+  char* code_;
+
+ public:
+  [[nodiscard]] bool init(uint32_t id, uint32_t offset, char* description,
+                          uint32_t numSuccessors) {
+    id_ = id;
+    offset_ = offset;
+    description_ = description;
+    numSuccessors_ = numSuccessors;
+    if (numSuccessors) {
+      successors_ = js_pod_calloc<uint32_t>(numSuccessors);
+      if (!successors_) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  void destroy() {
+    js_free(description_);
+    js_free(successors_);
+    js_free(code_);
+  }
+
+  uint32_t id() const { return id_; }
+
+  uint32_t offset() const { return offset_; }
+
+  const char* description() const { return description_; }
+
+  size_t numSuccessors() const { return numSuccessors_; }
+
+  void setSuccessor(size_t i, uint32_t id) {
+    MOZ_ASSERT(i < numSuccessors_);
+    successors_[i] = id;
+  }
+
+  uint32_t successor(size_t i) const {
+    MOZ_ASSERT(i < numSuccessors_);
+    return successors_[i];
+  }
+
+  uint64_t* addressOfHitCount() { return &hitCount_; }
+
+  uint64_t hitCount() const { return hitCount_; }
+
+  void setCode(const char* code) {
+    char* ncode = js_pod_malloc<char>(strlen(code) + 1);
+    if (ncode) {
+      strcpy(ncode, code);
+      code_ = ncode;
+    }
+  }
+
+  const char* code() const { return code_; }
+
+  size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+    return mallocSizeOf(description_) + mallocSizeOf(successors_) +
+           mallocSizeOf(code_);
+  }
+};
+
+// Execution information for a compiled script which may persist after the
+// IonScript is destroyed, for use during profiling.
+struct IonScriptCounts {
+ private:
+  // Any previous invalidated compilation(s) for the script.
+  IonScriptCounts* previous_ = nullptr;
+
+  // Information about basic blocks in this script.
+  size_t numBlocks_ = 0;
+  IonBlockCounts* blocks_ = nullptr;
+
+ public:
+  IonScriptCounts() = default;
+
+  ~IonScriptCounts() {
+    for (size_t i = 0; i < numBlocks_; i++) {
+      blocks_[i].destroy();
+    }
+    js_free(blocks_);
+    // The list can be long in some corner cases (bug 1140084), so
+    // unroll the recursion.
+    IonScriptCounts* victims = previous_;
+    while (victims) {
+      IonScriptCounts* victim = victims;
+      victims = victim->previous_;
+      victim->previous_ = nullptr;
+      js_delete(victim);
+    }
+  }
+
+  [[nodiscard]] bool init(size_t numBlocks) {
+    blocks_ = js_pod_calloc<IonBlockCounts>(numBlocks);
+    if (!blocks_) {
+      return false;
+    }
+
+    numBlocks_ = numBlocks;
+    return true;
+  }
+
+  size_t numBlocks() const { return numBlocks_; }
+
+  IonBlockCounts& block(size_t i) {
+    MOZ_ASSERT(i < numBlocks_);
+    return blocks_[i];
+  }
+
+  void setPrevious(IonScriptCounts* previous) { previous_ = previous; }
+
+  IonScriptCounts* previous() const { return previous_; }
+
+  size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+    size_t size = 0;
+    auto currCounts = this;
+    do {
+      size += currCounts->sizeOfOneIncludingThis(mallocSizeOf);
+      currCounts = currCounts->previous_;
+    } while (currCounts);
+    return size;
+  }
+
+  size_t sizeOfOneIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+    size_t size = mallocSizeOf(this) + mallocSizeOf(blocks_);
+    for (size_t i = 0; i < numBlocks_; i++) {
+      blocks_[i].sizeOfExcludingThis(mallocSizeOf);
+    }
+    return size;
+  }
+};
+
+}  // namespace jit
+}  // namespace js
+
+namespace JS {
+
+template <>
+struct DeletePolicy<js::jit::IonScript> {
+  explicit DeletePolicy(JSRuntime* rt) : rt_(rt) {}
+  void operator()(const js::jit::IonScript* script);
+
+ private:
+  JSRuntime* rt_;
+};
+
+}  // namespace JS
+
+#endif /* jit_IonScript_h */